Building Feature Flags System: From Simple Toggles to Complex Rules
Built a feature flags system handling 50M+ evaluations per day. From simple boolean toggles to complex targeting rules. Here's the complete production implementation.
Key Takeaways
- Scale: Processing 50M+ flag evaluations per day with <1ms latency
- Cost Savings: Built custom solution for $480/month vs $15,000+ for commercial services
- Performance: 99.5% cache hit rate with smart caching strategy
- Reliability: Fail-safe defaults ensure system stability during outages
Why We Needed Our Own Feature Flags
LaunchDarkly wanted $15,000/month for our scale. Split wanted $8,000. We needed something that could:
- Handle 50M+ flag evaluations per day
- Respond in <1ms for local evaluations
- Support complex targeting rules
- Work with 300+ microservices
- Cost less than $500/month total
Built our own system in 3 weeks. It's been running production for 2 years.
Architecture Overview
// Core flag evaluation engine
type FlagEngine struct {
mu sync.RWMutex
flags map[string]*Flag
cache *cache.LRU[string, bool]
// Metrics
evaluations int64
cacheHits int64
cacheMisses int64
// Configuration
refreshRate time.Duration
cacheSize int
remote FlagProvider
}
type Flag struct {
Key string
Enabled bool
Rules []Rule
DefaultRule Rule
UpdatedAt time.Time
}
type Rule struct {
Conditions []Condition
Action Action
Weight int // for rollout percentages
}
type Condition struct {
Field string
Operator string
Values []string
}
type Action struct {
Type string // "enable", "disable", "rollout"
Value interface{}
}The Core Engine
The flag engine is the heart of our system. It needs to be fast, thread-safe, and handle complex rules.
func NewFlagEngine(provider FlagProvider, opts ...Option) *FlagEngine {
engine := &FlagEngine{
flags: make(map[string]*Flag),
cache: cache.NewLRU[string, bool](10000),
refreshRate: 30 * time.Second,
cacheSize: 10000,
remote: provider,
}
for _, opt := range opts {
opt(engine)
}
// Start flag refresh goroutine
go engine.refreshLoop()
return engine
}
func (e *FlagEngine) IsEnabled(ctx context.Context, flagKey string, user *User) bool {
// Generate cache key
cacheKey := e.generateCacheKey(flagKey, user)
// Try cache first
if result, ok := e.cache.Get(cacheKey); ok {
atomic.AddInt64(&e.cacheHits, 1)
return result
}
atomic.AddInt64(&e.cacheMisses, 1)
// Evaluate flag
result := e.evaluate(flagKey, user)
// Cache the result
e.cache.Set(cacheKey, result)
atomic.AddInt64(&e.evaluations, 1)
return result
}
func (e *FlagEngine) evaluate(flagKey string, user *User) bool {
e.mu.RLock()
flag, exists := e.flags[flagKey]
e.mu.RUnlock()
if !exists {
return false // Fail-safe: unknown flags are disabled
}
if !flag.Enabled {
return false
}
// Evaluate rules in order
for _, rule := range flag.Rules {
if e.matchesRule(rule, user) {
return e.executeAction(rule.Action, user)
}
}
// Use default rule
return e.executeAction(flag.DefaultRule.Action, user)
}Rule Evaluation Engine
This is where the magic happens. Complex targeting rules get evaluated here.
func (e *FlagEngine) matchesRule(rule Rule, user *User) bool {
for _, condition := range rule.Conditions {
if !e.matchesCondition(condition, user) {
return false // All conditions must match
}
}
return true
}
func (e *FlagEngine) matchesCondition(condition Condition, user *User) bool {
userValue := e.getUserField(condition.Field, user)
switch condition.Operator {
case "equals":
return e.containsString(condition.Values, userValue)
case "not_equals":
return !e.containsString(condition.Values, userValue)
case "contains":
return e.stringContainsAny(userValue, condition.Values)
case "starts_with":
return e.stringStartsWithAny(userValue, condition.Values)
case "in":
return e.containsString(condition.Values, userValue)
case "matches_regex":
return e.matchesRegex(condition.Values[0], userValue)
case "semver_gt":
return e.semverCompare(userValue, condition.Values[0]) > 0
case "semver_lt":
return e.semverCompare(userValue, condition.Values[0]) < 0
default:
return false
}
}
func (e *FlagEngine) executeAction(action Action, user *User) bool {
switch action.Type {
case "enable":
return true
case "disable":
return false
case "rollout":
percentage := action.Value.(float64)
return e.isInRollout(user, percentage)
default:
return false
}
}
// Consistent rollout based on user ID hash
func (e *FlagEngine) isInRollout(user *User, percentage float64) bool {
hash := fnv.New32a()
hash.Write([]byte(user.ID))
userHash := hash.Sum32()
// Convert to percentage (0-100)
userPercentage := float64(userHash%100)
return userPercentage < percentage
}Storage and Synchronization
Flags need to be stored somewhere and synchronized across services. We use Redis with pub/sub for real-time updates.
type RedisProvider struct {
client redis.UniversalClient
pubsub *redis.PubSub
keyPrefix string
}
func NewRedisProvider(client redis.UniversalClient) *RedisProvider {
return &RedisProvider{
client: client,
keyPrefix: "flags:",
}
}
func (r *RedisProvider) GetAllFlags(ctx context.Context) (map[string]*Flag, error) {
pattern := r.keyPrefix + "*"
keys, err := r.client.Keys(ctx, pattern).Result()
if err != nil {
return nil, err
}
flags := make(map[string]*Flag)
if len(keys) > 0 {
values, err := r.client.MGet(ctx, keys...).Result()
if err != nil {
return nil, err
}
for i, key := range keys {
if values[i] == nil {
continue
}
flagKey := strings.TrimPrefix(key, r.keyPrefix)
var flag Flag
if err := json.Unmarshal([]byte(values[i].(string)), &flag); err != nil {
continue
}
flags[flagKey] = &flag
}
}
return flags, nil
}
func (r *RedisProvider) Subscribe(ctx context.Context, callback func(string, *Flag)) error {
r.pubsub = r.client.Subscribe(ctx, "flag_updates")
defer r.pubsub.Close()
for {
select {
case <-ctx.Done():
return ctx.Err()
case msg := <-r.pubsub.Channel():
var update struct {
Key string `json:"key"`
Flag *Flag `json:"flag"`
}
if err := json.Unmarshal([]byte(msg.Payload), &update); err != nil {
continue
}
callback(update.Key, update.Flag)
}
}
}Background Refresh
Flags are refreshed periodically and on updates via pub/sub.
func (e *FlagEngine) refreshLoop() {
ticker := time.NewTicker(e.refreshRate)
defer ticker.Stop()
ctx := context.Background()
// Subscribe to real-time updates
go e.remote.Subscribe(ctx, e.handleFlagUpdate)
for {
select {
case <-ticker.C:
if err := e.refreshFlags(ctx); err != nil {
log.Printf("Failed to refresh flags: %v", err)
}
}
}
}
func (e *FlagEngine) refreshFlags(ctx context.Context) error {
flags, err := e.remote.GetAllFlags(ctx)
if err != nil {
return err
}
e.mu.Lock()
e.flags = flags
e.mu.Unlock()
// Clear cache on flag updates
e.cache.Clear()
return nil
}
func (e *FlagEngine) handleFlagUpdate(key string, flag *Flag) {
e.mu.Lock()
if flag == nil {
delete(e.flags, key)
} else {
e.flags[key] = flag
}
e.mu.Unlock()
// Clear cache entries for this flag
e.cache.ClearPrefix(key + ":")
}Advanced Features
Gradual Rollout
Roll out features to a percentage of users with consistent hashing.
type GradualRollout struct {
Percentage float64 `json:"percentage"`
Salt string `json:"salt"` // For different rollouts of same feature
}
func (e *FlagEngine) isInGradualRollout(user *User, rollout GradualRollout) bool {
// Use salt + user ID for consistent but different hashing
hash := fnv.New64a()
hash.Write([]byte(rollout.Salt + user.ID))
userHash := hash.Sum64()
// Convert to percentage (0-100)
userPercentage := float64(userHash % 10000) / 100.0
return userPercentage < rollout.Percentage
}
// Example flag configuration
var flagConfig = Flag{
Key: "new_checkout_flow",
Enabled: true,
Rules: []Rule{
{
Conditions: []Condition{
{Field: "email", Operator: "contains", Values: []string{"@company.com"}},
},
Action: Action{Type: "enable"},
},
{
Conditions: []Condition{
{Field: "country", Operator: "in", Values: []string{"US", "CA"}},
},
Action: Action{
Type: "rollout",
Value: GradualRollout{
Percentage: 25.0,
Salt: "checkout_v2",
},
},
},
},
DefaultRule: Rule{
Action: Action{Type: "disable"},
},
}A/B Testing Integration
Feature flags can also serve as A/B test assignments.
type ABTest struct {
Name string `json:"name"`
Variants map[string]float64 `json:"variants"` // variant -> percentage
TrackingEnabled bool `json:"tracking_enabled"`
}
func (e *FlagEngine) getABTestVariant(user *User, test ABTest) string {
hash := fnv.New64a()
hash.Write([]byte(test.Name + user.ID))
userHash := hash.Sum64()
percentage := float64(userHash % 10000) / 100.0
var cumulative float64
for variant, weight := range test.Variants {
cumulative += weight
if percentage < cumulative {
if test.TrackingEnabled {
e.trackABTestAssignment(user, test.Name, variant)
}
return variant
}
}
return "control" // fallback
}
func (e *FlagEngine) trackABTestAssignment(user *User, testName, variant string) {
// Send to analytics service
go func() {
event := ABTestEvent{
UserID: user.ID,
TestName: testName,
Variant: variant,
Timestamp: time.Now(),
}
e.analytics.Track(event)
}()
}Performance Optimization
Smart Caching Strategy
Cache keys include all relevant user attributes to avoid incorrect results.
func (e *FlagEngine) generateCacheKey(flagKey string, user *User) string {
var keyParts []string
keyParts = append(keyParts, flagKey)
// Include user attributes that might affect flag evaluation
if user != nil {
keyParts = append(keyParts, user.ID)
keyParts = append(keyParts, user.Country)
keyParts = append(keyParts, user.Plan)
// Add other attributes that flags might use
}
return strings.Join(keyParts, ":")
}
// Cache with TTL for memory management
type TTLCache struct {
mu sync.RWMutex
items map[string]*cacheItem
ttl time.Duration
maxSize int
}
type cacheItem struct {
value bool
expiresAt time.Time
}
func (c *TTLCache) Get(key string) (bool, bool) {
c.mu.RLock()
defer c.mu.RUnlock()
item, exists := c.items[key]
if !exists || time.Now().After(item.expiresAt) {
return false, false
}
return item.value, true
}
func (c *TTLCache) Set(key string, value bool) {
c.mu.Lock()
defer c.mu.Unlock()
// Evict expired items if at capacity
if len(c.items) >= c.maxSize {
c.evictExpired()
}
c.items[key] = &cacheItem{
value: value,
expiresAt: time.Now().Add(c.ttl),
}
}Bulk Evaluation
For batch processing, evaluate multiple flags at once.
func (e *FlagEngine) EvaluateBatch(ctx context.Context, flagKeys []string, user *User) map[string]bool {
results := make(map[string]bool, len(flagKeys))
// Try to get all from cache first
var uncachedKeys []string
for _, key := range flagKeys {
cacheKey := e.generateCacheKey(key, user)
if result, ok := e.cache.Get(cacheKey); ok {
results[key] = result
atomic.AddInt64(&e.cacheHits, 1)
} else {
uncachedKeys = append(uncachedKeys, key)
}
}
// Evaluate uncached flags
if len(uncachedKeys) > 0 {
e.mu.RLock()
for _, key := range uncachedKeys {
result := e.evaluate(key, user)
results[key] = result
// Cache the result
cacheKey := e.generateCacheKey(key, user)
e.cache.Set(cacheKey, result)
atomic.AddInt64(&e.cacheMisses, 1)
atomic.AddInt64(&e.evaluations, 1)
}
e.mu.RUnlock()
}
return results
}HTTP API and Client SDK
Services need an easy way to evaluate flags via HTTP and native Go clients.
type FlagServer struct {
engine *FlagEngine
router *chi.Mux
}
func NewFlagServer(engine *FlagEngine) *FlagServer {
s := &FlagServer{
engine: engine,
router: chi.NewRouter(),
}
s.setupRoutes()
return s
}
func (s *FlagServer) setupRoutes() {
s.router.Use(middleware.Logger)
s.router.Use(middleware.Recoverer)
s.router.Use(middleware.Timeout(5 * time.Second))
s.router.Get("/flags/{flagKey}", s.evaluateFlag)
s.router.Post("/flags/batch", s.evaluateBatch)
s.router.Get("/health", s.health)
s.router.Get("/metrics", s.metrics)
}
func (s *FlagServer) evaluateFlag(w http.ResponseWriter, r *http.Request) {
flagKey := chi.URLParam(r, "flagKey")
if flagKey == "" {
http.Error(w, "flag key required", http.StatusBadRequest)
return
}
user := s.parseUser(r)
result := s.engine.IsEnabled(r.Context(), flagKey, user)
response := map[string]interface{}{
"flag": flagKey,
"enabled": result,
"user": user.ID,
}
w.Header().Set("Content-Type", "application/json")
json.NewEncoder(w).Encode(response)
}
func (s *FlagServer) parseUser(r *http.Request) *User {
return &User{
ID: r.Header.Get("X-User-ID"),
Country: r.Header.Get("X-User-Country"),
Plan: r.Header.Get("X-User-Plan"),
Email: r.Header.Get("X-User-Email"),
}
}
// Client SDK
type Client struct {
baseURL string
httpClient *http.Client
userID string
headers map[string]string
}
func NewClient(baseURL, userID string) *Client {
return &Client{
baseURL: baseURL,
httpClient: &http.Client{
Timeout: 2 * time.Second,
},
userID: userID,
headers: make(map[string]string),
}
}
func (c *Client) IsEnabled(ctx context.Context, flagKey string) bool {
url := fmt.Sprintf("%s/flags/%s", c.baseURL, flagKey)
req, err := http.NewRequestWithContext(ctx, "GET", url, nil)
if err != nil {
return false // Fail safe
}
req.Header.Set("X-User-ID", c.userID)
for k, v := range c.headers {
req.Header.Set(k, v)
}
resp, err := c.httpClient.Do(req)
if err != nil {
return false // Fail safe
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
return false
}
var result struct {
Enabled bool `json:"enabled"`
}
if err := json.NewDecoder(resp.Body).Decode(&result); err != nil {
return false
}
return result.Enabled
}Production Metrics
Here's what 2 years of production data taught us:
- 50M+ evaluations/day across 300+ services
- 99.5% cache hit rate with 5-minute TTL
- <1ms p99 latency for cached evaluations
- <5ms p99 latency for uncached evaluations
- 600+ active feature flags at peak
- $480/month total cost (Redis + compute)
Key Lessons Learned
- Fail-safe is critical: Unknown flags always return false
- Cache invalidation is hard: Use TTL + pub/sub for updates
- Consistent hashing matters: Users should get same experience
- Monitoring is essential: Track evaluation latency and cache hit rates
- Keep it simple: Complex rules are hard to debug
Deployment and Operations
// Metrics collection
func (e *FlagEngine) GetMetrics() map[string]interface{} {
return map[string]interface{}{
"evaluations": atomic.LoadInt64(&e.evaluations),
"cache_hits": atomic.LoadInt64(&e.cacheHits),
"cache_misses": atomic.LoadInt64(&e.cacheMisses),
"cache_hit_rate": float64(atomic.LoadInt64(&e.cacheHits)) /
float64(atomic.LoadInt64(&e.cacheHits) + atomic.LoadInt64(&e.cacheMisses)),
"active_flags": len(e.flags),
"cache_size": e.cache.Len(),
"memory_usage_mb": e.getMemoryUsage() / 1024 / 1024,
}
}
// Health check
func (s *FlagServer) health(w http.ResponseWriter, r *http.Request) {
ctx, cancel := context.WithTimeout(r.Context(), 5*time.Second)
defer cancel()
// Check Redis connectivity
if err := s.engine.remote.Ping(ctx); err != nil {
http.Error(w, "Redis unhealthy", http.StatusServiceUnavailable)
return
}
w.Header().Set("Content-Type", "application/json")
json.NewEncoder(w).Encode(map[string]string{"status": "healthy"})
}
// Graceful shutdown
func (s *FlagServer) Shutdown(ctx context.Context) error {
return s.server.Shutdown(ctx)
}Security Considerations
Feature flag systems require careful security design to prevent unauthorized access and manipulation.
Authentication and Authorization
type SecurityConfig struct {
APIKeys map[string]*APIKey
RateLimiting RateLimitConfig
IPWhitelist []string
RequireHTTPS bool
}
type APIKey struct {
ID string
Service string
Permissions []Permission
RateLimit int
ExpiresAt time.Time
LastUsed time.Time
}
type Permission struct {
Resource string // "flags", "metrics", "admin"
Action string // "read", "write", "delete"
Scope string // "*" or specific flag patterns
}
func (s *FlagServer) authenticateRequest(r *http.Request) (*APIKey, error) {
apiKey := r.Header.Get("X-API-Key")
if apiKey == "" {
return nil, errors.New("missing API key")
}
key, exists := s.security.APIKeys[apiKey]
if !exists {
return nil, errors.New("invalid API key")
}
if time.Now().After(key.ExpiresAt) {
return nil, errors.New("expired API key")
}
// Check IP whitelist
if len(s.security.IPWhitelist) > 0 {
clientIP := getClientIP(r)
if !s.isIPWhitelisted(clientIP) {
return nil, errors.New("IP not whitelisted")
}
}
// Update last used
key.LastUsed = time.Now()
return key, nil
}
func (s *FlagServer) checkPermission(key *APIKey, resource, action, target string) bool {
for _, perm := range key.Permissions {
if perm.Resource == resource && perm.Action == action {
if perm.Scope == "*" || strings.Contains(target, perm.Scope) {
return true
}
}
}
return false
}Rate Limiting and DoS Protection
type RateLimiter struct {
mu sync.RWMutex
requests map[string]*RequestBucket
config RateLimitConfig
}
type RequestBucket struct {
Count int
ResetTime time.Time
}
type RateLimitConfig struct {
RequestsPerMinute int
BurstSize int
CleanupInterval time.Duration
}
func NewRateLimiter(config RateLimitConfig) *RateLimiter {
rl := &RateLimiter{
requests: make(map[string]*RequestBucket),
config: config,
}
// Start cleanup goroutine
go rl.cleanup()
return rl
}
func (rl *RateLimiter) Allow(identifier string) bool {
rl.mu.Lock()
defer rl.mu.Unlock()
now := time.Now()
bucket, exists := rl.requests[identifier]
if !exists || now.After(bucket.ResetTime) {
rl.requests[identifier] = &RequestBucket{
Count: 1,
ResetTime: now.Add(time.Minute),
}
return true
}
if bucket.Count >= rl.config.RequestsPerMinute {
return false
}
bucket.Count++
return true
}
func (s *FlagServer) rateLimitMiddleware(next http.Handler) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
identifier := getClientIP(r)
// Use API key if available for more specific limiting
if apiKey := r.Header.Get("X-API-Key"); apiKey != "" {
identifier = apiKey
}
if !s.rateLimiter.Allow(identifier) {
http.Error(w, "Rate limit exceeded", http.StatusTooManyRequests)
return
}
next.ServeHTTP(w, r)
})
}Audit Logging and Monitoring
type AuditEvent struct {
Timestamp time.Time `json:"timestamp"`
RequestID string `json:"request_id"`
UserID string `json:"user_id"`
Service string `json:"service"`
Action string `json:"action"`
Resource string `json:"resource"`
Success bool `json:"success"`
Error string `json:"error,omitempty"`
IPAddress string `json:"ip_address"`
UserAgent string `json:"user_agent"`
Metadata interface{} `json:"metadata,omitempty"`
}
type AuditLogger struct {
events chan AuditEvent
file *os.File
mu sync.Mutex
}
func NewAuditLogger(logPath string) (*AuditLogger, error) {
file, err := os.OpenFile(logPath, os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0644)
if err != nil {
return nil, err
}
al := &AuditLogger{
events: make(chan AuditEvent, 10000),
file: file,
}
go al.processEvents()
return al, nil
}
func (al *AuditLogger) Log(event AuditEvent) {
select {
case al.events <- event:
default:
// Log channel full - this is a critical issue
log.Printf("CRITICAL: Audit log channel full, dropping event")
}
}
func (al *AuditLogger) processEvents() {
encoder := json.NewEncoder(al.file)
for event := range al.events {
al.mu.Lock()
if err := encoder.Encode(event); err != nil {
log.Printf("Failed to write audit log: %v", err)
}
al.mu.Unlock()
}
}Testing Strategy
Feature flag systems require comprehensive testing to ensure reliability and performance.
Unit Testing Core Components
func TestFlagEvaluation(t *testing.T) {
engine := NewFlagEngine(nil)
flag := &Flag{
Key: "test_flag",
Enabled: true,
Rules: []Rule{
{
Conditions: []Condition{
{Field: "country", Operator: "equals", Values: []string{"US"}},
},
Action: Action{Type: "enable"},
},
},
DefaultRule: Rule{
Action: Action{Type: "disable"},
},
}
engine.flags["test_flag"] = flag
tests := []struct {
name string
user *User
expected bool
}{
{
name: "user_in_US",
user: &User{ID: "user1", Country: "US"},
expected: true,
},
{
name: "user_not_in_US",
user: &User{ID: "user2", Country: "CA"},
expected: false,
},
{
name: "nil_user",
user: nil,
expected: false,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
result := engine.IsEnabled(context.Background(), "test_flag", tt.user)
assert.Equal(t, tt.expected, result)
})
}
}
func TestGradualRollout(t *testing.T) {
engine := NewFlagEngine(nil)
rollout := GradualRollout{
Percentage: 50.0,
Salt: "test_rollout",
}
enabledCount := 0
totalUsers := 1000
for i := 0; i < totalUsers; i++ {
user := &User{ID: fmt.Sprintf("user_%d", i)}
if engine.isInGradualRollout(user, rollout) {
enabledCount++
}
}
// Should be approximately 50% (allow some variance)
expectedEnabled := float64(totalUsers) * 0.5
tolerance := float64(totalUsers) * 0.05 // 5% tolerance
assert.InDelta(t, expectedEnabled, float64(enabledCount), tolerance,
"Rollout percentage should be approximately 50%%")
}
func TestCacheKeyGeneration(t *testing.T) {
engine := NewFlagEngine(nil)
user1 := &User{ID: "user1", Country: "US", Plan: "premium"}
user2 := &User{ID: "user1", Country: "CA", Plan: "premium"}
user3 := &User{ID: "user2", Country: "US", Plan: "premium"}
key1 := engine.generateCacheKey("test_flag", user1)
key2 := engine.generateCacheKey("test_flag", user2)
key3 := engine.generateCacheKey("test_flag", user3)
assert.NotEqual(t, key1, key2, "Different countries should generate different cache keys")
assert.NotEqual(t, key1, key3, "Different user IDs should generate different cache keys")
}Integration Testing
func TestRedisProvider(t *testing.T) {
// Start Redis container for testing
redisContainer := startRedisContainer(t)
defer redisContainer.Terminate(context.Background())
client := redis.NewClient(&redis.Options{
Addr: redisContainer.GetConnectionString(),
})
provider := NewRedisProvider(client)
// Test flag storage and retrieval
testFlag := &Flag{
Key: "integration_test",
Enabled: true,
Rules: []Rule{},
DefaultRule: Rule{
Action: Action{Type: "enable"},
},
}
err := provider.SetFlag(context.Background(), "integration_test", testFlag)
require.NoError(t, err)
flags, err := provider.GetAllFlags(context.Background())
require.NoError(t, err)
retrievedFlag, exists := flags["integration_test"]
assert.True(t, exists)
assert.Equal(t, testFlag.Key, retrievedFlag.Key)
assert.Equal(t, testFlag.Enabled, retrievedFlag.Enabled)
}
func TestHTTPAPI(t *testing.T) {
engine := NewFlagEngine(nil)
engine.flags["test_api_flag"] = &Flag{
Key: "test_api_flag",
Enabled: true,
DefaultRule: Rule{
Action: Action{Type: "enable"},
},
}
server := NewFlagServer(engine)
testServer := httptest.NewServer(server.router)
defer testServer.Close()
// Test flag evaluation endpoint
req, err := http.NewRequest("GET", testServer.URL+"/flags/test_api_flag", nil)
require.NoError(t, err)
req.Header.Set("X-User-ID", "test_user")
resp, err := http.DefaultClient.Do(req)
require.NoError(t, err)
defer resp.Body.Close()
assert.Equal(t, http.StatusOK, resp.StatusCode)
var result map[string]interface{}
err = json.NewDecoder(resp.Body).Decode(&result)
require.NoError(t, err)
assert.Equal(t, "test_api_flag", result["flag"])
assert.Equal(t, true, result["enabled"])
}Performance and Load Testing
func BenchmarkFlagEvaluation(b *testing.B) {
engine := NewFlagEngine(nil)
// Setup test flag
engine.flags["benchmark_flag"] = &Flag{
Key: "benchmark_flag",
Enabled: true,
Rules: []Rule{
{
Conditions: []Condition{
{Field: "country", Operator: "equals", Values: []string{"US"}},
},
Action: Action{Type: "enable"},
},
},
DefaultRule: Rule{
Action: Action{Type: "disable"},
},
}
user := &User{ID: "benchmark_user", Country: "US"}
b.ResetTimer()
b.RunParallel(func(pb *testing.PB) {
for pb.Next() {
engine.IsEnabled(context.Background(), "benchmark_flag", user)
}
})
}
func TestConcurrentEvaluations(t *testing.T) {
engine := NewFlagEngine(nil)
// Setup multiple flags
for i := 0; i < 100; i++ {
flagKey := fmt.Sprintf("concurrent_flag_%d", i)
engine.flags[flagKey] = &Flag{
Key: flagKey,
Enabled: true,
DefaultRule: Rule{
Action: Action{Type: "enable"},
},
}
}
const numGoroutines = 100
const evaluationsPerGoroutine = 1000
var wg sync.WaitGroup
errors := make(chan error, numGoroutines)
for i := 0; i < numGoroutines; i++ {
wg.Add(1)
go func(goroutineID int) {
defer wg.Done()
user := &User{ID: fmt.Sprintf("user_%d", goroutineID)}
for j := 0; j < evaluationsPerGoroutine; j++ {
flagKey := fmt.Sprintf("concurrent_flag_%d", j%100)
result := engine.IsEnabled(context.Background(), flagKey, user)
// All flags should be enabled for this test
if !result {
errors <- fmt.Errorf("flag %s unexpectedly disabled", flagKey)
return
}
}
}(i)
}
wg.Wait()
close(errors)
// Check for any errors
for err := range errors {
t.Errorf("Concurrent evaluation error: %v", err)
}
// Verify metrics
metrics := engine.GetMetrics()
totalEvaluations := numGoroutines * evaluationsPerGoroutine
assert.Equal(t, int64(totalEvaluations),
metrics["evaluations"].(int64) + metrics["cache_hits"].(int64))
}Conclusion
Building your own feature flags system isn't trivial, but it's doable. Once you're past a handful of flags and a few services, the per-seat pricing of commercial platforms adds up quickly — our implementation costs us infrastructure and maintenance time instead, and gives us complete control over the system.
Key takeaways:
- Start simple: boolean toggles first, complex rules later
- Performance matters: cache aggressively, measure everything
- Reliability is critical: fail-safe defaults and graceful degradation
- Operations are key: monitoring, health checks, graceful shutdown
The complete implementation is ~2,000 lines of Go code and handles our entire feature flag needs. Sometimes the build vs buy decision isn't just about features—it's about cost, control, and understanding your domain.
Our feature flag system demonstrates that with careful design and implementation, you can build production-grade infrastructure that rivals commercial solutions while maintaining full control and significantly reducing costs.