Go & Backend October 26, 2025 25 min read

eBPF and Go in Production: Network Monitoring and Security

Built production eBPF programs in Go for network monitoring and security. Processing 1B+ packets/day with kernel-level visibility. Here's what works and what doesn't.

Key Takeaways

Performance: Process 1B+ packets/day with zero-copy kernel processing
Visibility: Kernel-level monitoring without kernel modules
Safety: eBPF verifier ensures production-safe programs
Flexibility: Dynamic loading and updating without restarts

Why We Adopted eBPF

Traditional monitoring missed 40% of network traffic. DDoS attacks went undetected for minutes. Application profiling was expensive and intrusive. We needed kernel-level visibility without kernel modules.

eBPF gave us:

Zero-copy packet processing
Kernel-level network monitoring
Application performance tracing
Security policy enforcement
System call monitoring
Sub-microsecond latency
No kernel compilation required
Production-safe sandboxing

Built 5 production eBPF programs over 18 months. Here's what we learned.

eBPF Program Architecture

eBPF programs run in kernel space, but we control them from Go userspace programs.

type EBPFManager struct {
    programs    map[string]*EBPFProgram
    maps        map[string]*EBPFMap
    
    // Program compilation and loading
    compiler    *EBPFCompiler
    loader      *EBPFLoader
    
    // Event processing
    eventReader *PerfEventReader
    ringBuffer  *RingBuffer
    
    // Statistics
    stats       *EBPFStats
    
    mu          sync.RWMutex
}

type EBPFProgram struct {
    Name        string
    Type        ProgramType
    Code        []byte
    
    // Kernel objects
    fd          int
    maps        map[string]*EBPFMap
    
    // Attachment info
    attachment  *Attachment
    
    // Statistics
    runCount    uint64
    runTime     uint64
    
    mu          sync.RWMutex
}

type EBPFMap struct {
    Name        string
    Type        MapType
    KeySize     int
    ValueSize   int
    MaxEntries  int
    
    // Kernel file descriptor
    fd          int
    
    // Go-side cache
    cache       map[string]interface{}
    
    mu          sync.RWMutex
}

type ProgramType int

const (
    TypeXDP ProgramType = iota
    TypeTracePoint
    TypeKprobe
    TypeUprobe
    TypeCGroup
    TypeSocketFilter
)

type MapType int

const (
    MapTypeArray MapType = iota
    MapTypeHash
    MapTypeProgArray
    MapTypePerfEventArray
    MapTypeRingBuffer
)

Network Monitoring with XDP

XDP (eXpress Data Path) processes packets at the earliest point in the kernel, before the network stack.

// network_monitor.c - eBPF program
#include 
#include 
#include 
#include 
#include 
#include 
#include 
#include 

// Maps for sharing data with userspace
struct {
    __uint(type, BPF_MAP_TYPE_PERCPU_ARRAY);
    __uint(max_entries, 256);
    __type(key, __u32);
    __type(value, __u64);
} packet_stats SEC(".maps");

struct {
    __uint(type, BPF_MAP_TYPE_HASH);
    __uint(max_entries, 10000);
    __type(key, __u32); // Source IP
    __type(value, struct traffic_info);
} traffic_map SEC(".maps");

struct {
    __uint(type, BPF_MAP_TYPE_PERF_EVENT_ARRAY);
    __uint(key_size, sizeof(__u32));
    __uint(value_size, sizeof(__u32));
} events SEC(".maps");

struct traffic_info {
    __u64 bytes;
    __u64 packets;
    __u64 last_seen;
    __u32 flags;
};

struct packet_event {
    __u32 src_ip;
    __u32 dst_ip;
    __u16 src_port;
    __u16 dst_port;
    __u8  protocol;
    __u16 length;
    __u64 timestamp;
};

SEC("xdp")
int xdp_monitor(struct xdp_md *ctx) {
    void *data = (void *)(long)ctx->data;
    void *data_end = (void *)(long)ctx->data_end;
    
    struct ethhdr *eth = data;
    if ((void *)(eth + 1) > data_end)
        return XDP_PASS;
    
    if (eth->h_proto != bpf_htons(ETH_P_IP))
        return XDP_PASS;
    
    struct iphdr *ip = (void *)(eth + 1);
    if ((void *)(ip + 1) > data_end)
        return XDP_PASS;
    
    // Update packet statistics
    __u32 protocol = ip->protocol;
    __u64 *count = bpf_map_lookup_elem(&packet_stats, &protocol);
    if (count) {
        __sync_fetch_and_add(count, 1);
    } else {
        __u64 init_val = 1;
        bpf_map_update_elem(&packet_stats, &protocol, &init_val, BPF_ANY);
    }
    
    // Update traffic info
    __u32 src_ip = ip->saddr;
    struct traffic_info *info = bpf_map_lookup_elem(&traffic_map, &src_ip);
    if (info) {
        __sync_fetch_and_add(&info->bytes, bpf_ntohs(ip->tot_len));
        __sync_fetch_and_add(&info->packets, 1);
        info->last_seen = bpf_ktime_get_ns();
    } else {
        struct traffic_info new_info = {
            .bytes = bpf_ntohs(ip->tot_len),
            .packets = 1,
            .last_seen = bpf_ktime_get_ns(),
            .flags = 0,
        };
        bpf_map_update_elem(&traffic_map, &src_ip, &new_info, BPF_ANY);
    }
    
    // Send event for suspicious traffic
    if (is_suspicious_traffic(ip, data_end)) {
        struct packet_event event = {
            .src_ip = ip->saddr,
            .dst_ip = ip->daddr,
            .protocol = ip->protocol,
            .length = bpf_ntohs(ip->tot_len),
            .timestamp = bpf_ktime_get_ns(),
        };
        
        // Get port information based on protocol
        if (ip->protocol == IPPROTO_TCP) {
            struct tcphdr *tcp = (void *)ip + (ip->ihl * 4);
            if ((void *)(tcp + 1) <= data_end) {
                event.src_port = bpf_ntohs(tcp->source);
                event.dst_port = bpf_ntohs(tcp->dest);
            }
        } else if (ip->protocol == IPPROTO_UDP) {
            struct udphdr *udp = (void *)ip + (ip->ihl * 4);
            if ((void *)(udp + 1) <= data_end) {
                event.src_port = bpf_ntohs(udp->source);
                event.dst_port = bpf_ntohs(udp->dest);
            }
        }
        
        bpf_perf_event_output(ctx, &events, BPF_F_CURRENT_CPU, &event, sizeof(event));
    }
    
    return XDP_PASS;
}

static __always_inline bool is_suspicious_traffic(struct iphdr *ip, void *data_end) {
    // Simple DDoS detection: too many packets from same IP
    __u32 src_ip = ip->saddr;
    struct traffic_info *info = bpf_map_lookup_elem(&traffic_map, &src_ip);
    if (!info)
        return false;
    
    // Check packet rate (simplified)
    __u64 now = bpf_ktime_get_ns();
    __u64 time_diff = now - info->last_seen;
    
    // If more than 1000 packets per second, flag as suspicious
    if (time_diff < 1000000000 && info->packets > 1000) {
        return true;
    }
    
    return false;
}

char _license[] SEC("license") = "GPL";

Go Userspace Controller

The Go program loads the eBPF program and processes events.

type NetworkMonitor struct {
    manager     *EBPFManager
    program     *EBPFProgram
    
    // Maps
    packetStats *EBPFMap
    trafficMap  *EBPFMap
    eventsMap   *EBPFMap
    
    // Event processing
    eventReader *perf.Reader
    
    // Metrics
    metrics     *NetworkMetrics
    
    // Configuration
    config      *MonitorConfig
    
    mu          sync.RWMutex
}

type NetworkEvent struct {
    SrcIP     net.IP    `json:"src_ip"`
    DstIP     net.IP    `json:"dst_ip"`
    SrcPort   uint16    `json:"src_port"`
    DstPort   uint16    `json:"dst_port"`
    Protocol  uint8     `json:"protocol"`
    Length    uint16    `json:"length"`
    Timestamp time.Time `json:"timestamp"`
}

func NewNetworkMonitor(config *MonitorConfig) (*NetworkMonitor, error) {
    // Load eBPF program
    spec, err := LoadCollectionSpec("network_monitor.o")
    if err != nil {
        return nil, fmt.Errorf("failed to load eBPF spec: %w", err)
    }
    
    coll, err := NewCollectionWithOptions(spec, CollectionOptions{
        Programs: ProgramOptions{
            LogSize: 2 * 1024 * 1024,
        },
    })
    if err != nil {
        return nil, fmt.Errorf("failed to create collection: %w", err)
    }
    
    monitor := &NetworkMonitor{
        config:  config,
        metrics: NewNetworkMetrics(),
    }
    
    // Get maps
    monitor.packetStats = coll.Maps["packet_stats"]
    monitor.trafficMap = coll.Maps["traffic_map"]
    monitor.eventsMap = coll.Maps["events"]
    
    // Get program
    monitor.program = coll.Programs["xdp_monitor"]
    
    return monitor, nil
}

func (nm *NetworkMonitor) Start(ctx context.Context, interfaceName string) error {
    // Attach XDP program to network interface
    link, err := LinkXDP(LinkXDPOptions{
        Program:   nm.program,
        Interface: interfaceName,
        Flags:     XDPGenericMode, // Use generic mode for compatibility
    })
    if err != nil {
        return fmt.Errorf("failed to attach XDP program: %w", err)
    }
    defer link.Close()
    
    // Set up event reader
    nm.eventReader, err = NewReader(nm.eventsMap, 4096)
    if err != nil {
        return fmt.Errorf("failed to create event reader: %w", err)
    }
    defer nm.eventReader.Close()
    
    log.Printf("Network monitor started on interface %s", interfaceName)
    
    // Start statistics collection
    go nm.collectStatistics(ctx)
    
    // Process events
    return nm.processEvents(ctx)
}

func (nm *NetworkMonitor) processEvents(ctx context.Context) error {
    for {
        select {
        case <-ctx.Done():
            return ctx.Err()
        default:
        }
        
        record, err := nm.eventReader.Read()
        if err != nil {
            if errors.Is(err, perf.ErrClosed) {
                return nil
            }
            log.Printf("Error reading event: %v", err)
            continue
        }
        
        if record.LostSamples > 0 {
            nm.metrics.LostEvents.Add(float64(record.LostSamples))
            log.Printf("Lost %d events", record.LostSamples)
            continue
        }
        
        // Parse event
        event, err := nm.parseNetworkEvent(record.RawSample)
        if err != nil {
            log.Printf("Failed to parse event: %v", err)
            continue
        }
        
        nm.handleNetworkEvent(event)
    }
}

func (nm *NetworkMonitor) parseNetworkEvent(data []byte) (*NetworkEvent, error) {
    if len(data) < 24 { // sizeof(struct packet_event)
        return nil, fmt.Errorf("event data too short: %d bytes", len(data))
    }
    
    event := &NetworkEvent{}
    
    // Parse binary data (little endian)
    event.SrcIP = net.IPv4(data[3], data[2], data[1], data[0])
    event.DstIP = net.IPv4(data[7], data[6], data[5], data[4])
    event.SrcPort = binary.LittleEndian.Uint16(data[8:10])
    event.DstPort = binary.LittleEndian.Uint16(data[10:12])
    event.Protocol = data[12]
    event.Length = binary.LittleEndian.Uint16(data[13:15])
    
    // Convert nanoseconds to time
    timestamp := binary.LittleEndian.Uint64(data[16:24])
    event.Timestamp = time.Unix(0, int64(timestamp))
    
    return event, nil
}

func (nm *NetworkMonitor) handleNetworkEvent(event *NetworkEvent) {
    nm.metrics.EventsProcessed.Inc()
    
    // Log suspicious traffic
    log.Printf("Suspicious traffic detected: %s:%d -> %s:%d (proto=%d, len=%d)",
        event.SrcIP, event.SrcPort,
        event.DstIP, event.DstPort,
        event.Protocol, event.Length)
    
    // Could send to alerting system, store in database, etc.
    nm.sendAlert(event)
}

func (nm *NetworkMonitor) collectStatistics(ctx context.Context) {
    ticker := time.NewTicker(10 * time.Second)
    defer ticker.Stop()
    
    for {
        select {
        case <-ctx.Done():
            return
        case <-ticker.C:
            nm.updatePacketStatistics()
            nm.updateTrafficStatistics()
        }
    }
}

func (nm *NetworkMonitor) updatePacketStatistics() {
    var key uint32
    var value uint64
    
    // Iterate through packet stats map
    iter := nm.packetStats.Iterate()
    for iter.Next(&key, &value) {
        protocol := getProtocolName(key)
        nm.metrics.PacketsByProtocol.WithLabelValues(protocol).Set(float64(value))
    }
    
    if err := iter.Err(); err != nil {
        log.Printf("Error iterating packet stats: %v", err)
    }
}

func (nm *NetworkMonitor) updateTrafficStatistics() {
    var key uint32
    var value TrafficInfo
    
    totalIPs := 0
    totalBytes := uint64(0)
    totalPackets := uint64(0)
    
    // Iterate through traffic map
    iter := nm.trafficMap.Iterate()
    for iter.Next(&key, &value) {
        totalIPs++
        totalBytes += value.Bytes
        totalPackets += value.Packets
        
        // Check for cleanup (IPs not seen for a while)
        if time.Since(time.Unix(0, int64(value.LastSeen))) > 5*time.Minute {
            nm.trafficMap.Delete(key)
        }
    }
    
    if err := iter.Err(); err != nil {
        log.Printf("Error iterating traffic map: %v", err)
        return
    }
    
    nm.metrics.UniqueIPs.Set(float64(totalIPs))
    nm.metrics.TotalBytes.Set(float64(totalBytes))
    nm.metrics.TotalPackets.Set(float64(totalPackets))
}

type TrafficInfo struct {
    Bytes    uint64
    Packets  uint64
    LastSeen uint64
    Flags    uint32
}

func getProtocolName(proto uint32) string {
    switch proto {
    case 1:
        return "icmp"
    case 6:
        return "tcp"
    case 17:
        return "udp"
    default:
        return "other"
    }
}

System Call Tracing

Monitor system calls to detect security issues and performance problems.

// syscall_tracer.c - eBPF program for system call tracing
#include 
#include 
#include 
#include 

struct {
    __uint(type, BPF_MAP_TYPE_HASH);
    __uint(max_entries, 10000);
    __type(key, __u32); // PID
    __type(value, struct process_info);
} process_map SEC(".maps");

struct {
    __uint(type, BPF_MAP_TYPE_HASH);
    __uint(max_entries, 1000);
    __type(key, __u32); // Syscall number
    __type(value, __u64); // Count
} syscall_stats SEC(".maps");

struct {
    __uint(type, BPF_MAP_TYPE_PERF_EVENT_ARRAY);
    __uint(key_size, sizeof(__u32));
    __uint(value_size, sizeof(__u32));
} syscall_events SEC(".maps");

struct process_info {
    __u32 pid;
    __u32 ppid;
    __u32 uid;
    __u64 start_time;
    char comm[16];
    __u32 syscall_count;
    __u64 last_syscall_time;
};

struct syscall_event {
    __u32 pid;
    __u32 ppid;
    __u32 uid;
    __u32 syscall_nr;
    __u64 duration;
    __u64 timestamp;
    char comm[16];
    long args[6];
};

SEC("tracepoint/raw_syscalls/sys_enter")
int trace_sys_enter(struct trace_event_raw_sys_enter *args) {
    __u32 pid = bpf_get_current_pid_tgid() >> 32;
    __u64 ts = bpf_ktime_get_ns();
    
    // Skip kernel threads
    if (pid == 0)
        return 0;
    
    // Update or create process info
    struct process_info *proc = bpf_map_lookup_elem(&process_map, &pid);
    if (!proc) {
        struct process_info new_proc = {0};
        new_proc.pid = pid;
        new_proc.ppid = bpf_get_current_pid_tgid() & 0xFFFFFFFF;
        new_proc.uid = bpf_get_current_uid_gid() & 0xFFFFFFFF;
        new_proc.start_time = ts;
        new_proc.syscall_count = 1;
        new_proc.last_syscall_time = ts;
        
        bpf_get_current_comm(&new_proc.comm, sizeof(new_proc.comm));
        bpf_map_update_elem(&process_map, &pid, &new_proc, BPF_ANY);
    } else {
        proc->syscall_count++;
        proc->last_syscall_time = ts;
    }
    
    // Update syscall statistics
    __u32 syscall_nr = args->id;
    __u64 *count = bpf_map_lookup_elem(&syscall_stats, &syscall_nr);
    if (count) {
        __sync_fetch_and_add(count, 1);
    } else {
        __u64 init_val = 1;
        bpf_map_update_elem(&syscall_stats, &syscall_nr, &init_val, BPF_ANY);
    }
    
    // Check for suspicious patterns
    if (is_suspicious_syscall(syscall_nr, proc)) {
        struct syscall_event event = {0};
        event.pid = pid;
        event.ppid = proc ? proc->ppid : 0;
        event.uid = proc ? proc->uid : 0;
        event.syscall_nr = syscall_nr;
        event.timestamp = ts;
        
        if (proc) {
            __builtin_memcpy(&event.comm, proc->comm, sizeof(event.comm));
        }
        
        // Copy syscall arguments
        for (int i = 0; i < 6; i++) {
            event.args[i] = args->args[i];
        }
        
        bpf_perf_event_output(args, &syscall_events, BPF_F_CURRENT_CPU, 
                              &event, sizeof(event));
    }
    
    return 0;
}

static __always_inline bool is_suspicious_syscall(__u32 syscall_nr, struct process_info *proc) {
    // Suspicious syscalls that might indicate malicious activity
    switch (syscall_nr) {
    case 2:   // open
    case 257: // openat
    case 85:  // creat
    case 87:  // unlink
    case 263: // unlinkat
    case 39:  // getpid (when called excessively)
    case 110: // getppid
        if (proc && proc->syscall_count > 1000) {
            // Too many syscalls from this process
            return true;
        }
        break;
    case 76:  // getrlimit
    case 165: // mount
    case 166: // umount2
        // Always flag these as potentially suspicious
        return true;
    }
    
    return false;
}

char _license[] SEC("license") = "GPL";

Go System Call Monitor

type SyscallMonitor struct {
    program     *ebpf.Program
    
    // Maps
    processMap  *ebpf.Map
    syscallStats *ebpf.Map
    eventsMap   *ebpf.Map
    
    // Event processing
    eventReader *perf.Reader
    
    // Process tracking
    processes   map[uint32]*ProcessInfo
    
    // Metrics
    metrics     *SyscallMetrics
    
    mu          sync.RWMutex
}

type ProcessInfo struct {
    PID         uint32    `json:"pid"`
    PPID        uint32    `json:"ppid"`
    UID         uint32    `json:"uid"`
    StartTime   time.Time `json:"start_time"`
    Command     string    `json:"command"`
    SyscallCount uint32   `json:"syscall_count"`
    LastSyscall time.Time `json:"last_syscall"`
}

type SyscallEvent struct {
    PID       uint32    `json:"pid"`
    PPID      uint32    `json:"ppid"`
    UID       uint32    `json:"uid"`
    SyscallNr uint32    `json:"syscall_nr"`
    Duration  uint64    `json:"duration"`
    Timestamp time.Time `json:"timestamp"`
    Command   string    `json:"command"`
    Args      [6]int64  `json:"args"`
}

func NewSyscallMonitor() (*SyscallMonitor, error) {
    // Load eBPF program
    spec, err := LoadCollectionSpec("syscall_tracer.o")
    if err != nil {
        return nil, fmt.Errorf("failed to load eBPF spec: %w", err)
    }
    
    coll, err := NewCollection(spec)
    if err != nil {
        return nil, fmt.Errorf("failed to create collection: %w", err)
    }
    
    monitor := &SyscallMonitor{
        program:      coll.Programs["trace_sys_enter"],
        processMap:   coll.Maps["process_map"],
        syscallStats: coll.Maps["syscall_stats"],
        eventsMap:    coll.Maps["syscall_events"],
        processes:    make(map[uint32]*ProcessInfo),
        metrics:      NewSyscallMetrics(),
    }
    
    return monitor, nil
}

func (sm *SyscallMonitor) Start(ctx context.Context) error {
    // Attach to tracepoint
    link, err := LinkTracepoint(TracepointOptions{
        Group:   "raw_syscalls",
        Name:    "sys_enter",
        Program: sm.program,
    })
    if err != nil {
        return fmt.Errorf("failed to attach tracepoint: %w", err)
    }
    defer link.Close()
    
    // Set up event reader
    sm.eventReader, err = NewReader(sm.eventsMap, 4096)
    if err != nil {
        return fmt.Errorf("failed to create event reader: %w", err)
    }
    defer sm.eventReader.Close()
    
    log.Println("System call monitor started")
    
    // Start statistics collection
    go sm.collectStatistics(ctx)
    
    // Process events
    return sm.processEvents(ctx)
}

func (sm *SyscallMonitor) processEvents(ctx context.Context) error {
    for {
        select {
        case <-ctx.Done():
            return ctx.Err()
        default:
        }
        
        record, err := sm.eventReader.Read()
        if err != nil {
            if errors.Is(err, perf.ErrClosed) {
                return nil
            }
            log.Printf("Error reading syscall event: %v", err)
            continue
        }
        
        event, err := sm.parseSyscallEvent(record.RawSample)
        if err != nil {
            log.Printf("Failed to parse syscall event: %v", err)
            continue
        }
        
        sm.handleSyscallEvent(event)
    }
}

func (sm *SyscallMonitor) parseSyscallEvent(data []byte) (*SyscallEvent, error) {
    if len(data) < 80 { // sizeof(struct syscall_event)
        return nil, fmt.Errorf("event data too short")
    }
    
    event := &SyscallEvent{}
    
    // Parse binary data
    event.PID = binary.LittleEndian.Uint32(data[0:4])
    event.PPID = binary.LittleEndian.Uint32(data[4:8])
    event.UID = binary.LittleEndian.Uint32(data[8:12])
    event.SyscallNr = binary.LittleEndian.Uint32(data[12:16])
    event.Duration = binary.LittleEndian.Uint64(data[16:24])
    
    timestamp := binary.LittleEndian.Uint64(data[24:32])
    event.Timestamp = time.Unix(0, int64(timestamp))
    
    // Extract command name (null-terminated)
    commBytes := data[32:48]
    event.Command = string(bytes.TrimRight(commBytes, "\x00"))
    
    // Extract arguments
    for i := 0; i < 6; i++ {
        offset := 48 + (i * 8)
        event.Args[i] = int64(binary.LittleEndian.Uint64(data[offset : offset+8]))
    }
    
    return event, nil
}

func (sm *SyscallMonitor) handleSyscallEvent(event *SyscallEvent) {
    sm.metrics.EventsProcessed.Inc()
    
    syscallName := getSyscallName(event.SyscallNr)
    
    log.Printf("Suspicious syscall detected: PID=%d, Command=%s, Syscall=%s(%d), Args=%v",
        event.PID, event.Command, syscallName, event.SyscallNr, event.Args)
    
    // Send alert for security team
    sm.sendSecurityAlert(event)
    
    // Update metrics
    sm.metrics.SuspiciousSyscalls.WithLabelValues(syscallName).Inc()
}

func (sm *SyscallMonitor) collectStatistics(ctx context.Context) {
    ticker := time.NewTicker(30 * time.Second)
    defer ticker.Stop()
    
    for {
        select {
        case <-ctx.Done():
            return
        case <-ticker.C:
            sm.updateProcessStatistics()
            sm.updateSyscallStatistics()
        }
    }
}

func (sm *SyscallMonitor) updateSyscallStatistics() {
    var key uint32
    var value uint64
    
    iter := sm.syscallStats.Iterate()
    for iter.Next(&key, &value) {
        syscallName := getSyscallName(key)
        sm.metrics.SyscallCounts.WithLabelValues(syscallName).Set(float64(value))
    }
    
    if err := iter.Err(); err != nil {
        log.Printf("Error iterating syscall stats: %v", err)
    }
}

func getSyscallName(nr uint32) string {
    // Map common syscall numbers to names
    syscalls := map[uint32]string{
        0:   "read",
        1:   "write",
        2:   "open",
        3:   "close",
        39:  "getpid",
        57:  "fork",
        59:  "execve",
        97:  "getrlimit"
        85:  "creat",
        87:  "unlink",
        110: "getppid",
        165: "mount",
        166: "umount2",
        257: "openat",
        263: "unlinkat",
    }
    
    if name, exists := syscalls[nr]; exists {
        return name
    }
    
    return fmt.Sprintf("syscall_%d", nr)
}

Application Performance Tracing

Use uprobes to trace function calls in Go applications without modifying code.

type ApplicationTracer struct {
    programs    map[string]*ebpf.Program
    links       []link.Link
    symbols     []Symbol

    
    // Event processing
    eventReader *perf.Reader
    eventsMap   *ebpf.Map
    
    // Function tracking
    functions   map[string]*FunctionInfo
    
    // Metrics
    metrics     *TracingMetrics
    
    mu          sync.RWMutex
}

type FunctionInfo struct {
    Name      string
    Symbol    string
    Address   uint64
    CallCount uint64
    TotalTime time.Duration
    MinTime   time.Duration
    MaxTime   time.Duration
}

func NewApplicationTracer(binaryPath string) (*ApplicationTracer, error) {
    tracer := &ApplicationTracer{
        programs:  make(map[string]*ebpf.Program),
        functions: make(map[string]*FunctionInfo),
        metrics:   NewTracingMetrics(),
    }
    
    // Parse binary to find function symbols
    symbols, err := tracer.parseSymbols(binaryPath)
    if err != nil {
        return nil, fmt.Errorf("failed to parse symbols: %w", err)
    }
    
    // Create eBPF programs for function tracing
    if err := tracer.createTracingPrograms(symbols); err != nil {
        return nil, fmt.Errorf("failed to create tracing programs: %w", err)
    }
    
    return tracer, nil
}

func (at *ApplicationTracer) parseSymbols(binaryPath string) ([]Symbol, error) {
    file, err := elf.Open(binaryPath)
    if err != nil {
        return nil, err
    }
    defer file.Close()
    
    symbols, err := file.Symbols()
    if err != nil {
        return nil, err
    }
    
    var goSymbols []Symbol
    for _, sym := range symbols {
        // Only trace Go functions we're interested in
        if strings.HasPrefix(sym.Name, "main.") ||
           strings.HasPrefix(sym.Name, "github.com/mycompany/") {
            goSymbols = append(goSymbols, Symbol{
                Name:    sym.Name,
                Address: sym.Value,
                Size:    sym.Size,
            })
        }
    }
    
    return goSymbols, nil
}

type Symbol struct {
    Name    string
    Address uint64
    Size    uint64
}

func (at *ApplicationTracer) createTracingPrograms(symbols []Symbol) error {
    // Create eBPF program for function entry/exit tracing
    spec := &ebpf.ProgramSpec{
        Type:       ebpf.Kprobe,
        SectionName: "uprobe/function_entry",
        Instructions: asm.Instructions{
            // Simplified: real implementation would be more complex
            asm.LoadImm(asm.R0, 0, asm.DWord),
            asm.Return(),
        },
        License: "GPL",
    }
    
    program, err := ebpf.NewProgram(spec)
    if err != nil {
        return err
    }
    
    at.programs["function_tracer"] = program
    
    return nil
}

func (at *ApplicationTracer) AttachToProcess(ctx context.Context, pid int, binaryPath string) error {
    for _, symbol := range at.symbols {
        // Attach uprobe to function entry
        entryLink, err := link.Uprobe(link.UprobeOptions{
            PID:      pid,
            Offset:   symbol.Address,
            Symbol:   symbol.Name,
            Program:  at.programs["function_tracer"],
        })
        if err != nil {
            log.Printf("Failed to attach entry probe to %s: %v", symbol.Name, err)
            continue
        }
        at.links = append(at.links, entryLink)
        
        // Attach uretprobe to function exit
        exitLink, err := link.Uprobe(link.UprobeOptions{
            PID:      pid,
            Offset:   symbol.Address,
            Symbol:   symbol.Name,
            Program:  at.programs["function_tracer"],
            RetProbe: true,
        })
        if err != nil {
            log.Printf("Failed to attach exit probe to %s: %v", symbol.Name, err)
            entryLink.Close()
            continue
        }
        at.links = append(at.links, exitLink)
        
        log.Printf("Attached probes to function %s at offset 0x%x", symbol.Name, symbol.Address)
    }
    
    return nil
}

func (at *ApplicationTracer) Start(ctx context.Context) error {
    // Set up event reader
    var err error
    at.eventReader, err = perf.NewReader(at.eventsMap, 4096)
    if err != nil {
        return fmt.Errorf("failed to create event reader: %w", err)
    }
    defer at.eventReader.Close()
    
    log.Println("Application tracer started")
    
    // Start statistics collection
    go at.collectStatistics(ctx)
    
    // Process events
    return at.processEvents(ctx)
}

Security Policy Enforcement

Use eBPF to enforce security policies at the kernel level.

// security_enforcer.c - eBPF program for security policy enforcement
#include 
#include 
#include 
#include 
#include 
#include 
#include 

// Security policy map
struct {
    __uint(type, BPF_MAP_TYPE_HASH);
    __uint(max_entries, 10000);
    __type(key, struct rule_key);
    __type(value, struct rule_action);
} security_policies SEC(".maps");

// Blocked IPs map
struct {
    __uint(type, BPF_MAP_TYPE_HASH);
    __uint(max_entries, 100000);
    __type(key, __u32); // IP address
    __type(value, __u64); // Block until timestamp
} blocked_ips SEC(".maps");

// Rate limiting map
struct {
    __uint(type, BPF_MAP_TYPE_LRU_HASH);
    __uint(max_entries, 100000);
    __type(key, __u32); // Source IP
    __type(value, struct rate_limit_info);
} rate_limits SEC(".maps");

struct rule_key {
    __u32 src_ip;
    __u32 dst_ip;
    __u16 dst_port;
    __u8  protocol;
    __u8  padding;
};

struct rule_action {
    __u8 action;     // 0 = allow, 1 = drop, 2 = rate_limit
    __u8 log;        // 1 = log this action
    __u16 rate_limit; // packets per second
    __u32 flags;
};

struct rate_limit_info {
    __u64 last_time;
    __u32 packet_count;
    __u32 dropped_count;
};

#define ACTION_ALLOW     0
#define ACTION_DROP      1
#define ACTION_RATE_LIMIT 2

SEC("xdp")
int xdp_security_enforcer(struct xdp_md *ctx) {
    void *data = (void *)(long)ctx->data;
    void *data_end = (void *)(long)ctx->data_end;
    
    struct ethhdr *eth = data;
    if ((void *)(eth + 1) > data_end)
        return XDP_PASS;
    
    if (eth->h_proto != bpf_htons(ETH_P_IP))
        return XDP_PASS;
    
    struct iphdr *ip = (void *)(eth + 1);
    if ((void *)(ip + 1) > data_end)
        return XDP_PASS;
    
    __u32 src_ip = ip->saddr;
    __u32 dst_ip = ip->daddr;
    __u64 now = bpf_ktime_get_ns();
    
    // Check if source IP is blocked
    __u64 *block_until = bpf_map_lookup_elem(&blocked_ips, &src_ip);
    if (block_until && *block_until > now) {
        // IP is still blocked
        return XDP_DROP;
    }
    
    // Get destination port
    __u16 dst_port = 0;
    if (ip->protocol == IPPROTO_TCP) {
        struct tcphdr *tcp = (void *)ip + (ip->ihl * 4);
        if ((void *)(tcp + 1) <= data_end) {
            dst_port = bpf_ntohs(tcp->dest);
        }
    } else if (ip->protocol == IPPROTO_UDP) {
        struct udphdr *udp = (void *)ip + (ip->ihl * 4);
        if ((void *)(udp + 1) <= data_end) {
            dst_port = bpf_ntohs(udp->dest);
        }
    }
    
    // Check security policy
    struct rule_key key = {
        .src_ip = src_ip,
        .dst_ip = dst_ip,
        .dst_port = dst_port,
        .protocol = ip->protocol,
    };
    
    struct rule_action *action = bpf_map_lookup_elem(&security_policies, &key);
    if (action) {
        switch (action->action) {
        case ACTION_DROP:
            return XDP_DROP;
        case ACTION_RATE_LIMIT:
            if (check_rate_limit(src_ip, action->rate_limit, now)) {
                return XDP_DROP;
            }
            break;
        case ACTION_ALLOW:
        default:
            return XDP_PASS;
        }
    }
    
    // Check global rate limiting for unknown sources
    if (check_rate_limit(src_ip, 100, now)) { // 100 pps default limit
        // Block this IP for 60 seconds
        __u64 block_time = now + (60ULL * 1000000000ULL);
        bpf_map_update_elem(&blocked_ips, &src_ip, &block_time, BPF_ANY);
        return XDP_DROP;
    }
    
    return XDP_PASS;
}

static __always_inline bool check_rate_limit(__u32 src_ip, __u16 limit_pps, __u64 now) {
    struct rate_limit_info *info = bpf_map_lookup_elem(&rate_limits, &src_ip);
    if (!info) {
        struct rate_limit_info new_info = {
            .last_time = now,
            .packet_count = 1,
            .dropped_count = 0,
        };
        bpf_map_update_elem(&rate_limits, &src_ip, &new_info, BPF_ANY);
        return false;
    }
    
    // Check if we need to reset the window (1 second window)
    if (now - info->last_time > 1000000000ULL) {
        info->last_time = now;
        info->packet_count = 1;
        return false;
    }
    
    info->packet_count++;
    
    if (info->packet_count > limit_pps) {
        info->dropped_count++;
        return true; // Rate limit exceeded
    }
    
    return false;
}

char _license[] SEC("license") = "GPL";

Production Deployment and Monitoring

eBPF programs require careful deployment and monitoring in production.

type EBPFOrchestrator struct {
    programs    map[string]*ManagedProgram
    
    // Health monitoring
    healthCheck *HealthMonitor
    
    // Resource monitoring
    verifier    *VerifierMonitor
    
    // Deployment management
    deployer    *ProgramDeployer
    
    // Configuration
    config      *OrchestrationConfig
    
    mu          sync.RWMutex
}

type ManagedProgram struct {
    Name        string
    Program     *ebpf.Program
    Links       []link.Link
    
    // Health status
    Healthy     bool
    LastError   error
    
    // Resource usage
    CPUUsage    float64
    MemoryUsage uint64
    
    // Statistics
    LoadTime    time.Time
    RunCount    uint64
    ErrorCount  uint64
    
    mu          sync.RWMutex
}

type HealthMonitor struct {
    programs    map[string]*ManagedProgram
    
    // Monitoring configuration
    checkInterval time.Duration
    
    // Alerting
    alertManager *AlertManager
    
    mu          sync.RWMutex
}

func NewEBPFOrchestrator(config *OrchestrationConfig) *EBPFOrchestrator {
    orchestrator := &EBPFOrchestrator{
        programs: make(map[string]*ManagedProgram),
        config:   config,
    }
    
    orchestrator.healthCheck = NewHealthMonitor(orchestrator)
    orchestrator.verifier = NewVerifierMonitor()
    orchestrator.deployer = NewProgramDeployer()
    
    return orchestrator
}

func (eo *EBPFOrchestrator) DeployProgram(ctx context.Context, spec *ProgramSpec) error {
    log.Printf("Deploying eBPF program: %s", spec.Name)
    
    // Validate program before deployment
    if err := eo.validateProgram(spec); err != nil {
        return fmt.Errorf("program validation failed: %w", err)
    }
    
    // Load program
    program, err := eo.loadProgram(spec)
    if err != nil {
        return fmt.Errorf("failed to load program: %w", err)
    }
    
    // Create managed program
    managed := &ManagedProgram{
        Name:     spec.Name,
        Program:  program,
        Healthy:  true,
        LoadTime: time.Now(),
    }
    
    // Attach to appropriate hook points
    links, err := eo.attachProgram(program, spec)
    if err != nil {
        program.Close()
        return fmt.Errorf("failed to attach program: %w", err)
    }
    managed.Links = links
    
    eo.mu.Lock()
    eo.programs[spec.Name] = managed
    eo.mu.Unlock()
    
    log.Printf("Successfully deployed eBPF program: %s", spec.Name)
    return nil
}

func (eo *EBPFOrchestrator) validateProgram(spec *ProgramSpec) error {
    // Check resource limits
    if len(spec.Instructions) > eo.config.MaxInstructions {
        return fmt.Errorf("program too large: %d instructions (max %d)",
            len(spec.Instructions), eo.config.MaxInstructions)
    }
    
    // Check for dangerous operations
    if err := eo.checkSafetyConstraints(spec); err != nil {
        return fmt.Errorf("safety constraints violated: %w", err)
    }
    
    // Verify program with kernel verifier
    if err := eo.verifier.VerifyProgram(spec); err != nil {
        return fmt.Errorf("kernel verifier rejected program: %w", err)
    }
    
    return nil
}

func (eo *EBPFOrchestrator) MonitorPrograms(ctx context.Context) error {
    ticker := time.NewTicker(30 * time.Second)
    defer ticker.Stop()
    
    for {
        select {
        case <-ctx.Done():
            return ctx.Err()
        case <-ticker.C:
            eo.checkProgramHealth()
            eo.updateResourceMetrics()
        }
    }
}

func (eo *EBPFOrchestrator) checkProgramHealth() {
    eo.mu.RLock()
    programs := make([]*ManagedProgram, 0, len(eo.programs))
    for _, program := range eo.programs {
        programs = append(programs, program)
    }
    eo.mu.RUnlock()
    
    for _, program := range programs {
        if err := eo.healthCheckProgram(program); err != nil {
            program.mu.Lock()
            program.Healthy = false
            program.LastError = err
            program.ErrorCount++
            program.mu.Unlock()
            
            log.Printf("eBPF program %s unhealthy: %v", program.Name, err)
            
            // Attempt automatic recovery
            if err := eo.recoverProgram(program); err != nil {
                log.Printf("Failed to recover program %s: %v", program.Name, err)
                eo.healthCheck.alertManager.SendAlert(&Alert{
                    Type:    AlertTypeProgramFailure,
                    Program: program.Name,
                    Message: fmt.Sprintf("Program failed and recovery unsuccessful: %v", err),
                })
            }
        } else {
            program.mu.Lock()
            program.Healthy = true
            program.LastError = nil
            program.mu.Unlock()
        }
    }
}

func (eo *EBPFOrchestrator) healthCheckProgram(program *ManagedProgram) error {
    // Check if program is still loaded
    info, err := program.Program.Info()
    if err != nil {
        return fmt.Errorf("failed to get program info: %w", err)
    }
    
    // Check if program is still attached
    for _, link := range program.Links {
        if err := eo.checkLinkHealth(link); err != nil {
            return fmt.Errorf("link health check failed: %w", err)
        }
    }
    
    // Check resource usage
    program.mu.Lock()
    program.CPUUsage = eo.getProgramCPUUsage(program)
    program.MemoryUsage = info.MemLocked
    program.mu.Unlock()
    
    // Check for excessive resource usage
    if program.CPUUsage > eo.config.MaxCPUUsage {
        return fmt.Errorf("excessive CPU usage: %.2f%%", program.CPUUsage)
    }
    
    if program.MemoryUsage > eo.config.MaxMemoryUsage {
        return fmt.Errorf("excessive memory usage: %d bytes", program.MemoryUsage)
    }
    
    return nil
}

func (eo *EBPFOrchestrator) recoverProgram(program *ManagedProgram) error {
    log.Printf("Attempting to recover program %s", program.Name)
    
    // Detach existing links
    for _, link := range program.Links {
        link.Close()
    }
    
    // Close existing program
    program.Program.Close()
    
    // Reload and reattach (simplified - would need original spec)
    // This would require storing the original program specification
    
    return fmt.Errorf("program recovery not implemented")
}

// Resource monitoring and limits
type ResourceLimits struct {
    MaxInstructions uint32  `json:"max_instructions"`
    MaxMemoryUsage  uint64  `json:"max_memory_usage"`
    MaxCPUUsage     float64 `json:"max_cpu_usage"`
    MaxMaps         int     `json:"max_maps"`
}

// Deployment metrics
type DeploymentMetrics struct {
    ProgramsLoaded    prometheus.Gauge
    ProgramsHealthy   prometheus.Gauge
    TotalInstructions prometheus.Counter
    MemoryUsage       prometheus.Gauge
    CPUUsage          prometheus.Gauge
    VerifierErrors    prometheus.Counter
    AttachmentErrors  prometheus.Counter
}

func NewDeploymentMetrics() *DeploymentMetrics {
    return &DeploymentMetrics{
        ProgramsLoaded: prometheus.NewGauge(prometheus.GaugeOpts{
            Name: "ebpf_programs_loaded",
            Help: "Number of eBPF programs currently loaded",
        }),
        ProgramsHealthy: prometheus.NewGauge(prometheus.GaugeOpts{
            Name: "ebpf_programs_healthy",
            Help: "Number of healthy eBPF programs",
        }),
        TotalInstructions: prometheus.NewCounter(prometheus.CounterOpts{
            Name: "ebpf_instructions_total",
            Help: "Total number of eBPF instructions across all programs",
        }),
        MemoryUsage: prometheus.NewGauge(prometheus.GaugeOpts{
            Name: "ebpf_memory_usage_bytes",
            Help: "Total memory usage of eBPF programs in bytes",
        }),
        CPUUsage: prometheus.NewGauge(prometheus.GaugeOpts{
            Name: "ebpf_cpu_usage_percent",
            Help: "CPU usage percentage of eBPF programs",
        }),
        VerifierErrors: prometheus.NewCounter(prometheus.CounterOpts{
            Name: "ebpf_verifier_errors_total",
            Help: "Total number of eBPF verifier errors",
        }),
        AttachmentErrors: prometheus.NewCounter(prometheus.CounterOpts{
            Name: "ebpf_attachment_errors_total",
            Help: "Total number of eBPF attachment errors",
        }),
    }
}

Production Metrics and Lessons

After 18 months running eBPF in production:

1B+ packets/day processed by XDP programs
Sub-microsecond median latency for packet processing
5% CPU overhead vs traditional monitoring
99.99% program uptime with health monitoring
2MB memory usage per eBPF program average
Zero kernel crashes due to eBPF safety guarantees

Performance Impact

Use Case	Traditional Approach	eBPF Approach	Performance Gain
Network Monitoring	tcpdump + userspace	XDP	10x faster
System Call Tracing	strace	Tracepoints	100x less overhead
Function Tracing	Code instrumentation	uprobes	No code changes
Security Filtering	iptables	XDP	5x faster

Key Lessons Learned

Start simple: Begin with basic packet counting before complex logic
Kernel version matters: Newer kernels have more eBPF features
Map design is critical: Wrong map type kills performance
Verifier can be tricky: Some valid programs get rejected
Testing is essential: eBPF bugs can crash systems
Monitor everything: Resource usage, errors, performance
CO-RE saves time: Compile once, run everywhere

Security Considerations

eBPF programs run with kernel privileges, requiring careful security design.

eBPF Verifier Protection

Bounds checking: Verifier ensures all memory accesses are safe
Loop protection: Prevents infinite loops that could hang kernel
Function restrictions: Only whitelisted kernel functions allowed
Privilege checks: Programs inherit loader's capabilities

Map Security

type SecureMapConfig struct {
    MaxEntries  uint32
    KeySize     uint32
    ValueSize   uint32
    
    // Access controls
    ReadOnly    bool
    PrivilegedOnly bool
    
    // Resource limits
    MemoryLimit uint64
    RateLimit   uint32
}

func createSecureMap(config *SecureMapConfig) (*ebpf.Map, error) {
    if config.MaxEntries > MAX_SAFE_ENTRIES {
        return nil, errors.New("map too large")
    }
    
    if config.KeySize > MAX_KEY_SIZE {
        return nil, errors.New("key too large")
    }
    
    spec := &ebpf.MapSpec{
        Type:       ebpf.Hash,
        KeySize:    config.KeySize,
        ValueSize:  config.ValueSize,
        MaxEntries: config.MaxEntries,
        
        // Security flags
        Flags: ebpf.BPF_F_NO_PREALLOC, // Prevent memory exhaustion
    }
    
    return ebpf.NewMap(spec)
}

Input Validation

// Always validate packet bounds in eBPF programs
static __always_inline bool validate_packet(void *data, void *data_end, int offset) {
    return data + offset <= data_end;
}

SEC("xdp")
int secure_packet_processor(struct xdp_md *ctx) {
    void *data = (void *)(long)ctx->data;
    void *data_end = (void *)(long)ctx->data_end;
    
    // Validate ethernet header
    if (!validate_packet(data, data_end, sizeof(struct ethhdr))) {
        return XDP_DROP;
    }
    
    struct ethhdr *eth = data;
    
    // Validate IP header
    if (!validate_packet(data, data_end, sizeof(struct ethhdr) + sizeof(struct iphdr))) {
        return XDP_DROP;
    }
    
    // Continue processing...
    return XDP_PASS;
}

Privilege Management

type PrivilegeManager struct {
    capabilities []string
    uid          uint32
    gid          uint32
}

func (pm *PrivilegeManager) LoadProgram(spec *ebpf.ProgramSpec) error {
    // Check if current user has required capabilities
    if !pm.hasCapability("CAP_SYS_ADMIN") {
        return errors.New("insufficient privileges")
    }
    
    // Drop privileges after loading if possible
    defer pm.dropPrivileges()
    
    program, err := ebpf.NewProgram(spec)
    if err != nil {
        return fmt.Errorf("failed to load program: %w", err)
    }
    
    return nil
}

func (pm *PrivilegeManager) hasCapability(cap string) bool {
    for _, c := range pm.capabilities {
        if c == cap {
            return true
        }
    }
    return false
}

Testing Strategy

Testing eBPF programs requires specialized approaches due to kernel integration.

Unit Testing eBPF Programs

func TestXDPProgram(t *testing.T) {
    // Load test program
    spec, err := LoadCollectionSpec("test_program.o")
    require.NoError(t, err)
    
    coll, err := NewCollection(spec)
    require.NoError(t, err)
    defer coll.Close()
    
    program := coll.Programs["test_xdp"]
    
    tests := []struct {
        name     string
        packet   []byte
        expected int
    }{
        {
            name:     "valid_tcp_packet",
            packet:   createTCPPacket("192.168.1.1", "192.168.1.2", 80),
            expected: int(XDP_PASS),
        },
        {
            name:     "malformed_packet",
            packet:   []byte{0x00, 0x01}, // Too short
            expected: int(XDP_DROP),
        },
        {
            name:     "blocked_ip",
            packet:   createTCPPacket("10.0.0.1", "192.168.1.2", 80),
            expected: int(XDP_DROP),
        },
    }
    
    for _, tt := range tests {
        t.Run(tt.name, func(t *testing.T) {
            result := testXDPProgram(t, program, tt.packet)
            assert.Equal(t, tt.expected, result)
        })
    }
}

func testXDPProgram(t *testing.T, program *ebpf.Program, packet []byte) int {
    // Create test context
    ctx := &xdpTestContext{
        data:     packet,
        dataEnd:  len(packet),
        metadata: make([]byte, 256),
    }
    
    // Run program in test environment
    result, err := RunXDPTest(program, ctx)
    require.NoError(t, err)
    
    return result
}

func createTCPPacket(srcIP, dstIP string, port uint16) []byte {
    packet := make([]byte, 54) // Ethernet + IP + TCP headers
    
    // Ethernet header
    eth := (*EthernetHeader)(unsafe.Pointer(&packet[0]))
    eth.DstMAC = [6]byte{0xff, 0xff, 0xff, 0xff, 0xff, 0xff}
    eth.SrcMAC = [6]byte{0x00, 0x11, 0x22, 0x33, 0x44, 0x55}
    eth.EtherType = 0x0800 // IPv4
    
    // IP header
    ip := (*IPHeader)(unsafe.Pointer(&packet[14]))
    ip.Version = 4
    ip.HeaderLen = 5
    ip.TotalLen = 40 // IP + TCP headers
    ip.Protocol = 6  // TCP
    ip.SrcIP = inet_addr(srcIP)
    ip.DstIP = inet_addr(dstIP)
    
    // TCP header
    tcp := (*TCPHeader)(unsafe.Pointer(&packet[34]))
    tcp.SrcPort = 12345
    tcp.DstPort = port
    tcp.HeaderLen = 5 << 4
    
    return packet
}

Integration Testing

func TestNetworkMonitorIntegration(t *testing.T) {
    if testing.Short() {
        t.Skip("skipping integration test")
    }
    
    // Create test network namespace
    netns, err := createTestNamespace()
    require.NoError(t, err)
    defer netns.Close()
    
    // Set up test environment
    ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
    defer cancel()
    
    monitor, err := NewNetworkMonitor(&MonitorConfig{
        Interface: "veth-test",
        BufferSize: 4096,
    })
    require.NoError(t, err)
    
    // Start monitoring in background
    go func() {
        err := monitor.Start(ctx, "veth-test")
        assert.NoError(t, err)
    }()
    
    // Wait for program to attach
    time.Sleep(1 * time.Second)
    
    // Generate test traffic
    testTrafficGenerator := &TrafficGenerator{
        Interface: "veth-test-peer",
        Rate:      1000, // packets per second
    }
    
    err = testTrafficGenerator.GenerateTraffic(ctx, 5*time.Second)
    require.NoError(t, err)
    
    // Verify metrics
    metrics := monitor.GetMetrics()
    assert.Greater(t, metrics.PacketsProcessed, uint64(4000))
    assert.Less(t, metrics.PacketsDropped, uint64(100))
}

func createTestNamespace() (*NetNamespace, error) {
    // Create network namespace for isolated testing
    ns, err := netns.New()
    if err != nil {
        return nil, err
    }
    
    // Create veth pair for testing
    veth := &netlink.Veth{
        LinkAttrs: netlink.LinkAttrs{Name: "veth-test"},
        PeerName:  "veth-test-peer",
    }
    
    err = netlink.LinkAdd(veth)
    if err != nil {
        ns.Close()
        return nil, err
    }
    
    return &NetNamespace{
        handle: ns,
        veth:   veth,
    }, nil
}

Performance Testing

func BenchmarkXDPProcessing(b *testing.B) {
    program := loadTestProgram(b)
    packet := createTestPacket()
    
    b.ResetTimer()
    b.ReportAllocs()
    
    for i := 0; i < b.N; i++ {
        result := testXDPProgram(b, program, packet)
        if result != int(XDP_PASS) {
            b.Fatalf("unexpected result: %d", result)
        }
    }
}

func BenchmarkMapOperations(b *testing.B) {
    m, err := ebpf.NewMap(&ebpf.MapSpec{
        Type:       ebpf.Hash,
        KeySize:    4,
        ValueSize:  8,
        MaxEntries: 1000,
    })
    require.NoError(b, err)
    defer m.Close()
    
    key := uint32(123)
    value := uint64(456)
    
    b.Run("Put", func(b *testing.B) {
        for i := 0; i < b.N; i++ {
            err := m.Put(key, value)
            if err != nil {
                b.Fatal(err)
            }
        }
    })
    
    b.Run("Get", func(b *testing.B) {
        m.Put(key, value) // Ensure key exists
        
        for i := 0; i < b.N; i++ {
            var result uint64
            err := m.Lookup(key, &result)
            if err != nil {
                b.Fatal(err)
            }
        }
    })
}

Load Testing

func TestHighLoad(t *testing.T) {
    monitor, err := NewNetworkMonitor(&MonitorConfig{
        Interface: "eth0",
        BufferSize: 1024 * 1024, // 1MB buffer
    })
    require.NoError(t, err)
    
    ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
    defer cancel()
    
    // Start monitoring
    go monitor.Start(ctx, "eth0")
    time.Sleep(1 * time.Second)
    
    // Generate high load
    generators := make([]*TrafficGenerator, 10)
    for i := 0; i < 10; i++ {
        generators[i] = &TrafficGenerator{
            Rate:     10000, // 10k packets/second per generator
            Duration: 30 * time.Second,
        }
        
        go func(gen *TrafficGenerator) {
            err := gen.GenerateTraffic(ctx, 30*time.Second)
            assert.NoError(t, err)
        }(generators[i])
    }
    
    // Monitor system resources
    resourceMonitor := &ResourceMonitor{}
    go resourceMonitor.Monitor(ctx)
    
    // Wait for test completion
    <-ctx.Done()
    
    // Verify system stability
    metrics := monitor.GetMetrics()
    resources := resourceMonitor.GetMetrics()
    
    assert.Less(t, metrics.LostEvents, uint64(1000), "too many lost events")
    assert.Less(t, resources.CPUUsage, 80.0, "CPU usage too high")
    assert.Less(t, resources.MemoryUsage, 1024*1024*1024, "memory usage too high")
}

Best Practices and Gotchas

Always validate inputs: Check bounds before accessing packet data
Use appropriate map types: LRU for caches, per-CPU for hot paths
Minimize map operations: Each lookup has overhead
Handle edge cases: Packet parsing can fail in many ways
Test thoroughly: Use synthetic traffic and fuzzing
Monitor verifier logs: They contain valuable debugging info
Use BTF when possible: Improves portability and debugging
CO-RE (Compile Once - Run Everywhere): Modern approach enabling portable eBPF programs across kernel versions without recompilation

Conclusion

eBPF transformed our observability and security capabilities. Processing 1B+ packets daily with minimal overhead while providing unprecedented visibility into system behavior.

Key takeaways:

eBPF provides kernel-level visibility without kernel modules
Performance benefits are substantial for network and system monitoring
Safety guarantees make production deployment viable
Go's eBPF libraries make development manageable
Proper monitoring and deployment practices are essential

The investment in eBPF expertise paid off with better security posture, detailed observability, and significant performance improvements. eBPF is the future of kernel programming, and Go makes it accessible to more developers.