package main
 
import (
    "fmt"
    "log"
    
    "github.com/DataDog/datadog-api-client-go/v2/api/datadogV1"
)
 
type Service struct {
    Name              string
    Owner             string
    LatencyThreshold  float64
    ErrorRateThreshold float64
    ServiceType       string
}
 
func provisionMonitoring() error {
    // Fetch all active services from our service registry
    services, err := fetchActiveServices()
    if err != nil {
        return fmt.Errorf("failed to fetch services: %w", err)
    }
    
    log.Printf("Found %d active services to process", len(services))
    
    for _, svc := range services {
        // Check if monitoring already exists
        exists, err := monitorExists(svc.Name)
        if err != nil {
            log.Printf("Error checking monitor for %s: %v", svc.Name, err)
            continue
        }
        
        if !exists {
            if err := createDatadogMonitor(svc); err != nil {
                log.Printf("Failed to create monitor for %s: %v", svc.Name, err)
                continue
            }
            
            if err := configureBugsnag(svc); err != nil {
                log.Printf("Failed to configure Bugsnag for %s: %v", svc.Name, err)
                continue
            }
            
            log.Printf("Successfully provisioned monitoring for %s", svc.Name)
        }
    }
    
    return nil
}
 
func createDatadogMonitor(svc Service) error {
    monitor := datadogV1.Monitor{
        Name: fmt.Sprintf("[Auto] %s - Latency", svc.Name),
        Type: "metric alert",
        Query: fmt.Sprintf("avg(last_5m):avg:trace.servlet.request{service:%s} > %.2f", 
            svc.Name, svc.LatencyThreshold),
        Message: fmt.Sprintf("@%s High latency detected on %s", svc.Owner, svc.Name),
        Tags: []string{
            "auto-provisioned",
            fmt.Sprintf("service:%s", svc.Name),
            fmt.Sprintf("owner:%s", svc.Owner),
            fmt.Sprintf("type:%s", svc.ServiceType),
        },
        Options: &datadogV1.MonitorOptions{
            NotifyNoData:    true,
            NoDataTimeframe: 20,
            EvaluationDelay: 60,
        },
    }
    
    // Create the monitor using Datadog API client
    _, _, err := datadogClient.MonitorsApi.CreateMonitor(ctx, monitor)
    return err
}

// bugsnag-provisioning.js
import Bugsnag from '@bugsnag/js';
import BugsnagPluginExpress from '@bugsnag/plugin-express';
 
class BugsnagProvisioner {
    constructor(apiKey) {
        this.apiKey = apiKey;
        this.client = Bugsnag.start({
            apiKey: apiKey,
            enabledReleaseStages: ['production', 'staging'],
            appVersion: process.env.APP_VERSION,
        });
    }
    
    async setupServiceMonitoring(service) {
        // Configure project-level settings
        await this.configureProject(service);
        
        // Set up notification channels
        await this.configureAlerts(service);
        
        // Create initial dashboard
        await this.createDashboard(service);
        
        console.log(`Bugsnag monitoring configured for ${service.name}`);
    }
    
    async createDashboard(service) {
        // Fetch recent errors for the service
        const errors = await this.client.errors.list({
            filters: {
                'app.release_stage': 'production',
                'app.id': service.name,
            },
            sort: '-occurrences',
            limit: 100,
        });
        
        // Generate dashboard metrics
        const metrics = this.calculateErrorMetrics(errors);
        
        // Publish to internal dashboard service
        await this.publishMetrics(service.name, metrics);
        
        return metrics;
    }
    
    calculateErrorMetrics(errors) {
        const grouped = errors.reduce((acc, error) => {
            const key = error.error_class;
            if (!acc[key]) {
                acc[key] = { count: 0, users_affected: new Set() };
            }
            acc[key].count += error.occurrences;
            error.users.forEach(user => acc[key].users_affected.add(user.id));
            return acc;
        }, {});
        
        return Object.entries(grouped).map(([errorClass, data]) => ({
            error_class: errorClass,
            occurrences: data.count,
            users_affected: data.users_affected.size,
        }));
    }
}
 
export default BugsnagProvisioner;

package main
 
import (
    "context"
    "fmt"
    "log"
    "net/http"
    "net/http/httputil"
    "net/url"
    "time"
    
    "github.com/go-redis/redis/v8"
)
 
type RedisProxy struct {
    target           *url.URL
    proxy            *httputil.ReverseProxy
    redisClient      *redis.Client
    authService      AuthService
}
 
func NewRedisProxy(redisCommanderURL string, redisAddr string) (*RedisProxy, error) {
    target, err := url.Parse(redisCommanderURL)
    if err != nil {
        return nil, err
    }
    
    rp := &RedisProxy{
        target: target,
        proxy:  httputil.NewSingleHostReverseProxy(target),
        redisClient: redis.NewClient(&redis.Options{
            Addr:     redisAddr,
            Password: "", // configured via environment
            DB:       0,
        }),
        authService: NewAuthService(),
    }
    
    return rp, nil
}
 
func (rp *RedisProxy) ServeHTTP(w http.ResponseWriter, r *http.Request) {
    // Authenticate the engineer
    engineer, err := rp.authService.AuthenticateRequest(r)
    if err != nil {
        http.Error(w, "Unauthorised: Invalid credentials", http.StatusUnauthorized)
        rp.logAccessAttempt(r, false, err)
        return
    }
    
    // Check if engineer has Redis access permissions
    if !rp.authService.HasRedisAccess(engineer) {
        http.Error(w, "Forbidden: Insufficient permissions", http.StatusForbidden)
        rp.logAccessAttempt(r, false, fmt.Errorf("insufficient permissions"))
        return
    }
    
    // Log successful access
    rp.logAccessAttempt(r, true, nil)
    
    // Inject engineer context for audit trail
    r.Header.Set("X-Engineer-Email", engineer.Email)
    r.Header.Set("X-Engineer-Team", engineer.Team)
    
    // Proxy the request to Redis Commander
    rp.proxy.ServeHTTP(w, r)
}
 
func (rp *RedisProxy) logAccessAttempt(r *http.Request, success bool, err error) {
    log.Printf(
        "Redis access attempt - IP: %s, Path: %s, Success: %v, Error: %v",
        r.RemoteAddr, r.URL.Path, success, err,
    )
}

// redis-dashboard.jsx
import React, { useState, useEffect } from 'react';
import { LineChart, Line, BarChart, Bar, XAxis, YAxis, CartesianGrid, Tooltip, Legend } from 'recharts';
 
const RedisDashboard = () => {
    const [metrics, setMetrics] = useState(null);
    const [keyStats, setKeyStats] = useState([]);
    const [loading, setLoading] = useState(true);
    
    useEffect(() => {
        fetchRedisMetrics();
        const interval = setInterval(fetchRedisMetrics, 30000); // Update every 30s
        
        return () => clearInterval(interval);
    }, []);
    
    const fetchRedisMetrics = async () => {
        try {
            const response = await fetch('/api/redis/metrics');
            const data = await response.json();
            
            setMetrics(data.current);
            setKeyStats(data.key_statistics);
            setLoading(false);
        } catch (error) {
            console.error('Failed to fetch Redis metrics:', error);
        }
    };
    
    const renderKeyTable = () => {
        return (
            <div className="key-statistics">
                <h3>Top Keys by Memory Usage</h3>
                <table>
                    <thead>
                        <tr>
                            <th>Key Pattern</th>
                            <th>Count</th>
                            <th>TTL (avg)</th>
                            <th>Memory</th>
                        </tr>
                    </thead>
                    <tbody>
                        {keyStats.map((stat, index) => (
                            <tr key={index}>
                                <td><code>{stat.pattern}</code></td>
                                <td>{stat.count.toLocaleString()}</td>
                                <td>{formatTTL(stat.avg_ttl)}</td>
                                <td>{formatMemory(stat.total_memory)}</td>
                            </tr>
                        ))}
                    </tbody>
                </table>
            </div>
        );
    };
    
    if (loading) return <div>Loading Redis metrics...</div>;
    
    return (
        <div className="redis-dashboard">
            <h2>Redis Cluster Overview</h2>
            
            <div className="metrics-summary">
                <div className="metric-card">
                    <h4>Memory Usage</h4>
                    <p className="metric-value">{formatMemory(metrics.used_memory)}</p>
                    <p className="metric-label">Peak: {formatMemory(metrics.used_memory_peak)}</p>
                </div>
                
                <div className="metric-card">
                    <h4>Hit Rate</h4>
                    <p className="metric-value">
                        {((metrics.keyspace_hits / (metrics.keyspace_hits + metrics.keyspace_misses)) * 100).toFixed(2)}%
                    </p>
                </div>
            </div>
            
            {renderKeyTable()}
        </div>
    );
};
 
export default RedisDashboard;

package main
 
import (
    "bytes"
    "fmt"
    "text/template"
    "time"
)
 
type Incident struct {
    ID              string
    Service         string
    Date            time.Time
    Severity        string
    DetectionTime   time.Duration
    ResolutionTime  time.Duration
    RootCause       string
    ImpactSummary   string
    ResolutionSteps []ResolutionStep
    LessonsLearned  []Lesson
    Participants    []string
    Tags            []string
}
 
type ResolutionStep struct {
    Step        int
    Action      string
    Command     string
    ExpectedResult string
    TimeToComplete string
}
 
const runbookTemplate = `# Runbook: {{.Service}} - {{.RootCause}}
 
**Incident ID**: {{.ID}}  
**Date**: {{.Date.Format "2006-01-02"}}  
**Severity**: {{.Severity}}  
**Detection Time**: {{.DetectionTime}}  
**Resolution Time**: {{.ResolutionTime}}
 
## Overview
 
{{.ImpactSummary}}
 
## Root Cause
 
{{.RootCause}}
 
## Resolution Steps
 
{{range .ResolutionSteps}}
### Step {{.Step}}: {{.Action}}
 
{{if .Command}}
` + "```bash" + `
{{.Command}}
` + "```" + `
{{end}}
 
**Expected Result**: {{.ExpectedResult}}  
**Time to Complete**: ~{{.TimeToComplete}}
 
{{end}}
 
---
 
*This runbook was automatically generated from incident {{.ID}}.*
`
 
func generateRunbook(incident Incident) (string, error) {
    tmpl, err := template.New("runbook").Parse(runbookTemplate)
    if err != nil {
        return "", fmt.Errorf("failed to parse template: %w", err)
    }
    
    var buf bytes.Buffer
    if err := tmpl.Execute(&buf, incident); err != nil {
        return "", fmt.Errorf("failed to execute template: %w", err)
    }
    
    return buf.String(), nil
}

package main
 
import (
    "context"
    "fmt"
    "time"
    
    metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
    "k8s.io/client-go/kubernetes"
)
 
type ScalingController struct {
    k8sClient   *kubernetes.Clientset
    metricsAPI  MetricsAPI
    config      ScalingConfig
}
 
type ScalingConfig struct {
    CheckInterval      time.Duration
    LatencyThreshold   time.Duration
    CPUThreshold       float64
    MemoryThreshold    float64
    MinReplicas        int32
    MaxReplicas        int32
    ScaleUpCooldown    time.Duration
    ScaleDownCooldown  time.Duration
}
 
func (sc *ScalingController) Run(ctx context.Context) error {
    ticker := time.NewTicker(sc.config.CheckInterval)
    defer ticker.Stop()
    
    for {
        select {
        case <-ctx.Done():
            return ctx.Err()
        case <-ticker.C:
            if err := sc.evaluateAndScale(); err != nil {
                fmt.Printf("Scaling evaluation failed: %v\n", err)
            }
        }
    }
}
 
func (sc *ScalingController) evaluateAndScale() error {
    deployments, err := sc.k8sClient.AppsV1().Deployments("").List(
        context.Background(),
        metav1.ListOptions{
            LabelSelector: "auto-scale=enabled",
        },
    )
    if err != nil {
        return fmt.Errorf("failed to list deployments: %w", err)
    }
    
    for _, deploy := range deployments.Items {
        serviceName := deploy.Labels["service"]
        if serviceName == "" {
            continue
        }
        
        // Get current metrics
        metrics, err := sc.metricsAPI.GetServiceMetrics(serviceName)
        if err != nil {
            fmt.Printf("Failed to get metrics for %s: %v\n", serviceName, err)
            continue
        }
        
        // Determine if scaling is needed
        action := sc.determineScalingAction(deploy, metrics)
        
        if action.ShouldScale {
            if err := sc.executeScaling(deploy, action); err != nil {
                fmt.Printf("Failed to scale %s: %v\n", deploy.Name, err)
                continue
            }
            
            fmt.Printf("Scaled %s: %d -> %d replicas (reason: %s)\n",
                deploy.Name, deploy.Spec.Replicas, action.TargetReplicas, action.Reason)
        }
    }
    
    return nil
}

func (sc *ScalingController) determineScalingAction(
    deploy appsv1.Deployment,
    metrics ServiceMetrics,
) ScalingAction {
    currentReplicas := *deploy.Spec.Replicas
    
    // Check if we should scale up
    if metrics.LatencyP95 > sc.config.LatencyThreshold {
        targetReplicas := currentReplicas + int32(float64(currentReplicas)*0.3)
        if targetReplicas > sc.config.MaxReplicas {
            targetReplicas = sc.config.MaxReplicas
        }
        
        return ScalingAction{
            ShouldScale:    true,
            TargetReplicas: targetReplicas,
            Reason:         fmt.Sprintf("High latency: %.2fms > %.2fms", 
                float64(metrics.LatencyP95.Milliseconds()),
                float64(sc.config.LatencyThreshold.Milliseconds())),
        }
    }
    
    if metrics.CPUUtilization > sc.config.CPUThreshold {
        targetReplicas := currentReplicas + int32(float64(currentReplicas)*0.5)
        if targetReplicas > sc.config.MaxReplicas {
            targetReplicas = sc.config.MaxReplicas
        }
        
        return ScalingAction{
            ShouldScale:    true,
            TargetReplicas: targetReplicas,
            Reason:         fmt.Sprintf("High CPU: %.1f%% > %.1f%%", 
                metrics.CPUUtilization*100, sc.config.CPUThreshold*100),
        }
    }
    
    // Check if we can scale down
    if metrics.LatencyP95 < sc.config.LatencyThreshold/2 && 
       metrics.CPUUtilization < sc.config.CPUThreshold/2 {
        targetReplicas := currentReplicas - int32(float64(currentReplicas)*0.2)
        if targetReplicas < sc.config.MinReplicas {
            targetReplicas = sc.config.MinReplicas
        }
        
        if targetReplicas < currentReplicas {
            return ScalingAction{
                ShouldScale:    true,
                TargetReplicas: targetReplicas,
                Reason:         "Low resource utilisation, scaling down",
            }
        }
    }
    
    return ScalingAction{ShouldScale: false}
}

// pre-deployment-validation.js
import fetch from 'node-fetch';
 
class DeploymentValidator {
    constructor(config) {
        this.datadogAPI = config.datadogAPI;
        this.sloThresholds = config.sloThresholds;
        this.lookbackWindow = config.lookbackWindow || 3600; // 1 hour
    }
    
    async validateDeployment(service, targetEnvironment) {
        console.log(`Validating deployment for ${service} to ${targetEnvironment}...`);
        
        const validations = [
            this.validateLatencySLO(service),
            this.validateErrorRateSLO(service),
            this.validateDependencyHealth(service),
            this.validateErrorBudget(service),
            this.validateRecentIncidents(service),
        ];
        
        const results = await Promise.allSettled(validations);
        
        const failures = results
            .map((result, index) => ({ result, index }))
            .filter(({ result }) => result.status === 'rejected' || !result.value.passed)
            .map(({ result, index }) => ({
                check: validations[index].name,
                reason: result.status === 'rejected' 
                    ? result.reason.message 
                    : result.value.reason,
            }));
        
        if (failures.length > 0) {
            this.logValidationFailure(service, failures);
            throw new DeploymentBlockedError(
                `Deployment validation failed for ${service}`,
                failures
            );
        }
        
        console.log(`✓ All validations passed for ${service}`);
        return { passed: true, service, timestamp: new Date().toISOString() };
    }
    
    async validateErrorBudget(service) {
        const budget = await this.fetchErrorBudget(service);
        
        // Block deployment if less than 10% error budget remaining
        if (budget.remaining < 0.1) {
            return {
                passed: false,
                reason: `Insufficient error budget: ${(budget.remaining * 100).toFixed(1)}% remaining`,
                current: budget.remaining,
                threshold: 0.1,
            };
        }
        
        // Warn if less than 30% error budget remaining
        if (budget.remaining < 0.3) {
            console.warn(
                `⚠ Warning: Low error budget for ${service}: ${(budget.remaining * 100).toFixed(1)}% remaining`
            );
        }
        
        return { passed: true };
    }
}

# .github/workflows/deploy-production.yml
name: Deploy to Production
 
on:
  push:
    branches: [main]
  workflow_dispatch:
 
env:
  SERVICE_NAME: ${{ github.event.repository.name }}
  TARGET_ENVIRONMENT: production
 
jobs:
  validate-slo:
    runs-on: ubuntu-latest
    steps:
      - name: Checkout code
        uses: actions/checkout@v3
      
      - name: Setup Node.js
        uses: actions/setup-node@v3
        with:
          node-version: '18'
      
      - name: Run pre-deployment validation
        env:
          DATADOG_API_KEY: ${{ secrets.DATADOG_API_KEY }}
          DATADOG_APP_KEY: ${{ secrets.DATADOG_APP_KEY }}
        run: node scripts/pre-deployment-validation.js
  
  deploy:
    needs: validate-slo
    runs-on: ubuntu-latest
    steps:
      - name: Deploy to Kubernetes
        run: |
          kubectl set image deployment/$SERVICE_NAME \
            $SERVICE_NAME=$ECR_REGISTRY/$SERVICE_NAME:$IMAGE_TAG \
            --namespace=production
          
          kubectl rollout status deployment/$SERVICE_NAME \
            --namespace=production \
            --timeout=5m

package main
 
import (
    "context"
    "fmt"
    "time"
)
 
type ChaosExperiment struct {
    Name           string
    Description    string
    TargetService  string
    Duration       time.Duration
    Hypothesis     string
    Blast          BlastRadius
    Executor       ExperimentExecutor
}
 
type BlastRadius struct {
    Environment    string
    Namespace      string
    PodSelector    string
    MaxConcurrent  int
}
 
type ExperimentExecutor interface {
    Execute(ctx context.Context) error
    Validate(ctx context.Context) (bool, error)
    Rollback(ctx context.Context) error
}
 
// Pod Failure Experiment
type PodFailureExperiment struct {
    k8sClient      *kubernetes.Clientset
    targetService  string
    namespace      string
    failureCount   int
}
 
func (e *PodFailureExperiment) Execute(ctx context.Context) error {
    fmt.Printf("Starting pod failure experiment for %s\n", e.targetService)
    
    // Get pods for the service
    pods, err := e.k8sClient.CoreV1().Pods(e.namespace).List(ctx, metav1.ListOptions{
        LabelSelector: fmt.Sprintf("app=%s", e.targetService),
    })
    if err != nil {
        return fmt.Errorf("failed to list pods: %w", err)
    }
    
    if len(pods.Items) < e.failureCount {
        return fmt.Errorf("insufficient pods: have %d, need %d", len(pods.Items), e.failureCount)
    }
    
    // Delete specified number of pods
    for i := 0; i < e.failureCount; i++ {
        podName := pods.Items[i].Name
        fmt.Printf("Deleting pod: %s\n", podName)
        
        err := e.k8sClient.CoreV1().Pods(e.namespace).Delete(
            ctx,
            podName,
            metav1.DeleteOptions{},
        )
        
        if err != nil {
            return fmt.Errorf("failed to delete pod %s: %w", podName, err)
        }
    }
    
    return nil
}
 
func (e *PodFailureExperiment) Validate(ctx context.Context) (bool, error) {
    // Wait for pods to be recreated
    time.Sleep(30 * time.Second)
    
    // Check if new pods are running
    pods, err := e.k8sClient.CoreV1().Pods(e.namespace).List(ctx, metav1.ListOptions{
        LabelSelector: fmt.Sprintf("app=%s", e.targetService),
    })
    if err != nil {
        return false, fmt.Errorf("failed to list pods: %w", err)
    }
    
    runningPods := 0
    for _, pod := range pods.Items {
        if pod.Status.Phase == corev1.PodRunning {
            runningPods++
        }
    }
    
    fmt.Printf("Running pods after experiment: %d\n", runningPods)
    
    // Check service metrics
    metrics, err := getServiceMetrics(e.targetService)
    if err != nil {
        return false, fmt.Errorf("failed to get metrics: %w", err)
    }
    
    // Validate that service continued operating within SLO
    if metrics.ErrorRate > 0.05 { // 5% error rate threshold
        return false, fmt.Errorf("error rate %.2f%% exceeds threshold", metrics.ErrorRate*100)
    }
    
    if metrics.LatencyP95 > 500*time.Millisecond {
        return false, fmt.Errorf("p95 latency %v exceeds threshold", metrics.LatencyP95)
    }
    
    return true, nil
}
 
// Network Latency Injection Experiment
type LatencyInjectionExperiment struct {
    targetService  string
    latencyMS      int
    jitterMS       int
    duration       time.Duration
}
 
func (e *LatencyInjectionExperiment) Execute(ctx context.Context) error {
    fmt.Printf("Injecting %dms latency (+/- %dms jitter) to %s\n", 
        e.latencyMS, e.jitterMS, e.targetService)
    
    // Use tc (traffic control) to inject latency
    cmd := fmt.Sprintf(
        "kubectl exec -n staging deployment/%s -- tc qdisc add dev eth0 root netem delay %dms %dms",
        e.targetService,
        e.latencyMS,
        e.jitterMS,
    )
    
    if err := executeCommand(cmd); err != nil {
        return fmt.Errorf("failed to inject latency: %w", err)
    }
    
    return nil
}
 
func (e *LatencyInjectionExperiment) Validate(ctx context.Context) (bool, error) {
    // Monitor service behaviour during latency injection
    ticker := time.NewTicker(10 * time.Second)
    defer ticker.Stop()
    
    timeout := time.After(e.duration)
    
    for {
        select {
        case <-timeout:
            return true, nil
            
        case <-ticker.C:
            metrics, err := getServiceMetrics(e.targetService)
            if err != nil {
                return false, err
            }
            
            // Check if circuit breakers activated
            if metrics.CircuitBreakerState == "open" {
                fmt.Println("✓ Circuit breaker correctly opened")
            }
            
            // Ensure service degraded gracefully
            if metrics.ErrorRate > 0.10 {
                return false, fmt.Errorf("service failed to degrade gracefully: %.2f%% error rate", 
                    metrics.ErrorRate*100)
            }
        }
    }
}
 
// Redis Failure Experiment
type RedisFailureExperiment struct {
    redisCluster   string
    failoverTest   bool
}
 
func (e *RedisFailureExperiment) Execute(ctx context.Context) error {
    if e.failoverTest {
        fmt.Printf("Testing Redis failover for cluster: %s\n", e.redisCluster)
        
        // Trigger a controlled failover
        cmd := fmt.Sprintf(
            "redis-cli -h %s-master CLUSTER FAILOVER",
            e.redisCluster,
        )
        
        return executeCommand(cmd)
    }
    
    return fmt.Errorf("not implemented")
}
 
func (e *RedisFailureExperiment) Validate(ctx context.Context) (bool, error) {
    // Monitor application behaviour during Redis failover
    time.Sleep(5 * time.Second)
    
    // Check that failover completed successfully
    cmd := fmt.Sprintf("redis-cli -h %s-master ROLE", e.redisCluster)
    output, err := executeCommandWithOutput(cmd)
    if err != nil {
        return false, err
    }
    
    if !contains(output, "master") {
        return false, fmt.Errorf("failover did not complete successfully")
    }
    
    // Check application metrics
    dependentServices := getServicesDependingOnRedis(e.redisCluster)
    
    for _, svc := range dependentServices {
        metrics, err := getServiceMetrics(svc)
        if err != nil {
            return false, err
        }
        
        // Ensure services handled failover gracefully
        if metrics.ErrorRate > 0.05 {
            return false, fmt.Errorf("service %s error rate %.2f%% during Redis failover", 
                svc, metrics.ErrorRate*100)
        }
    }
    
    return true, nil
}
 
// Experiment Runner
type ChaosRunner struct {
    experiments []ChaosExperiment
    results     []ExperimentResult
}
 
type ExperimentResult struct {
    Experiment    string
    Success       bool
    Duration      time.Duration
    Observations  []string
    Error         error
}
 
func (cr *ChaosRunner) RunExperiments(ctx context.Context) error {
    for _, experiment := range cr.experiments {
        fmt.Printf("\n%s\n", strings.Repeat("=", 60))
        fmt.Printf("Running experiment: %s\n", experiment.Name)
        fmt.Printf("Hypothesis: %s\n", experiment.Hypothesis)
        fmt.Printf("%s\n\n", strings.Repeat("=", 60))
        
        startTime := time.Now()
        
        // Execute the experiment
        if err := experiment.Executor.Execute(ctx); err != nil {
            cr.recordResult(ExperimentResult{
                Experiment: experiment.Name,
                Success:    false,
                Duration:   time.Since(startTime),
                Error:      err,
            })
            continue
        }
        
        // Validate hypothesis
        success, err := experiment.Executor.Validate(ctx)
        
        // Always attempt rollback
        if rollbackErr := experiment.Executor.Rollback(ctx); rollbackErr != nil {
            fmt.Printf("Warning: Rollback failed: %v\n", rollbackErr)
        }
        
        duration := time.Since(startTime)
        
        result := ExperimentResult{
            Experiment: experiment.Name,
            Success:    success,
            Duration:   duration,
            Error:      err,
        }
        
        cr.recordResult(result)
        
        if success {
            fmt.Printf("\n✓ Experiment PASSED in %v\n", duration)
        } else {
            fmt.Printf("\n✗ Experiment FAILED: %v\n", err)
        }
    }
    
    cr.generateReport()
    return nil
}

# Incident: booking-service High Latency
 
## Quick Context
- **Service**: booking-service
- **Current P95**: 1,250ms (threshold: 300ms)
- **Error Rate**: 2.3% (threshold: 1%)
- **Affected Users**: ~450/minute
- **Similar Incidents**: INC-2024-08-723, INC-2024-07-612
 
## Immediate Actions (5 minutes)
 
### 1. Check Redis Performance
```bash
# SSH into Redis dashboard or use web UI
kubectl port-forward svc/redis-commander 8081:8081 -n cache
 
# Check for:
- High memory usage (>80%)
- Key eviction rate
- Connection count

kubectl logs -n bookings deployment/booking-service --tail=100 | grep "pool"

# Payment service health
curl https://payment-service.internal/health
 
# Availability service health  
curl https://availability-service.internal/health

# Current replica count
kubectl get deployment booking-service -n bookings
 
# Scale up by 50%
kubectl scale deployment booking-service --replicas=15 -n bookings

# Check recent deployments
kubectl rollout history deployment/booking-service -n bookings
 
# Compare current vs previous version
git diff <previous-sha> <current-sha>

-- Check for slow queries
SELECT query, calls, mean_exec_time, max_exec_time
FROM pg_stat_statements
WHERE query LIKE '%booking%'
ORDER BY mean_exec_time DESC
LIMIT 10;

# Rollback to previous version
kubectl rollout undo deployment/booking-service -n bookings
 
# Monitor rollout
kubectl rollout status deployment/booking-service -n bookings

# Update ConfigMap
kubectl edit configmap booking-service-config -n bookings
 
# Restart pods to pick up changes
kubectl rollout restart deployment/booking-service -n bookings

# Runbook: High Latency Response
 
## Quick Reference
**Severity**: HIGH  
**MTTR Target**: 15 minutes  
**On-call**: @booking-team  
 
## Symptoms
- P95 latency > 500ms
- User-reported slowness
- Increased timeout errors
 
## Immediate Actions (First 5 Minutes)
 
### 1. Check System Health
```bash
kubectl get pods -n <namespace> -l app=<service>
kubectl top pods -n <namespace> -l app=<service>
kubectl logs -n <namespace> deployment/<service> --tail=100

# Redis health
curl https://redis.internal/health
 
# Database health
psql -h db.internal -c "SELECT 1;"

# Scale up pods (if CPU/memory high)
kubectl scale deployment/<service> --replicas=<current+50%>
 
# Restart pods (if memory leak suspected)
kubectl rollout restart deployment/<service>

kubectl rollout undo deployment/<service>
kubectl rollout status deployment/<service>

CURRENT=$(kubectl get deployment <service> -o jsonpath='{.spec.replicas}')
TARGET=$((CURRENT * 3 / 2))
kubectl scale deployment/<service> --replicas=$TARGET

# Get pod status with custom columns
kubectl get pods -n <namespace> \
  -o custom-columns=NAME:.metadata.name,STATUS:.status.phase,RESTARTS:.status.containerStatuses[0].restartCount
 
# Get resource usage
kubectl top pods -n <namespace> --sort-by=memory
 
# Get logs from all pods matching label
kubectl logs -n <namespace> -l app=<service> --tail=100 -f
 
# Port forward to service
kubectl port-forward -n <namespace> svc/<service> 8080:80
 
# Get deployment rollout status
kubectl rollout status deployment/<name> -n <namespace>
 
# Rollback deployment
kubectl rollout undo deployment/<name> -n <namespace>
 
# Scale deployment
kubectl scale deployment/<name> --replicas=<count> -n <namespace>

# Connect to Redis
redis-cli -h <host> -p <port>
 
# Get memory usage
redis-cli INFO memory
 
# Scan for keys
redis-cli --scan --pattern 'user:*'
 
# Monitor commands
redis-cli MONITOR
 
# Get slow log
redis-cli SLOWLOG GET 10

-- Check connection count
SELECT state, COUNT(*) 
FROM pg_stat_activity 
WHERE datname = '<database>'
GROUP BY state;
 
-- Find slow queries
SELECT 
    query,
    mean_exec_time,
    calls
FROM pg_stat_statements
WHERE mean_exec_time > 1000
ORDER BY mean_exec_time DESC
LIMIT 20;