Intelligent Error Recovery and Self-Healing Workflows

Overview

As Claude Code agents take on more autonomous responsibilities, building robust error recovery and self-healing capabilities becomes critical. This guide presents comprehensive patterns for creating resilient agents that can detect, diagnose, and recover from failures automatically, minimizing downtime and human intervention.

Core Self-Healing Architecture

Three-Pillar Framework

graph TD
    A[Detection] --> B[Diagnosis]
    B --> C[Recovery]
    C --> D[Learning]
    D --> A

1. Detection: Continuous monitoring and anomaly identification
2. Diagnosis: Root cause analysis and impact assessment
3. Recovery: Automated remediation and validation
4. Learning: Pattern recognition and prevention

Error Detection Patterns

1. Proactive Health Monitoring

// Health check configuration
interface HealthCheck {
  name: string;
  interval: number; // seconds
  timeout: number;
  check: () => Promise<HealthStatus>;
  onFailure: (error: Error) => Promise<void>;
}
 
class AgentHealthMonitor {
  private checks: HealthCheck[] = [];
  private status = new Map<string, HealthStatus>();
  
  async runHealthChecks() {
    for (const check of this.checks) {
      try {
        const result = await Promise.race([
          check.check(),
          this.timeout(check.timeout)
        ]);
        this.status.set(check.name, result);
      } catch (error) {
        await this.handleFailure(check, error);
      }
    }
  }
  
  private async handleFailure(check: HealthCheck, error: Error) {
    console.error(`Health check failed: ${check.name}`, error);
    await check.onFailure(error);
    await this.triggerRecovery(check.name, error);
  }
}

2. Pattern-Based Anomaly Detection

// Anomaly detection using statistical analysis
class AnomalyDetector {
  private metrics: Map<string, number[]> = new Map();
  private thresholds = {
    responseTime: { mean: 1000, stdDev: 2 },
    errorRate: { threshold: 0.05 },
    memoryUsage: { max: 0.85 }
  };
  
  async detectAnomalies(metric: string, value: number): Promise<Anomaly[]> {
    const history = this.metrics.get(metric) || [];
    history.push(value);
    
    if (history.length > 100) {
      history.shift(); // Keep sliding window
    }
    
    const anomalies = [];
    
    // Statistical anomaly detection
    const stats = this.calculateStats(history);
    if (this.isStatisticalAnomaly(value, stats)) {
      anomalies.push({
        type: 'statistical',
        metric,
        value,
        severity: this.calculateSeverity(value, stats)
      });
    }
    
    // Threshold-based detection
    if (this.isThresholdViolation(metric, value)) {
      anomalies.push({
        type: 'threshold',
        metric,
        value,
        severity: 'high'
      });
    }
    
    return anomalies;
  }
}

3. Error Pattern Recognition

# Error pattern definitions
error_patterns:
  - name: "API Rate Limit"
    indicators:
      - status_code: 429
      - headers.contains: "X-RateLimit-Remaining: 0"
    recovery:
      strategy: "exponential_backoff"
      max_retries: 5
      
  - name: "Database Connection Lost"
    indicators:
      - error_message.contains: "ECONNREFUSED"
      - service: "database"
    recovery:
      strategy: "connection_pool_reset"
      fallback: "cache_mode"
      
  - name: "Memory Leak"
    indicators:
      - memory_usage.trend: "increasing"
      - gc_frequency: "high"
    recovery:
      strategy: "graceful_restart"
      preserve_state: true

Intelligent Diagnosis

1. Root Cause Analysis

// Automated root cause analysis
class RootCauseAnalyzer {
  private dependencyGraph: DependencyGraph;
  private eventLog: EventLog;
  
  async analyzeFailure(error: Error): Promise<RootCause> {
    // Trace error through dependency graph
    const affectedServices = this.traceImpact(error);
    
    // Correlate with recent events
    const correlatedEvents = await this.correlateEvents(
      error.timestamp,
      affectedServices
    );
    
    // Apply causal inference
    const probableCauses = this.inferCausality(
      error,
      correlatedEvents,
      affectedServices
    );
    
    return {
      primaryCause: probableCauses[0],
      contributingFactors: probableCauses.slice(1),
      affectedServices,
      recommendedActions: this.generateRecoveryPlan(probableCauses[0])
    };
  }
  
  private inferCausality(
    error: Error, 
    events: Event[], 
    services: Service[]
  ): Cause[] {
    // Machine learning-based causality inference
    const features = this.extractFeatures(error, events, services);
    const predictions = this.causalModel.predict(features);
    
    return predictions
      .filter(p => p.confidence > 0.7)
      .sort((a, b) => b.confidence - a.confidence);
  }
}

2. Impact Assessment

// Assess the blast radius of failures
interface ImpactAssessment {
  severity: 'low' | 'medium' | 'high' | 'critical';
  affectedComponents: string[];
  userImpact: number; // Percentage of users affected
  dataIntegrity: 'intact' | 'at_risk' | 'compromised';
  estimatedRecoveryTime: number; // minutes
}
 
async function assessImpact(failure: Failure): Promise<ImpactAssessment> {
  const dependencies = await traceDependencies(failure.component);
  const activeUsers = await getActiveUserCount();
  const affectedUsers = await estimateAffectedUsers(failure, dependencies);
  
  return {
    severity: calculateSeverity(failure, affectedUsers / activeUsers),
    affectedComponents: dependencies.map(d => d.name),
    userImpact: (affectedUsers / activeUsers) * 100,
    dataIntegrity: await checkDataIntegrity(failure),
    estimatedRecoveryTime: estimateRecoveryTime(failure)
  };
}

Self-Healing Strategies

1. Automated Recovery Actions

// Recovery strategy implementation
class RecoveryEngine {
  private strategies = new Map<string, RecoveryStrategy>();
  
  async executeRecovery(
    diagnosis: Diagnosis
  ): Promise<RecoveryResult> {
    const strategy = this.selectStrategy(diagnosis);
    
    // Pre-recovery snapshot
    const snapshot = await this.createSnapshot();
    
    try {
      // Execute recovery steps
      const result = await strategy.execute(diagnosis);
      
      // Validate recovery
      if (await this.validateRecovery(result)) {
        return { status: 'success', ...result };
      } else {
        throw new Error('Recovery validation failed');
      }
    } catch (error) {
      // Rollback on failure
      await this.rollback(snapshot);
      return { status: 'failed', error, fallback: true };
    }
  }
  
  private selectStrategy(diagnosis: Diagnosis): RecoveryStrategy {
    // ML-based strategy selection
    const features = this.extractDiagnosisFeatures(diagnosis);
    const recommendedStrategy = this.strategyModel.predict(features);
    
    return this.strategies.get(recommendedStrategy) || 
           this.strategies.get('generic');
  }
}

2. Common Recovery Patterns

// Recovery pattern implementations
const recoveryPatterns = {
  // Circuit breaker pattern
  circuitBreaker: {
    async execute(service: Service) {
      const breaker = new CircuitBreaker(service, {
        timeout: 3000,
        errorThreshold: 50,
        resetTimeout: 30000
      });
      
      breaker.on('open', () => {
        console.log('Circuit opened, using fallback');
        return this.fallbackService();
      });
      
      return breaker;
    }
  },
  
  // Graceful degradation
  gracefulDegradation: {
    async execute(failure: Failure) {
      const criticalFeatures = await identifyCriticalFeatures();
      const optionalFeatures = await identifyOptionalFeatures();
      
      // Disable non-critical features
      for (const feature of optionalFeatures) {
        await feature.disable();
      }
      
      // Ensure critical features remain operational
      for (const feature of criticalFeatures) {
        await feature.enableFallbackMode();
      }
    }
  },
  
  // State restoration
  stateRestoration: {
    async execute(component: Component) {
      const lastHealthyState = await getLastHealthyState(component);
      const currentState = await getCurrentState(component);
      
      const stateDiff = calculateStateDiff(lastHealthyState, currentState);
      
      if (stateDiff.isRecoverable) {
        await applyStatePatch(component, stateDiff.patch);
      } else {
        await fullStateRestore(component, lastHealthyState);
      }
    }
  }
};

3. Self-Repair Mechanisms

// Automated code fixing
class SelfRepairEngine {
  async repairCode(error: CodeError): Promise<Repair> {
    // Analyze error context
    const context = await this.analyzeErrorContext(error);
    
    // Generate fix candidates
    const candidates = await this.generateFixCandidates(context);
    
    // Test each candidate
    for (const candidate of candidates) {
      const testResult = await this.testFix(candidate);
      
      if (testResult.success) {
        // Apply fix
        await this.applyFix(candidate);
        
        // Verify fix
        if (await this.verifyFix(error, candidate)) {
          return {
            status: 'repaired',
            fix: candidate,
            confidence: testResult.confidence
          };
        }
      }
    }
    
    return { status: 'manual_intervention_required' };
  }
  
  private async generateFixCandidates(
    context: ErrorContext
  ): Promise<Fix[]> {
    const fixes = [];
    
    // Pattern-based fixes
    const knownPatterns = await this.matchKnownPatterns(context);
    fixes.push(...knownPatterns);
    
    // AI-generated fixes
    const aiSuggestions = await this.aiGenerateFixes(context);
    fixes.push(...aiSuggestions);
    
    // Historical fixes
    const historicalFixes = await this.findSimilarHistoricalFixes(context);
    fixes.push(...historicalFixes);
    
    return this.rankFixes(fixes);
  }
}

Claude Code Specific Implementations

1. Hook-Based Error Recovery

// Error recovery hook
export const errorRecoveryHook = {
  name: "self-healing-error-recovery",
  on: ["error", "command.failed"],
  
  async execute(context) {
    const { error, command } = context;
    
    // Categorize error
    const errorType = categorizeError(error);
    
    // Select recovery strategy
    const strategy = selectRecoveryStrategy(errorType);
    
    // Execute recovery
    const result = await executeRecovery(strategy, {
      error,
      command,
      context: context.getState()
    });
    
    if (result.success) {
      // Retry original command
      return context.retry();
    } else {
      // Escalate to user
      return context.requestUserIntervention({
        error,
        attemptedRecovery: result,
        suggestions: generateSuggestions(error)
      });
    }
  }
};

2. Persistent State Management

# .claude/recovery-state.yml
recovery:
  checkpoints:
    enabled: true
    interval: 300  # seconds
    retention: 24  # hours
    
  state_tracking:
    - file_modifications
    - command_history
    - conversation_context
    - tool_usage
    
  rollback:
    enabled: true
    confirmation_required: false
    preserve_user_edits: true

3. Intelligent Retry Logic

// Smart retry with learning
class IntelligentRetry {
  private retryHistory = new Map<string, RetryRecord[]>();
  
  async retry(
    operation: () => Promise<any>,
    context: OperationContext
  ): Promise<any> {
    const history = this.retryHistory.get(context.operationType) || [];
    const strategy = this.determineStrategy(history, context);
    
    let lastError;
    for (let attempt = 0; attempt < strategy.maxAttempts; attempt++) {
      try {
        // Apply pre-retry modifications
        await this.applyLearnings(context, history);
        
        // Execute with timeout
        const result = await Promise.race([
          operation(),
          this.timeout(strategy.timeout)
        ]);
        
        // Record success
        this.recordSuccess(context, attempt);
        return result;
        
      } catch (error) {
        lastError = error;
        
        // Record failure
        this.recordFailure(context, error, attempt);
        
        // Check if retry is worthwhile
        if (!this.shouldRetry(error, attempt, strategy)) {
          break;
        }
        
        // Wait before retry
        await this.wait(strategy.backoff(attempt));
      }
    }
    
    throw new RetryExhaustedError(lastError, context);
  }
  
  private determineStrategy(
    history: RetryRecord[],
    context: OperationContext
  ): RetryStrategy {
    // ML-based strategy selection
    const features = this.extractFeatures(history, context);
    const prediction = this.strategyModel.predict(features);
    
    return {
      maxAttempts: prediction.attempts,
      timeout: prediction.timeout,
      backoff: this.selectBackoffFunction(prediction.backoffType)
    };
  }
}

Monitoring and Alerting

1. Self-Healing Dashboard

// Real-time self-healing metrics
interface SelfHealingMetrics {
  totalIncidents: number;
  autoResolved: number;
  manualInterventions: number;
  averageRecoveryTime: number;
  preventedIncidents: number;
  healthScore: number;
  
  byCategory: {
    [category: string]: {
      count: number;
      successRate: number;
      avgRecoveryTime: number;
    };
  };
}
 
// Dashboard configuration
const dashboardConfig = {
  widgets: [
    {
      type: 'health-score',
      refreshInterval: 5000,
      thresholds: { good: 90, warning: 70, critical: 50 }
    },
    {
      type: 'incident-timeline',
      timeRange: '24h',
      showRecoveryActions: true
    },
    {
      type: 'pattern-analysis',
      displayTop: 10,
      includePreventionSuggestions: true
    }
  ]
};

2. Predictive Failure Detection

// Predict failures before they occur
class PredictiveMonitor {
  private model: FailurePredictionModel;
  private threshold = 0.8;
  
  async predictFailures(
    metrics: SystemMetrics
  ): Promise<PredictionResult[]> {
    const features = this.extractPredictiveFeatures(metrics);
    const predictions = await this.model.predict(features);
    
    return predictions
      .filter(p => p.probability > this.threshold)
      .map(p => ({
        component: p.component,
        failureType: p.type,
        probability: p.probability,
        estimatedTimeToFailure: p.ttf,
        preventiveActions: this.generatePreventiveActions(p)
      }));
  }
  
  private generatePreventiveActions(
    prediction: Prediction
  ): PreventiveAction[] {
    const actions = [];
    
    switch (prediction.type) {
      case 'memory_exhaustion':
        actions.push({
          name: 'preemptive_gc',
          timing: prediction.ttf * 0.5,
          command: 'claude-code gc --aggressive'
        });
        break;
        
      case 'rate_limit':
        actions.push({
          name: 'throttle_requests',
          timing: 'immediate',
          command: 'claude-code throttle --rate=0.5'
        });
        break;
        
      // More cases...
    }
    
    return actions;
  }
}

Best Practices

1. Fail Fast, Recover Faster

// Quick failure detection and recovery
const quickRecoveryConfig = {
  healthChecks: {
    interval: 5000,  // 5 seconds
    timeout: 1000,   // 1 second
    fastFail: true
  },
  
  recovery: {
    maxDuration: 30000,  // 30 seconds max recovery time
    parallelStrategies: true,
    abortOnTimeout: true
  }
};

2. Learning from Failures

// Continuous improvement through failure analysis
class FailureLearningSystem {
  async learnFromIncident(incident: Incident) {
    // Extract patterns
    const patterns = await this.extractPatterns(incident);
    
    // Update prevention rules
    await this.updatePreventionRules(patterns);
    
    // Improve recovery strategies
    await this.optimizeRecoveryStrategies(incident);
    
    // Generate postmortem
    const postmortem = await this.generatePostmortem(incident);
    
    // Share learnings
    await this.distributeLearnings(postmortem);
  }
}

3. Graceful Degradation Levels

# Degradation levels configuration
degradation_levels:
  - level: 0
    name: "full_service"
    features: ["all"]
    
  - level: 1
    name: "reduced_performance"
    features: ["core", "essential"]
    disabled: ["analytics", "recommendations"]
    
  - level: 2
    name: "essential_only"
    features: ["core"]
    disabled: ["all_optional"]
    
  - level: 3
    name: "read_only"
    features: ["read"]
    disabled: ["write", "modify", "delete"]
    
  - level: 4
    name: "maintenance"
    features: []
    message: "System under maintenance"

Common Failure Scenarios

1. API Rate Limiting

// Intelligent rate limit handling
const rateLimitRecovery = {
  detect: (error) => error.status === 429,
  
  recover: async (context) => {
    const retryAfter = parseRetryAfter(context.response);
    const backoffTime = Math.max(retryAfter, 60000);
    
    // Switch to alternative API if available
    if (context.hasAlternativeAPI()) {
      return context.useAlternativeAPI();
    }
    
    // Queue requests for batch processing
    await context.queueRequest();
    
    // Wait and retry with reduced rate
    await wait(backoffTime);
    return context.retryWithReducedRate(0.5);
  }
};

2. Memory Exhaustion

// Memory pressure recovery
const memoryRecovery = {
  detect: () => process.memoryUsage().heapUsed / 
               process.memoryUsage().heapTotal > 0.9,
  
  recover: async () => {
    // Clear caches
    await clearAllCaches();
    
    // Force garbage collection
    if (global.gc) global.gc();
    
    // Reduce concurrent operations
    await throttleConcurrency(0.5);
    
    // If still high, restart gracefully
    if (stillHighMemory()) {
      await gracefulRestart();
    }
  }
};

Testing Self-Healing Systems

Chaos Engineering

# Inject failures to test recovery
claude-code chaos inject \
  --type=network-partition \
  --duration=60s \
  --target=api-service
 
claude-code chaos inject \
  --type=cpu-spike \
  --severity=high \
  --duration=120s
 
claude-code chaos inject \
  --type=memory-leak \
  --rate=10mb/s \
  --max=500mb

Recovery Testing

// Automated recovery testing
describe('Self-Healing System', () => {
  it('should recover from API failures', async () => {
    // Inject failure
    mockAPI.failNext(3);
    
    // Execute operation
    const result = await systemUnderTest.execute();
    
    // Verify recovery
    expect(result).toBeSuccessful();
    expect(mockAPI.calls).toBe(4); // 3 failures + 1 success
  });
  
  it('should prevent cascade failures', async () => {
    // Trigger failure in one component
    await componentA.fail();
    
    // Verify other components remain operational
    expect(await componentB.health()).toBe('healthy');
    expect(await componentC.health()).toBe('degraded');
  });
});

Related Resources

• Error Recovery Patterns
• Remote Agent Supervision
• Observability Patterns
• Microsoft AIOpsLab Framework
• Self-Healing AI Systems Research

---

This guide represents cutting-edge patterns in self-healing systems for Claude Code agents, incorporating the latest research and production experiences from 2025.