Alert Management

Active Prometheus Alert Rules

groups:
  - name: trading.rules
    rules:
    - alert: HighExecutionLatencyP99
      expr: histogram_quantile(0.99,
              rate(order_execution_latency_seconds_bucket[1m])) > 0.03
      for: 1m
      labels:
        severity: warning
      annotations:
        summary: "P99 latency above 30ms"

    - alert: PodCrashLoopBackOff
      expr: kube_pod_container_status_waiting_reason{
              namespace=~"trading|streaming",
              reason="CrashLoopBackOff"} == 1
      for: 2m
      labels:
        severity: critical

    - alert: KafkaConsumerLag
      expr: kafka_consumergroup_lag{namespace="trading"} > 100
      for: 30s
      labels:
        severity: warning

    - alert: HighRejectionRate
      expr: rate(orders_rejected_total[5m])
              / rate(orders_total[5m]) > 0.01
      for: 2m
      labels:
        severity: warning