groups:
  - name: roster.recording
    interval: 30s
    rules:
      - record: roster:http_requests:rate5m
        expr: sum by (http_route, roster_outcome) (rate(roster_http_server_request_count_total[5m]))
      - record: roster:http_errors:ratio5m
        expr: (sum(rate(roster_http_server_request_count_total{roster_outcome="error"}[5m])) or vector(0)) / clamp_min(sum(rate(roster_http_server_request_count_total[5m])), 1e-9)
      - record: roster:http_duration:p95_5m
        expr: histogram_quantile(0.95, sum by (le, http_route) (rate(roster_http_server_request_duration_seconds_bucket[5m])))
      - record: roster:resolve_requests:rate5m
        expr: sum by (roster_outcome, roster_resolve_mode, roster_surface) (rate(roster_resolve_request_count_total[5m]))
      - record: roster:resolve_duration:p95_5m
        expr: histogram_quantile(0.95, sum by (le, roster_resolve_mode, roster_surface) (rate(roster_resolve_request_duration_seconds_bucket[5m])))
      - record: roster:resolve_errors:ratio5m
        expr: (sum(rate(roster_resolve_request_count_total{roster_outcome="error"}[5m])) or vector(0)) / clamp_min(sum(rate(roster_resolve_request_count_total[5m])), 1e-9)
      - record: roster:resolve_yield:ratio5m
        expr: (sum(rate(roster_resolve_request_count_total{roster_outcome="success"}[5m])) or vector(0)) / clamp_min(sum(rate(roster_resolve_request_count_total[5m])), 1e-9)
      - record: roster:llm_tokens:rate5m
        expr: sum by (gen_ai_provider_name, gen_ai_token_type) (rate(roster_llm_token_usage_total[5m]))

  - name: roster.operational-alerts
    rules:
      - alert: RosterWorkerBacklogGrowing
        expr: max(roster_worker_queue_depth_job{roster_queue_state="queued"}) > 100 and max(roster_worker_queue_oldest_age_seconds) > 900
        for: 15m
        labels:
          severity: warning
        annotations:
          summary: Roster worker backlog is growing
          description: More than 100 jobs have been queued and the oldest queued job is over 15 minutes old.
      - alert: RosterWorkerLeaseLosses
        expr: increase(roster_worker_lease_loss_count_total[10m]) > 0
        for: 2m
        labels:
          severity: warning
        annotations:
          summary: Roster workers are losing job leases
          description: Investigate database latency, worker pauses, or lease configuration.
      - alert: RosterTelemetryExportFailures
        expr: increase(roster_telemetry_export_error_count_total[15m]) > 0
        for: 5m
        labels:
          severity: warning
        annotations:
          summary: Roster reported OTLP delivery failures
          description: This confirms failed delivery attempts to the configured OTLP endpoint; it does not prove downstream persistence.