Метрики

Счётчики API в формате Prometheus (/api/metrics/prometheus). В Docker Compose подняты сервисы prometheus (:9090), grafana (:3030, логин admin / admin) и loki (:3100). Поток логов в Loki: docker compose --profile logs up -d (Linux Engine, см. .env.example). Единый вход через Caddy: docker compose --profile edge up -d → https://tenderai.mrboroai.ru.

Снимок Prometheus (из API)

Состояние: ok

# HELP tender_runs_completed_total Total tender runs completed successfully.
# TYPE tender_runs_completed_total counter
tender_runs_completed_total 0
# HELP tender_runs_failed_total Total tender runs ended with PARTIAL/FAILED stages.
# TYPE tender_runs_failed_total counter
tender_runs_failed_total 0
# HELP bitrix_webhook_accepted_total Bitrix webhook accepted (non-duplicate).
# TYPE bitrix_webhook_accepted_total counter
bitrix_webhook_accepted_total 3
# HELP tender_pipeline_metrics_snapshot_timestamp_seconds Unix time of the last successful pipeline operational snapshot.
# TYPE tender_pipeline_metrics_snapshot_timestamp_seconds gauge
tender_pipeline_metrics_snapshot_timestamp_seconds 1791486950
# HELP tender_pipeline_metrics_snapshot_healthy 1 when the last pipeline operational snapshot succeeded, 0 when it failed.
# TYPE tender_pipeline_metrics_snapshot_healthy gauge
tender_pipeline_metrics_snapshot_healthy 1
# HELP tender_pipeline_stage_pending_backlog Latest-run stages waiting to start, per stage.
# TYPE tender_pipeline_stage_pending_backlog gauge
# HELP tender_pipeline_stage_pending_oldest_age_seconds Age of the oldest stage waiting to start, per stage.
# TYPE tender_pipeline_stage_pending_oldest_age_seconds gauge
# HELP tender_pipeline_stage_active Latest-run stages currently RUNNING/RETRYING, per stage.
# TYPE tender_pipeline_stage_active gauge
# HELP tender_pipeline_stage_active_oldest_age_seconds Age since the oldest active stage started, per stage.
# TYPE tender_pipeline_stage_active_oldest_age_seconds gauge
# HELP tender_pipeline_stage_active_oldest_progress_age_seconds Age since the oldest active stage last made progress (EXTRACT uses the newest DEEPSEEK_PRODUCT_PASSPORT_IO artifact), per stage.
# TYPE tender_pipeline_stage_active_oldest_progress_age_seconds gauge
tender_pipeline_stage_pending_backlog{stage="INTAKE"} 0
tender_pipeline_stage_pending_oldest_age_seconds{stage="INTAKE"} 0
tender_pipeline_stage_active{stage="INTAKE"} 0
tender_pipeline_stage_active_oldest_age_seconds{stage="INTAKE"} 0
tender_pipeline_stage_active_oldest_progress_age_seconds{stage="INTAKE"} 0
tender_pipeline_stage_pending_backlog{stage="DOCUMENT_ACQUISITION"} 0
tender_pipeline_stage_pending_oldest_age_seconds{stage="DOCUMENT_ACQUISITION"} 0
tender_pipeline_stage_active{stage="DOCUMENT_ACQUISITION"} 0
tender_pipeline_stage_active_oldest_age_seconds{stage="DOCUMENT_ACQUISITION"} 0
tender_pipeline_stage_active_oldest_progress_age_seconds{stage="DOCUMENT_ACQUISITION"} 0
tender_pipeline_stage_pending_backlog{stage="ROUTING"} 0
tender_pipeline_stage_pending_oldest_age_seconds{stage="ROUTING"} 0
tender_pipeline_stage_active{stage="ROUTING"} 0
tender_pipeline_stage_active_oldest_age_seconds{stage="ROUTING"} 0
tender_pipeline_stage_active_oldest_progress_age_seconds{stage="ROUTING"} 0
tender_pipeline_stage_pending_backlog{stage="PARSE"} 0
tender_pipeline_stage_pending_oldest_age_seconds{stage="PARSE"} 0
tender_pipeline_stage_active{stage="PARSE"} 0
tender_pipeline_stage_active_oldest_age_seconds{stage="PARSE"} 0
tender_pipeline_stage_active_oldest_progress_age_seconds{stage="PARSE"} 0
tender_pipeline_stage_pending_backlog{stage="EXTRACT"} 0
tender_pipeline_stage_pending_oldest_age_seconds{stage="EXTRACT"} 0
tender_pipeline_stage_active{stage="EXTRACT"} 0
tender_pipeline_stage_active_oldest_age_seconds{stage="EXTRACT"} 0
tender_pipeline_stage_active_oldest_progress_age_seconds{stage="EXTRACT"} 0
tender_pipeline_stage_pending_backlog{stage="CONSOLIDATE"} 0
tender_pipeline_stage_pending_oldest_age_seconds{stage="CONSOLIDATE"} 0
tender_pipeline_stage_active{stage="CONSOLIDATE"} 0
tender_pipeline_stage_active_oldest_age_seconds{stage="CONSOLIDATE"} 0
tender_pipeline_stage_active_oldest_progress_age_seconds{stage="CONSOLIDATE"} 0
tender_pipeline_stage_pending_backlog{stage="CATALOG_MATCH"} 0
tender_pipeline_stage_pending_oldest_age_seconds{stage="CATALOG_MATCH"} 0
tender_pipeline_stage_active{stage="CATALOG_MATCH"} 0
tender_pipeline_stage_active_oldest_age_seconds{stage="CATALOG_MATCH"} 0
tender_pipeline_stage_active_oldest_progress_age_seconds{stage="CATALOG_MATCH"} 0
tender_pipeline_stage_pending_backlog{stage="EXTERNAL_SEARCH"} 0
tender_pipeline_stage_pending_oldest_age_seconds{stage="EXTERNAL_SEARCH"} 0
tender_pipeline_stage_active{stage="EXTERNAL_SEARCH"} 0
tender_pipeline_stage_active_oldest_age_seconds{stage="EXTERNAL_SEARCH"} 0
tender_pipeline_stage_active_oldest_progress_age_seconds{stage="EXTERNAL_SEARCH"} 0
tender_pipeline_stage_pending_backlog{stage="DECISION"} 0
tender_pipeline_stage_pending_oldest_age_seconds{stage="DECISION"} 0
tender_pipeline_stage_active{stage="DECISION"} 0
tender_pipeline_stage_active_oldest_age_seconds{stage="DECISION"} 0
tender_pipeline_stage_active_oldest_progress_age_seconds{stage="DECISION"} 0
tender_pipeline_stage_pending_backlog{stage="WRITEBACK"} 0
tender_pipeline_stage_pending_oldest_age_seconds{stage="WRITEBACK"} 0
tender_pipeline_stage_active{stage="WRITEBACK"} 0
tender_pipeline_stage_active_oldest_age_seconds{stage="WRITEBACK"} 0
tender_pipeline_stage_active_oldest_progress_age_seconds{stage="WRITEBACK"} 0
# HELP tender_writeback_outbox_undelivered Bitrix writeback outbox rows still undelivered, excluding intentional dry-runs.
# TYPE tender_writeback_outbox_undelivered gauge
tender_writeback_outbox_undelivered{status="PENDING"} 0
tender_writeback_outbox_undelivered{status="FAILED"} 3
tender_writeback_outbox_undelivered{status="PARTIAL"} 0
# HELP tender_writeback_outbox_oldest_undelivered_age_seconds Age of the oldest undelivered writeback outbox row, excluding intentional dry-runs.
# TYPE tender_writeback_outbox_oldest_undelivered_age_seconds gauge
tender_writeback_outbox_oldest_undelivered_age_seconds 25546.115