Add alerting for CTFE V2.

alertserver/alerts.cfg

 # This file defines alerts to be triggered by the server.
 
 #
 # SkiaPerf and SkiaGold
 #
 
 [[rule]]
 name = "Perf Alerts"
 message = "At least one perf alert has been found. Please visit https://perf.skia.org/alerts/ to triage."
 query = "select value from /skiaperf.skia-perf.alerting.new.value/ limit 1"
@@ skipping 371 matching lines @@
 [[rule]]
 name = "Low Disk Space (skia-status /mnt/pd0)"
 message = "Free space has fallen below 5GB on skia-status (/mnt/pd0)."
 query = "select mean(value) from /collectd.skia-status.df-mnt-pd0.df_complex-free/ where time > now() - 5m;"
 category = "infra"
 condition = "x <= 5e9"
 actions = ["Email(infra-alerts@skia.org)"]
 auto-dismiss = true
 nag = "1h"
 
+[[rule]]
+name = "Low Disk Space (skia-ctfe)"
+message = "Free space has fallen below 1GB on skia-ctfe (root)."
+query = "select mean(value) from /collectd.skia-ctfe.df-root.df_complex-free/ where time > now() - 5m;"
+category = "infra"
+condition = "x <= 1e9"
+actions = ["Email(infra-alerts@skia.org)"]
+auto-dismiss = true
+nag = "1h"
+
 #
 # Skia Status
 #
 
 [[rule]]
 name = "Skia Status Prober (main page)"
 message = "The main page at https://status.skia.org has failed."
 query = "select mean(value) from /prober.skiastatus.failure.value/ where time > now() - 10m;"
 category = "infra"
 condition = "x >= 1"
@@ skipping 115 matching lines @@
 
 [[rule]]
 name = "Datahopper Buildbot Ingestion Stalled (client.skia.fyi)"
 message = "Buildbot Ingestion in Datahopper has failed to run in at least 10 minutes for client.skia.fyi."
 query = "select mean(value) from /datahopper.skia-datahopper.buildbot-ingest-client.skia.fyi.time-since-last-successful-update.value/ where time > now() - 10m"
 category = "infra"
 condition = "x >= 600"
 actions = ["Email(infra-alerts@skia.org)"]
 auto-dismiss = true
 nag = "1h"
+
+#
+# CTFE
+#
+
+[[rule]]
+name = "CTFE Staging Prober (main page)"
+message = "The main page at http://ct-staging.skia.org is unavailable."
+query = "select mean(value) from /prober.ctfe_staging.failure.value/ where time > now() - 10m;"
+category = "infra"
+condition = "x >= 1"
+actions = ["Email(infra-alerts@skia.org)"]
+auto-dismiss = true
+nag = "1h"
+
+[[rule]]
+name = "CTFE Staging Prober (chromium_perf)"
+message = "The page at https://ct-staging.skia.org/chromium_perf/ is unavailable."
+query = "select mean(value) from /prober.ctfe_staging_chromium_perf.failure.value/ where time > now() - 10m;"
+category = "infra"
+condition = "x >= 1"
+actions = ["Email(infra-alerts@skia.org)"]
+auto-dismiss = true
+nag = "1h"
+
+[[rule]]
+name = "CTFE Staging Prober (capture_skps)"
+message = "The page at https://ct-staging.skia.org/capture_skps/ is unavailable."
+query = "select mean(value) from /prober.ctfe_staging_capture_skps.failure.value/ where time > now() - 10m;"
+category = "infra"
+condition = "x >= 1"
+actions = ["Email(infra-alerts@skia.org)"]
+auto-dismiss = true
+nag = "1h"
+
+[[rule]]
+name = "CTFE Staging Prober (lua_script)"
+message = "The page at https://ct-staging.skia.org/lua_script/ is unavailable."
+query = "select mean(value) from /prober.ctfe_staging_lua_script.failure.value/ where time > now() - 10m;"
+category = "infra"
+condition = "x >= 1"
+actions = ["Email(infra-alerts@skia.org)"]
+auto-dismiss = true
+nag = "1h"
+
+[[rule]]
+name = "CTFE Staging Prober (chromium_builds)"
+message = "The page at https://ct-staging.skia.org/chromium_builds/ is unavailable."
+query = "select mean(value) from /prober.ctfe_staging_chromium_builds.failure.value/ where time > now() - 10m;"
+category = "infra"
+condition = "x >= 1"
+actions = ["Email(infra-alerts@skia.org)"]
+auto-dismiss = true
+nag = "1h"
+
+[[rule]]
+name = "CTFE Staging Prober (admin_tasks)"
+message = "The page at https://ct-staging.skia.org/admin_tasks/ is unavailable."
+query = "select mean(value) from /prober.ctfe_staging_admin_tasks.failure.value/ where time > now() - 10m;"
+category = "infra"
+condition = "x >= 1"
+actions = ["Email(infra-alerts@skia.org)"]
+auto-dismiss = true
+nag = "1h"
+
+[[rule]]
+name = "CTFE Staging Prober (queue)"
+message = "The page at https://ct-staging.skia.org/queue/ is unavailable."
+query = "select mean(value) from /prober.ctfe_staging_queue.failure.value/ where time > now() - 10m;"
+category = "infra"
+condition = "x >= 1"
+actions = ["Email(infra-alerts@skia.org)"]
+auto-dismiss = true
+nag = "1h"
+
+[[rule]]
+name = "CTFE Staging Prober (history)"
+message = "The page at https://ct-staging.skia.org/history/ is unavailable."
+query = "select mean(value) from /prober.ctfe_staging_history.failure.value/ where time > now() - 10m;"
+category = "infra"
+condition = "x >= 1"
+actions = ["Email(infra-alerts@skia.org)"]
+auto-dismiss = true
+nag = "1h"
+
+[[rule]]
+name = "CTFE Staging Prober (chromium_perf_runs)"
+message = "The page at https://ct-staging.skia.org/chromium_perf_runs/ is unavailable."
+query = "select mean(value) from /prober.ctfe_staging_chromium_perf_runs.failure.value/ where time > now() - 10m;"
+category = "infra"
+condition = "x >= 1"
+actions = ["Email(infra-alerts@skia.org)"]
+auto-dismiss = true
+nag = "1h"
+
+[[rule]]
+name = "CTFE Staging Prober (capture_skp_runs)"
+message = "The page at https://ct-staging.skia.org/capture_skp_runs/ is unavailable."
+query = "select mean(value) from /prober.ctfe_staging_capture_skp_runs.failure.value/ where time > now() - 10m;"
+category = "infra"
+condition = "x >= 1"
+actions = ["Email(infra-alerts@skia.org)"]
+auto-dismiss = true
+nag = "1h"
+
+[[rule]]
+name = "CTFE Staging Prober (lua_script_runs)"
+message = "The page at https://ct-staging.skia.org/lua_script_runs/ is unavailable."
+query = "select mean(value) from /prober.ctfe_staging_lua_script_runs.failure.value/ where time > now() - 10m;"
+category = "infra"
+condition = "x >= 1"
+actions = ["Email(infra-alerts@skia.org)"]
+auto-dismiss = true
+nag = "1h"
+
+[[rule]]
+name = "CTFE Staging Prober (chromium_builds_runs)"
+message = "The page at https://ct-staging.skia.org/chromium_builds_runs/ is unavailable."
+query = "select mean(value) from /prober.ctfe_staging_chromium_builds_runs.failure.value/ where time > now() - 10m;"
+category = "infra"
+condition = "x >= 1"
+actions = ["Email(infra-alerts@skia.org)"]
+auto-dismiss = true
+nag = "1h"
+
+[[rule]]
+name = "CTFE Staging Prober (recreate_page_sets_runs)"
+message = "The page at https://ct-staging.skia.org/recreate_page_sets_runs/ is unavailable."
+query = "select mean(value) from /prober.ctfe_staging_recreate_page_sets_runs.failure.value/ where time > now() - 10m;"
+category = "infra"
+condition = "x >= 1"
+actions = ["Email(infra-alerts@skia.org)"]
+auto-dismiss = true
+nag = "1h"
+
+[[rule]]
+name = "CTFE Staging Prober (recreate_webpage_archives_runs)"
+message = "The page at https://ct-staging.skia.org/recreate_webpage_archives_runs/ is unavailable."
+query = "select mean(value) from /prober.ctfe_staging_recreate_webpage_archives_runs.failure.value/ where time > now() - 10m;"
+category = "infra"
+condition = "x >= 1"
+actions = ["Email(infra-alerts@skia.org)"]
+auto-dismiss = true
+nag = "1h"
+
+[[rule]]
+name = "CTFE Staging Prober (chromium_perf_parameters)"
+message = "The JSON endpoint at https://ct-staging.skia.org/_/chromium_perf/ is unavailable or returning unexpected data."
+query = "select mean(value) from /prober.ctfe_staging_chromium_perf_parameters.failure.value/ where time > now() - 10m;"
+category = "infra"
+condition = "x >= 1"
+actions = ["Email(infra-alerts@skia.org)"]
+auto-dismiss = true
+nag = "1h"
+
+[[rule]]
+name = "CTFE Staging Prober (chromium_rev_data)"
+message = "The JSON endpoint at https://ct-staging.skia.org/_/chromium_rev_data?rev=LKGR is unavailable or returning unexpected data."
+query = "select mean(value) from /prober.ctfe_staging_chromium_rev_data.failure.value/ where time > now() - 10m;"
+category = "infra"
+condition = "x >= 1"
+actions = ["Email(infra-alerts@skia.org)"]
+auto-dismiss = true
+nag = "1h"
+
+[[rule]]
+name = "CTFE Staging Prober (skia_rev_data)"
+message = "The JSON endpoint at https://ct-staging.skia.org/_/skia_rev_data?rev=LKGR is unavailable or returning unexpected data."
+query = "select mean(value) from /prober.ctfe_staging_skia_rev_data.failure.value/ where time > now() - 10m;"
+category = "infra"
+condition = "x >= 1"
+actions = ["Email(infra-alerts@skia.org)"]
+auto-dismiss = true
+nag = "1h"
+
+[[rule]]
+name = "CTFE Staging Prober (get_chromium_perf_tasks)"
+message = "The JSON endpoint at https://ct-staging.skia.org/_/get_chromium_perf_tasks?size=2 is unavailable or returning unexpected data."
+query = "select mean(value) from /prober.ctfe_staging_get_chromium_perf_tasks.failure.value/ where time > now() - 10m;"
+category = "infra"
+condition = "x >= 1"
+actions = ["Email(infra-alerts@skia.org)"]
+auto-dismiss = true
+nag = "1h"
+
+[[rule]]
+name = "CTFE Staging Prober (get_capture_skp_tasks)"
+message = "The JSON endpoint at https://ct-staging.skia.org/_/get_capture_skp_tasks?size=2 is unavailable or returning unexpected data."
+query = "select mean(value) from /prober.ctfe_staging_get_capture_skp_tasks.failure.value/ where time > now() - 10m;"
+category = "infra"
+condition = "x >= 1"
+actions = ["Email(infra-alerts@skia.org)"]
+auto-dismiss = true
+nag = "1h"
+
+[[rule]]
+name = "CTFE Staging Prober (get_lua_script_tasks)"
+message = "The JSON endpoint at https://ct-staging.skia.org/_/get_lua_script_tasks?size=2 is unavailable or returning unexpected data."
+query = "select mean(value) from /prober.ctfe_staging_get_lua_script_tasks.failure.value/ where time > now() - 10m;"
+category = "infra"
+condition = "x >= 1"
+actions = ["Email(infra-alerts@skia.org)"]
+auto-dismiss = true
+nag = "1h"
+
+[[rule]]
+name = "CTFE Staging Prober (get_chromium_build_tasks)"
+message = "The JSON endpoint at https://ct-staging.skia.org/_/get_chromium_build_tasks?size=2 is unavailable or returning unexpected data."
+query = "select mean(value) from /prober.ctfe_staging_get_chromium_build_tasks.failure.value/ where time > now() - 10m;"
+category = "infra"
+condition = "x >= 1"
+actions = ["Email(infra-alerts@skia.org)"]
+auto-dismiss = true
+nag = "1h"
+
+[[rule]]
+name = "CTFE Staging Prober (get_recreate_page_sets_tasks)"
+message = "The JSON endpoint at https://ct-staging.skia.org/_/get_recreate_page_sets_tasks?size=2 is unavailable or returning unexpected data."
+query = "select mean(value) from /prober.ctfe_staging_get_recreate_page_sets_tasks.failure.value/ where time > now() - 10m;"
+category = "infra"
+condition = "x >= 1"
+actions = ["Email(infra-alerts@skia.org)"]
+auto-dismiss = true
+nag = "1h"
+
+[[rule]]
+name = "CTFE Staging Prober (get_recreate_webpage_archives_tasks)"
+message = "The JSON endpoint at https://ct-staging.skia.org/_/get_recreate_webpage_archives_tasks?size=2 is unavailable or returning unexpected data."
+query = "select mean(value) from /prober.ctfe_staging_get_recreate_webpage_archives_tasks.failure.value/ where time > now() - 10m;"
+category = "infra"
+condition = "x >= 1"
+actions = ["Email(infra-alerts@skia.org)"]
+auto-dismiss = true
+nag = "1h"
+
+[[rule]]
+name = "CTFE Staging Prober (get_oldest_pending_task)"
+message = "The JSON endpoint at https://ct-staging.skia.org/_/get_oldest_pending_task is unavailable or returning unexpected data."
+query = "select mean(value) from /prober.ctfe_staging_get_oldest_pending_task.failure.value/ where time > now() - 10m;"
+category = "infra"
+condition = "x >= 1"
+actions = ["Email(infra-alerts@skia.org)"]
+auto-dismiss = true
+nag = "1h"
+
+[[rule]]
+name = "CTFE Staging Prober (any_skp_repository_available)"
+message = "There are no SKP repositories available for running Lua scripts."
+query = "select mean(value) from /prober.ctfe_staging_any_skp_repository_available.failure.value/ where time > now() - 60m;"
+category = "infra"
+condition = "x >= 1"
+actions = ["Email(infra-alerts@skia.org)"]
+auto-dismiss = true
+nag = "24h"
+
+[[rule]]
+name = "CTFE Staging Prober (any_chromium_builds_available)"
+message = "There are no Chromium builds available for running tasks."
+query = "select mean(value) from /prober.ctfe_staging_any_chromium_builds_available.failure.value/ where time > now() - 60m;"
+category = "infra"
+condition = "x >= 1"
+actions = ["Email(infra-alerts@skia.org)"]
+auto-dismiss = true
+nag = "24h"
+
+[[rule]]
+name = "CTFE Staging Pending Task Count"
+message = "There are a lot of pending tasks."
+query = "select mean(value) from /ctfe.ct-staging-skia-org.num-pending-tasks.value/ where time > now() - 10m;"
+category = "infra"
+condition = "x >= 100"
+actions = ["Email(infra-alerts@skia.org)"]
+auto-dismiss = true
+nag = "1h"
+
+[[rule]]
+name = "CTFE Staging Pending Task Status"
+message = "A task has been waiting to be executed for a while and it's still not started."
+query = "select mean(value) from /ctfe.ct-staging-skia-org.oldest-pending-task-status.value/ where time > now() - 10m;"
+category = "infra"
+condition = "x >= 2"
+actions = ["Email(infra-alerts@skia.org)"]
+auto-dismiss = true
+nag = "1h"
+
+[[rule]]
+name = "CTFE Staging Last Metrics Update"
+message = "No recent update from the CTFE metrics goroutine."
+query = "select count(value) from /ctfe.ct-staging-skia-org.oldest-pending-task-status.value/ where time > now() - 10m;"
+category = "infra"
+condition = "x < 1"
+actions = ["Email(infra-alerts@skia.org)"]
+auto-dismiss = true
+nag = "1h"
+
+[[rule]]
+name = "CTFE Staging Number of Goroutines"
+message = "There are more goroutines running than expected."
+query = "select mean(value) from /ctfe.ct-staging-skia-org.runtime.NumGoroutine.value/ where time > now() - 10m;"
+category = "infra"
+condition = "x >= 100"
+actions = ["Email(infra-alerts@skia.org)"]
+auto-dismiss = true
+nag = "1h"
+
+[[rule]]
+name = "CTFE Staging Error Rate"
+message = "The error rate is too high."
+query = "select derivative(value) from /^logserver.skia-ctfe.skia-ctfe.logserver.ctfe.ERROR.value$/ where time > now() - 10m"
+category = "infra"
+condition = "x >= 5"
+actions = ["Email(infra-alerts@skia.org)"]
+auto-dismiss = false
+nag = "1h"