{ "meta": { "version": "v1", "generated_at": "2026-10-19T07:37:05.046Z", "cache_ttl": 3600 }, "data": { "generated_at": "2026-10-19T07:21:44Z", "score_as_of": "2026-10-19T07:20:47Z", "system": { "frozen_forecasts": 2872, "matured_forecasts": 2735, "scored_forecasts": 2729, "matured_unscored_backlog": 6, "raw_chain_rows": 2824, "hash_chain_mismatches": 0, "exact_model_benchmarks": 3, "alerts": [ "6 matured forecast windows are waiting for scoring." ], "rollup_violations": [], "ledger_violations": [] }, "hazards": [ { "key": "eq", "hazard": "earthquake", "model_version": "eq_operational_S2_gear1_v2-ecb87f020f99", "verification_status": "logging_waiting_maturity", "status_badge": "Waiting", "verification_status_label": "The published model version's forecasts are being recorded; none of its 30-day windows has closed yet (scored windows of earlier versions: 668, each version's record is kept under its own name).", "metric_source": "retrospective_holdout_exact_model", "metric_source_label": "Pre-registered programme (docs/EARTHQUAKE_FORECAST_PROGRAM.md): a declared second read (reported, never used to decide) of 2023-2025 (153 issue times, 1384 M6+ cell-windows); +2.61 nats of information per M6+ cell-window over a uniform map. This version's live record starts when its first 30-day windows mature.", "auc": 0.9687, "brier": null, "brier_skill_score": 0.0219, "homepage_line": "AUC 0.969, pre-registered second read of years already scored", "forecast_storage": { "n_replay_artifacts": 789, "n_matured_forecasts": 668, "n_scored_forecasts": 668, "n_backlog": 0, "first_issued_at": "2026-03-13T03:00:00Z", "last_issued_at": "2026-10-19T06:00:00Z", "last_forecast_id": "eq_fcst_20261019_0600", "latest_replay_artifact": "/data/replay/eq_fcst_20261019_0600.json", "forecast_horizon": "30 days", "next_mature_at": "2026-10-19T11:00:00Z" }, "ledger": { "supported": true, "path": "/data/earthquake-ledger.jsonl", "n_rows": 800, "prev_hash_mismatches": 0 }, "prospective": { "summary_path": "results/earthquake_prospective/prospective_summary.json", "status": "ok", "scored_as_of": "2026-10-19T04:41:53Z", "message": null, "top_5_hit_rate": null, "this_model_version": null }, "exact_model_benchmark": { "availability": "exact_model_benchmark", "label": "Pre-registered programme (docs/EARTHQUAKE_FORECAST_PROGRAM.md): a declared second read (reported, never used to decide) of 2023-2025 (153 issue times, 1384 M6+ cell-windows); +2.61 nats of information per M6+ cell-window over a uniform map.", "model_version": "eq_operational_S2_gear1_v2-ecb87f020f99", "second_read": true, "auc": 0.9687, "auc_ci95": [ 0.9645, 0.9726 ], "brier_skill_score": 0.0219, "information_gain_per_positive_cell_window": 2.6134, "n_cases": 1790100 }, "related_benchmark": { "availability": "related_research_benchmark", "label": "Related research benchmark exists, but it is not yet bound to the current live model version.", "model_version": "earthquake_honest_regional_suite", "source_updated_at": null, "global_auc": 0.9074433326526443, "global_brier": 0.11605939643081976, "same_location_auc": 0.797277556440903, "source_files": [ "results/earthquake_honest/v4_regional_honest_results.json", "results/earthquake_honest/same_location_auc.json" ] }, "recommended_action": "Keep freezing every earthquake forecast. Once the first 30-day windows mature, run the prospective scorer and use those scores to tune thresholds and calibration." }, { "key": "hu", "hazard": "hurricane", "model_version": "hurricane_ri_v8_3", "verification_status": "prospective_scored", "status_badge": "Scored", "verification_status_label": "Prospective live scoring is active for matured hurricane forecasts. 177 storm forecasts scored against NHC best track; 28 rapid-intensification events observed.", "metric_source": "retrospective_holdout_exact_model", "metric_source_label": "Exact-model retrospective benchmark is available.", "auc": null, "brier": null, "brier_skill_score": null, "homepage_line": "117 matured forecasts scored", "forecast_storage": { "n_replay_artifacts": 125, "n_matured_forecasts": 121, "n_scored_forecasts": 117, "n_backlog": 4, "first_issued_at": "2026-03-13T03:00:00Z", "last_issued_at": "2026-10-19T03:36:35Z", "last_forecast_id": "hu_fcst_20261019_0336", "latest_replay_artifact": "/data/replay/hu_fcst_20261019_0336.json", "forecast_horizon": "24 hours", "next_mature_at": "2026-10-19T09:33:52Z" }, "ledger": { "supported": true, "path": "/data/hurricane-ledger.jsonl", "n_rows": 66, "prev_hash_mismatches": 0, "since": "2026-10-03 (earlier hurricane forecasts are kept as replay files, unchained)" }, "prospective": { "summary_path": "results/hurricane_prospective/prospective_summary.json", "status": "ok", "scored_as_of": "2026-10-19T04:44:45Z", "n_scored_forecasts": 117, "stale": false, "message": null, "unit": "storm-cycle: one scored record per storm and synoptic time t, the first made at or after t + 3 h 30 min (docs/HURRICANE_RI_V9_PROGRAM.md amendment 7, rule 1)", "pooled": { "n_storm_cycles": 177, "n_events": 28, "n_storms": 14, "brier": 0.097375, "mean_forecast": 0.100498, "observed_rate": 0.158192, "auc": 0.884468, "auc_withheld": null }, "by_model_version": { "hurricane_ri_stack_v1-35938d6c09b7": { "n_storm_cycles": 105, "n_events": 18, "n_storms": 8, "brier": 0.096883, "mean_forecast": 0.099905, "observed_rate": 0.171429, "auc": 0.919221, "auc_withheld": null }, "hurricane_ri_v8_2": { "n_storm_cycles": 33, "n_events": 6, "n_storms": 2, "brier": 0.173276, "mean_forecast": 0.125715, "observed_rate": 0.181818, "auc": null, "auc_withheld": "the cycles span 2 storms (< 5)" }, "hurricane_ri_v8_3": { "n_storm_cycles": 34, "n_events": 4, "n_storms": 5, "brier": 0.039544, "mean_forecast": 0.092635, "observed_rate": 0.117647, "auc": 1, "auc_withheld": null }, "jtwc_warning_ingest": { "n_storm_cycles": 5, "n_events": 0, "n_storms": 1, "brier": 0, "mean_forecast": 0, "observed_rate": 0, "auc": null, "auc_withheld": "only one outcome occurred: a ranking score is undefined" } }, "total_storms_scored": 177, "total_ri_events": 28 }, "exact_model_benchmark": { "availability": "exact_model_benchmark", "label": "Retrospective benchmark available for the current live model version.", "model_version": "hurricane_ri_v8_3", "source_updated_at": null, "auc": null, "auc_ci95": null, "brier": null, "brier_skill_score": null, "reliability_slope": null, "n_cases": null }, "related_benchmark": null, "recommended_action": "Keep scoring matured hurricane forecasts against NHC best track; live AUC stays undefined until at least one rapid-intensification event is observed, so promotion decisions still rest on the retrospective benchmark." }, { "key": "to", "hazard": "tornado", "model_version": "tornado_v3-05a06c843c87", "verification_status": "prospective_scored", "status_badge": "Scored", "verification_status_label": "Prospective live scoring is active for matured tornado forecasts.", "metric_source": "retrospective_holdout_exact_model", "metric_source_label": "Pre-registered programme (docs/TORNADO_MODEL_PROGRAM.md): every choice made on 2023, development test 2024, scored once on every 2025 storm observation (1,469,977; 1,579 tornadic). Live record of this version: 15,294 storm forecasts from 185 closed live forecasts, 30 followed by a tornado (SPC reports); Brier skill 0.14 against the base rate of outcomes that had closed before each forecast was issued.", "auc": 0.9702, "brier": null, "brier_skill_score": 0.1067, "brier_skill_score_reference": "test_year_base_rate", "live_this_version": { "auc": 0.9905, "brier": 0.001698, "brier_skill_score": 0.135, "n_storm_forecasts": 15294, "n_events": 30 }, "homepage_line": "AUC 0.970 pre-registered 2025 test ยท live BSS 0.14", "forecast_storage": { "n_replay_artifacts": 1958, "n_matured_forecasts": 1946, "n_scored_forecasts": 1944, "n_backlog": 2, "first_issued_at": "2026-03-13T03:00:00Z", "last_issued_at": "2026-10-19T07:20:47Z", "last_forecast_id": "to_fcst_20261019_0720", "latest_replay_artifact": "/data/replay/to_fcst_20261019_0720.json", "forecast_horizon": "24 hours", "next_mature_at": "2026-10-19T09:05:57Z" }, "ledger": { "supported": true, "path": "/data/tornado-ledger.jsonl", "n_rows": 2024, "prev_hash_mismatches": 0, "content_hash_verified": 1854, "content_hash_verified_without_forecast_id": 170, "content_hash_legacy_note": "170 rows (2026-03-22T04:11:35.843464Z to 2026-04-13T02:14:12.996567Z) were hashed before forecast_id was added to them; each verifies with forecast_id removed", "content_hash_mismatches": 0, "content_hash_mismatched_rows": [] }, "prospective": { "summary_path": "results/tornado_prospective/prospective_summary.json", "status": "ok", "scored_as_of": "2026-10-19T04:42:37Z", "n_scored_forecasts": 1944, "stale": false, "message": null, "outcome_time_convention": "SPC daily files are convective days (12 UTC to 12 UTC); HHMM is UTC; rows before 1200 are placed on the next UTC date", "pooled": { "n_storm_forecasts": 15294, "n_positive": 30, "base_rate": 0.00196155, "mean_forecast_probability": 0.00072575, "brier": 0.001698, "auc": 0.9905, "ece": 0.001491, "brier_sample_climatology": 0.00195771, "bss_vs_sample_climatology": 0.1327, "n_with_causal_reference": 14355, "brier_causal_climatology": 0.00209138, "bss_vs_causal_climatology": 0.135, "label_after_issue_only": { "n_storm_forecasts": 15294, "n_positive": 28, "base_rate": 0.00183078, "brier": 0.00159076, "auc": 0.9909 }, "n_repeat_storm_forecasts_dropped": 832, "n_forecasts": 185, "first_issued_at": "2026-10-03T03:39:53Z", "last_issued_at": "2026-10-18T00:51:38Z" }, "pooled_by_served_mode": null, "skill_reference": "bss_vs_causal_climatology: reference = base rate of the SAME model version's storm-forecasts whose 24 h window matured before the forecast was issued (no outcome leak; versions differ in event definition). bss_vs_sample_climatology: reference = pooled base rate of the scored set itself (in-sample, shown for comparison). Both are null when no event was observed. Scored on the model's full-precision probability.", "pooled_model_version": "tornado_v3-05a06c843c87" }, "exact_model_benchmark": { "availability": "exact_model_benchmark", "label": "Pre-registered programme (docs/TORNADO_MODEL_PROGRAM.md): every choice made on 2023, development test 2024, scored once on every 2025 storm observation (1,469,977; 1,579 tornadic).", "model_version": "tornado_v3-05a06c843c87", "auc": 0.9702, "auc_ci95": [ 0.9636, 0.9758 ], "brier_skill_score": 0.1067, "n_cases": 1469977 }, "related_benchmark": { "availability": "related_research_benchmark", "label": "A historical 2024 holdout benchmark exists for a related tornado GBT family, but not yet as an exact score for the live storm-object model. Its Brier/BSS were measured at a downsampled base rate of 0.003 and are not comparable to live skill at the natural base rate.", "model_version": "hazardpulse_tornado_definitive_v2", "source_updated_at": "2026-10-02T05:38:43Z", "evaluated_base_rate": 0.0032204083298574162, "auc": 0.939537523974802, "brier": 0.0030295057150957643, "brier_skill_score": 0.056239715640696075, "source_files": [ "results/definitive/definitive_results.json" ] }, "recommended_action": "Judge the live tornado model on the pooled BSS against causal climatology (and its served-probability split), not on per-forecast means or the downsampled research holdout." } ] } }