{
  "benchmark": "hm-1000-nested-future-holdout",
  "benchmark_type": "internal_production_forecast_validation",
  "dataset": {
    "name": "H&M Personalized Fashion Recommendations transactions",
    "source_sha256": "a2da6313d55c8ce3a981782aa496282335c40e28e21c327346bd3db4e26a699d",
    "source_rows": 2181762,
    "source_series": 104195,
    "selected_series": 2000,
    "selection_rule": "highest total demand, then positive weeks, then unique_id; all values computed through training_end only",
    "minimum_training_periods": 61,
    "pipeline_input_rows": 174550,
    "pipeline_input_sha256": "a3882bf0cdd7b964b9f126cbde2982efe2e0c1a5de6beeecd6f0b32647850225",
    "future_holdout_rows": 24000,
    "future_actuals_sha256": "8c80b4766ca5f1b4f60e4c145db6b8de152a045b28650539651f42e67f34355e"
  },
  "calendar": {
    "frequency": "W-MON",
    "horizon": 12,
    "training_end": "2020-03-30",
    "first_holdout_start": "2020-04-06",
    "first_holdout_end": "2020-06-22",
    "future_holdout_start": "2020-06-29",
    "future_holdout_end": "2020-09-14"
  },
  "protocol": {
    "rolling_cv_windows": 4,
    "first_holdout_role": "ordinary untouched pipeline evaluation",
    "future_holdout_role": "never passed to pipeline; scores production future_forecasts.csv",
    "future_refit_history_end": "2020-06-22",
    "ensemble_weight_source": "rolling CV before holdout 1",
    "raw_data_versioned": false
  },
  "configuration": {
    "profile": "quick",
    "jobs": 8,
    "stats_jobs": 4,
    "ml_jobs": 4,
    "parallel_families": true,
    "family_workers": 2,
    "ensemble_method": "best_cv",
    "primary_metric": "vn1",
    "feature_discovery_enabled": false,
    "refit_selected_only": false,
    "cv_cache": true
  },
  "runtime_seconds": 145.8813419030048,
  "pipeline_runtime_seconds": 135.5415316899307,
  "cache": {
    "cv_hit": true,
    "cv_key": "ac80781208ef072910b301d63ab4d91d7f3457aeadc46778f6134c060390604b",
    "conformal_hit": true,
    "conformal_key": "c48ed36c6a152735670d66771cc3c7eafdf1679a933964d939512205aa91c5e6"
  },
  "model_errors": {},
  "ensemble_method_scores": {
    "legacy_slsqp_horizon": 0.8744793461351357,
    "slsqp_horizon": 0.8736404125676102,
    "median_horizon": 1.0936985844160403
  },
  "artifact_sha256": {
    "cv_predictions": "60abfb80601377e3ddb3135f72d9a961ca0cddc7e2f2fed7e2543f207ba3f6a8",
    "holdout_predictions": "926649bb3ceadcf8402e9a2291582e81612e7693d045c17bbf1174548a50a871",
    "future_forecasts": "b9170ddcb9a1a6eb3f725532dcb14c710fa5c4e840aee4e021f1f729858ba246",
    "ensemble_weights": "4e4eb5ae8db0bb5588d7187307e17add8197fc79de2281da7f8903c2fceeea2a"
  },
  "software": {
    "python": "3.12.13",
    "pandas": "2.3.3",
    "numpy": "2.4.6",
    "lightgbm": "4.6.0",
    "catboost": "1.2.10",
    "mlforecast": "1.1.0",
    "statsforecast": "2.0.3"
  },
  "roles": {
    "predeclared_production_ensemble": "Ensemble_Horizon",
    "cv_best_all": "Basic_LGBM_reg90",
    "cv_best_ml": "Basic_LGBM_reg90",
    "cv_best_stats": "stat__tsb_alpha_0p7",
    "cv_best_naive": "Naive",
    "holdout_1_best_diagnostic": "Ensemble_Horizon",
    "future_holdout_2_best_oracle": "Custom_CatBoost_MAE"
  },
  "holdout_1": {
    "ensemble": {
      "stage": "holdout_1",
      "model": "Ensemble_Horizon",
      "wape": 0.6983028873767521,
      "mae": 9.611440941853616,
      "rmse": 32.9349822870676,
      "smape": 1.5598664902211452,
      "bias": -5.596095374198357,
      "pbias": -0.40657478743085995,
      "competition_score": 1.1048776748076121,
      "vn1": 1.1048776748076121,
      "rows": 24000
    },
    "cv_best_ml": {
      "stage": "holdout_1",
      "model": "Basic_LGBM_reg90",
      "wape": 0.7088211719436475,
      "mae": 9.756214610632364,
      "rmse": 33.51394099792896,
      "smape": 1.6155503176501018,
      "bias": -6.398008448599826,
      "pbias": -0.4648364173641257,
      "competition_score": 1.1736575893077732,
      "vn1": 1.1736575893077732,
      "rows": 24000
    },
    "interval_coverage_80": 0.4915,
    "interval_coverage_95": 0.511875
  },
  "future_holdout_2": {
    "ensemble": {
      "stage": "future_holdout_2",
      "model": "Ensemble_Horizon",
      "wape": 0.6996371197856935,
      "mae": 7.777836709110897,
      "rmse": 27.074425226293908,
      "smape": 1.5488843050566214,
      "bias": -0.8870481402781917,
      "pbias": -0.07979234190510968,
      "competition_score": 0.7794294616908032,
      "vn1": 0.7794294616908032,
      "rows": 24000
    },
    "cv_best_ml": {
      "stage": "future_holdout_2",
      "model": "Basic_LGBM_reg90",
      "wape": 0.6789270972911948,
      "mae": 7.547604251957159,
      "rmse": 27.17705814789663,
      "smape": 1.5595025940323248,
      "bias": -1.6028456790382848,
      "pbias": -0.14418023626411164,
      "competition_score": 0.8231073335553065,
      "vn1": 0.8231073335553065,
      "rows": 24000
    },
    "cv_selected_naive": {
      "stage": "future_holdout_2",
      "model": "Naive",
      "wape": 1.123572469987669,
      "mae": 12.490708333333334,
      "rmse": 39.94104353085098,
      "smape": 0.6160121101541338,
      "bias": 6.3585416666666665,
      "pbias": 0.5719677519705255,
      "competition_score": 1.6955402219581945,
      "vn1": 1.6955402219581945,
      "rows": 24000
    },
    "interval_coverage_80": 0.5377083333333333,
    "interval_coverage_95": 0.564375
  },
  "ensemble_weights": {
    "method": "slsqp_horizon",
    "active_model_count_by_horizon": {
      "1": 1,
      "2": 2,
      "3": 1,
      "4": 1,
      "5": 1,
      "6": 7,
      "7": 6,
      "8": 5,
      "9": 5,
      "10": 1,
      "11": 1,
      "12": 1
    },
    "weight_sum_by_horizon": {
      "1": 1.0,
      "2": 1.0,
      "3": 1.0,
      "4": 1.0,
      "5": 1.0,
      "6": 1.0000000000000002,
      "7": 1.0,
      "8": 1.0,
      "9": 0.9999999999999999,
      "10": 1.0,
      "11": 1.0,
      "12": 1.0
    }
  },
  "operational_gate": {
    "passed": true,
    "expected_rows": 24000,
    "forecast_rows": 24000,
    "missing_forecasts": 0,
    "unexpected_forecasts": 0,
    "duplicate_forecast_keys": 0,
    "all_model_forecasts_finite": true,
    "all_point_forecasts_nonnegative": true,
    "missing_model_columns": [],
    "unexpected_model_columns": [],
    "ensemble_recalculation_max_abs_delta": 5.684341886080802e-14,
    "maximum_weight_sum_error": 2.220446049250313e-16
  },
  "quality_stability_gate": {
    "passed": true,
    "maximum_allowed_relative_wape_degradation": 0.1,
    "ensemble_relative_wape_change": 0.001910678636821432,
    "cv_best_ml_relative_wape_change": -0.04217435346983297,
    "ensemble_future_wape_no_worse_than_cv_selected_naive": true,
    "ensemble_wape_stability_passed": true
  }
}