{
  "benchmark": "hm-1000-nested-future-holdout",
  "benchmark_type": "internal_production_forecast_validation",
  "dataset": {
    "name": "H&M Personalized Fashion Recommendations transactions",
    "source_sha256": "a2da6313d55c8ce3a981782aa496282335c40e28e21c327346bd3db4e26a699d",
    "source_rows": 2181762,
    "source_series": 104195,
    "selected_series": 2000,
    "selection_rule": "highest total demand, then positive weeks, then unique_id; all values computed through training_end only",
    "minimum_training_periods": 61,
    "pipeline_input_rows": 174550,
    "pipeline_input_sha256": "a3882bf0cdd7b964b9f126cbde2982efe2e0c1a5de6beeecd6f0b32647850225",
    "future_holdout_rows": 24000,
    "future_actuals_sha256": "8c80b4766ca5f1b4f60e4c145db6b8de152a045b28650539651f42e67f34355e"
  },
  "calendar": {
    "frequency": "W-MON",
    "horizon": 12,
    "training_end": "2020-03-30",
    "first_holdout_start": "2020-04-06",
    "first_holdout_end": "2020-06-22",
    "future_holdout_start": "2020-06-29",
    "future_holdout_end": "2020-09-14"
  },
  "protocol": {
    "rolling_cv_windows": 4,
    "first_holdout_role": "ordinary untouched pipeline evaluation",
    "future_holdout_role": "never passed to pipeline; scores production future_forecasts.csv",
    "future_refit_history_end": "2020-06-22",
    "ensemble_weight_source": "rolling CV before holdout 1",
    "raw_data_versioned": false
  },
  "configuration": {
    "profile": "balanced",
    "jobs": 8,
    "stats_jobs": 4,
    "ml_jobs": 4,
    "parallel_families": true,
    "family_workers": 2,
    "ensemble_method": "best_cv",
    "primary_metric": "vn1",
    "feature_discovery_enabled": false,
    "refit_selected_only": false,
    "cv_cache": true
  },
  "runtime_seconds": 384.48984938301146,
  "pipeline_runtime_seconds": 373.1745997290127,
  "cache": {
    "cv_hit": false,
    "cv_key": "4936a7f5344400cf76e0005435ccd7985b49b1c983ff885c12737d3fea4d19d9",
    "conformal_hit": false,
    "conformal_key": "ef43eeb697edad3965f8b911c5b1323b2b6e2a8672f0d31fc45bb0e5d1352b18"
  },
  "model_errors": {},
  "ensemble_method_scores": {
    "legacy_slsqp_horizon": 0.8772040344137323,
    "slsqp_horizon": 0.8766543768944073,
    "median_horizon": 1.1079218108862026
  },
  "artifact_sha256": {
    "cv_predictions": "725147bd1e8cf0f13fa9cf4e98fbab5b0e7abe397ac1b2eeed1ee869831a4b1c",
    "holdout_predictions": "4b1d5a8a5b31d0315830099a9de6a8de85c69cf9b5a729369282e3794cb540a7",
    "future_forecasts": "8f4ba96f21459a2b6ff327692be1ab52f78e01fdf821c6e4624e150c9d12bed0",
    "ensemble_weights": "469826f8f691296f2b5ac758555c15bb67f412df5535733ac6b62283f58345f8"
  },
  "software": {
    "python": "3.12.13",
    "pandas": "2.3.3",
    "numpy": "2.4.6",
    "lightgbm": "4.6.0",
    "catboost": "1.2.10",
    "mlforecast": "1.1.0",
    "statsforecast": "2.0.3"
  },
  "roles": {
    "predeclared_production_ensemble": "Ensemble_Horizon",
    "cv_best_all": "Basic_LGBM_reg90",
    "cv_best_ml": "Basic_LGBM_reg90",
    "cv_best_stats": "stat__tsb_alpha_0p7",
    "cv_best_naive": "Naive",
    "holdout_1_best_diagnostic": "Ensemble_Horizon",
    "future_holdout_2_best_oracle": "Custom_CatBoost_MAE"
  },
  "holdout_1": {
    "ensemble": {
      "stage": "holdout_1",
      "model": "Ensemble_Horizon",
      "wape": 0.6984523494423746,
      "mae": 9.613498137724843,
      "rmse": 32.945728836679834,
      "smape": 1.5590613469383343,
      "bias": -5.598952608451695,
      "pbias": -0.4067823749238372,
      "competition_score": 1.1052347243662117,
      "vn1": 1.1052347243662117,
      "rows": 24000
    },
    "cv_best_ml": {
      "stage": "holdout_1",
      "model": "Basic_LGBM_reg90",
      "wape": 0.7088211719436475,
      "mae": 9.756214610632364,
      "rmse": 33.51394099792896,
      "smape": 1.6155503176501018,
      "bias": -6.398008448599826,
      "pbias": -0.4648364173641257,
      "competition_score": 1.1736575893077732,
      "vn1": 1.1736575893077732,
      "rows": 24000
    },
    "interval_coverage_80": 0.49254166666666666,
    "interval_coverage_95": 0.5124583333333333
  },
  "future_holdout_2": {
    "ensemble": {
      "stage": "future_holdout_2",
      "model": "Ensemble_Horizon",
      "wape": 0.7016913937643113,
      "mae": 7.800673987336443,
      "rmse": 27.151221370095197,
      "smape": 1.5487142919161505,
      "bias": -0.8426679943412066,
      "pbias": -0.07580022961987115,
      "competition_score": 0.7774916233841824,
      "vn1": 0.7774916233841824,
      "rows": 24000
    },
    "cv_best_ml": {
      "stage": "future_holdout_2",
      "model": "Basic_LGBM_reg90",
      "wape": 0.6789270972911948,
      "mae": 7.547604251957159,
      "rmse": 27.17705814789663,
      "smape": 1.5595025940323248,
      "bias": -1.6028456790382848,
      "pbias": -0.14418023626411164,
      "competition_score": 0.8231073335553065,
      "vn1": 0.8231073335553065,
      "rows": 24000
    },
    "cv_selected_naive": {
      "stage": "future_holdout_2",
      "model": "Naive",
      "wape": 1.123572469987669,
      "mae": 12.490708333333334,
      "rmse": 39.94104353085098,
      "smape": 0.6160121101541338,
      "bias": 6.3585416666666665,
      "pbias": 0.5719677519705255,
      "competition_score": 1.6955402219581945,
      "vn1": 1.6955402219581945,
      "rows": 24000
    },
    "interval_coverage_80": 0.5382083333333333,
    "interval_coverage_95": 0.5644583333333333
  },
  "ensemble_weights": {
    "method": "slsqp_horizon",
    "active_model_count_by_horizon": {
      "1": 1,
      "2": 2,
      "3": 1,
      "4": 1,
      "5": 1,
      "6": 8,
      "7": 6,
      "8": 5,
      "9": 5,
      "10": 1,
      "11": 1,
      "12": 1
    },
    "weight_sum_by_horizon": {
      "1": 1.0,
      "2": 1.0,
      "3": 1.0,
      "4": 1.0,
      "5": 1.0,
      "6": 0.9999999999999999,
      "7": 1.0,
      "8": 1.0,
      "9": 0.9999999999999999,
      "10": 1.0,
      "11": 1.0,
      "12": 1.0
    }
  },
  "operational_gate": {
    "passed": true,
    "expected_rows": 24000,
    "forecast_rows": 24000,
    "missing_forecasts": 0,
    "unexpected_forecasts": 0,
    "duplicate_forecast_keys": 0,
    "all_model_forecasts_finite": true,
    "all_point_forecasts_nonnegative": true,
    "missing_model_columns": [],
    "unexpected_model_columns": [],
    "ensemble_recalculation_max_abs_delta": 5.684341886080802e-14,
    "maximum_weight_sum_error": 1.1102230246251565e-16
  },
  "quality_stability_gate": {
    "passed": true,
    "maximum_allowed_relative_wape_degradation": 0.1,
    "ensemble_relative_wape_change": 0.004637459269086494,
    "cv_best_ml_relative_wape_change": -0.04217435346983297,
    "ensemble_future_wape_no_worse_than_cv_selected_naive": true,
    "ensemble_wape_stability_passed": true
  }
}