{
  "$schema": "http://json-schema.org/draft-07/schema#",
  "$id": "https://duellab.org/benchmark/leaderboard_data.schema.json",
  "title": "Public benchmark leaderboard.json v11",
  "description": "Machine-readable leaderboard export for the generated public benchmark site; produced by boardgame_cli (see crates/boardgame_cli/src/public_benchmark/site/leaderboard_data.rs).",
  "type": "object",
  "required": [
    "schema_version",
    "public_release_id",
    "generated_at",
    "benchmark_line",
    "snapshot_id",
    "results_cut_id",
    "suite_line_id",
    "suite_version",
    "comparability_tuples",
    "rows"
  ],
  "additionalProperties": false,
  "properties": {
    "schema_version": { "enum": [7, 8, 9, 10, 11] },
    "public_release_id": { "type": "string", "pattern": "^gb2-[0-9]{8}-[a-f0-9]{12}$" },
    "generated_at": { "type": "string", "format": "date-time" },
    "benchmark_line": { "const": "gamebench2" },
    "snapshot_id": { "type": "string", "minLength": 1 },
    "results_cut_id": { "type": "string", "minLength": 1 },
    "suite_line_id": { "type": "string", "minLength": 1 },
    "suite_version": { "type": "string" },
    "comparability_tuples": {
      "type": "array",
      "items": { "$ref": "#/definitions/comparabilityTuple" }
    },
    "rows": {
      "type": "array",
      "items": { "$ref": "#/definitions/leaderboardRow" }
    }
  },
  "definitions": {
    "leaderboardRow": {
      "type": "object",
      "required": [
        "snapshot_id",
        "results_cut_id",
        "suite_line_id",
        "suite_version",
        "scoring_policy_version",
        "rating_method",
        "row_kind",
        "track_id",
        "track_name",
        "game_alias",
        "canonical_public_row_identity",
        "evaluated_setting_identity",
        "eligibility_flag",
        "rank",
        "ranking_status",
        "ineligibility_reason_codes",
        "model_slug",
        "model_display_name",
        "provider",
        "provider_key",
        "reasoning_level",
        "reasoning_bracket",
        "reasoning_setting",
        "measured_at",
        "last_measured_at",
        "first_seen_at",
        "is_recent_addition",
        "rank_movement",
        "rank_movement_baseline_at",
        "score",
        "conditional_success_score",
        "plugin_success_rate",
        "score_basis",
        "min_score",
        "max_score",
        "raw_elo",
        "rated_matches",
        "attempted_matches",
        "capped_draws",
        "unilateral_faults",
        "distinct_rated_opponents",
        "wins",
        "losses",
        "draws",
        "uncertainty",
        "entry_count",
        "contributing_entrants",
        "codegen_evaluated_cells",
        "admitted_plugins",
        "first_pass_admitted",
        "repaired_admitted",
        "model_code_failures",
        "model_code_failure_rate",
        "first_pass_rate",
        "repaired_rate",
        "infra_failures_excluded",
        "failed_game_aliases",
        "infra_excluded_game_aliases",
        "codegen_cost_usd",
        "reasoning_provenance",
        "generation_policy",
        "generation_timing"
      ],
      "additionalProperties": false,
      "properties": {
        "snapshot_id": { "type": "string", "minLength": 1 },
        "results_cut_id": { "type": "string", "minLength": 1 },
        "suite_line_id": { "type": "string", "minLength": 1 },
        "suite_version": { "type": "string" },
        "scoring_policy_version": { "type": "string", "minLength": 1 },
        "rating_method": { "type": "string", "minLength": 1 },
        "row_kind": { "enum": ["track", "game"] },
        "track_id": { "type": "string", "minLength": 1 },
        "track_name": { "type": "string", "minLength": 1 },
        "game_alias": { "type": ["string", "null"] },
        "canonical_public_row_identity": { "type": "string", "pattern": "^publication-(?:row|game-row)\\.v1:" },
        "evaluated_setting_identity": {
          "anyOf": [
            { "type": "null" },
            { "$ref": "#/definitions/evaluatedSettingIdentityV1" }
          ]
        },
        "eligibility_flag": {
          "type": "boolean",
          "description": "True only for complete-evidence ranked rows. Partial Overall rows remain false even though they receive an official rank."
        },
        "rank": {
          "type": ["integer", "null"],
          "minimum": 1,
          "description": "Official rank. In schema v11, ranked and partial Overall rows share one contiguous score ordering; provisional Reasoning Variants remain null."
        },
        "ranking_status": { "enum": ["ranked", "provisional", "partial", "not_applicable"] },
        "ineligibility_reason_codes": {
          "type": "array",
          "uniqueItems": true,
          "items": {
            "enum": [
              "missing_bracket_scores",
              "unknown_capability_evidence",
              "missing_or_non_distinct_settings",
              "untested_or_ineligible_settings",
              "missing_generation_quality",
              "below_generation_quality_threshold"
            ]
          }
        },
        "model_slug": { "type": ["string", "null"] },
        "model_display_name": { "type": "string", "minLength": 1 },
        "provider": { "type": "string" },
        "provider_key": { "type": "string" },
        "reasoning_level": { "type": ["string", "null"] },
        "reasoning_bracket": {
          "type": ["string", "null"],
          "enum": ["baseline", "balanced", "intensive", "mixed", null]
        },
        "reasoning_setting": { "type": ["string", "null"] },
        "measured_at": { "type": ["string", "null"] },
        "last_measured_at": { "type": ["string", "null"] },
        "first_seen_at": { "type": ["string", "null"] },
        "is_recent_addition": { "type": "boolean" },
        "rank_movement": { "type": ["integer", "null"] },
        "rank_movement_baseline_at": { "type": ["string", "null"] },
        "score": { "type": "number" },
        "conditional_success_score": { "type": ["number", "null"], "minimum": 0, "maximum": 100 },
        "plugin_success_rate": { "type": ["number", "null"], "minimum": 0, "maximum": 1 },
        "score_basis": { "type": "string", "minLength": 1 },
        "min_score": { "type": ["number", "null"] },
        "max_score": { "type": ["number", "null"] },
        "raw_elo": { "type": ["number", "null"] },
        "rated_matches": { "type": ["integer", "null"], "minimum": 0 },
        "attempted_matches": { "type": ["integer", "null"], "minimum": 0 },
        "capped_draws": { "type": ["integer", "null"], "minimum": 0 },
        "unilateral_faults": { "type": ["integer", "null"], "minimum": 0 },
        "distinct_rated_opponents": { "type": ["integer", "null"], "minimum": 0 },
        "wins": { "type": ["integer", "null"], "minimum": 0 },
        "losses": { "type": ["integer", "null"], "minimum": 0 },
        "draws": { "type": ["integer", "null"], "minimum": 0 },
        "uncertainty": { "type": ["number", "null"] },
        "entry_count": { "type": ["integer", "null"], "minimum": 0 },
        "contributing_entrants": { "type": ["integer", "null"], "minimum": 0 },
        "codegen_evaluated_cells": { "type": ["integer", "null"], "minimum": 0 },
        "admitted_plugins": { "type": ["integer", "null"], "minimum": 0 },
        "first_pass_admitted": { "type": ["integer", "null"], "minimum": 0 },
        "repaired_admitted": { "type": ["integer", "null"], "minimum": 0 },
        "model_code_failures": { "type": ["integer", "null"], "minimum": 0 },
        "model_code_failure_rate": { "type": ["number", "null"], "minimum": 0, "maximum": 1 },
        "first_pass_rate": { "type": ["number", "null"], "minimum": 0, "maximum": 1 },
        "repaired_rate": { "type": ["number", "null"], "minimum": 0, "maximum": 1 },
        "infra_failures_excluded": { "type": ["integer", "null"], "minimum": 0 },
        "failed_game_aliases": {
          "type": "array",
          "items": { "type": "string", "minLength": 1 },
          "uniqueItems": true
        },
        "infra_excluded_game_aliases": {
          "type": "array",
          "items": { "type": "string", "minLength": 1 },
          "uniqueItems": true
        },
        "codegen_cost_usd": { "$ref": "#/definitions/codegenCostUsdNullable" },
        "reasoning_provenance": { "$ref": "#/definitions/reasoningProvenance" },
        "generation_policy": { "$ref": "#/definitions/generationPolicy" },
        "generation_timing": { "$ref": "#/definitions/generationTiming" }
      }
    },
    "evaluatedSettingIdentityV1": {
      "type": "object",
      "additionalProperties": false,
      "required": [
        "schema_version",
        "canonical_provider_route",
        "provider_model",
        "request_mode",
        "effective_request_value",
        "prompt_identity",
        "prompt_profile_identity",
        "output_mode",
        "reasoning_policy_version",
        "reasoning_catalog_sha256",
        "resolution_status"
      ],
      "properties": {
        "schema_version": { "const": "evaluated_setting_identity.v1" },
        "canonical_provider_route": { "type": "string", "minLength": 1 },
        "provider_model": { "type": "string", "minLength": 1 },
        "request_mode": { "type": "string", "minLength": 1 },
        "effective_request_value": { "type": "string", "minLength": 1 },
        "prompt_identity": { "type": "string", "minLength": 1 },
        "prompt_profile_identity": { "type": "string", "minLength": 1 },
        "output_mode": { "type": "string", "minLength": 1 },
        "reasoning_policy_version": { "type": "string", "minLength": 1 },
        "reasoning_catalog_sha256": { "type": "string", "minLength": 1 },
        "resolution_status": { "enum": ["exact", "opaque_historical"] }
      }
    },
    "comparabilityTuple": {
      "type": "object",
      "required": [
        "evaluation_family",
        "track_id",
        "track_kind",
        "reasoning_effort",
        "suite_line_id",
        "suite_version",
        "scoring_policy_version",
        "rating_method",
        "snapshot_id"
      ],
      "additionalProperties": false,
      "properties": {
        "evaluation_family": { "type": "string", "minLength": 1 },
        "track_id": { "type": "string", "minLength": 1 },
        "track_kind": { "type": "string", "minLength": 1 },
        "reasoning_effort": { "type": "string", "minLength": 1 },
        "suite_line_id": { "type": "string", "minLength": 1 },
        "suite_version": { "type": "string" },
        "scoring_policy_version": { "type": "string", "minLength": 1 },
        "coverage_policy_identity": { "type": ["string", "null"], "minLength": 1 },
        "rating_method": { "type": "string", "minLength": 1 },
        "snapshot_id": { "type": "string", "minLength": 1 },
        "results_cut_id": { "type": ["string", "null"] }
      }
    },
    "codegenCostUsdNullable": {
      "anyOf": [{ "type": "null" }, { "$ref": "#/definitions/codegenCostUsd" }]
    },
    "codegenCostUsd": {
      "type": "object",
      "required": ["mean", "min", "max", "total", "run_count"],
      "additionalProperties": false,
      "properties": {
        "mean": { "type": "number", "minimum": 0 },
        "median": { "type": "number", "minimum": 0 },
        "min": { "type": "number", "minimum": 0 },
        "max": { "type": "number", "minimum": 0 },
        "total": { "type": "number", "minimum": 0 },
        "cell_count": { "type": "integer", "minimum": 0 },
        "run_count": { "type": "integer", "minimum": 0 },
        "admitted_cell_count": { "type": "integer", "minimum": 0 },
        "failed_cell_count": { "type": "integer", "minimum": 0 },
        "recorded_attempt_count": { "type": "integer", "minimum": 0 },
        "token_priced_attempt_count": { "type": "integer", "minimum": 0 },
        "legacy_label_count": { "type": "integer", "minimum": 0 },
        "cost_basis_counts": {
          "type": "object",
          "additionalProperties": { "type": "integer", "minimum": 0 }
        },
        "pricing_table_versions": {
          "type": "object",
          "additionalProperties": { "type": "integer", "minimum": 0 }
        },
        "evidence_basis": { "type": "string" }
      }
    },
    "generationSettingsPublic": {
      "type": "object",
      "required": ["temperature", "max_tokens", "stream"],
      "additionalProperties": false,
      "properties": {
        "temperature": { "type": ["number", "null"] },
        "max_tokens": { "type": ["integer", "null"], "minimum": 0 },
        "stream": { "type": ["boolean", "null"] }
      }
    },
    "reasoningProvenance": {
      "type": "object",
      "required": [
        "reasoning_bucket",
        "reasoning_bucket_label",
        "provider",
        "provider_model",
        "requested_reasoning_effort",
        "provider_setting_label",
        "provider_reasoning_params",
        "prompt_identity",
        "prompt_profile",
        "code_prompt_version",
        "code_output_mode",
        "generation_settings_public"
      ],
      "additionalProperties": false,
      "properties": {
        "reasoning_bucket": { "type": "string", "minLength": 1 },
        "reasoning_bucket_label": { "type": "string", "minLength": 1 },
        "reasoning_bracket": { "type": ["string", "null"], "enum": ["baseline", "balanced", "intensive", "mixed", null] },
        "reasoning_policy_version": { "type": "string", "minLength": 1 },
        "reasoning_catalog_sha256": { "type": "string", "pattern": "^[a-f0-9]{64}$" },
        "provider": { "type": "string", "minLength": 1 },
        "provider_model": { "type": "string", "minLength": 1 },
        "provider_route": { "type": "string", "minLength": 1 },
        "requested_reasoning_effort": { "type": ["string", "null"] },
        "provider_setting_label": { "type": ["string", "null"] },
        "reasoning_request_mode": { "type": ["string", "null"], "enum": ["effort", "toggle", "omitted", "budget", null] },
        "provider_reasoning_params": { "type": "object" },
        "prompt_identity": { "type": "string", "minLength": 1 },
        "prompt_profile": { "type": "string", "minLength": 1 },
        "code_prompt_version": { "type": "string", "minLength": 1 },
        "code_output_mode": { "type": "string", "minLength": 1 },
        "generation_settings_public": { "$ref": "#/definitions/generationSettingsPublic" }
      }
    },
    "generationPolicy": {
      "type": "object",
      "required": [
        "generation_policy",
        "timeout_mode",
        "stream_idle_timeout_seconds",
        "strict_wall_clock_timeout_seconds",
        "streaming_enabled"
      ],
      "additionalProperties": false,
      "properties": {
        "generation_policy": {
          "enum": [
            "quality_generation_policy",
            "latency_bounded_generation_policy",
            "diagnostic_generation_policy",
            "unknown_legacy"
          ]
        },
        "timeout_mode": { "type": "string", "minLength": 1 },
        "stream_idle_timeout_seconds": { "type": ["integer", "null"], "minimum": 0 },
        "strict_wall_clock_timeout_seconds": { "type": ["integer", "null"], "minimum": 0 },
        "streaming_enabled": { "type": ["boolean", "null"] }
      }
    },
    "generationTiming": {
      "type": "object",
      "required": [
        "generation_started_at",
        "generation_completed_at",
        "generation_wall_clock_seconds",
        "stream_progress_event_count",
        "first_progress_after_seconds",
        "finish_reason"
      ],
      "additionalProperties": false,
      "properties": {
        "generation_started_at": { "type": ["string", "null"] },
        "generation_completed_at": { "type": ["string", "null"] },
        "generation_wall_clock_seconds": { "type": ["number", "null"], "minimum": 0 },
        "stream_progress_event_count": { "type": ["integer", "null"], "minimum": 0 },
        "first_progress_after_seconds": { "type": ["number", "null"], "minimum": 0 },
        "finish_reason": { "type": "string", "minLength": 1 }
      }
    }
  }
}