{ "$schema": "http://json-schema.org/draft-07/schema#", "$id": "https://duellab.org/benchmark/leaderboard_data.schema.json", "title": "Public benchmark leaderboard.json (v7-v11 single file; v12 index and parts)", "description": "Machine-readable leaderboard export for the generated public benchmark site; produced by boardgame_cli (see crates/boardgame_cli/src/public_benchmark/site/leaderboard_data.rs). Download schema v12 splits rows into leaderboard/.json and leaderboard//.json parts listed by the leaderboard.json index; row fields are unchanged.", "oneOf": [ { "$ref": "#/definitions/legacyLeaderboard" }, { "$ref": "#/definitions/splitIndex" }, { "$ref": "#/definitions/splitPart" } ], "definitions": { "legacyLeaderboard": { "type": "object", "required": [ "schema_version", "public_release_id", "generated_at", "benchmark_line", "snapshot_id", "results_cut_id", "suite_line_id", "suite_version", "comparability_tuples", "rows" ], "additionalProperties": false, "properties": { "schema_version": { "enum": [7, 8, 9, 10, 11] }, "public_release_id": { "type": "string", "pattern": "^gb2-[0-9]{8}-[a-f0-9]{12}$" }, "generated_at": { "type": "string", "format": "date-time" }, "benchmark_line": { "const": "gamebench2" }, "snapshot_id": { "type": "string", "minLength": 1 }, "results_cut_id": { "type": "string", "minLength": 1 }, "suite_line_id": { "type": "string", "minLength": 1 }, "suite_version": { "type": "string" }, "comparability_tuples": { "type": "array", "items": { "$ref": "#/definitions/comparabilityTuple" } }, "rows": { "type": "array", "items": { "$ref": "#/definitions/leaderboardRow" } } }, "description": "Single-file leaderboard.json (download schema v7-v11). Historical releases keep this layout." }, "splitIndex": { "type": "object", "description": "Download schema v12 leaderboard.json: an index of part files; rows are not inline.", "required": [ "schema_version", "layout", "public_release_id", "generated_at", "benchmark_line", "snapshot_id", "results_cut_id", "suite_line_id", "suite_version", "comparability_tuples", "full_exports", "parts" ], "additionalProperties": false, "properties": { "schema_version": { "const": 12 }, "layout": { "const": "split" }, "public_release_id": { "type": "string", "pattern": "^gb2-[0-9]{8}-[a-f0-9]{12}$", "description": "Release identity; cite it with the immutable releases// copy." }, "generated_at": { "type": "string", "format": "date-time" }, "benchmark_line": { "const": "gamebench2" }, "snapshot_id": { "type": "string", "minLength": 1 }, "results_cut_id": { "type": "string", "minLength": 1 }, "suite_line_id": { "type": "string", "minLength": 1 }, "suite_version": { "type": "string" }, "comparability_tuples": { "type": "array", "items": { "$ref": "#/definitions/comparabilityTuple" } }, "full_exports": { "type": "object", "required": ["csv", "parquet"], "additionalProperties": false, "properties": { "csv": { "const": "leaderboard.csv" }, "parquet": { "const": "leaderboard.parquet" } } }, "parts": { "type": "array", "minItems": 1, "items": { "$ref": "#/definitions/splitPartEntry" } } } }, "splitPartEntry": { "type": "object", "required": [ "path", "track_id", "identity_component", "row_kind", "game_alias", "row_count", "bytes", "sha256" ], "additionalProperties": false, "properties": { "path": { "type": "string", "pattern": "^leaderboard/[a-z0-9-]+(/[a-z0-9-]+)?\\.json$" }, "track_id": { "type": "string", "minLength": 1 }, "identity_component": { "type": "string", "minLength": 1 }, "row_kind": { "enum": ["track", "game"] }, "game_alias": { "type": ["string", "null"] }, "row_count": { "type": "integer", "minimum": 1 }, "bytes": { "type": "integer", "minimum": 1 }, "sha256": { "type": "string", "pattern": "^[a-f0-9]{64}$" } } }, "splitPart": { "type": "object", "description": "Download schema v12 part file under leaderboard/: rows of one track summary or one track-game detail.", "required": [ "schema_version", "public_release_id", "snapshot_id", "track_id", "row_kind", "game_alias", "rows" ], "additionalProperties": false, "properties": { "schema_version": { "const": 12 }, "public_release_id": { "type": "string", "pattern": "^gb2-[0-9]{8}-[a-f0-9]{12}$" }, "snapshot_id": { "type": "string", "minLength": 1 }, "track_id": { "type": "string", "minLength": 1 }, "row_kind": { "enum": ["track", "game"] }, "game_alias": { "type": ["string", "null"] }, "rows": { "type": "array", "minItems": 1, "items": { "$ref": "#/definitions/leaderboardRow" } } } }, "leaderboardRow": { "type": "object", "required": [ "snapshot_id", "results_cut_id", "suite_line_id", "suite_version", "scoring_policy_version", "rating_method", "row_kind", "track_id", "track_name", "game_alias", "canonical_public_row_identity", "evaluated_setting_identity", "eligibility_flag", "rank", "ranking_status", "ineligibility_reason_codes", "model_slug", "model_display_name", "provider", "provider_key", "reasoning_level", "reasoning_bracket", "reasoning_setting", "measured_at", "last_measured_at", "first_seen_at", "is_recent_addition", "rank_movement", "rank_movement_baseline_at", "score", "conditional_success_score", "plugin_success_rate", "score_basis", "min_score", "max_score", "raw_elo", "rated_matches", "attempted_matches", "capped_draws", "unilateral_faults", "distinct_rated_opponents", "wins", "losses", "draws", "uncertainty", "entry_count", "contributing_entrants", "codegen_evaluated_cells", "admitted_plugins", "first_pass_admitted", "repaired_admitted", "model_code_failures", "model_code_failure_rate", "first_pass_rate", "repaired_rate", "infra_failures_excluded", "failed_game_aliases", "infra_excluded_game_aliases", "codegen_cost_usd", "reasoning_provenance", "generation_policy", "generation_timing" ], "additionalProperties": false, "properties": { "snapshot_id": { "type": "string", "minLength": 1, "description": "Internal results-cut identity (legacy name)." }, "results_cut_id": { "type": "string", "minLength": 1, "description": "In downloads built by the public site, the release identity (equal to public_release_id). snapshot_id carries the internal results-cut identity." }, "suite_line_id": { "type": "string", "minLength": 1 }, "suite_version": { "type": "string" }, "scoring_policy_version": { "type": "string", "minLength": 1 }, "rating_method": { "type": "string", "minLength": 1 }, "row_kind": { "enum": ["track", "game"] }, "track_id": { "type": "string", "minLength": 1, "description": "overall (one row per model); full_freedom/mixed, Reasoning Variants (one row per evaluated request; use it for single settings); full_freedom/baseline, balanced or intensive (one row per model and group, which can pool several requests)." }, "track_name": { "type": "string", "minLength": 1 }, "game_alias": { "type": ["string", "null"] }, "canonical_public_row_identity": { "description": "Public key of this row within one release. Reference an imported result by public_release_id plus this key; a key can change between releases without any change in the result (Overall keys include the reasoning catalog digest; variant numbers are positional).", "type": "string", "pattern": "^publication-(?:row|game-row)\\.v1:" }, "evaluated_setting_identity": { "description": "The evaluated request. Classify a Reasoning Variants row's setting from request_mode and effective_request_value, not from labels.", "anyOf": [{ "type": "null" }, { "$ref": "#/definitions/evaluatedSettingIdentityV1" }] }, "eligibility_flag": { "type": "boolean", "description": "True only for complete-evidence ranked rows. Partial Overall rows remain false even though they receive an official rank." }, "rank": { "type": ["integer", "null"], "minimum": 1, "description": "Official rank. In schema v11, ranked and partial Overall rows share one contiguous score ordering; provisional Reasoning Variants remain null." }, "ranking_status": { "enum": ["ranked", "provisional", "partial", "not_applicable"], "description": "ranked: official rank, complete evidence. partial: official Overall rank, incomplete evidence (eligibility_flag false). provisional: no rank. not_applicable: game rows." }, "ineligibility_reason_codes": { "type": "array", "uniqueItems": true, "items": { "enum": [ "missing_bracket_scores", "unknown_capability_evidence", "missing_or_non_distinct_settings", "untested_or_ineligible_settings", "missing_generation_quality", "generation_incomplete", "below_generation_quality_threshold", "fixed_coverage_incomplete", "grouped_stability_incomplete" ] } }, "model_slug": { "type": ["string", "null"], "description": "Model identity within a release. Dated provider versions have their own slugs and rows." }, "model_display_name": { "type": "string", "minLength": 1, "description": "Display name only; not a join key and not unique across dated versions." }, "provider": { "type": "string" }, "provider_key": { "type": "string" }, "reasoning_level": { "type": ["string", "null"], "description": "Display label of the setting (Overall Models on Overall rows): the current capability catalog's wording for the recorded value, so it can change between releases. Not a join key." }, "reasoning_bracket": { "description": "Reasoning group: baseline, balanced or intensive. Null on Overall rows and on a row whose request served more than one group.", "type": ["string", "null"], "enum": ["baseline", "balanced", "intensive", "mixed", null] }, "reasoning_setting": { "type": ["string", "null"], "description": "Reasoning Variants rows: the release's public setting key (for example historical-provider-setting-intensive, with -variant-N numbered by position), not the literal request; the literal is evaluated_setting_identity.effective_request_value. Baseline, Balanced and Intensive rows: the tested setting the row reports." }, "measured_at": { "type": ["string", "null"] }, "last_measured_at": { "type": ["string", "null"] }, "first_seen_at": { "type": ["string", "null"] }, "is_recent_addition": { "type": "boolean" }, "rank_movement": { "type": ["integer", "null"] }, "rank_movement_baseline_at": { "type": ["string", "null"] }, "score": { "type": "number" }, "conditional_success_score": { "type": ["number", "null"], "minimum": 0, "maximum": 100 }, "plugin_success_rate": { "type": ["number", "null"], "minimum": 0, "maximum": 1 }, "score_basis": { "type": "string", "minLength": 1, "description": "How score is computed. From scoring v17 a game row's score is working-program strength (failed generation attempts not counted); the headline per-game values are the track row's headline_games." }, "headline_games": { "type": "object", "description": "Scoring v17 track rows only: each game's headline value keyed by public game alias. score is the equal-weighted mean of their score, except on Overall rows, whose score is the mean of the setting scores and whose values are each game's mean over the settings (for display). Absent before v17 and on game rows.", "additionalProperties": { "$ref": "#/definitions/headlineGameValue" } }, "min_score": { "type": ["number", "null"] }, "max_score": { "type": ["number", "null"] }, "raw_elo": { "type": ["number", "null"] }, "rated_matches": { "type": ["integer", "null"], "minimum": 0 }, "attempted_matches": { "type": ["integer", "null"], "minimum": 0 }, "capped_draws": { "type": ["integer", "null"], "minimum": 0 }, "unilateral_faults": { "type": ["integer", "null"], "minimum": 0 }, "distinct_rated_opponents": { "type": ["integer", "null"], "minimum": 0 }, "wins": { "type": ["integer", "null"], "minimum": 0 }, "losses": { "type": ["integer", "null"], "minimum": 0 }, "draws": { "type": ["integer", "null"], "minimum": 0 }, "uncertainty": { "type": ["number", "null"] }, "entry_count": { "type": ["integer", "null"], "minimum": 0 }, "contributing_entrants": { "type": ["integer", "null"], "minimum": 0 }, "working_attempts": { "type": "integer", "minimum": 0, "description": "Scoring v17 game rows only: working attempts behind the row's programs in this game (a program two attempts produced identically counts twice). With counted_attempts it reads \"N working of M attempts\". Absent on track rows and before v17." }, "counted_attempts": { "type": "integer", "minimum": 0, "description": "Scoring v17 game rows only: counted attempts in this game, the working attempts plus the attempts that failed in model-written code (too-slow programs and provider, API or test-machine faults are not counted). Absent on track rows and before v17." }, "program_raw_elo_low": { "type": "number", "description": "Scoring v17 game rows with two or more rated programs: the lowest program raw rating; raw_elo is the programs' attempt-weighted mean and wins, losses and draws their combined record. A description of the programs, not a confidence interval." }, "program_raw_elo_high": { "type": "number", "description": "Scoring v17 game rows with two or more rated programs: the highest program raw rating (see program_raw_elo_low)." }, "codegen_evaluated_cells": { "type": ["integer", "null"], "minimum": 0 }, "admitted_plugins": { "type": ["integer", "null"], "minimum": 0 }, "first_pass_admitted": { "type": ["integer", "null"], "minimum": 0 }, "repaired_admitted": { "type": ["integer", "null"], "minimum": 0 }, "model_code_failures": { "type": ["integer", "null"], "minimum": 0 }, "model_code_failure_rate": { "type": ["number", "null"], "minimum": 0, "maximum": 1 }, "first_pass_rate": { "type": ["number", "null"], "minimum": 0, "maximum": 1 }, "repaired_rate": { "type": ["number", "null"], "minimum": 0, "maximum": 1 }, "infra_failures_excluded": { "type": ["integer", "null"], "minimum": 0 }, "failed_game_aliases": { "type": "array", "items": { "type": "string", "minLength": 1 }, "uniqueItems": true }, "infra_excluded_game_aliases": { "type": "array", "items": { "type": "string", "minLength": 1 }, "uniqueItems": true }, "performance_rejections": { "type": ["integer", "null"], "minimum": 0 }, "performance_rejected_game_aliases": { "type": "array", "items": { "type": "string", "minLength": 1 }, "uniqueItems": true }, "codegen_cost_usd": { "$ref": "#/definitions/codegenCostUsdNullable" }, "reasoning_provenance": { "$ref": "#/definitions/reasoningProvenance" }, "generation_policy": { "$ref": "#/definitions/generationPolicy" }, "generation_timing": { "$ref": "#/definitions/generationTiming" } } }, "headlineGameValue": { "type": "object", "description": "One game's headline value (scoring v17): the rated working programs' scores weighted by the attempts that produced them, plus 0 for each model-caused failed generation attempt, divided by those attempts.", "required": ["score", "attempts"], "additionalProperties": false, "properties": { "score": { "type": "number", "minimum": 0, "maximum": 100 }, "playable": { "type": "number", "minimum": 0, "maximum": 100, "description": "Attempt-weighted mean of the rated working programs; absent without one." }, "attempts": { "type": "integer", "minimum": 1 }, "failed_attempts": { "type": "integer", "minimum": 1, "description": "Model-caused failed attempts; absent when there were none." } } }, "evaluatedSettingIdentityV1": { "type": "object", "additionalProperties": false, "required": [ "schema_version", "canonical_provider_route", "provider_model", "request_mode", "effective_request_value", "prompt_identity", "prompt_profile_identity", "output_mode", "reasoning_policy_version", "reasoning_catalog_sha256", "resolution_status" ], "properties": { "schema_version": { "const": "evaluated_setting_identity.v1" }, "canonical_provider_route": { "type": "string", "minLength": 1, "description": "Provider route. For a row rebuilt without a stored identity, the current catalog's route for the model, not a recorded route." }, "provider_model": { "type": "string", "minLength": 1, "description": "The provider's model identifier for this request." }, "request_mode": { "type": "string", "minLength": 1, "description": "How the request set reasoning: effort, toggle, omitted, budget, or unknown_legacy." }, "effective_request_value": { "type": "string", "minLength": 1, "description": "The recorded value for request_mode, for example xhigh, none, enabled or default. A historical-... value or unknown_legacy means no row-level value was recorded; Overall rows use aggregate." }, "prompt_identity": { "type": "string", "minLength": 1 }, "prompt_profile_identity": { "type": "string", "minLength": 1 }, "output_mode": { "type": "string", "minLength": 1 }, "reasoning_policy_version": { "type": "string", "minLength": 1 }, "reasoning_catalog_sha256": { "type": "string", "minLength": 1, "description": "Digest of the reasoning capability catalog bound to the request; on Overall rows, the catalog the site was built with." }, "resolution_status": { "enum": ["exact", "opaque_historical"], "description": "exact when every identity field was retained; opaque_historical when at least one was not, often the prompt or output identity. It does not say whether the effort is known." } } }, "comparabilityTuple": { "type": "object", "required": [ "evaluation_family", "track_id", "track_kind", "reasoning_effort", "suite_line_id", "suite_version", "scoring_policy_version", "rating_method", "snapshot_id" ], "additionalProperties": false, "properties": { "evaluation_family": { "type": "string", "minLength": 1 }, "track_id": { "type": "string", "minLength": 1 }, "track_kind": { "type": "string", "minLength": 1 }, "reasoning_effort": { "type": "string", "minLength": 1 }, "suite_line_id": { "type": "string", "minLength": 1 }, "suite_version": { "type": "string" }, "scoring_policy_version": { "type": "string", "minLength": 1 }, "coverage_policy_identity": { "type": ["string", "null"], "minLength": 1 }, "rating_method": { "type": "string", "minLength": 1 }, "snapshot_id": { "type": "string", "minLength": 1 }, "results_cut_id": { "type": ["string", "null"], "description": "Internal results-cut identity (equal to snapshot_id)." } } }, "codegenCostUsdNullable": { "anyOf": [{ "type": "null" }, { "$ref": "#/definitions/codegenCostUsd" }] }, "codegenCostUsd": { "type": "object", "required": ["mean", "min", "max", "total", "run_count"], "additionalProperties": false, "properties": { "mean": { "type": "number", "minimum": 0 }, "median": { "type": "number", "minimum": 0 }, "min": { "type": "number", "minimum": 0 }, "max": { "type": "number", "minimum": 0 }, "total": { "type": "number", "minimum": 0 }, "cell_count": { "type": "integer", "minimum": 0 }, "run_count": { "type": "integer", "minimum": 0 }, "admitted_cell_count": { "type": "integer", "minimum": 0 }, "failed_cell_count": { "type": "integer", "minimum": 0 }, "recorded_attempt_count": { "type": "integer", "minimum": 0 }, "token_priced_attempt_count": { "type": "integer", "minimum": 0 }, "legacy_label_count": { "type": "integer", "minimum": 0 }, "cost_basis_counts": { "type": "object", "additionalProperties": { "type": "integer", "minimum": 0 } }, "pricing_table_versions": { "type": "object", "additionalProperties": { "type": "integer", "minimum": 0 } }, "evidence_basis": { "type": "string" }, "cost_is_complete": { "type": "boolean" } } }, "generationSettingsPublic": { "type": "object", "required": ["temperature", "max_tokens", "stream"], "additionalProperties": false, "properties": { "temperature": { "type": ["number", "null"] }, "max_tokens": { "type": ["integer", "null"], "minimum": 0 }, "stream": { "type": ["boolean", "null"] } } }, "reasoningProvenance": { "type": "object", "required": [ "reasoning_bucket", "reasoning_bucket_label", "provider", "provider_model", "requested_reasoning_effort", "provider_setting_label", "provider_reasoning_params", "prompt_identity", "prompt_profile", "code_prompt_version", "code_output_mode", "generation_settings_public" ], "additionalProperties": false, "properties": { "reasoning_bucket": { "type": "string", "minLength": 1, "description": "Legacy transport bucket or public setting key; not a comparison key." }, "reasoning_bucket_label": { "type": "string", "minLength": 1 }, "reasoning_bracket": { "type": ["string", "null"], "enum": ["baseline", "balanced", "intensive", "mixed", null] }, "reasoning_policy_version": { "type": "string", "minLength": 1 }, "reasoning_catalog_sha256": { "type": "string", "pattern": "^[a-f0-9]{64}$" }, "provider": { "type": "string", "minLength": 1 }, "provider_model": { "type": "string", "minLength": 1 }, "provider_route": { "type": "string", "minLength": 1 }, "requested_reasoning_effort": { "type": ["string", "null"], "description": "Reasoning Variants rows: the same public setting key as reasoning_setting, not the literal request (see evaluated_setting_identity)." }, "provider_setting_label": { "type": ["string", "null"], "description": "Display label of the setting." }, "reasoning_request_mode": { "type": ["string", "null"], "enum": ["effort", "toggle", "omitted", "budget", null] }, "provider_reasoning_params": { "type": "object", "description": "Untyped request details. On Reasoning Variants rows, evidence_status and bracket_eligible follow evaluated_setting_identity.resolution_status, so they do not show whether the request was recorded; on Baseline, Balanced and Intensive rows they show whether the setting is the model's documented one for its group." }, "prompt_identity": { "type": "string", "minLength": 1 }, "prompt_profile": { "type": "string", "minLength": 1 }, "code_prompt_version": { "type": "string", "minLength": 1 }, "code_output_mode": { "type": "string", "minLength": 1 }, "generation_settings_public": { "$ref": "#/definitions/generationSettingsPublic" } } }, "generationPolicy": { "type": "object", "required": [ "generation_policy", "timeout_mode", "stream_idle_timeout_seconds", "strict_wall_clock_timeout_seconds", "streaming_enabled" ], "additionalProperties": false, "properties": { "generation_policy": { "enum": [ "quality_generation_policy", "latency_bounded_generation_policy", "diagnostic_generation_policy", "unknown_legacy" ] }, "timeout_mode": { "type": "string", "minLength": 1 }, "stream_idle_timeout_seconds": { "type": ["integer", "null"], "minimum": 0 }, "strict_wall_clock_timeout_seconds": { "type": ["integer", "null"], "minimum": 0 }, "streaming_enabled": { "type": ["boolean", "null"] } } }, "generationTiming": { "type": "object", "required": [ "generation_started_at", "generation_completed_at", "generation_wall_clock_seconds", "stream_progress_event_count", "first_progress_after_seconds", "finish_reason" ], "additionalProperties": false, "properties": { "generation_started_at": { "type": ["string", "null"] }, "generation_completed_at": { "type": ["string", "null"] }, "generation_wall_clock_seconds": { "type": ["number", "null"], "minimum": 0 }, "stream_progress_event_count": { "type": ["integer", "null"], "minimum": 0 }, "first_progress_after_seconds": { "type": ["number", "null"], "minimum": 0 }, "finish_reason": { "type": "string", "minLength": 1 } } } } }