{
  "name": "AI Music Detector Public Benchmark",
  "version": "v1.0",
  "updated": "2026-08-15",
  "engine_build": "ensemble-1",
  "status": "protocol_published_results_pending",
  "license": "CC BY 4.0",
  "url": "https://aimusicdetector.co/benchmark",
  "engine_facts": [
    {
      "id": "models",
      "label": "Models in the ensemble",
      "value": "4"
    },
    {
      "id": "weights",
      "label": "Vote weights",
      "value": "spectral-artefact 2.2 · embedding 1.6 · acoustic 1.0 · extended-features 0.9"
    },
    {
      "id": "output-cap",
      "label": "Reported probability range",
      "value": "15–85%"
    },
    {
      "id": "shrinkage",
      "label": "Shrinkage toward 50%",
      "value": "0.94"
    },
    {
      "id": "confidence-bands",
      "label": "Confidence bands",
      "value": "moderate ≥ 0.50 · high ≥ 0.75"
    },
    {
      "id": "disagreement",
      "label": "Disagreement penalty",
      "value": "threshold 0.20 · penalty 1.1×"
    },
    {
      "id": "processing",
      "label": "Where analysis runs",
      "value": "In the browser"
    }
  ],
  "metrics": [
    {
      "id": "tpr",
      "name": "Detection rate (TPR)",
      "definition": "Share of known AI-generated tracks scored above 50% by the engine, counted per generator rather than pooled."
    },
    {
      "id": "fpr",
      "name": "False-positive rate (FPR)",
      "definition": "Share of verified human-produced tracks scored above 50%. This is the metric we treat as primary, because a false accusation costs a musician more than a missed detection."
    },
    {
      "id": "inconclusive",
      "name": "Inconclusive rate",
      "definition": "Share of tracks where combined confidence falls below the moderate band and no verdict is issued. Abstentions are reported, not hidden inside accuracy."
    },
    {
      "id": "calibration",
      "name": "Calibration error",
      "definition": "Mean absolute gap between the reported probability and the observed frequency of AI origin within that probability bucket. A detector that says 70% should be right about 70% of the time."
    },
    {
      "id": "stability",
      "name": "Encode stability",
      "definition": "Mean absolute change in reported probability for the same master rendered at 320, 192, 128 and 64 kbps against its lossless reading."
    }
  ],
  "corpus": {
    "planned_tracks": 690,
    "generator_arms": [
      {
        "id": "suno",
        "generator": "Suno",
        "version": "v4 / v4.5",
        "planned": 60,
        "tpr": null,
        "inconclusive": null
      },
      {
        "id": "udio",
        "generator": "Udio",
        "version": "v1.5",
        "planned": 60,
        "tpr": null,
        "inconclusive": null
      },
      {
        "id": "stable-audio",
        "generator": "Stable Audio",
        "version": "2.0",
        "planned": 40,
        "tpr": null,
        "inconclusive": null
      },
      {
        "id": "elevenlabs",
        "generator": "ElevenLabs Music",
        "version": "current",
        "planned": 40,
        "tpr": null,
        "inconclusive": null
      },
      {
        "id": "riffusion",
        "generator": "Riffusion",
        "version": "FUZZ",
        "planned": 40,
        "tpr": null,
        "inconclusive": null
      },
      {
        "id": "seed-music",
        "generator": "Seed Music",
        "version": "current",
        "planned": 30,
        "tpr": null,
        "inconclusive": null
      },
      {
        "id": "minimax",
        "generator": "MiniMax Music",
        "version": "current",
        "planned": 30,
        "tpr": null,
        "inconclusive": null
      },
      {
        "id": "mureka",
        "generator": "Mureka",
        "version": "current",
        "planned": 30,
        "tpr": null,
        "inconclusive": null
      },
      {
        "id": "held-out",
        "generator": "Held-out generator",
        "version": "undisclosed until publication",
        "planned": 40,
        "tpr": null,
        "inconclusive": null
      }
    ],
    "control_arms": [
      {
        "id": "studio",
        "source": "Commercially released studio recordings",
        "planned": 80,
        "rationale": "Heavily processed human music is the most common false-positive scenario.",
        "fpr": null
      },
      {
        "id": "bedroom",
        "source": "Independent bedroom productions",
        "planned": 80,
        "rationale": "In-the-box production with stock plugins looks superficially synthetic.",
        "fpr": null
      },
      {
        "id": "acoustic",
        "source": "Live acoustic and single-room recordings",
        "planned": 60,
        "rationale": "The easiest control case; a failure here would be disqualifying.",
        "fpr": null
      },
      {
        "id": "electronic",
        "source": "Fully synthetic human-authored electronic music",
        "planned": 60,
        "rationale": "Quantised, synthesiser-only human work is the hardest control case.",
        "fpr": null
      },
      {
        "id": "hybrid",
        "source": "Hybrid human/AI tracks (AI stems in a human arrangement)",
        "planned": 40,
        "rationale": "Reported separately; there is no correct binary answer for these.",
        "fpr": null
      }
    ]
  },
  "limitations": [
    {
      "id": "lim-compression",
      "title": "Lossy compression is the dominant error source",
      "body": "Several features are computed in exactly the frequency region an encoder discards first. Results are reported per source condition, never pooled across bitrates."
    },
    {
      "id": "lim-drift",
      "title": "Generator versions move faster than benchmarks",
      "body": "A figure measured against one model release says little about the next. Every number is stamped with the generator version and the engine build that produced it."
    },
    {
      "id": "lim-hybrid",
      "title": "Hybrid tracks have no ground truth",
      "body": "A human arrangement built around a generated stem is neither class. These are reported as a separate arm and excluded from headline figures."
    },
    {
      "id": "lim-adversarial",
      "title": "The benchmark is not adversarial",
      "body": "Deliberate evasion — re-recording, heavy remastering, aggressive time and pitch manipulation — is a separate study. Do not read these figures as robustness against an attacker."
    },
    {
      "id": "lim-scope",
      "title": "No claim about individual tracks",
      "body": "A population-level rate does not transfer to a single file. No figure on this page should be used as evidence about a specific song or person."
    }
  ],
  "note": "Null result fields are unmeasured. They are never populated with estimates."
}