{
  "schema_version": 1,
  "revised": "2026-11-03",
  "benchmarks": [
    {
      "id": "pendulum-ideal-v1",
      "title": "Ideal pendulum: period and energy convergence",
      "short_title": "Ideal pendulum",
      "category": "Numerical verification",
      "question": "Does the implementation approach the prescribed equations as the timestep decreases, without introducing unacceptable energy error?",
      "status": "protocol-ready",
      "rationale": "A compact first check with a mathematical reference independent of the simulator. The equations and reference mathematics are established; the amplitudes, run lengths, metrics, and reporting rules below are a proposed jBOK protocol, not an upstream benchmark standard. This checks numerical implementation, not agreement with a real pendulum.",
      "reference": {
        "kind": "analytical",
        "description": "Undamped point mass on a massless rigid rod: theta'' = -(g/L) sin(theta), released from rest. For amplitude theta0 below pi, T = 4 sqrt(L/g) K(sin(theta0/2)). K uses the modulus convention; libraries that accept the parameter m require m = sin(theta0/2)^2. Independently compute K with the NIST arithmetic-geometric-mean identity. E = 0.5 m L^2 omega^2 + m g L (1 - cos(theta)) is constant.",
        "uncertainty": "No experimental uncertainty is represented. The local evaluator cross-checks its binary64 AGM reference against numerical quadrature and a 80-digit power-series calculation. It separately reports a conditional crossing-resolution bound; that bound is not a statistical confidence interval or a bound on solver error."
      },
      "protocol": {
        "version": "1.0-draft",
        "scenario": "Use m = 1 kg, L = 1 m, prescribed g = 9.81 m/s^2, zero damping and actuation. Run separate release amplitudes 0.1, 1.0, and 2.0 rad with zero initial angular velocity, each for 20 analytical periods. Compare dt = T/100, T/200, T/400, and T/800. Each amplitude/timestep combination is a separate comparison group.",
        "controls": [
          "Record solver and implementation revision, floating-point precision, coordinate convention, timestep, substeps, constraint settings, and output sampling. Do not silently substitute the small-angle equation.",
          "Use the same initial state and stop time across implementations. Detect downward zero crossings with the same documented interpolation; discard the first complete cycle and average subsequent full periods. No time warping or post-hoc phase alignment.",
          "Evaluate energy from the recorded angle and angular velocity using the stated formula at every integration step; normalize against initial energy, which is nonzero for all selected amplitudes.",
          "For timing, exclude initialization and compilation; disclose warm-up, repetition count, hardware, threads, synchronization, and timed region. Compare elapsed time only within matching scenarios and hardware conditions.",
          "Show error versus timestep. A finer timestep is not automatically a better result if roundoff dominates; investigate the convergence range instead of assigning a single aggregate trust score."
        ],
        "metrics": [
          {
            "id": "period_relative_error",
            "label": "Absolute relative period error",
            "unit": "%",
            "direction": "lower",
            "description": "100 times abs(mean simulated period - analytical period) / analytical period, using the fixed crossing rule. Report amplitude, timestep, and number of complete cycles."
          },
          {
            "id": "energy_max_drift",
            "label": "Maximum absolute energy drift",
            "unit": "%",
            "direction": "lower",
            "description": "100 times max over the run of abs(E(t) - E(0)) / E(0). This measures deviation from the prescribed conservative system, not real-world accuracy."
          },
          {
            "id": "elapsed_ms",
            "label": "Median simulation time",
            "unit": "ms",
            "direction": "lower",
            "description": "Median wall-clock duration of the documented simulation region after warm-up. Report repeat count and spread alongside this scalar; interpret together with accuracy."
          }
        ],
        "failure_criteria": [
          "Reject a result as non-comparable if equations, amplitude, timestep, run length, period extraction, or energy convention differ from the declared comparison group.",
          "Mark runs invalid for nonfinite states, missing required crossings, or incomplete output. Investigate a failure to converge under timestep refinement; it is not sufficient to report one favorable timestep.",
          "No universal numerical pass threshold is asserted. Before using a result for a decision, declare that use case's allowable period and energy errors and evaluate all prescribed cases against them."
        ],
        "limitations": [
          "An ideal one-degree-of-freedom oscillator does not test contact, friction, uncertainty calibration, perception, learning, or physical validation.",
          "Period and energy can hide trajectory defects; retain full trajectories for review. A correct period alone does not establish correct dynamics.",
          "No benchmark scores are public. A local trajectory evaluator and a benchmark-specific JWM adapter are implemented for private use; their existence does not establish physical validity or validate a general-purpose solver. Public independent reproduction is not yet available."
        ]
      },
      "sources": [
        {
          "title": "MIT Underactuated Robotics — The Simple Pendulum",
          "url": "https://underactuated.mit.edu/pend.html",
          "role": "Governing equations and conservative energy; intellectual source, not validation of jBOK."
        },
        {
          "title": "Cornell Computational Methods — Pendulum Exercise",
          "url": "https://cac.cornell.edu/myers/teaching/computationalmethods/computerexercises/pendulum/index.html",
          "role": "Pedagogical precedent for comparing integrators, timestep effects, and energy behavior."
        },
        {
          "title": "NIST DLMF §19.8 — Arithmetic-geometric mean and elliptic integrals",
          "url": "https://dlmf.nist.gov/19.8",
          "role": "Independent mathematical method for evaluating the elliptic-integral period reference."
        },
        {
          "title": "NIST DLMF §19.5 — Maclaurin expansions of elliptic integrals",
          "url": "https://dlmf.nist.gov/19.5",
          "role": "A distinct power-series formula used at high precision to cross-check the evaluator's AGM reference."
        }
      ],
      "alternatives": [
        {
          "name": "Linear harmonic oscillator",
          "reason": "Simpler exact trajectory, but omits amplitude-dependent nonlinear period.",
          "switch_when": "Use first when debugging integration or testing a model that explicitly assumes small angles; keep its results in a separate benchmark."
        },
        {
          "name": "Measured physical pendulum",
          "reason": "Would assess model-to-reality agreement, but introduces damping, geometry, initial-state, and measurement uncertainty.",
          "switch_when": "Use when physical validation is the question and calibrated apparatus data are available."
        }
      ],
      "readiness": {
        "data": "No external dataset required. Generate the prescribed initial conditions and retain all output trajectories.",
        "evaluator": "Local trajectory evaluator implemented with independent reference checks, complete-grid validation, period/energy metrics, qualified timing, and source hashes. Evaluator source, adapters, and run artifacts remain in the private research workspace; no scores are published.",
        "license": "Citations only. No upstream code, text, or dataset is redistributed; any later code reuse requires its own license review."
      },
      "coverage": [
        "Numerical verification",
        "Energy conservation",
        "Timestep convergence",
        "Computational cost"
      ]
    },
    {
      "id": "planar-pushing-v1",
      "title": "Measured planar pushing: contact and friction prediction",
      "short_title": "Planar pushing",
      "category": "Physical validation",
      "question": "Can a model predict held-out object motion under a measured pushing action, with uncertainty and preprocessing made explicit?",
      "status": "candidate",
      "rationale": "MIT MCube provides a primary experimental reference with measured motion and contact forces. It complements the pendulum by confronting a simulator with physical measurements. jBOK proposes a narrow, reproducible subset rather than claiming to reproduce the authors' published evaluation. The subset, split, uncertainty treatment, and evaluator must be frozen before scoring.",
      "reference": {
        "kind": "measured",
        "description": "MIT MCube planar pushing data contain timestamped pusher/object poses and interaction forces. The project's repeated-push validation set is a candidate for evaluating both prediction error and repeatability. Use measured pusher motion as input and held-out object motion as the target.",
        "uncertainty": "Repeated trials expose variability but do not alone establish calibrated measurement uncertainty. Extract sensor, synchronization, pose-processing, and parameter uncertainty from the primary documentation before defining acceptance thresholds; unknown components must remain explicit."
      },
      "protocol": {
        "version": "1.0-draft",
        "scenario": "Candidate scope: the rect1 object in the MIT catalog's Validation dataset for modeling [2], associated with Bauza and Rodriguez (ICRA 2017): 1 cm straight pushes at 20 mm/s with repeated trials. This is distinct from the catalog's 2016 repeated pushing collection. Confirm the surface metadata and select one surface for the first release. Freeze exact filenames, exclusions, checksums, coordinate transforms, time grid, and calibration/evaluation assignments in a versioned manifest before accepting results.",
        "controls": [
          "Keep complete repeated trials intact when splitting data. Separate parameter calibration and model selection from final evaluation; never fit friction or uncertainty parameters on evaluation targets.",
          "Distinguish held-out repetitions of familiar conditions from held-out contact locations or directions. Report them as separate comparison groups; success on repetitions does not demonstrate transfer to new conditions.",
          "Replay measured pusher motion. Supply only the initial measured object state, documented geometry, and permitted calibration parameters; do not feed evaluation object trajectories back into the predictor.",
          "Pin preprocessing and resampling rules, orientation unwrapping, coordinate transforms, known bad-record exclusions, and the shared evaluation interval before scoring. Retain raw-to-processed traceability.",
          "Keep geometry, material parameter provenance, action inputs, evaluation trials, timestep, and output grid identical for a comparison. Report per-trial metrics and aggregate with uncertainty across whole trials.",
          "Record implementation revision, precision, hardware, timing region, warm-up, and repetition count. Runtime comparisons require matching execution conditions."
        ],
        "metrics": [
          {
            "id": "position_rmse",
            "label": "Planar position RMSE",
            "unit": "mm",
            "direction": "lower",
            "description": "Per trial, sqrt(mean of squared Euclidean x/y position error on the frozen time grid), converted to millimeters. Publish the median across held-out trials with trial count and an interval; retain per-trial values."
          },
          {
            "id": "orientation_rmse",
            "label": "Orientation RMSE",
            "unit": "rad",
            "direction": "lower",
            "description": "Per trial, root mean square of the wrapped angular difference in [-pi, pi]. Publish the median across held-out trials with trial count and an interval."
          },
          {
            "id": "endpoint_error",
            "label": "Final position error",
            "unit": "mm",
            "direction": "lower",
            "description": "Euclidean x/y error at the common final evaluation time, converted to millimeters. Publish the median across held-out trials; do not substitute this for trajectory error."
          },
          {
            "id": "elapsed_ms",
            "label": "Median prediction time",
            "unit": "ms",
            "direction": "lower",
            "description": "Median wall-clock time for predicting one complete held-out trial after warm-up. State trial duration distribution, repeat count, spread, and included operations."
          }
        ],
        "failure_criteria": [
          "Do not accept scores until the data manifest, license basis, split, preprocessing, and evaluator are pinned. Do not rank records from different manifests or evaluation tracks together.",
          "Reject results that tune parameters on final evaluation targets, omit failed trials without a declared exclusion rule, use target trajectories as prediction inputs, or silently change alignment to reduce error.",
          "Report nonfinite or incomplete predictions and the number of failed trials separately from errors on successful trials.",
          "Declare decision-specific error tolerances before inspecting evaluation outcomes. Physical agreement is limited to the tested conditions and reference uncertainty; no universal pass threshold is claimed."
        ],
        "limitations": [
          "This is a candidate protocol, not a completed benchmark or a claim of JWM validation. Upstream download links are published, but the archive has not been retrieved and pinned for this release.",
          "Planar rigid-object pushing does not establish fidelity for impacts, deformables, fluids, 3D manipulation, or all friction regimes.",
          "Unknown measurement uncertainty and insufficient metadata may limit interpretation even when trajectory errors are small.",
          "Uncertainty-calibration scores are intentionally deferred until interval definitions and the trial split are fixed; low RMSE is not evidence of calibrated confidence."
        ]
      },
      "sources": [
        {
          "title": "MIT MCube — Push Dataset",
          "url": "https://web.mit.edu/mcube/push-dataset/",
          "role": "Primary data catalog, formats, geometry, validation collection, download links, and known bad-record list."
        },
        {
          "title": "Yu, Bauza, Fazeli and Rodriguez — More than a Million Ways to Be Pushed (IROS 2016)",
          "url": "https://arxiv.org/abs/1604.04038",
          "role": "Experimental design and friction variability; source for assessing measurement assumptions."
        },
        {
          "title": "MIT MCube — pdproc",
          "url": "https://github.com/mcubelab/pdproc",
          "role": "Upstream preprocessing reference, including resampling and orientation handling. No code is copied into this release."
        },
        {
          "title": "Bauza and Rodriguez — A probabilistic data-driven model for planar pushing (ICRA 2017)",
          "url": "https://arxiv.org/abs/1704.03033v2",
          "role": "Primary modeling paper associated with the selected repeated validation collection; distinguishes its provenance from the 2016 dataset paper."
        },
        {
          "title": "GAUGE — project repository",
          "url": "https://github.com/InternRobotics/GAUGE",
          "role": "Alternative under consideration; inspected release status, not an adopted dependency or source of benchmark scores."
        },
        {
          "title": "InternRobotics — GAUGE repository",
          "url": "https://github.com/InternRobotics/GAUGE",
          "role": "Alternative under review; inspected README still describes the evaluation toolkit as forthcoming."
        }
      ],
      "alternatives": [
        {
          "name": "Analytical Coulomb-friction block slide",
          "reason": "Easy to implement and valuable for checking equations, but agreement with a prescribed friction law is not physical validation.",
          "switch_when": "Use as an additional numerical verification case while measured data are being prepared."
        },
        {
          "name": "GAUGE",
          "reason": "Targets measured physical fidelity across more regimes. Its inspected repository still labels the evaluation toolkit as forthcoming; this release does not rely on it.",
          "switch_when": "Reassess when a specific task's data, metadata, evaluator, version, and reuse terms can be retrieved and independently checked."
        }
      ],
      "readiness": {
        "data": "Primary catalog and download links verified; archive retrieval, immutable subset, exclusion list, and checksums pending. A published link is not a verified local dataset.",
        "evaluator": "Candidate protocol only. Data adapter, fixed split, uncertainty analysis, and independently checked scorer remain to be built.",
        "license": "Dataset redistribution and upstream code reuse terms have not been established for this release. Link to the owners; do not bundle their data or code until resolved."
      },
      "coverage": [
        "Physical validation",
        "Contact and friction",
        "Repeatability",
        "Computational cost"
      ]
    }
  ],
  "results": []
}
