feat: add explicit retrospective v2 computation contracts

2026-09-08 19:40:19 +08:00
parent 68dd68392a
commit 2b0e8ee637
19 changed files with 7688 additions and 4 deletions
@@ -0,0 +1,311 @@
+"""Fresh synthetic artifact assembly; no reuse or relabelling of real outputs."""
+
+from __future__ import annotations
+
+import hashlib
+import json
+from dataclasses import replace
+from typing import Any
+
+import pandas as pd
+import pytest
+
+from quant_engine.artifact import (
+    EvidenceQualification,
+    PerformanceEvidenceError,
+    ResearchRunArtifact,
+    build_research_run_artifact,
+    build_backtest_evidence_manifest,
+    build_performance_evidence,
+)
+from quant_engine.execution import ExecutionConfig
+from quant_engine.factor_contracts import FactorContractError
+from quant_engine.governed_pipeline import BacktestContractError
+from quant_engine.research_pipeline import run_factor_backtest_research
+from quant_engine.retrospective_artifact_contracts import (
+    build_retrospective_backtest_evidence_manifest,
+    build_retrospective_performance_evidence,
+    RetrospectiveBacktestEvidenceManifest,
+    RetrospectivePerformanceEvidence,
+)
+from quant_engine.retrospective_backtest_contracts import RetrospectiveBacktestRunRef
+from test_retrospective_backtest_contracts import run_arguments
+from test_retrospective_data_contracts import identify, replace_at
+
+CONTRACT_ERRORS = (FactorContractError, BacktestContractError, PerformanceEvidenceError)
+
+
+def synthetic_artifact(run: RetrospectiveBacktestRunRef) -> ResearchRunArtifact:
+    # Artifact-envelope tests, not an end-to-end proof of factor/source authenticity.
+    # The existing financial methods receive new, in-memory synthetic matrices.
+    dates = pd.date_range("2018-01-02", periods=4, freq="B")
+    scores = pd.DataFrame({"SIM0": [2.0, 0.0], "SIM1": [1.0, 3.0]}, index=dates[:2])
+    opens = pd.DataFrame(
+        {"SIM0": [10.0, 10.0, 15.0, 15.0], "SIM1": [20.0, 20.0, 20.0, 21.0]}, index=dates
+    )
+    closes = pd.DataFrame(
+        {"SIM0": [10.0, 12.0, 15.0, 15.0], "SIM1": [20.0, 20.0, 18.0, 21.0]}, index=dates
+    )
+    result = run_factor_backtest_research(
+        scores,
+        opens,
+        closes,
+        top_k=1,
+        execution_price_field="open",
+        valuation_price_field="close",
+        initial_cash=1000.0,
+        config=ExecutionConfig(
+            commission_bps=0, stamp_tax_bps=0, slippage_bps=0, min_trade_amount=0
+        ),
+    )
+    benchmark = pd.Series(
+        [0.0, 0.01, -0.01, 0.02], index=result.returns.index, name="benchmark_return"
+    )
+    return build_research_run_artifact(
+        result,
+        run_id=run.run_id,
+        strategy_id=run.strategy_id,
+        strategy_name="Synthetic Top 1",
+        strategy_version=run.strategy_version,
+        engine_version="0.1.0",
+        code_revision=run.code_revision,
+        data_snapshot_id=run.dataset_snapshot_id,
+        calendar="CN-A",
+        timezone="Asia/Shanghai",
+        started_at=run.evaluation_at,
+        finished_at=run.computed_at,
+        parameters={"lag_sessions": 1, "top_k": 1},
+        benchmark_id="synthetic.benchmark",
+        benchmark_returns=benchmark,
+    )
+
+
+def test_manifest_closes_all_nine_existing_tables_without_changing_their_schema() -> None:
+    run = RetrospectiveBacktestRunRef.create(**run_arguments())
+    artifact = synthetic_artifact(run)
+    manifest = build_retrospective_backtest_evidence_manifest(
+        run, artifact, artifact_available_at="2026-09-08T01:11:00Z"
+    )
+    wire = manifest.to_dict()
+    assert wire["schema_version"] == "2.0.0"
+    assert wire["artifact_schema_version"] == "1.1.0"
+    assert wire["run_id"] == run.run_id
+    assert wire["usage"] == "retrospective_research"
+    assert wire["historical_availability"] == "not_established"
+    assert wire["execution_validation"] == "not_validated"
+    assert wire["decision_eligible"] is False
+    assert manifest.manifest_id.startswith("rhbacktestevidencev2:sha256:")
+    assert len({table.logical_name for item in manifest.evidence for table in item.tables}) == 9
+
+
+def test_performance_v2_keeps_existing_metric_methods_and_binds_all_upstreams() -> None:
+    run = RetrospectiveBacktestRunRef.create(**run_arguments())
+    artifact = synthetic_artifact(run)
+    manifest = build_retrospective_backtest_evidence_manifest(
+        run, artifact, artifact_available_at="2026-09-08T01:11:00Z"
+    )
+    evidence = build_retrospective_performance_evidence(artifact, run, manifest)
+    wire = evidence.to_dict()
+    assert wire["schema_version"] == "researchhub.performance-evidence.v2"
+    assert wire["methodology_id"] == "researchhub.quant-performance-methodology.v1"
+    assert wire["metric_schema_id"] == "researchhub.quant-performance-metrics.v1"
+    assert wire["research_artifact_schema_version"] == "1.1.0"
+    assert wire["backtest_run_ref_id"] == run.run_id
+    assert wire["backtest_evidence_manifest_id"] == manifest.manifest_id
+    assert wire["historical_availability"] == "not_established"
+    assert wire["usage"] == "retrospective_research"
+    assert wire["start_date"] == "2018-01-02"
+    assert wire["end_date"] == "2018-01-05"
+    assert wire["artifact_available_at"] == "2026-09-08T01:11:00Z"
+    assert evidence.run_id == run.run_id
+    assert evidence.performance_evidence_id.startswith("rhperformancev2:sha256:")
+    for metric in evidence.metrics:
+        if metric.value is not None:
+            assert metric.value == artifact.performance.iloc[0][metric.source_column]
+    assert (
+        RetrospectivePerformanceEvidence.from_json(
+            evidence.to_json(), artifact=artifact, run_ref=run, evidence_manifest=manifest
+        )
+        == evidence
+    )
+    assert (
+        RetrospectiveBacktestEvidenceManifest.from_json(
+            manifest.to_json(), artifact=artifact, backtest_run_ref=run
+        )
+        == manifest
+    )
+
+
+@pytest.mark.parametrize(
+    ("path", "value"),
+    [
+        ("schema_version", "1.0.0"),
+        ("run_id", "rhbacktestrunv2:sha256:" + "0" * 64),
+        ("profile", "offline_research_v1"),
+        ("historical_availability", "established"),
+        ("decision_eligible", True),
+        ("execution_validation", "validated"),
+        ("evidence_scope", "real_data"),
+        ("artifact_available_at", "2026-09-08T01:09:00Z"),
+        ("artifact_schema_version", "2.0.0"),
+        ("qualification", "legacy_exploratory"),
+        ("evidence_digest", "sha256:" + "0" * 64),
+        ("evidence.0.tables.0.row_count", True),
+        ("evidence.0.tables.0.content_digest", "sha256:" + "0" * 64),
+        ("run_reference.value.factor_set_id", "rhfactorsetv2:sha256:" + "0" * 64),
+    ],
+)
+def test_manifest_rejects_reidentified_claims_without_actual_table_closure(
+    path: str, value: Any
+) -> None:
+    run = RetrospectiveBacktestRunRef.create(**run_arguments())
+    artifact = synthetic_artifact(run)
+    row = build_retrospective_backtest_evidence_manifest(
+        run, artifact, artifact_available_at="2026-09-08T01:11:00Z"
+    ).to_dict()
+    replace_at(row, path, value)
+    identify(row, "manifest_id", "rhbacktestevidencev2:")
+    with pytest.raises(CONTRACT_ERRORS):
+        RetrospectiveBacktestEvidenceManifest.from_dict(
+            row, artifact=artifact, backtest_run_ref=run
+        )
+
+
+@pytest.mark.parametrize(
+    ("table", "column", "value"),
+    [
+        ("run", "data_snapshot_id", "rhdsv2:sha256:" + "0" * 64),
+        ("run", "config_hash", "0" * 64),
+        ("run", "code_revision", "0" * 40),
+        ("run", "started_at", "2018-01-02T07:00:00Z"),
+        ("run", "finished_at", "2026-09-08T01:12:00Z"),
+        ("signals", "asset_id", "/private/data.csv"),
+        ("nav", "run_id", "old.run"),
+        ("performance", "run_id", "old.run"),
+    ],
+)
+def test_manifest_rejects_artifact_identity_time_or_private_data_mismatch(
+    table: str, column: str, value: Any
+) -> None:
+    run = RetrospectiveBacktestRunRef.create(**run_arguments())
+    artifact = synthetic_artifact(run)
+    frame = getattr(artifact, table)
+    frame.loc[frame.index[0], column] = value
+    forged = replace(artifact, **{"_" + table: frame})
+    with pytest.raises(CONTRACT_ERRORS):
+        build_retrospective_backtest_evidence_manifest(
+            run, forged, artifact_available_at="2026-09-08T01:13:00Z"
+        )
+
+
+def test_each_table_is_reconciled_and_legacy_apis_cannot_accept_v2() -> None:
+    run = RetrospectiveBacktestRunRef.create(**run_arguments())
+    artifact = synthetic_artifact(run)
+    manifest = build_retrospective_backtest_evidence_manifest(
+        run, artifact, artifact_available_at="2026-09-08T01:11:00Z"
+    )
+    for item in manifest.evidence:
+        for table in item.tables:
+            with pytest.raises(CONTRACT_ERRORS):
+                build_retrospective_backtest_evidence_manifest(
+                    run,
+                    artifact,
+                    artifact_available_at="2026-09-08T01:11:00Z",
+                    expected_table_digests={table.logical_name: "sha256:" + "0" * 64},
+                )
+    with pytest.raises(CONTRACT_ERRORS):
+        build_backtest_evidence_manifest(
+            run, artifact, artifact_available_at="2026-09-08T01:11:00Z"
+        )
+    with pytest.raises(CONTRACT_ERRORS):
+        build_performance_evidence(artifact, run, manifest)
+    with pytest.raises(CONTRACT_ERRORS):
+        build_retrospective_backtest_evidence_manifest(
+            run,
+            artifact,
+            artifact_available_at="2026-09-08T01:11:00Z",
+            qualification=EvidenceQualification.LEGACY_EXPLORATORY,
+        )
+
+
+def seal_performance(row: dict[str, Any]) -> None:
+    def sha(document: Any) -> str:
+        return (
+            "sha256:"
+            + hashlib.sha256(
+                json.dumps(
+                    document,
+                    ensure_ascii=False,
+                    sort_keys=True,
+                    separators=(",", ":"),
+                    allow_nan=False,
+                ).encode()
+            ).hexdigest()
+        )
+
+    row.pop("document_sha256", None)
+    row.pop("performance_evidence_id", None)
+    row["performance_evidence_id"] = "rhperformancev2:" + sha(row)
+    row["document_sha256"] = sha(row)
+
+
+@pytest.mark.parametrize(
+    ("path", "value"),
+    [
+        ("schema_version", "researchhub.performance-evidence.v1"),
+        ("scope", "live"),
+        ("historical_availability", "established"),
+        ("decision_eligible", True),
+        ("evidence_scope", "real_data"),
+        ("dataset_snapshot_id", "rhdsv2:sha256:" + "0" * 64),
+        ("backtest_run_ref_document_sha256", "sha256:" + "0" * 64),
+        ("backtest_evidence_manifest_id", "rhbacktestevidencev2:sha256:" + "0" * 64),
+        ("methodology.periods_per_year", 365),
+        ("metric_schema_id", "new.metric"),
+        ("metrics.0.value", 0.0),
+        ("metrics.0.nullable", True),
+        ("start_date", "2017-01-01"),
+        ("artifact_available_at", "2018-01-02T07:00:00Z"),
+    ],
+)
+def test_performance_never_accepts_reidentified_changed_facts(path: str, value: Any) -> None:
+    run = RetrospectiveBacktestRunRef.create(**run_arguments())
+    artifact = synthetic_artifact(run)
+    manifest = build_retrospective_backtest_evidence_manifest(
+        run, artifact, artifact_available_at="2026-09-08T01:11:00Z"
+    )
+    row = build_retrospective_performance_evidence(artifact, run, manifest).to_dict()
+    replace_at(row, path, value)
+    seal_performance(row)
+    with pytest.raises(CONTRACT_ERRORS):
+        RetrospectivePerformanceEvidence.from_dict(
+            row, artifact=artifact, run_ref=run, evidence_manifest=manifest
+        )
+
+
+def test_performance_has_immutable_finite_canonical_payload_and_current_tables() -> None:
+    run = RetrospectiveBacktestRunRef.create(**run_arguments())
+    artifact = synthetic_artifact(run)
+    manifest = build_retrospective_backtest_evidence_manifest(
+        run, artifact, artifact_available_at="2026-09-08T01:11:00Z"
+    )
+    evidence = build_retrospective_performance_evidence(artifact, run, manifest)
+    assert evidence.document_sha256.startswith("sha256:")
+    exported = evidence.to_dict()
+    exported["metrics"][0]["value"] = 9.0
+    assert evidence.to_dict()["metrics"][0]["value"] != 9.0
+    for data in (
+        evidence.to_json() + "\n",
+        '{"schema_version":"x",' + evidence.to_json()[1:],
+        "null",
+        "{bad",
+    ):
+        with pytest.raises(CONTRACT_ERRORS):
+            RetrospectivePerformanceEvidence.from_json(
+                data, artifact=artifact, run_ref=run, evidence_manifest=manifest
+            )
+    frame = artifact.performance
+    frame.loc[0, "total_ret"] = 0.0
+    forged = replace(artifact, _performance=frame)
+    with pytest.raises(CONTRACT_ERRORS):
+        build_retrospective_performance_evidence(forged, run, manifest)