diff --git a/scripts/validate-reviewer-routes.py b/scripts/validate-reviewer-routes.py index 61d2d69..1ad12b7 100644 --- a/scripts/validate-reviewer-routes.py +++ b/scripts/validate-reviewer-routes.py @@ -208,8 +208,8 @@ "local_manifest_only", "sbom_present", "policy_matched", - "enrichment_mocked", - "enrichment_live", + "enrichment_recorded", + "provenance_recorded", "not a package safety verdict", "not a CVE result", ), diff --git a/tools/sbom-diff-and-risk/docs/report-schema.md b/tools/sbom-diff-and-risk/docs/report-schema.md index 78b26b9..4fd20bb 100644 --- a/tools/sbom-diff-and-risk/docs/report-schema.md +++ b/tools/sbom-diff-and-risk/docs/report-schema.md @@ -130,8 +130,8 @@ not a CVE result. The same value appears at top level, in `summary`, and in | `local_manifest_only` | The report was produced from local manifest-style inputs without SBOM input, policy matches, or enrichment evidence. | | `sbom_present` | At least one input is an SBOM format such as CycloneDX JSON or SPDX JSON. | | `policy_matched` | Local policy evaluation produced at least one blocking, warning, or suppressed policy match. | -| `enrichment_mocked` | Enrichment-shaped evidence is present without recorded live network access, or the report explicitly marks constructed snapshot evidence as mocked. | -| `enrichment_live` | Opt-in enrichment recorded live network access for PyPI provenance or OpenSSF Scorecard evidence. | +| `enrichment_recorded` | Optional non-provenance enrichment evidence, such as OpenSSF Scorecard evidence, was recorded for the report. | +| `provenance_recorded` | PyPI provenance evidence or provenance-enrichment metadata was recorded for the report. | `summary.policy` appears only when a policy is applied. Absence of `summary.policy` means policy was not used, not that policy evaluation failed. diff --git a/tools/sbom-diff-and-risk/docs/summary-json-ci-cookbook.md b/tools/sbom-diff-and-risk/docs/summary-json-ci-cookbook.md index 4bcdd25..ca4bf77 100644 --- a/tools/sbom-diff-and-risk/docs/summary-json-ci-cookbook.md +++ b/tools/sbom-diff-and-risk/docs/summary-json-ci-cookbook.md @@ -81,7 +81,7 @@ if ($newPackageCount -gt $maxNewPackages) { - `summary.enrichment` appears only when PyPI or Scorecard enrichment is used. - `summary.evidence_confidence` is always present and can be `local_manifest_only`, `sbom_present`, `policy_matched`, - `enrichment_mocked`, or `enrichment_live`. + `enrichment_recorded`, or `provenance_recorded`. - `unchanged` is absent because unchanged components are not modeled. - Absence of `summary.policy` or `summary.enrichment` means the feature was not used, not that it failed. diff --git a/tools/sbom-diff-and-risk/examples/sample-provenance-report.json b/tools/sbom-diff-and-risk/examples/sample-provenance-report.json index 4aa918a..4ad94bb 100644 --- a/tools/sbom-diff-and-risk/examples/sample-provenance-report.json +++ b/tools/sbom-diff-and-risk/examples/sample-provenance-report.json @@ -12,7 +12,7 @@ "suspicious_source": 0, "not_evaluated": 0 }, - "evidence_confidence": "enrichment_mocked", + "evidence_confidence": "provenance_recorded", "policy": { "status": "fail", "blocking": 2, @@ -33,7 +33,7 @@ } } }, - "evidence_confidence": "enrichment_mocked", + "evidence_confidence": "provenance_recorded", "components": { "added": [ { @@ -746,7 +746,7 @@ }, "exit_code": 1 }, - "evidence_confidence": "enrichment_mocked", + "evidence_confidence": "provenance_recorded", "enrichment": { "mode": "opt_in_pypi", "pypi_enabled": true, diff --git a/tools/sbom-diff-and-risk/examples/sample-provenance-report.md b/tools/sbom-diff-and-risk/examples/sample-provenance-report.md index 294ed7e..8c3e910 100644 --- a/tools/sbom-diff-and-risk/examples/sample-provenance-report.md +++ b/tools/sbom-diff-and-risk/examples/sample-provenance-report.md @@ -6,7 +6,7 @@ - Added: 2 - Removed: 0 - Version changes: 1 -- Evidence confidence: enrichment_mocked +- Evidence confidence: provenance_recorded ## Risk buckets - new_package: 2 diff --git a/tools/sbom-diff-and-risk/examples/sample-scorecard-report.json b/tools/sbom-diff-and-risk/examples/sample-scorecard-report.json index 455259f..b0747ce 100644 --- a/tools/sbom-diff-and-risk/examples/sample-scorecard-report.json +++ b/tools/sbom-diff-and-risk/examples/sample-scorecard-report.json @@ -12,7 +12,7 @@ "suspicious_source": 0, "not_evaluated": 0 }, - "evidence_confidence": "enrichment_mocked", + "evidence_confidence": "enrichment_recorded", "policy": { "status": "warn", "blocking": 0, @@ -32,7 +32,7 @@ } } }, - "evidence_confidence": "enrichment_mocked", + "evidence_confidence": "enrichment_recorded", "components": { "added": [ { @@ -575,7 +575,7 @@ }, "exit_code": 0 }, - "evidence_confidence": "enrichment_mocked", + "evidence_confidence": "enrichment_recorded", "enrichment": { "mode": "opt_in_scorecard", "pypi_enabled": false, diff --git a/tools/sbom-diff-and-risk/examples/sample-scorecard-report.md b/tools/sbom-diff-and-risk/examples/sample-scorecard-report.md index d910ba6..c53bdae 100644 --- a/tools/sbom-diff-and-risk/examples/sample-scorecard-report.md +++ b/tools/sbom-diff-and-risk/examples/sample-scorecard-report.md @@ -6,7 +6,7 @@ - Added: 2 - Removed: 0 - Version changes: 1 -- Evidence confidence: enrichment_mocked +- Evidence confidence: enrichment_recorded ## Risk buckets - new_package: 2 diff --git a/tools/sbom-diff-and-risk/src/sbom_diff_risk/evidence_confidence.py b/tools/sbom-diff-and-risk/src/sbom_diff_risk/evidence_confidence.py index 77c5f3a..96d2c47 100644 --- a/tools/sbom-diff-and-risk/src/sbom_diff_risk/evidence_confidence.py +++ b/tools/sbom-diff-and-risk/src/sbom_diff_risk/evidence_confidence.py @@ -9,10 +9,11 @@ def evidence_confidence_for_report(report: CompareReport) -> EvidenceConfidence: if report.metadata.evidence_confidence is not None: return report.metadata.evidence_confidence - if _has_enrichment_evidence(report): - if _has_live_enrichment(report): - return EvidenceConfidence.ENRICHMENT_LIVE - return EvidenceConfidence.ENRICHMENT_MOCKED + if _has_provenance_record(report): + return EvidenceConfidence.PROVENANCE_RECORDED + + if _has_enrichment_record(report): + return EvidenceConfidence.ENRICHMENT_RECORDED if _has_policy_match(report): return EvidenceConfidence.POLICY_MATCHED @@ -27,21 +28,20 @@ def evidence_confidence_value(report: CompareReport) -> str: return evidence_confidence_for_report(report).value -def _has_enrichment_evidence(report: CompareReport) -> bool: +def _has_provenance_record(report: CompareReport) -> bool: metadata = report.metadata.enrichment - if metadata.pypi_enabled or metadata.scorecard_enabled: + if metadata.pypi_enabled: return True - return any(component.provenance is not None or component.scorecard is not None for component in _all_components(report)) + return any(component.provenance is not None for component in _all_components(report)) -def _has_live_enrichment(report: CompareReport) -> bool: +def _has_enrichment_record(report: CompareReport) -> bool: metadata = report.metadata.enrichment - return ( - metadata.network_access_performed - or metadata.pypi_network_access_performed - or metadata.scorecard_network_access_performed - ) + if metadata.scorecard_enabled: + return True + + return any(component.scorecard is not None for component in _all_components(report)) def _has_policy_match(report: CompareReport) -> bool: diff --git a/tools/sbom-diff-and-risk/src/sbom_diff_risk/models.py b/tools/sbom-diff-and-risk/src/sbom_diff_risk/models.py index 23ca64e..9776145 100644 --- a/tools/sbom-diff-and-risk/src/sbom_diff_risk/models.py +++ b/tools/sbom-diff-and-risk/src/sbom_diff_risk/models.py @@ -22,8 +22,8 @@ class EvidenceConfidence(StrEnum): LOCAL_MANIFEST_ONLY = "local_manifest_only" SBOM_PRESENT = "sbom_present" POLICY_MATCHED = "policy_matched" - ENRICHMENT_MOCKED = "enrichment_mocked" - ENRICHMENT_LIVE = "enrichment_live" + ENRICHMENT_RECORDED = "enrichment_recorded" + PROVENANCE_RECORDED = "provenance_recorded" class ProvenanceStatus(StrEnum): diff --git a/tools/sbom-diff-and-risk/tests/test_cli_no_enrichment_regression.py b/tools/sbom-diff-and-risk/tests/test_cli_no_enrichment_regression.py index 588b4ea..7169d43 100644 --- a/tools/sbom-diff-and-risk/tests/test_cli_no_enrichment_regression.py +++ b/tools/sbom-diff-and-risk/tests/test_cli_no_enrichment_regression.py @@ -131,7 +131,7 @@ def __init__(self, *args, **kwargs) -> None: # noqa: ANN002, ANN003 assert payload["metadata"]["enrichment"]["mode"] == "opt_in_pypi" assert payload["metadata"]["enrichment"]["pypi_enabled"] is True assert payload["metadata"]["enrichment"]["pypi_timeout_seconds"] == 2.5 - assert payload["evidence_confidence"] == "enrichment_mocked" + assert payload["evidence_confidence"] == "provenance_recorded" assert payload["notes"][1] == "PyPI provenance enrichment was requested explicitly." assert payload["trust_signal_notes"] == [] @@ -200,5 +200,5 @@ def build_report_metadata(self) -> ReportEnrichmentMetadata: assert payload["metadata"]["enrichment"]["mode"] == "opt_in_scorecard" assert payload["metadata"]["enrichment"]["scorecard_enabled"] is True assert payload["metadata"]["enrichment"]["scorecard_timeout_seconds"] == 4.25 - assert payload["evidence_confidence"] == "enrichment_mocked" + assert payload["evidence_confidence"] == "enrichment_recorded" assert payload["notes"][1] == "OpenSSF Scorecard enrichment was requested explicitly." diff --git a/tools/sbom-diff-and-risk/tests/test_cli_summary_json.py b/tools/sbom-diff-and-risk/tests/test_cli_summary_json.py index 8f23369..c52871a 100644 --- a/tools/sbom-diff-and-risk/tests/test_cli_summary_json.py +++ b/tools/sbom-diff-and-risk/tests/test_cli_summary_json.py @@ -161,7 +161,7 @@ def build_report_metadata(self) -> ReportEnrichmentMetadata: }, }, } - assert payload["evidence_confidence"] == "enrichment_mocked" + assert payload["evidence_confidence"] == "provenance_recorded" assert "policy" not in payload @@ -224,7 +224,7 @@ def build_report_metadata(self) -> ReportEnrichmentMetadata: }, }, } - assert payload["evidence_confidence"] == "enrichment_mocked" + assert payload["evidence_confidence"] == "enrichment_recorded" assert "policy" not in payload diff --git a/tools/sbom-diff-and-risk/tests/test_evidence_confidence.py b/tools/sbom-diff-and-risk/tests/test_evidence_confidence.py index 4ab8532..4e43289 100644 --- a/tools/sbom-diff-and-risk/tests/test_evidence_confidence.py +++ b/tools/sbom-diff-and-risk/tests/test_evidence_confidence.py @@ -13,6 +13,16 @@ from sbom_diff_risk.policy_models import PolicyEvaluation, PolicyLevel, PolicyViolation +def test_evidence_confidence_values_are_release_facing_labels() -> None: + assert {item.value for item in EvidenceConfidence} == { + "local_manifest_only", + "sbom_present", + "policy_matched", + "enrichment_recorded", + "provenance_recorded", + } + + def test_evidence_confidence_defaults_to_local_manifest_only() -> None: report = _minimal_report(before_format="requirements-txt", after_format="requirements-txt") @@ -43,7 +53,7 @@ def test_evidence_confidence_marks_policy_matched() -> None: assert evidence_confidence_for_report(report) is EvidenceConfidence.POLICY_MATCHED -def test_evidence_confidence_marks_mocked_enrichment_without_network() -> None: +def test_evidence_confidence_marks_provenance_recorded_when_pypi_enrichment_is_used() -> None: report = _minimal_report( before_format="requirements-txt", after_format="requirements-txt", @@ -55,29 +65,29 @@ def test_evidence_confidence_marks_mocked_enrichment_without_network() -> None: ), ) - assert evidence_confidence_for_report(report) is EvidenceConfidence.ENRICHMENT_MOCKED + assert evidence_confidence_for_report(report) is EvidenceConfidence.PROVENANCE_RECORDED -def test_evidence_confidence_marks_live_enrichment_when_network_access_was_performed() -> None: +def test_evidence_confidence_marks_enrichment_recorded_when_scorecard_enrichment_is_used() -> None: report = _minimal_report( before_format="requirements-txt", after_format="requirements-txt", enrichment=ReportEnrichmentMetadata( - mode="opt_in_pypi", - pypi_enabled=True, - pypi_network_access_performed=True, + mode="opt_in_scorecard", + scorecard_enabled=True, + scorecard_network_access_performed=True, network_access_performed=True, ), ) - assert evidence_confidence_for_report(report) is EvidenceConfidence.ENRICHMENT_LIVE + assert evidence_confidence_for_report(report) is EvidenceConfidence.ENRICHMENT_RECORDED -def test_evidence_confidence_allows_explicit_mock_override_for_constructed_snapshots() -> None: +def test_evidence_confidence_allows_explicit_recorded_override_for_constructed_snapshots() -> None: report = _minimal_report( before_format="requirements-txt", after_format="requirements-txt", - evidence_confidence=EvidenceConfidence.ENRICHMENT_MOCKED, + evidence_confidence=EvidenceConfidence.PROVENANCE_RECORDED, enrichment=ReportEnrichmentMetadata( mode="opt_in_pypi", pypi_enabled=True, @@ -86,7 +96,7 @@ def test_evidence_confidence_allows_explicit_mock_override_for_constructed_snaps ), ) - assert evidence_confidence_for_report(report) is EvidenceConfidence.ENRICHMENT_MOCKED + assert evidence_confidence_for_report(report) is EvidenceConfidence.PROVENANCE_RECORDED def _minimal_report( diff --git a/tools/sbom-diff-and-risk/tests/test_provenance_reporting.py b/tools/sbom-diff-and-risk/tests/test_provenance_reporting.py index 2ef060c..e8aebdf 100644 --- a/tools/sbom-diff-and-risk/tests/test_provenance_reporting.py +++ b/tools/sbom-diff-and-risk/tests/test_provenance_reporting.py @@ -59,7 +59,7 @@ def test_provenance_report_json_includes_provenance_policy_summary() -> None: "warning": 1, "suppressed": 0, } - assert payload["summary"]["evidence_confidence"] == "enrichment_mocked" + assert payload["summary"]["evidence_confidence"] == "provenance_recorded" assert payload["summary"]["enrichment"] == { "status": "used", "mode": "opt_in_pypi", @@ -405,7 +405,7 @@ def _build_sample_provenance_report() -> tuple[CompareReport, Path, Path]: strict=False, stub=False, policy_evaluation=policy_evaluation, - evidence_confidence=EvidenceConfidence.ENRICHMENT_MOCKED, + evidence_confidence=EvidenceConfidence.PROVENANCE_RECORDED, enrichment=ReportEnrichmentMetadata( mode="opt_in_pypi", pypi_enabled=True, diff --git a/tools/sbom-diff-and-risk/tests/test_reports.py b/tools/sbom-diff-and-risk/tests/test_reports.py index c076f27..f35ca14 100644 --- a/tools/sbom-diff-and-risk/tests/test_reports.py +++ b/tools/sbom-diff-and-risk/tests/test_reports.py @@ -324,7 +324,7 @@ def test_reports_include_provenance_policy_details_for_v2_policy() -> None: strict=False, stub=False, policy_evaluation=policy_evaluation, - evidence_confidence=EvidenceConfidence.ENRICHMENT_MOCKED, + evidence_confidence=EvidenceConfidence.PROVENANCE_RECORDED, ), notes=["PyPI provenance enrichment was requested explicitly."], ) diff --git a/tools/sbom-diff-and-risk/tests/test_scorecard_reporting.py b/tools/sbom-diff-and-risk/tests/test_scorecard_reporting.py index 3bcbf45..b720fc7 100644 --- a/tools/sbom-diff-and-risk/tests/test_scorecard_reporting.py +++ b/tools/sbom-diff-and-risk/tests/test_scorecard_reporting.py @@ -48,7 +48,7 @@ def test_scorecard_report_json_summary_includes_enrichment_status() -> None: "warning": 1, "suppressed": 0, } - assert payload["summary"]["evidence_confidence"] == "enrichment_mocked" + assert payload["summary"]["evidence_confidence"] == "enrichment_recorded" assert payload["summary"]["enrichment"] == { "status": "used", "mode": "opt_in_scorecard", @@ -238,7 +238,7 @@ def _build_sample_scorecard_report() -> tuple[CompareReport, Path, Path]: strict=False, stub=False, policy_evaluation=policy_evaluation, - evidence_confidence=EvidenceConfidence.ENRICHMENT_MOCKED, + evidence_confidence=EvidenceConfidence.ENRICHMENT_RECORDED, enrichment=ReportEnrichmentMetadata( mode="opt_in_scorecard", scorecard_enabled=True,