testsAndMisc/linux_configuration/tests/test_validate_evidence.py

"""Tests for meta/scripts/validate_evidence.py (AI-evidence schema checker)."""

from __future__ import annotations

import json
from typing import TYPE_CHECKING, Any

import validate_evidence
from validate_evidence import main, validate

if TYPE_CHECKING:
    from pathlib import Path

    import pytest


def _valid() -> dict[str, Any]:
    """Return a minimal evidence artifact that satisfies every rule."""
    return {
        "intent": "do the thing",
        "scope": ["a"],
        "changes": ["a"],
        "risks": ["a"],
        "rollback": ["a"],
        "verification": [{"command": "c", "result": "passed", "evidence": "e"}],
    }


def _write(tmp_path: Path, obj: object) -> Path:
    """Write ``obj`` as JSON to a temp file and return its path."""
    path = tmp_path / "evidence.json"
    path.write_text(json.dumps(obj), encoding="utf-8")
    return path


class TestValidate:
    """The ``validate`` function returns a list of problems."""

    def test_valid_has_no_errors(self, tmp_path: Path) -> None:
        """A fully-populated artifact produces no errors."""
        assert validate(_write(tmp_path, _valid())) == []

    def test_missing_keys(self, tmp_path: Path) -> None:
        """Absent top-level keys are reported in one message."""
        errors = validate(_write(tmp_path, {"intent": "x"}))
        assert errors == [
            "missing required keys: scope, changes, verification, risks, rollback",
        ]

    def test_intent_must_be_nonempty_string(self, tmp_path: Path) -> None:
        """A blank intent is rejected."""
        data = _valid()
        data["intent"] = "   "
        assert "intent must be a non-empty string" in validate(_write(tmp_path, data))

    def test_string_list_not_a_list(self, tmp_path: Path) -> None:
        """A string-list field that is not a list is rejected."""
        data = _valid()
        data["scope"] = "nope"
        assert "scope must be a non-empty list" in validate(_write(tmp_path, data))

    def test_string_list_with_blank_entry(self, tmp_path: Path) -> None:
        """A blank entry inside a string-list field is rejected."""
        data = _valid()
        data["changes"] = ["ok", "  "]
        assert "changes entries must be non-empty strings" in validate(
            _write(tmp_path, data),
        )

    def test_verification_not_a_list(self, tmp_path: Path) -> None:
        """A non-list verification field is rejected."""
        data = _valid()
        data["verification"] = {}
        assert "verification must be a non-empty list" in validate(
            _write(tmp_path, data),
        )

    def test_verification_item_not_object(self, tmp_path: Path) -> None:
        """A non-object verification entry is rejected."""
        data = _valid()
        data["verification"] = ["nope"]
        assert "verification[0] must be an object" in validate(_write(tmp_path, data))

    def test_verification_missing_fields(self, tmp_path: Path) -> None:
        """Missing per-entry fields are reported."""
        data = _valid()
        data["verification"] = [{"command": "c"}]
        errors = validate(_write(tmp_path, data))
        assert "verification[0] missing fields: result, evidence" in errors

    def test_verification_blank_field(self, tmp_path: Path) -> None:
        """A blank per-entry field is rejected."""
        data = _valid()
        data["verification"] = [{"command": "c", "result": " ", "evidence": "e"}]
        errors = validate(_write(tmp_path, data))
        assert "verification[0].result must be a non-empty string" in errors

    def test_banned_phrase(self, tmp_path: Path) -> None:
        """A rationalization phrase anywhere in the file is rejected."""
        data = _valid()
        data["verification"] = [
            {"command": "c", "result": "should work", "evidence": "e"},
        ]
        errors = validate(_write(tmp_path, data))
        assert any("rationalization phrase 'should work'" in e for e in errors)

    def test_non_object_top_level(self, tmp_path: Path) -> None:
        """A non-object top-level JSON value is rejected."""
        assert validate(_write(tmp_path, [1, 2])) == [
            "top-level JSON value must be an object",
        ]

    def test_invalid_json(self, tmp_path: Path) -> None:
        """Malformed JSON is reported, not raised."""
        path = tmp_path / "bad.json"
        path.write_text("not json", encoding="utf-8")
        errors = validate(path)
        assert len(errors) == 1
        assert errors[0].startswith("invalid JSON")

    def test_unreadable_path(self, tmp_path: Path) -> None:
        """An unreadable path (a directory) is reported, not raised."""
        errors = validate(tmp_path)
        assert len(errors) == 1
        assert errors[0].startswith("cannot read file")


class TestMain:
    """The CLI entry point maps validation to an exit code."""

    def test_no_argument_returns_2(self, monkeypatch: pytest.MonkeyPatch) -> None:
        """Missing the path argument is a usage error (rc 2)."""
        monkeypatch.setattr(validate_evidence.sys, "argv", ["validate_evidence"])
        assert main() == 2

    def test_valid_returns_0(
        self,
        tmp_path: Path,
        monkeypatch: pytest.MonkeyPatch,
        capsys: pytest.CaptureFixture[str],
    ) -> None:
        """A valid artifact prints OK and returns 0."""
        path = _write(tmp_path, _valid())
        monkeypatch.setattr(
            validate_evidence.sys,
            "argv",
            ["validate_evidence", str(path)],
        )
        assert main() == 0
        assert "schema OK" in capsys.readouterr().out

    def test_invalid_returns_1(
        self,
        tmp_path: Path,
        monkeypatch: pytest.MonkeyPatch,
        capsys: pytest.CaptureFixture[str],
    ) -> None:
        """An invalid artifact writes the problem to stderr and returns 1."""
        path = _write(tmp_path, {"intent": "x"})
        monkeypatch.setattr(
            validate_evidence.sys,
            "argv",
            ["validate_evidence", str(path)],
        )
        assert main() == 1
        assert "missing required keys" in capsys.readouterr().err