#!/usr/bin/env python3 """Integration test runner for scc compiler. Reads test expectations from `expect.toml`, compiles each C source file with scc, executes the resulting binary, and validates the process return code or stdout. """ from __future__ import annotations import argparse import logging import os import subprocess import sys import tempfile import tomllib import uuid import time from dataclasses import dataclass from pathlib import Path from typing import Sequence # --------------------------------------------------------------------------- # Configuration # --------------------------------------------------------------------------- WORKSPACE = Path(__file__).resolve().parent CC_PATH = WORKSPACE / "../../build/dev/scc" CONFIG_PATH = WORKSPACE / "expect.toml" DEFAULT_TIMEOUT = 1 # seconds logger = logging.getLogger("scc-test") # --------------------------------------------------------------------------- # Data models # --------------------------------------------------------------------------- @dataclass class TestCase: """A single test case defined in expect.toml.""" source: Path expected: int | str # return code (int) or stdout (str) test_type: str # "return" or "stdout" origin_key: str # original TOML key for user-friendly display @property def description(self) -> str: return f"{self.test_type.upper():6s} {self.source}" # --------------------------------------------------------------------------- # Test runner # --------------------------------------------------------------------------- class Runner: """Compiles and executes test cases.""" def __init__( self, cc: Path, workspace: Path, timeout: float = DEFAULT_TIMEOUT, keep_temps: bool = False, ) -> None: self.cc = cc.resolve() self.workspace = workspace.resolve() self.timeout = timeout self.keep_temps = keep_temps self._temp_dir: tempfile.TemporaryDirectory | None = None @property def temp_dir(self) -> Path: if self._temp_dir is None: self._temp_dir = tempfile.TemporaryDirectory( prefix="scc-test-", dir=self.workspace ) return Path(self._temp_dir.name) def cleanup(self) -> None: if self._temp_dir is not None: self._temp_dir.cleanup() self._temp_dir = None def _unique_exe_path(self) -> Path: """Generate a unique executable path inside the temporary directory.""" return self.temp_dir / f"test_{uuid.uuid4().hex[:8]}.exe" def compile(self, source: Path, output: Path) -> tuple[bool, str]: """Compile *source* into *output*. Returns (success, stderr).""" cmd = [ str(self.cc), str(source), "-o", str(output), "--entry-point-symbol", "main", ] try: proc = subprocess.run( cmd, cwd=self.workspace, capture_output=True, text=True, timeout=self.timeout, ) except subprocess.TimeoutExpired: return False, "Compilation timed out" except OSError as exc: return False, f"Failed to invoke compiler: {exc}" if proc.returncode != 0 or not output.exists(): return False, proc.stderr.strip() or "(no error message)" return True, proc.stderr.strip() def run_exe(self, exe: Path) -> tuple[int, str, str]: """Execute a binary. Returns (returncode, stdout, stderr).""" # Make sure the file is executable (Unix) exe.chmod(exe.stat().st_mode | 0o111) try: proc = subprocess.run( [str(exe)], cwd=self.workspace, capture_output=True, text=True, timeout=self.timeout, ) except subprocess.TimeoutExpired: return -1, "", "Execution timed out" except OSError as exc: return -1, "", f"Failed to run executable: {exc}" return proc.returncode, proc.stdout, proc.stderr.strip() def run_one(self, test: TestCase) -> bool: """Run a single test case. Returns True if passed.""" logger.info("Testing %s", test.source) # 1. Compile exe_path = self._unique_exe_path() ok, stderr = self.compile(test.source, exe_path) if not ok: logger.error(" Compilation FAILED: %s", stderr) return False # 2. Execute returncode, stdout, run_err = self.run_exe(exe_path) # Runtime error detection: negative return code (signal) or timeout if run_err and returncode < 0: logger.error(" Runtime error: %s", run_err) self._remove(exe_path) return False # 3. Validate if test.test_type == "return": actual = returncode else: actual = stdout passed = actual == test.expected if passed: logger.info(" PASSED") else: logger.error( " FAILED: expected %r, got %r", test.expected, actual ) # 4. Cleanup self._remove(exe_path) return passed def _remove(self, path: Path) -> None: """Remove *path* unless keep_temps is set.""" if self.keep_temps: return try: path.unlink(missing_ok=True) except OSError: pass def run_all(self, tests: Sequence[TestCase], global_timeout: float | None = None) -> tuple[int, int]: """Run a sequence of tests. Returns (passed, total).""" passed = 0 tested = 0 start_time = time.monotonic() for idx, test in enumerate(tests, start=1): if global_timeout is not None and (time.monotonic() - start_time) >= global_timeout: logger.warning( "Global timeout (%.1fs) reached - skipping remaining %d tests.", global_timeout, len(tests) - tested ) break logger.info("(%d/%d) %s", idx, len(tests), test.description) if self.run_one(test): passed += 1 tested += 1 total_skipped = len(tests) - tested if total_skipped > 0: logger.warning("%d tests skipped due to global timeout.", total_skipped) return passed, tested # --------------------------------------------------------------------------- # Configuration loading # --------------------------------------------------------------------------- def load_config(config_path: Path) -> list[TestCase]: """Parse `expect.toml` and build list of TestCase objects.""" if not config_path.is_file(): logger.error("Config file not found: %s", config_path) sys.exit(1) with config_path.open("rb") as fh: config = tomllib.load(fh) cases: list[TestCase] = [] for test_file, expected in config.get("return_val_cases", {}).items(): cases.append( TestCase( source=WORKSPACE / test_file, expected=expected, test_type="return", origin_key=test_file, ) ) for test_file, expected in config.get("stdout_val_cases", {}).items(): cases.append( TestCase( source=WORKSPACE / test_file, expected=expected, test_type="stdout", origin_key=test_file, ) ) return cases # --------------------------------------------------------------------------- # CLI helpers # --------------------------------------------------------------------------- def build_arg_parser() -> argparse.ArgumentParser: parser = argparse.ArgumentParser( description="Integration test runner for scc", ) parser.add_argument( "tests", nargs="*", metavar="SOURCE", help="Specific test source files to run (as listed in expect.toml). " "If none given, all tests are executed.", ) parser.add_argument( "--list", action="store_true", help="List all available test cases from expect.toml and exit.", ) parser.add_argument( "--verbose", "-v", action="store_true", help="Enable debug-level logging.", ) parser.add_argument( "--keep-temps", action="store_true", help="Keep temporary executables (useful for debugging).", ) parser.add_argument( "--timeout", type=float, default=DEFAULT_TIMEOUT, help=f"Timeout in seconds for compilation/execution (default: {DEFAULT_TIMEOUT}).", ) parser.add_argument( "--cc", type=Path, default=CC_PATH, help=f"Path to scc compiler (default: {CC_PATH}).", ) return parser def select_tests(all_tests: list[TestCase], requested: list[str]) -> list[TestCase]: """Filter *all_tests* by user-provided source paths.""" if not requested: return all_tests selected: list[TestCase] = [] for name in requested: matched = [ t for t in all_tests if t.origin_key == name or str(t.source) == name or t.source.name == name ] if not matched: logger.warning("No test case matches '%s'", name) continue selected.extend(matched) return selected # --------------------------------------------------------------------------- # Main entry point # --------------------------------------------------------------------------- def main() -> None: parser = build_arg_parser() args = parser.parse_args() logging.basicConfig( level=logging.DEBUG if args.verbose else logging.INFO, format="%(message)s", ) # Load configuration all_tests = load_config(CONFIG_PATH) if args.list: print("Available test cases:") for i, test in enumerate(all_tests, start=1): print(f" {i:3d}: {test.description}") return # Filter tests selected = select_tests(all_tests, args.tests) if not selected: logger.warning("No tests to run.") return runner = Runner( cc=args.cc, workspace=WORKSPACE, timeout=args.timeout, keep_temps=args.keep_temps, ) global_timeout = DEFAULT_TIMEOUT * 3 passed, total = runner.run_all(selected, global_timeout) logger.info("=" * 40) if global_timeout and total < len(selected): logger.info("Summary: %d/%d passed (%d skipped due to timeout)", passed, total, len(selected) - total) else: logger.info("Summary: %d/%d passed", passed, total) runner.cleanup() if __name__ == "__main__": main()