315 lines
12 KiB
Python
315 lines
12 KiB
Python
#!/usr/bin/env python3
|
|
"""Enforce the complete native source-generation milestone gate."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import hashlib
|
|
import json
|
|
import re
|
|
import subprocess
|
|
import tempfile
|
|
import tomllib
|
|
from pathlib import Path
|
|
|
|
import check_api_coverage
|
|
import generate_api_shims
|
|
import generate_rust_mapping
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
INVENTORY_PATH = ROOT / "codegen/sources.json"
|
|
CATALOG_PATH = ROOT / "api/public-api.json"
|
|
CODEGEN_README = ROOT / "codegen/README.md"
|
|
API_REPORT = ROOT / "api/API-COVERAGE.md"
|
|
|
|
OUTPUT_OWNERS = {
|
|
"codegen/generated/source_manifest.rs": ("source-manifest", None),
|
|
"crates/libremetaverse/src/packet_catalog.rs": ("packets", "packets"),
|
|
"crates/libremetaverse/src/visual_catalog.rs": ("visual-params", "visual_params"),
|
|
"crates/libremetaverse/src/foliage_catalog.rs": ("foliage", "trees"),
|
|
"crates/libremetaverse/src/skeleton_catalog.rs": ("skeleton", "skeleton"),
|
|
"crates/libremetaverse/src/attention_catalog.rs": ("attentions", "attentions"),
|
|
"crates/libremetaverse/src/genepool_catalog.rs": ("genepool", "genepool"),
|
|
}
|
|
|
|
SOURCE_GENERATED_NATIVE_TYPES = {
|
|
"T:LibreMetaverse.ArchetypeParam",
|
|
"T:LibreMetaverse.AttentionData",
|
|
"T:LibreMetaverse.AttentionSet",
|
|
"T:LibreMetaverse.DrivenParamInfo",
|
|
"T:LibreMetaverse.Genepool",
|
|
"T:LibreMetaverse.GenepoolArchetype",
|
|
"T:LibreMetaverse.GrassDefinition",
|
|
"T:LibreMetaverse.GrassDefinitions",
|
|
"T:LibreMetaverse.LindenAttentions",
|
|
"T:LibreMetaverse.SkeletalBoneInfo",
|
|
"T:LibreMetaverse.TreeDefinition",
|
|
"T:LibreMetaverse.TreeDefinitions",
|
|
"T:LibreMetaverse.VisualAlphaParam",
|
|
"T:LibreMetaverse.VisualColorParam",
|
|
"T:LibreMetaverse.VisualParam",
|
|
"T:LibreMetaverse.VisualParams",
|
|
"T:LibreMetaverse.VolumeMorphInfo",
|
|
"T:LibreMetaverse.Rendering.CollisionVolume",
|
|
"T:LibreMetaverse.Rendering.Joint",
|
|
"T:LibreMetaverse.Rendering.JointBase",
|
|
"T:LibreMetaverse.Rendering.LindenSkeleton",
|
|
}
|
|
|
|
FORBIDDEN_RUNTIME_NAMES = ("dotnet", "roslyn", "microsoft.codeanalysis", "coreclr")
|
|
FORBIDDEN_RUNTIME_BYTES = tuple(value.encode() for value in FORBIDDEN_RUNTIME_NAMES)
|
|
|
|
|
|
def fail(message: str) -> None:
|
|
raise ValueError(message)
|
|
|
|
|
|
def sha256(path: Path) -> str:
|
|
return hashlib.sha256(path.read_bytes()).hexdigest()
|
|
|
|
|
|
def run_temp_generation() -> None:
|
|
with tempfile.TemporaryDirectory(prefix="metacrate-codegen-") as directory:
|
|
output_root = Path(directory)
|
|
subprocess.run(
|
|
[
|
|
"cargo",
|
|
"run",
|
|
"--locked",
|
|
"--quiet",
|
|
"-p",
|
|
"libremetaverse-codegen",
|
|
"--",
|
|
"generate-to",
|
|
str(output_root),
|
|
],
|
|
cwd=ROOT,
|
|
check=True,
|
|
)
|
|
actual_paths = {
|
|
path.relative_to(output_root).as_posix()
|
|
for path in output_root.rglob("*")
|
|
if path.is_file()
|
|
}
|
|
expected_paths = set(OUTPUT_OWNERS)
|
|
if actual_paths != expected_paths:
|
|
fail(
|
|
"temporary generation output mismatch: "
|
|
f"missing={sorted(expected_paths - actual_paths)}, "
|
|
f"unexpected={sorted(actual_paths - expected_paths)}"
|
|
)
|
|
for relative in sorted(expected_paths):
|
|
generated = output_root / relative
|
|
checked_in = ROOT / relative
|
|
if generated.read_bytes() != checked_in.read_bytes():
|
|
fail(f"{relative} differs from clean temporary regeneration")
|
|
|
|
|
|
def audit_inventory_and_headers() -> tuple[dict, dict]:
|
|
inventory = json.loads(INVENTORY_PATH.read_text())
|
|
catalog = json.loads(CATALOG_PATH.read_text())
|
|
if inventory.get("schema") != 1:
|
|
fail("unsupported codegen inventory schema")
|
|
if catalog["upstream"]["commit"] != inventory["upstream_commit"]:
|
|
fail("public API catalog and source inventory pin different upstream commits")
|
|
if catalog["upstream"]["repository"] != inventory["upstream_repository"]:
|
|
fail("public API catalog and source inventory pin different repositories")
|
|
|
|
generators = {item["id"]: item for item in inventory["generators"]}
|
|
inputs = {item["id"]: item for item in inventory["inputs"]}
|
|
if len(generators) != 6 or len(inputs) != len(inventory["inputs"]):
|
|
fail("source inventory must contain six unique generators and unique inputs")
|
|
public_generators = {item["name"] for item in catalog["source_generators"]}
|
|
inventoried_generators = {
|
|
Path(item["reference_source"]).stem for item in generators.values()
|
|
}
|
|
if public_generators != inventoried_generators:
|
|
fail("source inventory does not exactly cover the authoritative generator catalog")
|
|
|
|
referenced_inputs: set[str] = set()
|
|
for generator in generators.values():
|
|
if generator["license"] != "BSD-3-Clause":
|
|
fail(f"generator {generator['id']} has an unexpected license")
|
|
if not re.fullmatch(r"[0-9a-f]{64}", generator["sha256"]):
|
|
fail(f"generator {generator['id']} has an invalid SHA-256")
|
|
referenced_inputs.update(generator["inputs"])
|
|
if referenced_inputs != set(inputs):
|
|
fail("generator input lists do not exactly cover the input inventory")
|
|
for item in inputs.values():
|
|
if item["license"] != "BSD-3-Clause":
|
|
fail(f"input {item['id']} has an unexpected license")
|
|
path = ROOT / item["vendored_path"]
|
|
if sha256(path) != item["sha256"]:
|
|
fail(f"{item['vendored_path']} does not match its pinned SHA-256")
|
|
|
|
for relative, (header_name, owner_id) in OUTPUT_OWNERS.items():
|
|
text = (ROOT / relative).read_text()
|
|
if not text.startswith("// @generated by libremetaverse-codegen; DO NOT EDIT.\n"):
|
|
fail(f"{relative} has no generated-file header")
|
|
if f"// Generator: {header_name}\n" not in text[:1000]:
|
|
fail(f"{relative} has the wrong generator header")
|
|
expected_inputs = inputs.values() if owner_id is None else (
|
|
inputs[input_id] for input_id in generators[owner_id]["inputs"]
|
|
)
|
|
for item in expected_inputs:
|
|
source_line = (
|
|
f"// Source: {item['vendored_path']} sha256={item['sha256']} "
|
|
f"license={item['license']}"
|
|
)
|
|
if source_line not in text[:4000]:
|
|
fail(f"{relative} omits provenance for {item['id']}")
|
|
return inventory, catalog
|
|
|
|
|
|
def require_current(path: Path, expected: str, label: str) -> None:
|
|
if not path.exists() or path.read_text() != expected:
|
|
fail(f"{label} is stale: {path.relative_to(ROOT)}")
|
|
|
|
|
|
def audit_api_outputs(catalog: dict) -> tuple[int, int]:
|
|
for path, expected in generate_rust_mapping.generate().items():
|
|
require_current(path, expected, "Rust API mapping")
|
|
for path, expected in generate_api_shims.generate().items():
|
|
require_current(path, expected, "Rust API shim")
|
|
require_current(API_REPORT, check_api_coverage.audit(), "complete API coverage report")
|
|
|
|
catalog_types = {
|
|
item["doc_id"]
|
|
for assembly in catalog["assemblies"]
|
|
for item in assembly["types"]
|
|
}
|
|
missing_native = SOURCE_GENERATED_NATIVE_TYPES - catalog_types
|
|
if missing_native:
|
|
fail(f"source-generated native type catalog entries are missing: {sorted(missing_native)}")
|
|
unmapped_native = SOURCE_GENERATED_NATIVE_TYPES - set(generate_api_shims.NATIVE_TYPES)
|
|
if unmapped_native:
|
|
fail(f"source-generated types are not backed by native Rust: {sorted(unmapped_native)}")
|
|
|
|
packet_type = re.compile(
|
|
r"T:LibreMetaverse\.Packets\.[^.]+Packet(?:\.[^.]+Block)?"
|
|
)
|
|
packet_osd = re.compile(
|
|
r"M:LibreMetaverse\.Packets\.[^.]+Packet(?:\.[^.]+Block)?\."
|
|
r"(?:PacketToOSD|PacketFromOSD|ToOSD|FromOSD)(?:\([^)]*\))?"
|
|
)
|
|
expected_members = {
|
|
member["doc_id"]
|
|
for assembly in catalog["assemblies"]
|
|
for item in assembly["types"]
|
|
for member in item["members"]
|
|
if packet_type.fullmatch(item["doc_id"])
|
|
}
|
|
osd_members = {member for member in expected_members if packet_osd.fullmatch(member)}
|
|
generated = generate_api_shims.MAIN_TARGET.read_text()
|
|
markers = list(re.finditer(r"/// C# member: `([^`]+)`\.", generated))
|
|
blocks = {
|
|
match.group(1): generated[
|
|
match.start() : markers[index + 1].start() if index + 1 < len(markers) else len(generated)
|
|
]
|
|
for index, match in enumerate(markers)
|
|
}
|
|
if not osd_members or expected_members - set(blocks):
|
|
fail("packet source-generator API coverage is missing generated members")
|
|
for member_id in expected_members:
|
|
block = blocks[member_id]
|
|
if "not_implemented(" in block or "unimplemented_api(" in block:
|
|
fail(f"packet source-generator API remains a failure shim: {member_id}")
|
|
return len(expected_members), len(osd_members)
|
|
|
|
|
|
def dependency_names() -> set[str]:
|
|
names: set[str] = set()
|
|
lock = tomllib.loads((ROOT / "Cargo.lock").read_text())
|
|
names.update(package["name"].lower() for package in lock["package"])
|
|
for manifest in ROOT.rglob("Cargo.toml"):
|
|
if "target" in manifest.parts:
|
|
continue
|
|
data = tomllib.loads(manifest.read_text())
|
|
for table_name, value in data.items():
|
|
if table_name.endswith("dependencies") and isinstance(value, dict):
|
|
names.update(name.lower() for name in value)
|
|
for target in data.get("target", {}).values():
|
|
if isinstance(target, dict):
|
|
for table_name, value in target.items():
|
|
if table_name.endswith("dependencies") and isinstance(value, dict):
|
|
names.update(name.lower() for name in value)
|
|
return names
|
|
|
|
|
|
def audit_rust_only_release(release_dir: Path | None) -> int:
|
|
bad_dependencies = sorted(
|
|
name
|
|
for name in dependency_names()
|
|
if any(forbidden in name for forbidden in FORBIDDEN_RUNTIME_NAMES)
|
|
)
|
|
if bad_dependencies:
|
|
fail(f"managed-runtime dependencies found: {bad_dependencies}")
|
|
for build_script in ROOT.rglob("build.rs"):
|
|
if "target" in build_script.parts:
|
|
continue
|
|
lowered = build_script.read_bytes().lower()
|
|
if any(value in lowered for value in FORBIDDEN_RUNTIME_BYTES):
|
|
fail(f"managed-runtime invocation found in {build_script.relative_to(ROOT)}")
|
|
|
|
if release_dir is None:
|
|
return 0
|
|
if not release_dir.is_dir():
|
|
fail(f"release artifact directory does not exist: {release_dir}")
|
|
scanned = 0
|
|
for path in release_dir.rglob("*"):
|
|
if not path.is_file() or path.suffix in {".d", ".rmeta"}:
|
|
continue
|
|
lowered_name = path.name.lower()
|
|
if any(value in lowered_name for value in FORBIDDEN_RUNTIME_NAMES):
|
|
fail(f"managed-runtime release artifact found: {path}")
|
|
data = path.read_bytes().lower()
|
|
if any(value in data for value in FORBIDDEN_RUNTIME_BYTES):
|
|
fail(f"managed-runtime marker embedded in release artifact: {path}")
|
|
scanned += 1
|
|
if scanned == 0:
|
|
fail(f"no release artifacts found under {release_dir}")
|
|
return scanned
|
|
|
|
|
|
def audit_documentation(inventory: dict) -> None:
|
|
text = CODEGEN_README.read_text()
|
|
required = [
|
|
*(Path(path).name for path in OUTPUT_OWNERS),
|
|
*(Path(item["reference_source"]).stem for item in inventory["generators"]),
|
|
*(Path(item["vendored_path"]).name for item in inventory["inputs"]),
|
|
"generate-to",
|
|
"check_codegen_gate.py",
|
|
]
|
|
missing = sorted(value for value in required if value not in text)
|
|
if missing:
|
|
fail(f"codegen documentation omits audited inputs/outputs/commands: {missing}")
|
|
|
|
|
|
def main() -> None:
|
|
parser = argparse.ArgumentParser()
|
|
parser.add_argument(
|
|
"--release-dir",
|
|
type=Path,
|
|
help="scan a completed Rust release build for managed-runtime artifacts",
|
|
)
|
|
args = parser.parse_args()
|
|
|
|
run_temp_generation()
|
|
inventory, catalog = audit_inventory_and_headers()
|
|
packet_members, osd_members = audit_api_outputs(catalog)
|
|
audit_documentation(inventory)
|
|
artifacts = audit_rust_only_release(args.release_dir)
|
|
print(
|
|
"native codegen gate passed: "
|
|
f"{len(OUTPUT_OWNERS)} byte-identical outputs, "
|
|
f"{len(inventory['generators'])} generators, "
|
|
f"{len(inventory['inputs'])} inputs, "
|
|
f"{packet_members} native packet members, "
|
|
f"{osd_members} packet OSD members, "
|
|
f"{artifacts} release artifacts scanned"
|
|
)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|