Complete native extension milestone gate (#84)

tools/check_milestone_10.py

@@ -0,0 +1,252 @@
+#!/usr/bin/env python3
+"""Audit the complete native rendering, RLV, and LSL milestone boundary."""
+
+from __future__ import annotations
+
+import json
+import re
+import subprocess
+from pathlib import Path
+
+import generate_api_shims
+
+
+ROOT = Path(__file__).resolve().parents[1]
+ASSEMBLIES = {
+    "LibreMetaverse.Rendering.Simple": (1, 6),
+    "LibreMetaverse.Rendering.MeshFoundry": (1, 14),
+    "LibreMetaverse.RLV": (28, 499),
+    "LibreMetaverse.LslTools": (164, 768),
+}
+CRATES = (
+    "libremetaverse-rendering-simple",
+    "libremetaverse-rendering-mesh-foundry",
+    "libremetaverse-rlv",
+    "libremetaverse-lsl-tools",
+)
+RLV_GENERATED_TYPES = {
+    "T:LibreMetaverse.RLV.IRlvActionCallbacks",
+    "T:LibreMetaverse.RLV.IRlvQueryCallbacks",
+    "T:LibreMetaverse.RLV.RlvAttachmentPoint",
+    "T:LibreMetaverse.RLV.RlvGestureState",
+    "T:LibreMetaverse.RLV.RlvGetDebugType",
+    "T:LibreMetaverse.RLV.RlvGetEnvType",
+    "T:LibreMetaverse.RLV.RlvPermissionsService.HoverTextLocation",
+    "T:LibreMetaverse.RLV.RlvPermissionsService.ObjectLocation",
+    "T:LibreMetaverse.RLV.RlvPermissionsService.TouchLocation",
+    "T:LibreMetaverse.RLV.RlvRestrictionType",
+    "T:LibreMetaverse.RLV.RlvWearableType",
+}
+STUB_RE = re.compile(
+    r"\b(?:not_implemented|unimplemented_api)\b|\b(?:todo|unimplemented)!\s*\("
+)
+PLATFORM_RE = re.compile(r"unsafe\s*\{|unsafe\s+impl|target_os\s*=\s*\"macos\"")
+
+
+def require_markers(path: Path, markers: tuple[str, ...]) -> None:
+    text = path.read_text()
+    missing = [marker for marker in markers if marker not in text]
+    if missing:
+        raise SystemExit(f"{path}: audit evidence missing: " + ", ".join(missing))
+
+
+def catalog_assemblies() -> dict[str, dict]:
+    catalog = json.loads((ROOT / "api" / "public-api.json").read_text())
+    return {assembly["identity"]["name"]: assembly for assembly in catalog["assemblies"]}
+
+
+def member_is_native(item: dict, member: dict) -> bool:
+    doc_id = item["doc_id"]
+    return (
+        doc_id in generate_api_shims.NATIVE_TYPES
+        or doc_id in generate_api_shims.NATIVE_GENERATED_TYPES
+        or (
+            doc_id in generate_api_shims.NATIVE_DECLARATIONS
+            and member["kind"] in {"field", "constant"}
+        )
+        or generate_api_shims.native_member_body(member["doc_id"]) is not None
+    )
+
+
+def check_catalog_ownership() -> None:
+    assemblies = catalog_assemblies()
+    native_types = (
+        set(generate_api_shims.NATIVE_TYPES)
+        | set(generate_api_shims.NATIVE_DECLARATIONS)
+        | set(generate_api_shims.NATIVE_GENERATED_TYPES)
+    )
+    for name, expected in ASSEMBLIES.items():
+        assembly = assemblies[name]
+        actual = (
+            len(assembly["types"]),
+            sum(len(item["members"]) for item in assembly["types"]),
+        )
+        if actual != expected:
+            raise SystemExit(f"{name}: expected catalog counts {expected}, found {actual}")
+        missing_types = [
+            item["doc_id"] for item in assembly["types"] if item["doc_id"] not in native_types
+        ]
+        missing_members = [
+            member["doc_id"]
+            for item in assembly["types"]
+            for member in item["members"]
+            if not member_is_native(item, member)
+        ]
+        if missing_types or missing_members:
+            raise SystemExit(
+                f"{name}: non-native types={missing_types}, members={missing_members}"
+            )
+    if generate_api_shims.NATIVE_GENERATED_TYPES != RLV_GENERATED_TYPES:
+        raise SystemExit("milestone 10 generated-native RLV declaration set changed")
+
+
+def check_sources_and_manifests() -> None:
+    offenders: list[str] = []
+    platform_offenders: list[str] = []
+    for crate in CRATES:
+        crate_root = ROOT / "crates" / crate
+        manifest = (crate_root / "Cargo.toml").read_text()
+        if "[features]" in manifest:
+            raise SystemExit(f"{crate}: native implementation must be active without a feature")
+        if re.search(r"\b(?:dotnet|csc|mcs)\b", manifest, re.IGNORECASE):
+            raise SystemExit(f"{crate}: manifest depends on a C#/.NET tool")
+        for source in (crate_root / "src").glob("*.rs"):
+            text = source.read_text()
+            if STUB_RE.search(text):
+                offenders.append(str(source.relative_to(ROOT)))
+            if PLATFORM_RE.search(text):
+                platform_offenders.append(str(source.relative_to(ROOT)))
+    if offenders:
+        raise SystemExit("milestone 10 owned Rust stubs remain: " + ", ".join(offenders))
+    if platform_offenders:
+        raise SystemExit(
+            "milestone 10 unsafe or macOS-only sources remain: "
+            + ", ".join(platform_offenders)
+        )
+
+
+def check_parity() -> None:
+    catalog = json.loads((ROOT / "tests" / "upstream-tests.json").read_text())
+    related = [
+        test
+        for test in catalog["tests"]
+        if test["source"].startswith("LibreMetaverse.Tests/RLV/")
+        or test["source"].startswith("LibreMetaverse.Rendering.Tests/")
+    ]
+    if len(related) != 691:
+        raise SystemExit(f"milestone 10 expected 691 upstream parity cases, found {len(related)}")
+    bad = [
+        test["id"]
+        for test in related
+        if test["status"] != "translated" or test.get("semantic_review") != "reviewed"
+    ]
+    if bad:
+        raise SystemExit("milestone 10 parity cases are not reviewed: " + ", ".join(bad))
+    missing = sorted(
+        {
+            test["rust_file"]
+            for test in related
+            if not (ROOT / test["rust_file"]).is_file()
+        }
+    )
+    if missing:
+        raise SystemExit("milestone 10 translated files are missing: " + ", ".join(missing))
+
+
+def check_contract_evidence() -> None:
+    require_markers(
+        ROOT / "crates" / "libremetaverse-rendering-simple" / "src" / "simple_renderer.rs",
+        ("MAX_FACE_VERTICES", "checked_add", "u16::try_from"),
+    )
+    require_markers(
+        ROOT / "crates" / "libremetaverse-rendering-mesh-foundry" / "src" / "mesh_foundry.rs",
+        ("MAX_ASSET_BYTES", "MAX_TOTAL_VERTICES", "MAX_JOINTS", "checked_add"),
+    )
+    require_markers(
+        ROOT / "crates" / "libremetaverse-rlv" / "src" / "service.rs",
+        (
+            "MAX_RLV_MESSAGE_BYTES",
+            "MAX_RLV_COMMANDS",
+            "Arc<dyn IRlvQueryCallbacks>",
+            "Arc<dyn IRlvActionCallbacks>",
+            "throw_if_cancellation_requested",
+        ),
+    )
+    require_markers(
+        ROOT / "crates" / "libremetaverse-lsl-tools" / "src" / "parser.rs",
+        ("MAX_PARSER_STATES", "MAX_PARSER_STACK", "MAX_PARSER_STEPS", "MAX_RECOVERY_ERRORS"),
+    )
+    require_markers(
+        ROOT / "crates" / "libremetaverse-lsl-tools" / "src" / "lexer.rs",
+        ("MAX_SOURCE_UNITS", "MAX_TOKEN_UNITS"),
+    )
+    require_markers(
+        ROOT / "tests" / "compat" / "tests" / "milestone_10_integration.rs",
+        (
+            "native_extensions_consume_core_models_and_preserve_service_state",
+            "SimpleRenderer",
+            "MeshFoundry",
+            "generated_parser",
+            "RlvHarness",
+        ),
+    )
+    require_markers(
+        ROOT / "docs" / "extension-milestone-gate.md",
+        (
+            "Choosing an extension",
+            "Rendering migration",
+            "RLV host integration and teardown",
+            "LSL tooling migration",
+            "194 types",
+            "1,287 members",
+            "tools/test_milestone_10.py",
+            "ubuntu-latest",
+        ),
+    )
+    require_markers(
+        ROOT / "api" / "SHIM-COVERAGE.md",
+        tuple(
+            f"`{name}` | {types} | {members} | native implementation"
+            for name, (types, members) in ASSEMBLIES.items()
+        ),
+    )
+
+
+def check_generation_and_ci() -> None:
+    subprocess.run(
+        ["python3", "tools/generate_rust_mapping.py", "--check"], cwd=ROOT, check=True
+    )
+    subprocess.run(
+        ["python3", "tools/generate_api_shims.py", "--check"], cwd=ROOT, check=True
+    )
+    subprocess.run(
+        ["python3", "tools/generate_lsl_tables.py", "--check"], cwd=ROOT, check=True
+    )
+    workflow = ROOT / ".gitea" / "workflows" / "rust-workspace.yml"
+    require_markers(
+        workflow,
+        (
+            "python3 tools/check_milestone_10.py",
+            "python3 tools/test_milestone_10.py",
+            "runs-on: ubuntu-latest",
+        ),
+    )
+    if re.search(r"runs-on:\s*(?:macos|windows)", workflow.read_text(), re.IGNORECASE):
+        raise SystemExit("milestone 10 workflow is not ubuntu-only")
+
+
+def main() -> None:
+    check_catalog_ownership()
+    check_sources_and_manifests()
+    check_parity()
+    check_contract_evidence()
+    check_generation_and_ci()
+    print(
+        "milestone 10 audit: 194 types and 1,287 members are native; no owned "
+        "stubs remain; bounds, callback teardown, 691 reviewed parity cases, "
+        "integration, migration docs, deterministic generation, and ubuntu-only CI are present"
+    )
+
+
+if __name__ == "__main__":
+    main()

tools/check_milestone_10_issue_81.py

@@ -132,7 +132,7 @@ def main() -> None:
         raise SystemExit(f"issue 81 expected 27 focused fixtures, found {test_count}")
     require_markers(DOC, (
         "UTF-16 code-unit offsets", "maximum munch", "structured `Diagnostic`",
-        "64 Mi UTF-16", "16 Mi units", "Lexer::iter", "27 focused",
+        "64 Mi UTF-16", "16 Mi units", "Lexer::iter", "46 focused",
         "No C#, .NET runtime", "ubuntu-latest",
     ))
     require_markers(WORKFLOW, ("python3 tools/check_milestone_10_issue_81.py",))

tools/check_milestone_10_issue_82.py

@@ -122,7 +122,7 @@ def main() -> None:
     require_markers(EXTENSION_TESTS, ("assert!(YyParser::new().is_ok())",))
     require_markers(DOC, (
         "canonical LR(0)", "deterministic LALR(1)", "Shift/reduce conflicts",
-        "nonassociative", "1,048,576", "16,777,216", "37 focused",
+        "nonassociative", "1,048,576", "16,777,216", "46 focused",
         "issue 83", "ubuntu-latest",
     ))
     require_markers(WORKFLOW, ("python3 tools/check_milestone_10_issue_82.py",))

tools/generate_api_shims.py

@@ -601,6 +601,26 @@ NATIVE_DECLARATIONS = {
     "T:LibreMetaverse.Rendering.MeshFoundry": "crate::mesh_foundry::MeshFoundry",
 }
 
+# Catalog declarations which are themselves complete native Rust. Unlike
+# NATIVE_TYPES, these types intentionally remain in generated.rs: the RLV
+# enums are closed value tables and the callback interfaces are Rust traits
+# whose required methods are implemented by the host and default adapters.
+# Keeping this set explicit lets coverage distinguish concrete generated code
+# from a callable failure-only compatibility shell.
+NATIVE_GENERATED_TYPES = {
+    "T:LibreMetaverse.RLV.IRlvActionCallbacks",
+    "T:LibreMetaverse.RLV.IRlvQueryCallbacks",
+    "T:LibreMetaverse.RLV.RlvAttachmentPoint",
+    "T:LibreMetaverse.RLV.RlvGestureState",
+    "T:LibreMetaverse.RLV.RlvGetDebugType",
+    "T:LibreMetaverse.RLV.RlvGetEnvType",
+    "T:LibreMetaverse.RLV.RlvPermissionsService.HoverTextLocation",
+    "T:LibreMetaverse.RLV.RlvPermissionsService.ObjectLocation",
+    "T:LibreMetaverse.RLV.RlvPermissionsService.TouchLocation",
+    "T:LibreMetaverse.RLV.RlvRestrictionType",
+    "T:LibreMetaverse.RLV.RlvWearableType",
+}
+
 NATIVE_MEMBER_BODIES = {
     "M:LibreMetaverse.RLV.RlvCommon.TryGetAttachmentPointFromItemName(System.String,System.Nullable{LibreMetaverse.RLV.RlvAttachmentPoint}@)":
         "Self::native_try_get_attachment_point_from_item_name(item_name, attachment_point)",
@@ -3523,9 +3543,18 @@ def native_member_body(member_id: str) -> str | None:
     return None
 
 
-def render_body(signature: str, member_id: str, error_model: str, asyncness: str, trait: bool) -> str:
+def render_body(
+    signature: str,
+    member_id: str,
+    error_model: str,
+    asyncness: str,
+    trait: bool,
+    required_trait_method: bool = False,
+) -> str:
     if trait:
         signature = signature.removeprefix("pub ")
+    if required_trait_method:
+        return f"    {signature};"
     if body := native_member_body(member_id):
         if any(f"LibreMetaverse.{owner}." in member_id for owner in CLIENT_CORE_NATIVE_OWNERS):
             marker = " /* native client-core implementation */"
@@ -3647,6 +3676,7 @@ def render_type(item: dict, type_row: dict, member_rows: dict[str, dict[str, str
                     row["error_model"],
                     row["asyncness"],
                     trait,
+                    trait and item["doc_id"] in NATIVE_GENERATED_TYPES,
                 )
             )
     lines.append("}")
@@ -3844,7 +3874,9 @@ def coverage_report(catalog: dict, coverage: dict[str, tuple[int, int, bool]]) -
             native_type_ids = {
                 item["doc_id"]
                 for item in assembly["types"]
-                if item["doc_id"] in NATIVE_TYPES or item["doc_id"] in NATIVE_DECLARATIONS
+                if item["doc_id"] in NATIVE_TYPES
+                or item["doc_id"] in NATIVE_DECLARATIONS
+                or item["doc_id"] in NATIVE_GENERATED_TYPES
                 or (name == "LibreMetaverse.Types" and item["kind"] == "enum")
             }
             native_member_ids = {
@@ -3852,6 +3884,7 @@ def coverage_report(catalog: dict, coverage: dict[str, tuple[int, int, bool]]) -
                 for item in assembly["types"]
                 for member in item["members"]
                 if item["doc_id"] in NATIVE_TYPES
+                or item["doc_id"] in NATIVE_GENERATED_TYPES
                 or (name == "LibreMetaverse.Types" and item["kind"] == "enum")
                 or (
                     item["doc_id"] in NATIVE_DECLARATIONS

tools/generate_rust_mapping.py

@@ -960,7 +960,10 @@ def function_bodies(text: str) -> list[str]:
 
 
 def validate_generated_shims() -> None:
-    for path in sorted((ROOT / "crates").glob("*/src/generated*.rs")):
+    # Only generated.rs files are callable compatibility shim targets.
+    # Deterministic native artifacts such as LSL generated_tables.rs contain
+    # ordinary successful functions and are validated by their own generator.
+    for path in sorted((ROOT / "crates").glob("*/src/generated.rs")):
         text = path.read_text()
         default_types = set(
             re.findall(

tools/test_milestone_10.py

@@ -0,0 +1,77 @@
+#!/usr/bin/env python3
+"""Run only tests owned by the rendering, RLV, and LSL milestone."""
+
+from __future__ import annotations
+
+import json
+import os
+import subprocess
+from pathlib import Path
+
+
+ROOT = Path(__file__).resolve().parents[1]
+ENV = os.environ.copy()
+ENV.setdefault("CARGO_BUILD_JOBS", "1")
+ENV.setdefault("CARGO_INCREMENTAL", "0")
+ENV.setdefault("CARGO_PROFILE_DEV_DEBUG", "0")
+ENV.setdefault("CARGO_PROFILE_TEST_DEBUG", "0")
+ENV.pop("RUN_LIVE_TESTS", None)
+
+
+def run(command: list[str]) -> None:
+    print("+", " ".join(command), flush=True)
+    subprocess.run(command, cwd=ROOT, env=ENV, check=True)
+
+
+run(
+    [
+        "cargo",
+        "test",
+        "-p",
+        "libremetaverse-rendering-simple",
+        "-p",
+        "libremetaverse-rendering-mesh-foundry",
+        "-p",
+        "libremetaverse-rlv",
+        "-p",
+        "libremetaverse-lsl-tools",
+        "--locked",
+        "-j",
+        "1",
+    ]
+)
+
+catalog = json.loads((ROOT / "tests" / "upstream-tests.json").read_text())
+rlv_targets = sorted(
+    {
+        Path(test["rust_file"]).stem
+        for test in catalog["tests"]
+        if test["source"].startswith("LibreMetaverse.Tests/RLV/")
+    }
+)
+if len(rlv_targets) != 24:
+    raise SystemExit(f"milestone 10 expected 24 RLV test targets, found {rlv_targets}")
+
+compat = ["cargo", "test", "-p", "libremetaverse-compat-tests"]
+for target in (
+    "imaging_meshing_semantics",
+    "rendering_shims",
+    "extension_shims",
+    "milestone_10_integration",
+    *rlv_targets,
+):
+    compat.extend(("--test", target))
+compat.extend(("--locked", "-j", "1"))
+run(compat)
+
+run(
+    [
+        "cargo",
+        "check",
+        "--manifest-path",
+        "tests/api-compile/Cargo.toml",
+        "--locked",
+        "-j",
+        "1",
+    ]
+)