Initial MetaCrate rewrite shell
This commit is contained in:
346
tools/generate_surface.py
Normal file
346
tools/generate_surface.py
Normal file
@@ -0,0 +1,346 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Generate traceable Rust shims and test/sample parity ledgers from LibreMetaverse.
|
||||
|
||||
This is deliberately a source inventory generator, not a C# parser. It handles
|
||||
the declaration and NUnit forms present in the pinned upstream checkout and
|
||||
fails loudly when its expected counts drift.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import hashlib
|
||||
import json
|
||||
import keyword
|
||||
import re
|
||||
from collections import defaultdict
|
||||
from dataclasses import dataclass, field
|
||||
from pathlib import Path
|
||||
|
||||
UPSTREAM_COMMIT = "2aa70bb68513b39795da5d13c88f31b86e85a3ba"
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Project:
|
||||
source: str
|
||||
crate: str
|
||||
namespace: str
|
||||
|
||||
|
||||
PROJECTS = (
|
||||
Project("LibreMetaverse.Types", "libremetaverse-types", "LibreMetaverse"),
|
||||
Project("LibreMetaverse.StructuredData", "libremetaverse-structured-data", "LibreMetaverse.StructuredData"),
|
||||
Project("LibreMetaverse.Imaging.Abstractions", "libremetaverse-imaging", "LibreMetaverse.Imaging"),
|
||||
Project("LibreMetaverse.Imaging.Skia", "libremetaverse-imaging-skia", "LibreMetaverse.Imaging.Skia"),
|
||||
Project("PrimMesher", "libremetaverse-prim-mesher", "LibreMetaverse.PrimMesher"),
|
||||
Project("LibreMetaverse", "libremetaverse", "LibreMetaverse"),
|
||||
Project("LibreMetaverse.Rendering.Simple", "libremetaverse-rendering-simple", "LibreMetaverse.Rendering"),
|
||||
Project("LibreMetaverse.Rendering.MeshFoundry", "libremetaverse-rendering-mesh-foundry", "LibreMetaverse.Rendering"),
|
||||
Project("LibreMetaverse.LslTools", "libremetaverse-lsl-tools", "LibreMetaverse.LslTools"),
|
||||
Project("LibreMetaverse.RLV", "libremetaverse-rlv", "LibreMetaverse.RLV"),
|
||||
Project("LibreMetaverse.Utilities", "libremetaverse-utilities", "LibreMetaverse.Utilities"),
|
||||
Project("LibreMetaverse.Voice.Vivox", "libremetaverse-voice-vivox", "LibreMetaverse.Voice.Vivox"),
|
||||
Project("LibreMetaverse.Voice.WebRTC", "libremetaverse-voice-webrtc", "LibreMetaverse.Voice.WebRTC"),
|
||||
)
|
||||
|
||||
TYPE_RE = re.compile(
|
||||
r"^\s*public\s+(?:(?:static|abstract|sealed|partial|readonly|unsafe|ref|new)\s+)*"
|
||||
r"(?P<kind>class|struct|interface|enum|delegate|record(?:\s+(?:class|struct))?)\s+"
|
||||
r"(?P<name>@?[A-Za-z_][A-Za-z0-9_]*)(?P<generic><[^>{}()]+>)?",
|
||||
re.MULTILINE,
|
||||
)
|
||||
NAMESPACE_RE = re.compile(r"^\s*namespace\s+([A-Za-z_][A-Za-z0-9_.]*)", re.MULTILINE)
|
||||
TEST_ATTR_RE = re.compile(r"\[(Test|TestCase)(?:\((.*?)\))?\]", re.DOTALL)
|
||||
METHOD_RE = re.compile(
|
||||
r"\b(?:public|internal)\s+(?:static\s+)?(?:async\s+)?"
|
||||
r"(?:void|Task(?:\s*<[^>]+>)?|ValueTask(?:\s*<[^>]+>)?)\s+"
|
||||
r"([A-Za-z_][A-Za-z0-9_]*)\s*\(",
|
||||
re.DOTALL,
|
||||
)
|
||||
CLASS_RE = re.compile(r"\b(?:public|internal)\s+(?:sealed\s+|partial\s+)?class\s+([A-Za-z_][A-Za-z0-9_]*)")
|
||||
|
||||
|
||||
def snake(name: str) -> str:
|
||||
name = name.lstrip("@").replace("-", "_")
|
||||
name = re.sub(r"([A-Z]+)([A-Z][a-z])", r"\1_\2", name)
|
||||
name = re.sub(r"([a-z0-9])([A-Z])", r"\1_\2", name).lower()
|
||||
name = re.sub(r"[^a-z0-9_]", "_", name)
|
||||
if not name or name[0].isdigit():
|
||||
name = f"item_{name}"
|
||||
if keyword.iskeyword(name) or name in {"crate", "self", "super", "type", "match", "mod", "move", "ref", "use", "where", "loop", "async", "await", "dyn"}:
|
||||
name += "_"
|
||||
return name
|
||||
|
||||
|
||||
def rust_type_name(name: str) -> str:
|
||||
name = name.lstrip("@")
|
||||
replacements = {
|
||||
"Self": "SelfType",
|
||||
"box": "BoxShim",
|
||||
"type": "TypeShim",
|
||||
}
|
||||
if name in replacements:
|
||||
return replacements[name]
|
||||
return re.sub(r"[^A-Za-z0-9_]", "_", name)
|
||||
|
||||
|
||||
def source_files(root: Path) -> list[Path]:
|
||||
return sorted(p for p in root.rglob("*.cs") if "obj" not in p.parts and "bin" not in p.parts)
|
||||
|
||||
|
||||
def namespace_at(text: str, position: int) -> str:
|
||||
matches = [m for m in NAMESPACE_RE.finditer(text) if m.start() <= position]
|
||||
return matches[-1].group(1) if matches else ""
|
||||
|
||||
|
||||
def generic_arity(generic: str | None) -> int:
|
||||
if not generic:
|
||||
return 0
|
||||
return len([p for p in generic[1:-1].split(",") if p.strip()])
|
||||
|
||||
|
||||
@dataclass
|
||||
class ModuleNode:
|
||||
types: dict[str, tuple[str, int, str]] = field(default_factory=dict)
|
||||
children: dict[str, "ModuleNode"] = field(default_factory=dict)
|
||||
|
||||
|
||||
def add_type(root: ModuleNode, modules: list[str], name: str, kind: str, arity: int, csharp_name: str) -> None:
|
||||
node = root
|
||||
for module in modules:
|
||||
node = node.children.setdefault(module, ModuleNode())
|
||||
node.types.setdefault(name, (kind, arity, csharp_name))
|
||||
|
||||
|
||||
def render_type(name: str, kind: str, arity: int, csharp_name: str, indent: str) -> list[str]:
|
||||
rust_name = rust_type_name(name)
|
||||
kind_label = kind.replace(" ", "_")
|
||||
if kind == "interface":
|
||||
params = "" if arity == 0 else "<" + ", ".join(f"T{i} = ()" for i in range(arity)) + ">"
|
||||
return [
|
||||
f"{indent}/// Shim for C# `{csharp_name}`.",
|
||||
f"{indent}pub trait {rust_name}{params} {{",
|
||||
f"{indent} const CSHARP_NAME: &'static str = \"{csharp_name}\";",
|
||||
f"{indent}}}",
|
||||
]
|
||||
params = "" if arity == 0 else "<" + ", ".join(f"T{i} = ()" for i in range(arity)) + ">"
|
||||
marker = "" if arity == 0 else "(" + "std::marker::PhantomData<(" + ", ".join(f"T{i}" for i in range(arity)) + ",)>" + ")"
|
||||
declaration = f"pub struct {rust_name}{params}" + (";" if not marker else f"{marker};")
|
||||
return [
|
||||
f"{indent}/// `{kind}` shim for C# `{csharp_name}`; members remain intentionally unimplemented.",
|
||||
f"{indent}#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Hash)]",
|
||||
f"{indent}{declaration}",
|
||||
f"{indent}impl{('<' + ', '.join(f'T{i}' for i in range(arity)) + '>') if arity else ''} {rust_name}{('<' + ', '.join(f'T{i}' for i in range(arity)) + '>') if arity else ''} {{",
|
||||
f"{indent} pub const CSHARP_NAME: &'static str = \"{csharp_name}\";",
|
||||
f"{indent} pub const CSHARP_KIND: &'static str = \"{kind_label}\";",
|
||||
f"{indent}}}",
|
||||
]
|
||||
|
||||
|
||||
def render_node(node: ModuleNode, indent: str = "") -> list[str]:
|
||||
lines: list[str] = []
|
||||
for name, (kind, arity, csharp_name) in sorted(node.types.items()):
|
||||
lines.extend(render_type(name, kind, arity, csharp_name, indent))
|
||||
lines.append("")
|
||||
for name, child in sorted(node.children.items()):
|
||||
lines.append(f"{indent}pub mod {name} {{")
|
||||
lines.extend(render_node(child, indent + " "))
|
||||
lines.append(f"{indent}}}")
|
||||
lines.append("")
|
||||
return lines
|
||||
|
||||
|
||||
def generate_apis(upstream: Path, output: Path) -> tuple[int, int]:
|
||||
surface_lines = ["project\tfile\tline\tdeclaration_sha256\tdeclaration"]
|
||||
type_lines = ["crate\tnamespace\trust_path\tkind\tcsharp_name\tfile\tline"]
|
||||
total_types = 0
|
||||
total_members = 0
|
||||
for project in PROJECTS:
|
||||
project_root = upstream / project.source
|
||||
tree = ModuleNode()
|
||||
for path in source_files(project_root):
|
||||
text = path.read_text(encoding="utf-8-sig")
|
||||
relative = path.relative_to(upstream).as_posix()
|
||||
for line_no, line in enumerate(text.splitlines(), 1):
|
||||
if re.match(r"^\s*public\s+", line):
|
||||
declaration = " ".join(line.strip().split())
|
||||
digest = hashlib.sha256(declaration.encode()).hexdigest()
|
||||
surface_lines.append(f"{project.source}\t{relative}\t{line_no}\t{digest}\t{declaration}")
|
||||
total_members += 1
|
||||
for match in TYPE_RE.finditer(text):
|
||||
namespace = namespace_at(text, match.start())
|
||||
remainder = namespace
|
||||
if remainder == project.namespace:
|
||||
remainder = ""
|
||||
elif remainder.startswith(project.namespace + "."):
|
||||
remainder = remainder[len(project.namespace) + 1 :]
|
||||
modules = [snake(part) for part in remainder.split(".") if part]
|
||||
name = match.group("name").lstrip("@")
|
||||
kind = match.group("kind")
|
||||
arity = generic_arity(match.group("generic"))
|
||||
csharp_name = ".".join(filter(None, [namespace, name]))
|
||||
add_type(tree, modules, name, kind, arity, csharp_name)
|
||||
rust_path = "::".join(modules + [rust_type_name(name)])
|
||||
line_no = text.count("\n", 0, match.start()) + 1
|
||||
type_lines.append(f"{project.crate}\t{namespace}\t{rust_path}\t{kind}\t{csharp_name}\t{relative}\t{line_no}")
|
||||
total_types += 1
|
||||
generated = [
|
||||
"// @generated by tools/generate_surface.py; do not edit by hand.",
|
||||
f"// Source: LibreMetaverse {UPSTREAM_COMMIT}",
|
||||
"#![allow(non_camel_case_types)]",
|
||||
"",
|
||||
] + render_node(tree)
|
||||
crate_file = output / "crates" / project.crate / "src" / "generated.rs"
|
||||
crate_file.parent.mkdir(parents=True, exist_ok=True)
|
||||
crate_file.write_text("\n".join(generated).rstrip() + "\n")
|
||||
api_dir = output / "api"
|
||||
api_dir.mkdir(parents=True, exist_ok=True)
|
||||
(api_dir / "SURFACE.tsv").write_text("\n".join(surface_lines) + "\n")
|
||||
(api_dir / "TYPES.tsv").write_text("\n".join(type_lines) + "\n")
|
||||
return total_types, total_members
|
||||
|
||||
|
||||
def enclosing_class(text: str, position: int) -> str:
|
||||
matches = [m for m in CLASS_RE.finditer(text) if m.start() <= position]
|
||||
return matches[-1].group(1) if matches else "UnknownFixture"
|
||||
|
||||
|
||||
def method_body_hash(text: str, method_start: int) -> str:
|
||||
brace = text.find("{", method_start)
|
||||
arrow = text.find("=>", method_start)
|
||||
if arrow != -1 and (brace == -1 or arrow < brace):
|
||||
end = text.find(";", arrow)
|
||||
body = text[method_start : end + 1]
|
||||
elif brace != -1:
|
||||
depth = 0
|
||||
end = brace
|
||||
for end in range(brace, len(text)):
|
||||
if text[end] == "{":
|
||||
depth += 1
|
||||
elif text[end] == "}":
|
||||
depth -= 1
|
||||
if depth == 0:
|
||||
break
|
||||
body = text[method_start : end + 1]
|
||||
else:
|
||||
body = text[method_start : method_start + 500]
|
||||
return hashlib.sha256(body.encode()).hexdigest()
|
||||
|
||||
|
||||
def generate_tests(upstream: Path, output: Path) -> int:
|
||||
roots = (upstream / "LibreMetaverse.Tests", upstream / "LibreMetaverse.Rendering.Tests")
|
||||
tests: list[dict[str, object]] = []
|
||||
for root in roots:
|
||||
for path in source_files(root):
|
||||
text = path.read_text(encoding="utf-8-sig")
|
||||
relative = path.relative_to(upstream).as_posix()
|
||||
attrs = list(TEST_ATTR_RE.finditer(text))
|
||||
for attr_index, attr in enumerate(attrs):
|
||||
next_attr = attrs[attr_index + 1].start() if attr_index + 1 < len(attrs) else len(text)
|
||||
method = METHOD_RE.search(text, attr.end(), next_attr + 1200)
|
||||
if method is None:
|
||||
method = METHOD_RE.search(text, attr.end())
|
||||
if method is None:
|
||||
raise RuntimeError(f"No test method after {relative}:{text.count(chr(10), 0, attr.start()) + 1}")
|
||||
name = method.group(1)
|
||||
fixture = enclosing_class(text, method.start())
|
||||
attr_text = " ".join(attr.group(0).split())
|
||||
line = text.count("\n", 0, attr.start()) + 1
|
||||
tests.append(
|
||||
{
|
||||
"source": relative,
|
||||
"line": line,
|
||||
"fixture": fixture,
|
||||
"method": name,
|
||||
"attribute": attr_text,
|
||||
"body_sha256": method_body_hash(text, method.start()),
|
||||
}
|
||||
)
|
||||
seen: defaultdict[str, int] = defaultdict(int)
|
||||
rust_lines = [
|
||||
"// @generated by tools/generate_surface.py; do not edit by hand.",
|
||||
f"// Source: LibreMetaverse {UPSTREAM_COMMIT}",
|
||||
"use libremetaverse_compat_tests::pending;",
|
||||
"",
|
||||
]
|
||||
parity_lines = [
|
||||
"# NUnit to Rust parity ledger",
|
||||
"",
|
||||
f"Generated from LibreMetaverse `{UPSTREAM_COMMIT}`. Each row is one NUnit `[Test]` or `[TestCase]` invocation. The body hash covers the original C# method declaration and body.",
|
||||
"",
|
||||
"| Rust test | C# test | Source | Attribute | Body SHA-256 |",
|
||||
"|---|---|---|---|---|",
|
||||
]
|
||||
for test in tests:
|
||||
base = snake(f"{test['fixture']}_{test['method']}")
|
||||
seen[base] += 1
|
||||
rust_name = base if seen[base] == 1 else f"{base}_case_{seen[base]}"
|
||||
test["rust_test"] = rust_name
|
||||
identity = f"{test['fixture']}.{test['method']}"
|
||||
rust_lines.extend(
|
||||
[
|
||||
"#[test]",
|
||||
f"fn {rust_name}() {{",
|
||||
f" pending({json.dumps(test['source'])}, {test['line']}, {json.dumps(identity)}, {json.dumps(test['attribute'])}, {json.dumps(test['body_sha256'])});",
|
||||
"}",
|
||||
"",
|
||||
]
|
||||
)
|
||||
parity_lines.append(
|
||||
f"| `{rust_name}` | `{identity}` | `{test['source']}:{test['line']}` | `{str(test['attribute']).replace('|', '|')}` | `{test['body_sha256']}` |"
|
||||
)
|
||||
generated_dir = output / "tests" / "compat" / "tests"
|
||||
generated_dir.mkdir(parents=True, exist_ok=True)
|
||||
(generated_dir / "generated_parity.rs").write_text("\n".join(rust_lines))
|
||||
(output / "tests" / "PARITY.md").write_text("\n".join(parity_lines) + "\n")
|
||||
(output / "tests" / "upstream-tests.json").write_text(json.dumps({"upstream_commit": UPSTREAM_COMMIT, "tests": tests}, indent=2) + "\n")
|
||||
return len(tests)
|
||||
|
||||
|
||||
def generate_program_manifest(upstream: Path, output: Path) -> int:
|
||||
roots = [
|
||||
upstream / "Programs" / "VivoxTest",
|
||||
upstream / "Programs" / "WebRtcTest",
|
||||
*sorted((upstream / "Programs" / "examples").iterdir()),
|
||||
*sorted((upstream / "Programs" / "tools").iterdir()),
|
||||
]
|
||||
entries = []
|
||||
for root in roots:
|
||||
if not root.is_dir() or not list(root.glob("*.csproj")):
|
||||
continue
|
||||
files = []
|
||||
for path in source_files(root):
|
||||
data = path.read_bytes()
|
||||
files.append({"path": path.relative_to(upstream).as_posix(), "sha256": hashlib.sha256(data).hexdigest()})
|
||||
entries.append({"project": root.name, "files": files})
|
||||
(output / "programs" / "upstream-programs.json").write_text(json.dumps({"upstream_commit": UPSTREAM_COMMIT, "projects": entries}, indent=2) + "\n")
|
||||
command_files = source_files(upstream / "Programs" / "examples" / "TestClient" / "Commands")
|
||||
command_names = [path.stem for path in command_files]
|
||||
command_rs = [
|
||||
"// @generated by tools/generate_surface.py; do not edit by hand.",
|
||||
"pub const TEST_CLIENT_COMMANDS: &[&str] = &[",
|
||||
*[f" {json.dumps(name)}," for name in command_names],
|
||||
"];",
|
||||
"",
|
||||
]
|
||||
(output / "programs" / "src" / "commands.rs").write_text("\n".join(command_rs))
|
||||
return len(entries)
|
||||
|
||||
|
||||
def main() -> None:
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--upstream", type=Path, default=Path("../libremetaverse"))
|
||||
parser.add_argument("--output", type=Path, default=Path("."))
|
||||
args = parser.parse_args()
|
||||
upstream = args.upstream.resolve()
|
||||
output = args.output.resolve()
|
||||
types, members = generate_apis(upstream, output)
|
||||
tests = generate_tests(upstream, output)
|
||||
programs = generate_program_manifest(upstream, output)
|
||||
if tests != 1295:
|
||||
raise RuntimeError(f"Expected 1295 NUnit invocations at {UPSTREAM_COMMIT}, found {tests}")
|
||||
print(f"generated {types} type declarations, {members} public declaration lines, {tests} tests, and {programs} program manifests")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user