347 lines
15 KiB
Python
347 lines
15 KiB
Python
#!/usr/bin/env python3
|
|
"""Generate traceable Rust shims and test/sample parity ledgers from LibreMetaverse.
|
|
|
|
This is deliberately a source inventory generator, not a C# parser. It handles
|
|
the declaration and NUnit forms present in the pinned upstream checkout and
|
|
fails loudly when its expected counts drift.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import hashlib
|
|
import json
|
|
import keyword
|
|
import re
|
|
from collections import defaultdict
|
|
from dataclasses import dataclass, field
|
|
from pathlib import Path
|
|
|
|
UPSTREAM_COMMIT = "2aa70bb68513b39795da5d13c88f31b86e85a3ba"
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class Project:
|
|
source: str
|
|
crate: str
|
|
namespace: str
|
|
|
|
|
|
PROJECTS = (
|
|
Project("LibreMetaverse.Types", "libremetaverse-types", "LibreMetaverse"),
|
|
Project("LibreMetaverse.StructuredData", "libremetaverse-structured-data", "LibreMetaverse.StructuredData"),
|
|
Project("LibreMetaverse.Imaging.Abstractions", "libremetaverse-imaging", "LibreMetaverse.Imaging"),
|
|
Project("LibreMetaverse.Imaging.Skia", "libremetaverse-imaging-skia", "LibreMetaverse.Imaging.Skia"),
|
|
Project("PrimMesher", "libremetaverse-prim-mesher", "LibreMetaverse.PrimMesher"),
|
|
Project("LibreMetaverse", "libremetaverse", "LibreMetaverse"),
|
|
Project("LibreMetaverse.Rendering.Simple", "libremetaverse-rendering-simple", "LibreMetaverse.Rendering"),
|
|
Project("LibreMetaverse.Rendering.MeshFoundry", "libremetaverse-rendering-mesh-foundry", "LibreMetaverse.Rendering"),
|
|
Project("LibreMetaverse.LslTools", "libremetaverse-lsl-tools", "LibreMetaverse.LslTools"),
|
|
Project("LibreMetaverse.RLV", "libremetaverse-rlv", "LibreMetaverse.RLV"),
|
|
Project("LibreMetaverse.Utilities", "libremetaverse-utilities", "LibreMetaverse.Utilities"),
|
|
Project("LibreMetaverse.Voice.Vivox", "libremetaverse-voice-vivox", "LibreMetaverse.Voice.Vivox"),
|
|
Project("LibreMetaverse.Voice.WebRTC", "libremetaverse-voice-webrtc", "LibreMetaverse.Voice.WebRTC"),
|
|
)
|
|
|
|
TYPE_RE = re.compile(
|
|
r"^\s*public\s+(?:(?:static|abstract|sealed|partial|readonly|unsafe|ref|new)\s+)*"
|
|
r"(?P<kind>class|struct|interface|enum|delegate|record(?:\s+(?:class|struct))?)\s+"
|
|
r"(?P<name>@?[A-Za-z_][A-Za-z0-9_]*)(?P<generic><[^>{}()]+>)?",
|
|
re.MULTILINE,
|
|
)
|
|
NAMESPACE_RE = re.compile(r"^\s*namespace\s+([A-Za-z_][A-Za-z0-9_.]*)", re.MULTILINE)
|
|
TEST_ATTR_RE = re.compile(r"\[(Test|TestCase)(?:\((.*?)\))?\]", re.DOTALL)
|
|
METHOD_RE = re.compile(
|
|
r"\b(?:public|internal)\s+(?:static\s+)?(?:async\s+)?"
|
|
r"(?:void|Task(?:\s*<[^>]+>)?|ValueTask(?:\s*<[^>]+>)?)\s+"
|
|
r"([A-Za-z_][A-Za-z0-9_]*)\s*\(",
|
|
re.DOTALL,
|
|
)
|
|
CLASS_RE = re.compile(r"\b(?:public|internal)\s+(?:sealed\s+|partial\s+)?class\s+([A-Za-z_][A-Za-z0-9_]*)")
|
|
|
|
|
|
def snake(name: str) -> str:
|
|
name = name.lstrip("@").replace("-", "_")
|
|
name = re.sub(r"([A-Z]+)([A-Z][a-z])", r"\1_\2", name)
|
|
name = re.sub(r"([a-z0-9])([A-Z])", r"\1_\2", name).lower()
|
|
name = re.sub(r"[^a-z0-9_]", "_", name)
|
|
if not name or name[0].isdigit():
|
|
name = f"item_{name}"
|
|
if keyword.iskeyword(name) or name in {"crate", "self", "super", "type", "match", "mod", "move", "ref", "use", "where", "loop", "async", "await", "dyn"}:
|
|
name += "_"
|
|
return name
|
|
|
|
|
|
def rust_type_name(name: str) -> str:
|
|
name = name.lstrip("@")
|
|
replacements = {
|
|
"Self": "SelfType",
|
|
"box": "BoxShim",
|
|
"type": "TypeShim",
|
|
}
|
|
if name in replacements:
|
|
return replacements[name]
|
|
return re.sub(r"[^A-Za-z0-9_]", "_", name)
|
|
|
|
|
|
def source_files(root: Path) -> list[Path]:
|
|
return sorted(p for p in root.rglob("*.cs") if "obj" not in p.parts and "bin" not in p.parts)
|
|
|
|
|
|
def namespace_at(text: str, position: int) -> str:
|
|
matches = [m for m in NAMESPACE_RE.finditer(text) if m.start() <= position]
|
|
return matches[-1].group(1) if matches else ""
|
|
|
|
|
|
def generic_arity(generic: str | None) -> int:
|
|
if not generic:
|
|
return 0
|
|
return len([p for p in generic[1:-1].split(",") if p.strip()])
|
|
|
|
|
|
@dataclass
|
|
class ModuleNode:
|
|
types: dict[str, tuple[str, int, str]] = field(default_factory=dict)
|
|
children: dict[str, "ModuleNode"] = field(default_factory=dict)
|
|
|
|
|
|
def add_type(root: ModuleNode, modules: list[str], name: str, kind: str, arity: int, csharp_name: str) -> None:
|
|
node = root
|
|
for module in modules:
|
|
node = node.children.setdefault(module, ModuleNode())
|
|
node.types.setdefault(name, (kind, arity, csharp_name))
|
|
|
|
|
|
def render_type(name: str, kind: str, arity: int, csharp_name: str, indent: str) -> list[str]:
|
|
rust_name = rust_type_name(name)
|
|
kind_label = kind.replace(" ", "_")
|
|
if kind == "interface":
|
|
params = "" if arity == 0 else "<" + ", ".join(f"T{i} = ()" for i in range(arity)) + ">"
|
|
return [
|
|
f"{indent}/// Shim for C# `{csharp_name}`.",
|
|
f"{indent}pub trait {rust_name}{params} {{",
|
|
f"{indent} const CSHARP_NAME: &'static str = \"{csharp_name}\";",
|
|
f"{indent}}}",
|
|
]
|
|
params = "" if arity == 0 else "<" + ", ".join(f"T{i} = ()" for i in range(arity)) + ">"
|
|
marker = "" if arity == 0 else "(" + "std::marker::PhantomData<(" + ", ".join(f"T{i}" for i in range(arity)) + ",)>" + ")"
|
|
declaration = f"pub struct {rust_name}{params}" + (";" if not marker else f"{marker};")
|
|
return [
|
|
f"{indent}/// `{kind}` shim for C# `{csharp_name}`; members remain intentionally unimplemented.",
|
|
f"{indent}#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Hash)]",
|
|
f"{indent}{declaration}",
|
|
f"{indent}impl{('<' + ', '.join(f'T{i}' for i in range(arity)) + '>') if arity else ''} {rust_name}{('<' + ', '.join(f'T{i}' for i in range(arity)) + '>') if arity else ''} {{",
|
|
f"{indent} pub const CSHARP_NAME: &'static str = \"{csharp_name}\";",
|
|
f"{indent} pub const CSHARP_KIND: &'static str = \"{kind_label}\";",
|
|
f"{indent}}}",
|
|
]
|
|
|
|
|
|
def render_node(node: ModuleNode, indent: str = "") -> list[str]:
|
|
lines: list[str] = []
|
|
for name, (kind, arity, csharp_name) in sorted(node.types.items()):
|
|
lines.extend(render_type(name, kind, arity, csharp_name, indent))
|
|
lines.append("")
|
|
for name, child in sorted(node.children.items()):
|
|
lines.append(f"{indent}pub mod {name} {{")
|
|
lines.extend(render_node(child, indent + " "))
|
|
lines.append(f"{indent}}}")
|
|
lines.append("")
|
|
return lines
|
|
|
|
|
|
def generate_apis(upstream: Path, output: Path) -> tuple[int, int]:
|
|
surface_lines = ["project\tfile\tline\tdeclaration_sha256\tdeclaration"]
|
|
type_lines = ["crate\tnamespace\trust_path\tkind\tcsharp_name\tfile\tline"]
|
|
total_types = 0
|
|
total_members = 0
|
|
for project in PROJECTS:
|
|
project_root = upstream / project.source
|
|
tree = ModuleNode()
|
|
for path in source_files(project_root):
|
|
text = path.read_text(encoding="utf-8-sig")
|
|
relative = path.relative_to(upstream).as_posix()
|
|
for line_no, line in enumerate(text.splitlines(), 1):
|
|
if re.match(r"^\s*public\s+", line):
|
|
declaration = " ".join(line.strip().split())
|
|
digest = hashlib.sha256(declaration.encode()).hexdigest()
|
|
surface_lines.append(f"{project.source}\t{relative}\t{line_no}\t{digest}\t{declaration}")
|
|
total_members += 1
|
|
for match in TYPE_RE.finditer(text):
|
|
namespace = namespace_at(text, match.start())
|
|
remainder = namespace
|
|
if remainder == project.namespace:
|
|
remainder = ""
|
|
elif remainder.startswith(project.namespace + "."):
|
|
remainder = remainder[len(project.namespace) + 1 :]
|
|
modules = [snake(part) for part in remainder.split(".") if part]
|
|
name = match.group("name").lstrip("@")
|
|
kind = match.group("kind")
|
|
arity = generic_arity(match.group("generic"))
|
|
csharp_name = ".".join(filter(None, [namespace, name]))
|
|
add_type(tree, modules, name, kind, arity, csharp_name)
|
|
rust_path = "::".join(modules + [rust_type_name(name)])
|
|
line_no = text.count("\n", 0, match.start()) + 1
|
|
type_lines.append(f"{project.crate}\t{namespace}\t{rust_path}\t{kind}\t{csharp_name}\t{relative}\t{line_no}")
|
|
total_types += 1
|
|
generated = [
|
|
"// @generated by tools/generate_surface.py; do not edit by hand.",
|
|
f"// Source: LibreMetaverse {UPSTREAM_COMMIT}",
|
|
"#![allow(non_camel_case_types)]",
|
|
"",
|
|
] + render_node(tree)
|
|
crate_file = output / "crates" / project.crate / "src" / "generated.rs"
|
|
crate_file.parent.mkdir(parents=True, exist_ok=True)
|
|
crate_file.write_text("\n".join(generated).rstrip() + "\n")
|
|
api_dir = output / "api"
|
|
api_dir.mkdir(parents=True, exist_ok=True)
|
|
(api_dir / "SURFACE.tsv").write_text("\n".join(surface_lines) + "\n")
|
|
(api_dir / "TYPES.tsv").write_text("\n".join(type_lines) + "\n")
|
|
return total_types, total_members
|
|
|
|
|
|
def enclosing_class(text: str, position: int) -> str:
|
|
matches = [m for m in CLASS_RE.finditer(text) if m.start() <= position]
|
|
return matches[-1].group(1) if matches else "UnknownFixture"
|
|
|
|
|
|
def method_body_hash(text: str, method_start: int) -> str:
|
|
brace = text.find("{", method_start)
|
|
arrow = text.find("=>", method_start)
|
|
if arrow != -1 and (brace == -1 or arrow < brace):
|
|
end = text.find(";", arrow)
|
|
body = text[method_start : end + 1]
|
|
elif brace != -1:
|
|
depth = 0
|
|
end = brace
|
|
for end in range(brace, len(text)):
|
|
if text[end] == "{":
|
|
depth += 1
|
|
elif text[end] == "}":
|
|
depth -= 1
|
|
if depth == 0:
|
|
break
|
|
body = text[method_start : end + 1]
|
|
else:
|
|
body = text[method_start : method_start + 500]
|
|
return hashlib.sha256(body.encode()).hexdigest()
|
|
|
|
|
|
def generate_tests(upstream: Path, output: Path) -> int:
|
|
roots = (upstream / "LibreMetaverse.Tests", upstream / "LibreMetaverse.Rendering.Tests")
|
|
tests: list[dict[str, object]] = []
|
|
for root in roots:
|
|
for path in source_files(root):
|
|
text = path.read_text(encoding="utf-8-sig")
|
|
relative = path.relative_to(upstream).as_posix()
|
|
attrs = list(TEST_ATTR_RE.finditer(text))
|
|
for attr_index, attr in enumerate(attrs):
|
|
next_attr = attrs[attr_index + 1].start() if attr_index + 1 < len(attrs) else len(text)
|
|
method = METHOD_RE.search(text, attr.end(), next_attr + 1200)
|
|
if method is None:
|
|
method = METHOD_RE.search(text, attr.end())
|
|
if method is None:
|
|
raise RuntimeError(f"No test method after {relative}:{text.count(chr(10), 0, attr.start()) + 1}")
|
|
name = method.group(1)
|
|
fixture = enclosing_class(text, method.start())
|
|
attr_text = " ".join(attr.group(0).split())
|
|
line = text.count("\n", 0, attr.start()) + 1
|
|
tests.append(
|
|
{
|
|
"source": relative,
|
|
"line": line,
|
|
"fixture": fixture,
|
|
"method": name,
|
|
"attribute": attr_text,
|
|
"body_sha256": method_body_hash(text, method.start()),
|
|
}
|
|
)
|
|
seen: defaultdict[str, int] = defaultdict(int)
|
|
rust_lines = [
|
|
"// @generated by tools/generate_surface.py; do not edit by hand.",
|
|
f"// Source: LibreMetaverse {UPSTREAM_COMMIT}",
|
|
"use libremetaverse_compat_tests::pending;",
|
|
"",
|
|
]
|
|
parity_lines = [
|
|
"# NUnit to Rust parity ledger",
|
|
"",
|
|
f"Generated from LibreMetaverse `{UPSTREAM_COMMIT}`. Each row is one NUnit `[Test]` or `[TestCase]` invocation. The body hash covers the original C# method declaration and body.",
|
|
"",
|
|
"| Rust test | C# test | Source | Attribute | Body SHA-256 |",
|
|
"|---|---|---|---|---|",
|
|
]
|
|
for test in tests:
|
|
base = snake(f"{test['fixture']}_{test['method']}")
|
|
seen[base] += 1
|
|
rust_name = base if seen[base] == 1 else f"{base}_case_{seen[base]}"
|
|
test["rust_test"] = rust_name
|
|
identity = f"{test['fixture']}.{test['method']}"
|
|
rust_lines.extend(
|
|
[
|
|
"#[test]",
|
|
f"fn {rust_name}() {{",
|
|
f" pending({json.dumps(test['source'])}, {test['line']}, {json.dumps(identity)}, {json.dumps(test['attribute'])}, {json.dumps(test['body_sha256'])});",
|
|
"}",
|
|
"",
|
|
]
|
|
)
|
|
parity_lines.append(
|
|
f"| `{rust_name}` | `{identity}` | `{test['source']}:{test['line']}` | `{str(test['attribute']).replace('|', '|')}` | `{test['body_sha256']}` |"
|
|
)
|
|
generated_dir = output / "tests" / "compat" / "tests"
|
|
generated_dir.mkdir(parents=True, exist_ok=True)
|
|
(generated_dir / "generated_parity.rs").write_text("\n".join(rust_lines))
|
|
(output / "tests" / "PARITY.md").write_text("\n".join(parity_lines) + "\n")
|
|
(output / "tests" / "upstream-tests.json").write_text(json.dumps({"upstream_commit": UPSTREAM_COMMIT, "tests": tests}, indent=2) + "\n")
|
|
return len(tests)
|
|
|
|
|
|
def generate_program_manifest(upstream: Path, output: Path) -> int:
|
|
roots = [
|
|
upstream / "Programs" / "VivoxTest",
|
|
upstream / "Programs" / "WebRtcTest",
|
|
*sorted((upstream / "Programs" / "examples").iterdir()),
|
|
*sorted((upstream / "Programs" / "tools").iterdir()),
|
|
]
|
|
entries = []
|
|
for root in roots:
|
|
if not root.is_dir() or not list(root.glob("*.csproj")):
|
|
continue
|
|
files = []
|
|
for path in source_files(root):
|
|
data = path.read_bytes()
|
|
files.append({"path": path.relative_to(upstream).as_posix(), "sha256": hashlib.sha256(data).hexdigest()})
|
|
entries.append({"project": root.name, "files": files})
|
|
(output / "programs" / "upstream-programs.json").write_text(json.dumps({"upstream_commit": UPSTREAM_COMMIT, "projects": entries}, indent=2) + "\n")
|
|
command_files = source_files(upstream / "Programs" / "examples" / "TestClient" / "Commands")
|
|
command_names = [path.stem for path in command_files]
|
|
command_rs = [
|
|
"// @generated by tools/generate_surface.py; do not edit by hand.",
|
|
"pub const TEST_CLIENT_COMMANDS: &[&str] = &[",
|
|
*[f" {json.dumps(name)}," for name in command_names],
|
|
"];",
|
|
"",
|
|
]
|
|
(output / "programs" / "src" / "commands.rs").write_text("\n".join(command_rs))
|
|
return len(entries)
|
|
|
|
|
|
def main() -> None:
|
|
parser = argparse.ArgumentParser()
|
|
parser.add_argument("--upstream", type=Path, default=Path("../libremetaverse"))
|
|
parser.add_argument("--output", type=Path, default=Path("."))
|
|
args = parser.parse_args()
|
|
upstream = args.upstream.resolve()
|
|
output = args.output.resolve()
|
|
types, members = generate_apis(upstream, output)
|
|
tests = generate_tests(upstream, output)
|
|
programs = generate_program_manifest(upstream, output)
|
|
if tests != 1295:
|
|
raise RuntimeError(f"Expected 1295 NUnit invocations at {UPSTREAM_COMMIT}, found {tests}")
|
|
print(f"generated {types} type declarations, {members} public declaration lines, {tests} tests, and {programs} program manifests")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|