Generate Types and StructuredData API shims
This commit is contained in:
280
tools/generate_api_shims.py
Normal file
280
tools/generate_api_shims.py
Normal file
@@ -0,0 +1,280 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Generate callable, failure-only Rust shims from the authoritative catalog."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import csv
|
||||
import json
|
||||
import re
|
||||
import subprocess
|
||||
from collections import Counter
|
||||
from pathlib import Path
|
||||
|
||||
import generate_rust_mapping as mapping
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
CATALOG = ROOT / "api/public-api.json"
|
||||
LEDGER = ROOT / "api/RUST-MAPPING.tsv"
|
||||
COVERAGE = ROOT / "api/SHIM-COVERAGE.md"
|
||||
|
||||
TARGETS = {
|
||||
"LibreMetaverse.Types": ROOT / "crates/libremetaverse-types/src/generated.rs",
|
||||
"LibreMetaverse.StructuredData": ROOT / "crates/libremetaverse-structured-data/src/generated.rs",
|
||||
}
|
||||
|
||||
UNDERLYING = {
|
||||
"System.SByte": "i8", "System.Byte": "u8", "System.Int16": "i16", "System.UInt16": "u16",
|
||||
"System.Int32": "i32", "System.UInt32": "u32", "System.Int64": "i64", "System.UInt64": "u64",
|
||||
}
|
||||
|
||||
PRIVATE_LAYOUTS = {
|
||||
"T:LibreMetaverse.CacheDictionary`2": [("entries", "std::collections::HashMap<TKey, TValue>")],
|
||||
"T:LibreMetaverse.DoubleDictionary`3": [
|
||||
("by_first", "std::collections::HashMap<TKey1, TValue>"),
|
||||
("by_second", "std::collections::HashMap<TKey2, TValue>"),
|
||||
],
|
||||
"T:LibreMetaverse.EmptyRemovalStrategy`1": [("keys", "Vec<TKey>")],
|
||||
"T:LibreMetaverse.ExpiringCache`2": [("entries", "std::collections::HashMap<TKey, TValue>")],
|
||||
"T:LibreMetaverse.LruRemovalStrategy`1": [("keys", "Vec<TKey>")],
|
||||
"T:LibreMetaverse.MruRemovalStrategy`1": [("keys", "Vec<TKey>")],
|
||||
"T:LibreMetaverse.MultiValueDictionary`2": [("entries", "std::collections::HashMap<TKey, Vec<TValue>>")],
|
||||
"T:LibreMetaverse.TokenBucket": [("content", "i32"), ("max_burst", "i32"), ("drip_rate", "i32")],
|
||||
"T:LibreMetaverse.UUID": [("bytes", "[u8; 16]")],
|
||||
"T:LibreMetaverse.StructuredData.OSDArray": [("values", "Vec<OSD>")],
|
||||
"T:LibreMetaverse.StructuredData.OSDBinary": [("value", "Vec<u8>")],
|
||||
"T:LibreMetaverse.StructuredData.OSDBoolean": [("value", "bool")],
|
||||
"T:LibreMetaverse.StructuredData.OSDDate": [("value", "std::time::SystemTime")],
|
||||
"T:LibreMetaverse.StructuredData.OSDException": [("message", "String")],
|
||||
"T:LibreMetaverse.StructuredData.OSDInteger": [("value", "i32")],
|
||||
"T:LibreMetaverse.StructuredData.OSDMap": [("values", "std::collections::HashMap<String, OSD>")],
|
||||
"T:LibreMetaverse.StructuredData.OSDReal": [("value", "f64")],
|
||||
"T:LibreMetaverse.StructuredData.OSDString": [("value", "String")],
|
||||
"T:LibreMetaverse.StructuredData.OSDUUID": [("value", "libremetaverse_types::UUID")],
|
||||
"T:LibreMetaverse.StructuredData.OSDUri": [("value", "libremetaverse_types::compat::Uri")],
|
||||
}
|
||||
|
||||
|
||||
def generic_names(item: dict) -> list[str]:
|
||||
return [parameter["name"] for parameter in item.get("generic_parameters", [])]
|
||||
|
||||
|
||||
def format_rust(source: str) -> str:
|
||||
result = subprocess.run(
|
||||
["rustfmt", "--edition", "2024", "--emit", "stdout"],
|
||||
input=source,
|
||||
text=True,
|
||||
capture_output=True,
|
||||
check=True,
|
||||
)
|
||||
return result.stdout
|
||||
|
||||
|
||||
def type_name(type_row: dict) -> str:
|
||||
return type_row["rust_path"].rsplit("::", 1)[-1]
|
||||
|
||||
|
||||
def generic_suffix(names: list[str]) -> str:
|
||||
return "<" + ", ".join(names) + ">" if names else ""
|
||||
|
||||
|
||||
def flags_enum(item: dict) -> bool:
|
||||
return any(attribute["type"] == "System.FlagsAttribute" for attribute in item.get("attributes", []))
|
||||
|
||||
|
||||
def duplicate_enum_values(item: dict) -> bool:
|
||||
values = [member["value"]["value"] for member in item["members"] if member["kind"] == "enum_value"]
|
||||
return len(values) != len(set(values))
|
||||
|
||||
|
||||
def render_enum(item: dict, rust_name: str) -> list[str]:
|
||||
underlying = UNDERLYING[item["enum_underlying_type"]]
|
||||
values = [member for member in item["members"] if member["kind"] == "enum_value"]
|
||||
lines = [f"/// C# type: `{item['doc_id']}`."]
|
||||
if flags_enum(item) or duplicate_enum_values(item):
|
||||
lines += [
|
||||
"#[derive(Clone, Copy, Debug, Eq, Hash, PartialEq)]",
|
||||
"#[repr(transparent)]",
|
||||
f"pub struct {rust_name}(pub {underlying});",
|
||||
f"impl {rust_name} {{",
|
||||
]
|
||||
for member in values:
|
||||
name = mapping.snake(member["name"]).upper()
|
||||
value = member["value"]["value"]
|
||||
lines += [f" /// C# member: `{member['doc_id']}`.", f" pub const {name}: Self = Self({value});"]
|
||||
lines.append("}")
|
||||
return lines
|
||||
lines += [
|
||||
"#[derive(Clone, Copy, Debug, Eq, Hash, PartialEq)]",
|
||||
f"#[repr({underlying})]",
|
||||
f"pub enum {rust_name} {{",
|
||||
]
|
||||
for member in values:
|
||||
lines += [f" /// C# member: `{member['doc_id']}`.", f" {mapping.pascal(member['name'])} = {member['value']['value']},"]
|
||||
lines.append("}")
|
||||
return lines
|
||||
|
||||
|
||||
def render_body(signature: str, member_id: str, error_model: str, trait: bool) -> str:
|
||||
if trait:
|
||||
signature = signature.removeprefix("pub ")
|
||||
failure = (
|
||||
f"libremetaverse_types::not_implemented({json.dumps(member_id)})"
|
||||
if error_model.startswith("Result")
|
||||
else f"libremetaverse_types::unimplemented_api!({json.dumps(member_id)})"
|
||||
)
|
||||
return f" {signature} {{ {failure} }}"
|
||||
|
||||
|
||||
def render_type(item: dict, type_row: dict, member_rows: dict[str, dict[str, str]], mapper: mapping.Mapper) -> str:
|
||||
rust_name = type_name(type_row)
|
||||
if item["kind"] == "enum":
|
||||
return "\n".join(render_enum(item, rust_name))
|
||||
names = generic_names(item)
|
||||
suffix = generic_suffix(names)
|
||||
trait = item["kind"] == "interface"
|
||||
lines = [f"/// C# type: `{item['doc_id']}`."]
|
||||
if trait:
|
||||
lines.append(f"pub trait {rust_name}{suffix} {{")
|
||||
else:
|
||||
fields = [member for member in item["members"] if member["kind"] == "field" and not member.get("static")]
|
||||
private_fields = PRIVATE_LAYOUTS.get(item["doc_id"], [])
|
||||
if item["doc_id"] == "T:LibreMetaverse.StructuredData.OSD":
|
||||
lines += [
|
||||
"#[non_exhaustive]",
|
||||
"pub enum OSD {",
|
||||
" Undefined, Boolean(bool), Integer(i32), Real(f64), String(String),",
|
||||
" UUID(libremetaverse_types::UUID), Date(std::time::SystemTime),",
|
||||
" Uri(libremetaverse_types::compat::Uri), Binary(Vec<u8>),",
|
||||
" Array(Vec<OSD>), Map(std::collections::HashMap<String, OSD>), LlsdXml(String),",
|
||||
"}",
|
||||
]
|
||||
elif not fields and not names and not private_fields:
|
||||
lines.append(f"pub struct {rust_name};")
|
||||
else:
|
||||
lines.append(f"pub struct {rust_name}{suffix} {{")
|
||||
generics = set(names)
|
||||
for field in fields:
|
||||
field_type = mapper.type(field["type"], field.get("nullability"), generics)
|
||||
lines += [f" /// C# member: `{field['doc_id']}`.", f" pub {mapping.snake(field['name'])}: {field_type},"]
|
||||
for field_name, field_type in private_fields:
|
||||
lines.append(f" {field_name}: {field_type},")
|
||||
if names:
|
||||
tuple_type = "(" + ", ".join(names) + ("," if len(names) == 1 else "") + ")"
|
||||
lines.append(f" _marker: std::marker::PhantomData<{tuple_type}>,")
|
||||
lines.append("}")
|
||||
lines += [f"impl{suffix} {rust_name}{suffix} {{"]
|
||||
|
||||
for member in item["members"]:
|
||||
if member["kind"] in {"enum_value"} or (member["kind"] == "field" and not member.get("static")):
|
||||
continue
|
||||
row = member_rows[member["doc_id"]]
|
||||
lines.append(f" /// C# member: `{member['doc_id']}`.")
|
||||
if member["kind"] == "constant":
|
||||
value = member["value"]["value"]
|
||||
lines.append(f" {row['rust_signature']} = {value};")
|
||||
continue
|
||||
signatures = row["rust_signature"].split(" ; ")
|
||||
for index, signature in enumerate(signatures):
|
||||
if index:
|
||||
lines.append(f" /// Setter for C# member: `{member['doc_id']}`.")
|
||||
lines.append(render_body(signature, member["doc_id"], row["error_model"], trait))
|
||||
lines.append("}")
|
||||
if not trait:
|
||||
for interface in item.get("interfaces", []):
|
||||
base, arguments, _ = mapping.type_parts(interface)
|
||||
if base not in mapper.interfaces:
|
||||
continue
|
||||
mapped_arguments = [mapper.type(argument, None, set(names)) for argument in arguments]
|
||||
trait_path = mapper.resolved[base] + generic_suffix(mapped_arguments)
|
||||
lines.append(f"impl{suffix} {trait_path} for {rust_name}{suffix} {{}}")
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
def generate_sources(catalog: dict) -> tuple[dict[Path, str], dict[str, tuple[int, int]]]:
|
||||
type_rows, resolved = mapping.build_type_rows(catalog)
|
||||
type_by_id = {row["csharp_type_id"]: row for row in type_rows}
|
||||
interfaces = {mapping.type_parts(item["signature"])[0] for assembly in catalog["assemblies"] for item in assembly["types"] if item["kind"] == "interface"}
|
||||
mapper = mapping.Mapper(resolved, interfaces)
|
||||
member_rows = {row["csharp_id"]: row for row in csv.DictReader(LEDGER.open(), delimiter="\t")}
|
||||
outputs = {}
|
||||
coverage = {}
|
||||
for assembly in catalog["assemblies"]:
|
||||
name = assembly["identity"]["name"]
|
||||
if name not in TARGETS:
|
||||
continue
|
||||
chunks = [
|
||||
"// @generated by tools/generate_api_shims.py; do not edit by hand.",
|
||||
"#![allow(clippy::missing_errors_doc)]",
|
||||
"#![allow(clippy::must_use_candidate)]",
|
||||
"#![allow(clippy::all, clippy::pedantic, dead_code, unused_variables)]",
|
||||
"#![allow(non_camel_case_types)]",
|
||||
"#![allow(non_snake_case)]",
|
||||
"",
|
||||
]
|
||||
for item in assembly["types"]:
|
||||
chunks += [render_type(item, type_by_id[item["doc_id"]], member_rows, mapper), ""]
|
||||
source = format_rust("\n".join(chunks).rstrip() + "\n")
|
||||
expected_types = {item["doc_id"] for item in assembly["types"]}
|
||||
expected_members = {member["doc_id"] for item in assembly["types"] for member in item["members"]}
|
||||
emitted_types = set(re.findall(r"C# type: `(.*)`\.", source))
|
||||
emitted_members = set(re.findall(r"C# member: `(.*)`\.", source))
|
||||
if emitted_types != expected_types or emitted_members != expected_members:
|
||||
raise ValueError(
|
||||
f"generated catalog ID mismatch for {name}: "
|
||||
f"types missing/stale={len(expected_types - emitted_types)}/{len(emitted_types - expected_types)}, "
|
||||
f"members missing/stale={len(expected_members - emitted_members)}/{len(emitted_members - expected_members)}"
|
||||
)
|
||||
outputs[TARGETS[name]] = source
|
||||
coverage[name] = (len(assembly["types"]), sum(len(item["members"]) for item in assembly["types"]))
|
||||
return outputs, coverage
|
||||
|
||||
|
||||
def coverage_report(catalog: dict, coverage: dict[str, tuple[int, int]]) -> str:
|
||||
lines = [
|
||||
"# Callable shim coverage", "", "Generated by `python3 tools/generate_api_shims.py`; do not edit by hand.", "",
|
||||
"| Assembly | Types | Members | Status |", "|---|---:|---:|---|",
|
||||
]
|
||||
total_types = total_members = 0
|
||||
for assembly in catalog["assemblies"]:
|
||||
name = assembly["identity"]["name"]
|
||||
types = len(assembly["types"])
|
||||
members = sum(len(item["members"]) for item in assembly["types"])
|
||||
if name in coverage:
|
||||
got_types, got_members = coverage[name]
|
||||
if (types, members) != (got_types, got_members):
|
||||
raise ValueError(f"incomplete generated slice for {name}")
|
||||
status = "callable failure-only shim"
|
||||
total_types += types
|
||||
total_members += members
|
||||
else:
|
||||
status = "pending milestone issue"
|
||||
lines.append(f"| `{name}` | {types:,} | {members:,} | {status} |")
|
||||
lines += ["", f"Current callable coverage: **{total_types:,} types / {total_members:,} members**.", ""]
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
def generate() -> dict[Path, str]:
|
||||
catalog = json.loads(CATALOG.read_text())
|
||||
sources, coverage = generate_sources(catalog)
|
||||
sources[COVERAGE] = coverage_report(catalog, coverage)
|
||||
return sources
|
||||
|
||||
|
||||
def main() -> None:
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--check", action="store_true")
|
||||
args = parser.parse_args()
|
||||
outputs = generate()
|
||||
if args.check:
|
||||
stale = [str(path.relative_to(ROOT)) for path, text in outputs.items() if not path.exists() or path.read_text() != text]
|
||||
if stale:
|
||||
raise SystemExit("stale generated API shims: " + ", ".join(stale))
|
||||
else:
|
||||
for path, text in outputs.items():
|
||||
path.write_text(text)
|
||||
print("generated " + ", ".join(str(path.relative_to(ROOT)) for path in outputs))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -54,7 +54,7 @@ PRIMITIVES = {
|
||||
}
|
||||
|
||||
STDLIB = {
|
||||
"System.Array": "Vec<T>",
|
||||
"System.Array": "libremetaverse_types::compat::Array",
|
||||
"System.ArraySegment": "libremetaverse_types::compat::ArraySegment<T>",
|
||||
"System.Collections.BitArray": "Vec<bool>",
|
||||
"System.Collections.Generic.Dictionary": "std::collections::HashMap<K, V>",
|
||||
@@ -96,8 +96,8 @@ STDLIB = {
|
||||
"System.DateTime": "std::time::SystemTime",
|
||||
"System.DateTimeOffset": "std::time::SystemTime",
|
||||
"System.TimeSpan": "std::time::Duration",
|
||||
"System.Guid": "uuid::Uuid",
|
||||
"System.Uri": "url::Url",
|
||||
"System.Guid": "libremetaverse_types::compat::Guid",
|
||||
"System.Uri": "libremetaverse_types::compat::Uri",
|
||||
"System.Version": "libremetaverse_types::compat::Version",
|
||||
"System.Random": "libremetaverse_types::compat::RandomSource",
|
||||
"System.Exception": "libremetaverse_types::compat::ExternalError",
|
||||
@@ -114,7 +114,7 @@ STDLIB = {
|
||||
"System.ValueType": "libremetaverse_types::compat::Value",
|
||||
"System.Text.Encoding": "libremetaverse_types::compat::TextEncoding",
|
||||
"System.Text.StringBuilder": "String",
|
||||
"System.Text.RegularExpressions.Regex": "regex::Regex",
|
||||
"System.Text.RegularExpressions.Regex": "libremetaverse_types::compat::Regex",
|
||||
"System.IO.Stream": "Box<dyn libremetaverse_types::compat::ReadWrite + Send>",
|
||||
"System.IO.MemoryStream": "std::io::Cursor<Vec<u8>>",
|
||||
"System.IO.TextReader": "Box<dyn std::io::BufRead + Send>",
|
||||
@@ -142,8 +142,8 @@ STDLIB = {
|
||||
"System.ValueTuple": "(T1, T2)",
|
||||
"System.Xml.XmlNode": "libremetaverse_structured_data::xml::Node",
|
||||
"System.Xml.XmlElement": "libremetaverse_structured_data::xml::Element",
|
||||
"System.Xml.XmlReader": "libremetaverse_structured_data::xml::Reader<R>",
|
||||
"System.Xml.XmlWriter": "libremetaverse_structured_data::xml::Writer<W>",
|
||||
"System.Xml.XmlReader": "libremetaverse_structured_data::xml::Reader",
|
||||
"System.Xml.XmlWriter": "libremetaverse_structured_data::xml::Writer",
|
||||
"System.Text.Json.JsonElement": "libremetaverse_voice_webrtc::signaling::JsonValue",
|
||||
}
|
||||
|
||||
@@ -459,8 +459,9 @@ def build_type_rows(catalog: dict) -> tuple[list[dict[str, str]], dict[str, str]
|
||||
|
||||
|
||||
class Mapper:
|
||||
def __init__(self, resolved: dict[str, str]):
|
||||
def __init__(self, resolved: dict[str, str], interfaces: set[str] | None = None):
|
||||
self.resolved = resolved
|
||||
self.interfaces = interfaces or set()
|
||||
|
||||
def type(self, value: str, nullability: dict | None = None, generics: set[str] | None = None) -> str:
|
||||
base, arguments, suffix = type_parts(value)
|
||||
@@ -498,8 +499,10 @@ class Mapper:
|
||||
"System.ValueTuple", "System.Threading.Tasks.Task", "System.Threading.Tasks.ValueTask",
|
||||
}:
|
||||
target = substitute(target, mapped_arguments)
|
||||
if "<" not in target and target not in {"Ord", "Eq", "Clone", "std::fmt::Display"}:
|
||||
if "<" not in target and not target.startswith("(") and target not in {"Ord", "Eq", "Clone", "std::fmt::Display"}:
|
||||
target += "<" + ", ".join(mapped_arguments) + ">"
|
||||
if base in self.interfaces:
|
||||
target = f"Box<dyn {target}>"
|
||||
for _ in re.findall(r"\[[,]*\]", suffix):
|
||||
target = f"Vec<{target}>"
|
||||
if "*" in suffix:
|
||||
@@ -551,7 +554,7 @@ def map_parameter(mapper: Mapper, parameter: dict, generics: set[str]) -> tuple[
|
||||
def member_base_name(item: dict) -> str:
|
||||
if item["kind"] == "enum_value":
|
||||
return pascal(item["name"])
|
||||
if item["kind"] == "constant" or (item["kind"] == "field" and item.get("static")):
|
||||
if item["kind"] == "constant":
|
||||
return snake(item["name"]).upper()
|
||||
if item["kind"] == "constructor":
|
||||
return "new"
|
||||
@@ -566,6 +569,14 @@ def member_base_name(item: dict) -> str:
|
||||
return operators.get(item["name"], snake(item["name"].removesuffix("Async")))
|
||||
|
||||
|
||||
def parameter_slug(value: str) -> str:
|
||||
base, _, suffix = type_parts(value)
|
||||
slug = snake(base.rsplit(".", 1)[-1])
|
||||
if "[" in suffix:
|
||||
return "bytes" if base == "System.Byte" else f"{slug}_array"
|
||||
return slug
|
||||
|
||||
|
||||
def member_signature(mapper: Mapper, owner: dict, item: dict, rust_name: str) -> tuple[str, str, str, str, str]:
|
||||
generics = generic_names(item, owner)
|
||||
mapped_parameters = [map_parameter(mapper, parameter, generics) for parameter in item.get("parameters", [])]
|
||||
@@ -581,7 +592,8 @@ def member_signature(mapper: Mapper, owner: dict, item: dict, rust_name: str) ->
|
||||
elif kind == "constant":
|
||||
signature = f"pub const {rust_name}: {value_type}"
|
||||
elif item.get("static"):
|
||||
signature = f"pub static {rust_name}: {value_type}"
|
||||
signature = f"pub fn {rust_name}() -> {value_type}"
|
||||
return signature, "none", asyncness, error_model, "static_field_accessor"
|
||||
else:
|
||||
signature = f"pub {rust_name}: {value_type}"
|
||||
return signature, "owned_value", asyncness, "none", "direct"
|
||||
@@ -653,16 +665,22 @@ def build_member_rows(catalog: dict, mapper: Mapper) -> list[dict[str, str]]:
|
||||
for owner in assembly["types"]:
|
||||
_, module, owner_path = declared_type_path(assembly_name, owner)
|
||||
groups = Counter(member_base_name(member) for member in owner["members"])
|
||||
enum_values = [member.get("value", {}).get("value") for member in owner["members"] if member["kind"] == "enum_value"]
|
||||
enum_constants = owner["kind"] == "enum" and (
|
||||
any(attribute["type"] == "System.FlagsAttribute" for attribute in owner.get("attributes", []))
|
||||
or len(enum_values) != len(set(enum_values))
|
||||
)
|
||||
used = set()
|
||||
for item in owner["members"]:
|
||||
base_name = member_base_name(item)
|
||||
if item["kind"] == "enum_value" and enum_constants:
|
||||
base_name = snake(item["name"]).upper()
|
||||
rust_name = base_name
|
||||
decision = "direct_snake_case"
|
||||
if groups[base_name] > 1:
|
||||
suffixes = []
|
||||
for parameter in item.get("parameters", []):
|
||||
base, _, _ = type_parts(parameter["type"])
|
||||
suffixes.append(snake(base.rsplit(".", 1)[-1]))
|
||||
suffixes.append(parameter_slug(parameter["type"]))
|
||||
suffix = "_".join(suffixes) or item["kind"]
|
||||
rust_name = f"{base_name}_with_{suffix}"
|
||||
decision = "descriptive_overload_name"
|
||||
@@ -808,7 +826,8 @@ def coverage_report(catalog: dict, type_rows: list[dict[str, str]], member_rows:
|
||||
def generate() -> dict[Path, str]:
|
||||
catalog = json.loads(CATALOG.read_text())
|
||||
type_rows, resolved = build_type_rows(catalog)
|
||||
mapper = Mapper(resolved)
|
||||
interfaces = {type_parts(item["signature"])[0] for assembly in catalog["assemblies"] for item in assembly["types"] if item["kind"] == "interface"}
|
||||
mapper = Mapper(resolved, interfaces)
|
||||
member_rows = build_member_rows(catalog, mapper)
|
||||
validate(catalog, type_rows, member_rows, mapper)
|
||||
type_fields = ["csharp_type_id", "csharp_signature", "source_assembly", "source_kind", "rust_crate", "rust_module", "rust_path", "mapping_decision", "generic_bounds", "inheritance_mapping", "enum_mapping", "status", "notes"]
|
||||
|
||||
Reference in New Issue
Block a user