204 lines
7.4 KiB
Python
204 lines
7.4 KiB
Python
#!/usr/bin/env python3
|
|
"""Audit issue 83's deterministic native LSL parser/token generation."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
import re
|
|
from pathlib import Path
|
|
|
|
import generate_api_shims
|
|
import generate_lsl_tables
|
|
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
CATALOG = ROOT / "api" / "public-api.json"
|
|
INPUT = ROOT / "codegen" / "inputs" / "lsl_tools_grammar.json"
|
|
TABLES = ROOT / "crates" / "libremetaverse-lsl-tools" / "src" / "generated_tables.rs"
|
|
GENERATOR = ROOT / "crates" / "libremetaverse-lsl-tools" / "src" / "generator.rs"
|
|
GENERATED = ROOT / "crates" / "libremetaverse-lsl-tools" / "src" / "generated.rs"
|
|
TESTS = ROOT / "crates" / "libremetaverse-lsl-tools" / "tests" / "generator_compat.rs"
|
|
DOC = ROOT / "crates" / "libremetaverse-lsl-tools" / "README.md"
|
|
WORKFLOW = ROOT / ".gitea" / "workflows" / "ci.yml"
|
|
SCRIPT = ROOT / "tools" / "generate_lsl_tables.py"
|
|
STUB_RE = re.compile(
|
|
r"\b(?:not_implemented|unimplemented_api)\b|\b(?:todo|unimplemented)!\s*\("
|
|
)
|
|
|
|
|
|
def owned_types() -> dict[str, str]:
|
|
return {
|
|
doc_id: declaration
|
|
for doc_id, declaration in generate_api_shims.NATIVE_TYPES.items()
|
|
if declaration.startswith("crate::generated_tables::")
|
|
or declaration.startswith("crate::generator::")
|
|
}
|
|
|
|
|
|
def catalog_counts(doc_ids: set[str]) -> tuple[int, int]:
|
|
catalog = json.loads(CATALOG.read_text())
|
|
assembly = next(
|
|
value
|
|
for value in catalog["assemblies"]
|
|
if value["identity"]["name"] == "LibreMetaverse.LslTools"
|
|
)
|
|
found: dict[str, int] = {}
|
|
|
|
def visit(value: object) -> None:
|
|
if isinstance(value, dict):
|
|
doc_id = value.get("doc_id")
|
|
if doc_id in doc_ids and str(doc_id).startswith("T:"):
|
|
found[str(doc_id)] = len(value.get("members", []))
|
|
for child in value.values():
|
|
visit(child)
|
|
elif isinstance(value, list):
|
|
for child in value:
|
|
visit(child)
|
|
|
|
visit(assembly["types"])
|
|
if set(found) != doc_ids:
|
|
missing = sorted(doc_ids - set(found))
|
|
raise SystemExit(f"issue 83 catalog types are missing: {missing}")
|
|
return len(found), sum(found.values())
|
|
|
|
|
|
def generated_type_block(text: str, doc_id: str) -> str:
|
|
marker = f"/// C# type: `{doc_id}`."
|
|
start = text.find(marker)
|
|
if start < 0:
|
|
raise SystemExit(f"generated declaration is missing for {doc_id}")
|
|
next_type = text.find("/// C# type:", start + len(marker))
|
|
return text[start:] if next_type < 0 else text[start:next_type]
|
|
|
|
|
|
def require_markers(path: Path, markers: tuple[str, ...]) -> None:
|
|
text = path.read_text()
|
|
missing = [marker for marker in markers if marker not in text]
|
|
if missing:
|
|
raise SystemExit(f"{path.name}: audit evidence missing: " + ", ".join(missing))
|
|
|
|
|
|
def main() -> None:
|
|
types = owned_types()
|
|
type_count, member_count = catalog_counts(set(types))
|
|
if (type_count, member_count) != (120, 365):
|
|
raise SystemExit(
|
|
f"issue 83 expected 120 mapped types and 365 members, found "
|
|
f"{type_count} and {member_count}"
|
|
)
|
|
|
|
data = json.loads(INPUT.read_text())
|
|
if data["source"].get("license") != "BSD-3-Clause":
|
|
raise SystemExit("issue 83 reviewed grammar input has no BSD-3-Clause provenance")
|
|
if len(data["tokens"]) != 14 or len(data["nonterminals"]) != 8:
|
|
raise SystemExit("issue 83 grammar must contain 14 tokens and 8 nonterminals")
|
|
if len(data["productions"]) != 25:
|
|
raise SystemExit("issue 83 grammar must contain all 25 reviewed productions")
|
|
first = generate_lsl_tables.render(data).encode()
|
|
second = generate_lsl_tables.render(json.loads(INPUT.read_text())).encode()
|
|
if first != second or first != TABLES.read_bytes():
|
|
raise SystemExit("two clean LSL generations were not byte-identical")
|
|
|
|
for path in (TABLES, GENERATOR):
|
|
source = path.read_text()
|
|
if STUB_RE.search(source):
|
|
raise SystemExit(f"issue 83 owned Rust stubs remain in {path.name}")
|
|
if re.search(r"unsafe\s*\{|unsafe\s+impl|target_os\s*=\s*\"macos\"", source):
|
|
raise SystemExit(f"{path.name} contains unsafe or macOS-only code")
|
|
script = SCRIPT.read_text()
|
|
if re.search(r"\b(?:dotnet|csc|mcs)\b", script, re.IGNORECASE):
|
|
raise SystemExit("LSL regeneration depends on a C#/.NET process")
|
|
|
|
generated = GENERATED.read_text()
|
|
for doc_id in types:
|
|
if STUB_RE.search(generated_type_block(generated, doc_id)):
|
|
raise SystemExit(f"issue 83 generated stubs remain for {doc_id}")
|
|
|
|
require_markers(
|
|
TABLES,
|
|
(
|
|
"LSL_GENERATOR_SOURCE_COMMIT",
|
|
"LSL_GENERATOR_SOURCE_PARSER",
|
|
"LSL_GENERATOR_SOURCE_LEXER",
|
|
"LSL_GENERATOR_SOURCE_LICENSE",
|
|
"LSL_GENERATOR_SERIALIZATION_VERSION",
|
|
"GENERATED_PARSER_DATA",
|
|
"pub fn generated_parser()",
|
|
"pub fn generated_lexer_with_handler",
|
|
"pub struct GeneratedLexerToken",
|
|
"pub struct GeneratedParserSymbol",
|
|
"pub struct yycs0syntax",
|
|
"pub struct yycs0tokens",
|
|
"pub struct cs0syntax",
|
|
"pub struct cs0tokens",
|
|
),
|
|
)
|
|
require_markers(
|
|
GENERATOR,
|
|
(
|
|
"pub struct Serialiser",
|
|
"pub struct GenBase",
|
|
"pub struct SymbolsGen",
|
|
"pub struct TokensGen",
|
|
"pub struct Regex",
|
|
"pub struct Nfa",
|
|
"pub struct Sfactory",
|
|
"pub struct Tfactory",
|
|
"pub struct TokClassDef",
|
|
"pub struct ObjectListOListEnumerator",
|
|
"pub struct AddToFunc",
|
|
"MAX_GENERATED_DFA_STATES",
|
|
"MAX_REGEX_UNITS",
|
|
"System.Char",
|
|
),
|
|
)
|
|
require_markers(
|
|
TESTS,
|
|
(
|
|
"generated_lexer_preserves_tokens_reserved_words_and_eof",
|
|
"generated_parser_accepts_the_reviewed_class_body_grammar",
|
|
"parser_and_lexer_emission_is_byte_deterministic",
|
|
"generated_token_classes_have_live_identity_and_text",
|
|
"regex_and_nfa_build_a_functional_native_dfa",
|
|
"serialiser_round_trips_deterministically_with_versioning",
|
|
"compatibility_enumerator_and_generator_state_are_live",
|
|
"symbol_and_token_factories_execute_registered_rust_closures",
|
|
"symbols_paths_and_token_class_definitions_keep_native_state",
|
|
),
|
|
)
|
|
if TESTS.read_text().count("#[test]") != 9:
|
|
raise SystemExit("issue 83 expected 9 focused generator fixtures")
|
|
require_markers(
|
|
DOC,
|
|
(
|
|
"Checked-in generator contract",
|
|
"25-production grammar",
|
|
"84 generated syntax",
|
|
"13 generated token classes",
|
|
"python3 tools/generate_lsl_tables.py --check",
|
|
"46 focused",
|
|
"ubuntu-latest",
|
|
),
|
|
)
|
|
workflow = WORKFLOW.read_text()
|
|
require_markers(
|
|
WORKFLOW,
|
|
(
|
|
"required-gate",
|
|
),
|
|
)
|
|
if "runs-on: ubuntu-latest" not in workflow or re.search(
|
|
r"runs-on:\s*(?:macos|windows)", workflow, re.IGNORECASE
|
|
):
|
|
raise SystemExit("issue 83 workflow is not ubuntu-only")
|
|
|
|
print(
|
|
"issue 83 audit: 120 native mapped generator/table types and 365 members, "
|
|
"14 tokens, 8 nonterminals, 25 productions, two byte-identical generations, "
|
|
"9 focused fixtures, migration docs, and ubuntu-only CI are present"
|
|
)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|