Establish native deterministic codegen framework (#45)

tools/codegen/src/lib.rs

@@ -0,0 +1,365 @@
+//! Deterministic framework shared by the `LibreMetaverse` data generators.
+
+#![allow(clippy::missing_errors_doc)] // The CLI renders the complete error at its boundary.
+#![allow(clippy::must_use_candidate)] // Generator helpers are also exercised for validation.
+
+use serde::Deserialize;
+use sha2::{Digest, Sha256};
+use std::collections::BTreeSet;
+use std::fmt;
+use std::fmt::Write as _;
+use std::fs;
+use std::path::{Component, Path, PathBuf};
+
+pub const INVENTORY_PATH: &str = "codegen/sources.json";
+pub const MANIFEST_OUTPUT: &str = "codegen/generated/source_manifest.rs";
+
+#[derive(Clone, Debug, Deserialize, Eq, PartialEq)]
+pub struct Inventory {
+    pub schema: u32,
+    pub upstream_commit: String,
+    pub upstream_repository: String,
+    pub generators: Vec<GeneratorSpec>,
+    pub inputs: Vec<InputSpec>,
+}
+
+#[derive(Clone, Debug, Deserialize, Eq, PartialEq)]
+pub struct GeneratorSpec {
+    pub id: String,
+    pub reference_source: String,
+    pub sha256: String,
+    pub license: String,
+    pub inputs: Vec<String>,
+    #[serde(default)]
+    pub note: String,
+}
+
+#[derive(Clone, Debug, Deserialize, Eq, PartialEq)]
+pub struct InputSpec {
+    pub id: String,
+    pub reference_path: String,
+    pub vendored_path: String,
+    pub sha256: String,
+    pub license: String,
+    pub format: String,
+}
+
+#[derive(Clone, Copy, Debug, Eq, Ord, PartialEq, PartialOrd)]
+pub enum Severity {
+    Error,
+    Warning,
+}
+
+#[derive(Clone, Debug, Eq, Ord, PartialEq, PartialOrd)]
+pub struct Diagnostic {
+    pub code: &'static str,
+    pub severity: Severity,
+    pub path: String,
+    pub line: usize,
+    pub column: usize,
+    pub message: String,
+}
+
+impl fmt::Display for Diagnostic {
+    fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
+        let severity = match self.severity {
+            Severity::Error => "error",
+            Severity::Warning => "warning",
+        };
+        write!(
+            formatter,
+            "{}:{}:{}: {severity}[{}]: {}",
+            self.path, self.line, self.column, self.code, self.message
+        )
+    }
+}
+
+pub fn workspace_root() -> PathBuf {
+    Path::new(env!("CARGO_MANIFEST_DIR")).join("../..")
+}
+
+pub fn normalize_text(path: &str, bytes: &[u8]) -> Result<String, Diagnostic> {
+    let bytes = bytes.strip_prefix(&[0xef, 0xbb, 0xbf]).unwrap_or(bytes);
+    let text = std::str::from_utf8(bytes).map_err(|error| {
+        let valid = &bytes[..error.valid_up_to()];
+        let (line, column) = line_column(valid);
+        Diagnostic {
+            code: "CG001",
+            severity: Severity::Error,
+            path: path.replace('\\', "/"),
+            line,
+            column,
+            message: "input is not valid UTF-8".to_owned(),
+        }
+    })?;
+    if let Some(position) = text.as_bytes().iter().position(|byte| *byte == 0) {
+        let prefix = &text.as_bytes()[..position];
+        let (line, column) = line_column(prefix);
+        return Err(Diagnostic {
+            code: "CG002",
+            severity: Severity::Error,
+            path: path.replace('\\', "/"),
+            line,
+            column,
+            message: "input contains a NUL byte".to_owned(),
+        });
+    }
+    Ok(text.replace("\r\n", "\n").replace('\r', "\n"))
+}
+
+fn line_column(bytes: &[u8]) -> (usize, usize) {
+    let mut line = 1;
+    let mut column = 1;
+    for byte in bytes {
+        if *byte == b'\n' {
+            line += 1;
+            column = 1;
+        } else {
+            column += 1;
+        }
+    }
+    (line, column)
+}
+
+pub fn generated_rust(generator: &str, sources: &[&InputSpec], body: &str) -> Vec<u8> {
+    let mut output = String::new();
+    output.push_str("// @generated by libremetaverse-codegen; DO NOT EDIT.\n");
+    output.push_str("// Regenerate: cargo run -p libremetaverse-codegen -- generate\n");
+    let _ = writeln!(output, "// Generator: {generator}");
+    for source in sources {
+        let _ = writeln!(
+            output,
+            "// Source: {} sha256={} license={}",
+            source.vendored_path, source.sha256, source.license
+        );
+    }
+    output.push('\n');
+    output.push_str(body.trim_end());
+    output.push('\n');
+    output.into_bytes()
+}
+
+pub fn load_inventory(root: &Path) -> Result<Inventory, String> {
+    let path = root.join(INVENTORY_PATH);
+    let bytes = fs::read(&path).map_err(|error| format!("{}: {error}", path.display()))?;
+    let inventory: Inventory =
+        serde_json::from_slice(&bytes).map_err(|error| format!("{}: {error}", path.display()))?;
+    validate_inventory(&inventory)?;
+    Ok(inventory)
+}
+
+fn validate_inventory(inventory: &Inventory) -> Result<(), String> {
+    if inventory.schema != 1 || inventory.upstream_commit.len() != 40 {
+        return Err("unsupported or malformed source inventory header".to_owned());
+    }
+    let mut input_ids = BTreeSet::new();
+    for input in &inventory.inputs {
+        validate_relative(&input.reference_path)?;
+        validate_relative(&input.vendored_path)?;
+        validate_hash(&input.sha256)?;
+        if input.license != "BSD-3-Clause" || !input_ids.insert(&input.id) {
+            return Err(format!("invalid or duplicate input {}", input.id));
+        }
+    }
+    let mut generator_ids = BTreeSet::new();
+    for generator in &inventory.generators {
+        validate_relative(&generator.reference_source)?;
+        validate_hash(&generator.sha256)?;
+        if generator.license != "BSD-3-Clause" || !generator_ids.insert(&generator.id) {
+            return Err(format!("invalid or duplicate generator {}", generator.id));
+        }
+        for input in &generator.inputs {
+            if !input_ids.contains(input) {
+                return Err(format!(
+                    "generator {} references unknown input {input}",
+                    generator.id
+                ));
+            }
+        }
+    }
+    Ok(())
+}
+
+fn validate_relative(value: &str) -> Result<(), String> {
+    let path = Path::new(value);
+    if path.is_absolute()
+        || path
+            .components()
+            .any(|part| !matches!(part, Component::Normal(_)))
+    {
+        return Err(format!("unsafe inventory path {value}"));
+    }
+    Ok(())
+}
+
+fn validate_hash(value: &str) -> Result<(), String> {
+    if value.len() == 64 && value.bytes().all(|byte| byte.is_ascii_hexdigit()) {
+        Ok(())
+    } else {
+        Err(format!("invalid SHA-256 {value}"))
+    }
+}
+
+fn sha256(bytes: &[u8]) -> String {
+    const HEX: &[u8; 16] = b"0123456789abcdef";
+    let mut output = String::with_capacity(64);
+    for byte in Sha256::digest(bytes) {
+        output.push(char::from(HEX[usize::from(byte >> 4)]));
+        output.push(char::from(HEX[usize::from(byte & 0x0f)]));
+    }
+    output
+}
+
+pub fn verify_inputs(root: &Path, inventory: &Inventory) -> Result<(), String> {
+    for input in &inventory.inputs {
+        let path = root.join(&input.vendored_path);
+        let bytes = fs::read(&path).map_err(|error| format!("{}: {error}", path.display()))?;
+        let actual = sha256(&bytes);
+        if actual != input.sha256 {
+            return Err(format!(
+                "{}: SHA-256 mismatch: expected {}, found {actual}",
+                path.display(),
+                input.sha256
+            ));
+        }
+        normalize_text(&input.vendored_path, &bytes).map_err(|error| error.to_string())?;
+    }
+    Ok(())
+}
+
+pub fn vendor_inputs(root: &Path, reference: &Path) -> Result<(), String> {
+    let inventory = load_inventory(root)?;
+    for generator in &inventory.generators {
+        let source = reference.join(&generator.reference_source);
+        let bytes = fs::read(&source).map_err(|error| format!("{}: {error}", source.display()))?;
+        let actual = sha256(&bytes);
+        if actual != generator.sha256 {
+            return Err(format!(
+                "{}: pinned generator SHA-256 mismatch: expected {}, found {actual}",
+                source.display(),
+                generator.sha256
+            ));
+        }
+    }
+    let mut snapshots = Vec::with_capacity(inventory.inputs.len());
+    for input in &inventory.inputs {
+        let source = reference.join(&input.reference_path);
+        let bytes = fs::read(&source).map_err(|error| format!("{}: {error}", source.display()))?;
+        let actual = sha256(&bytes);
+        if actual != input.sha256 {
+            return Err(format!(
+                "{}: pinned SHA-256 mismatch: expected {}, found {actual}",
+                source.display(),
+                input.sha256
+            ));
+        }
+        snapshots.push((root.join(&input.vendored_path), bytes));
+    }
+    for (target, bytes) in snapshots {
+        fs::create_dir_all(target.parent().ok_or("vendored input has no parent")?)
+            .map_err(|error| error.to_string())?;
+        fs::write(&target, bytes).map_err(|error| format!("{}: {error}", target.display()))?;
+    }
+    Ok(())
+}
+
+pub fn source_manifest_bytes(root: &Path) -> Result<Vec<u8>, String> {
+    let inventory = load_inventory(root)?;
+    verify_inputs(root, &inventory)?;
+    let mut inputs: Vec<&InputSpec> = inventory.inputs.iter().collect();
+    inputs.sort_by_key(|input| &input.id);
+    let mut generators: Vec<&GeneratorSpec> = inventory.generators.iter().collect();
+    generators.sort_by_key(|generator| &generator.id);
+    let mut body = String::new();
+    let _ = writeln!(
+        body,
+        "pub const UPSTREAM_COMMIT: &str = {:?};",
+        inventory.upstream_commit
+    );
+    let _ = writeln!(
+        body,
+        "pub const UPSTREAM_REPOSITORY: &str = {:?};",
+        inventory.upstream_repository
+    );
+    body.push_str("pub const SOURCES: &[(&str, &str, &str, &str)] = &[\n");
+    for input in &inputs {
+        let _ = writeln!(
+            body,
+            "    ({:?}, {:?}, {:?}, {:?}),",
+            input.id, input.vendored_path, input.sha256, input.license
+        );
+    }
+    body.push_str("];\n");
+    body.push_str("pub const GENERATORS: &[(&str, &str, &str, &str)] = &[\n");
+    for generator in generators {
+        let _ = writeln!(
+            body,
+            "    ({:?}, {:?}, {:?}, {:?}),",
+            generator.id, generator.reference_source, generator.sha256, generator.license
+        );
+    }
+    body.push_str("];\n");
+    Ok(generated_rust("source-manifest", &inputs, &body))
+}
+
+pub fn regenerate(root: &Path, check: bool) -> Result<(), String> {
+    let expected = source_manifest_bytes(root)?;
+    let path = root.join(MANIFEST_OUTPUT);
+    if check {
+        let actual = fs::read(&path).map_err(|error| format!("{}: {error}", path.display()))?;
+        if actual != expected {
+            return Err(format!(
+                "{} is stale; run the regeneration command",
+                path.display()
+            ));
+        }
+    } else {
+        fs::create_dir_all(path.parent().ok_or("generated output has no parent")?)
+            .map_err(|error| error.to_string())?;
+        fs::write(&path, expected).map_err(|error| format!("{}: {error}", path.display()))?;
+    }
+    Ok(())
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn checked_in_inputs_and_manifest_are_current() {
+        let root = workspace_root();
+        let first = source_manifest_bytes(&root).expect("first deterministic generation");
+        let second = source_manifest_bytes(&root).expect("second deterministic generation");
+        assert_eq!(first, second);
+        regenerate(&root, true).expect("checked-in manifest");
+    }
+
+    #[test]
+    fn text_normalization_and_diagnostics_are_stable() {
+        assert_eq!(
+            normalize_text("a\\b.xml", b"\xef\xbb\xbfa\r\nb\r").unwrap(),
+            "a\nb\n"
+        );
+        let error = normalize_text("a\\b.xml", b"one\n\0two").unwrap_err();
+        assert_eq!(
+            error.to_string(),
+            "a/b.xml:2:1: error[CG002]: input contains a NUL byte"
+        );
+    }
+
+    #[test]
+    fn inventory_rejects_unsafe_paths_hashes_and_unknown_inputs() {
+        let root = workspace_root();
+        let mut inventory = load_inventory(&root).expect("pinned inventory");
+        inventory.inputs[0].vendored_path = "../escape".to_owned();
+        assert!(validate_inventory(&inventory).is_err());
+
+        let mut inventory = load_inventory(&root).expect("pinned inventory");
+        inventory.generators[0].sha256 = "not-a-hash".to_owned();
+        assert!(validate_inventory(&inventory).is_err());
+
+        let mut inventory = load_inventory(&root).expect("pinned inventory");
+        inventory.generators[0].inputs.push("missing".to_owned());
+        assert!(validate_inventory(&inventory).is_err());
+    }
+}