Complete Rust API and migration documentation (#102)

tools/ci-matrix/src/documentation.rs

@@ -0,0 +1,606 @@
+use super::{MatrixError, Result};
+use serde::Serialize;
+use std::collections::{BTreeMap, BTreeSet};
+use std::fs::{self, OpenOptions};
+use std::io::Write as _;
+use std::path::{Path, PathBuf};
+
+const MAPPING_PATH: &str = "api/RUST-MAPPING.tsv";
+const TYPES_PATH: &str = "api/RUST-TYPES.tsv";
+const GUIDE_PATH: &str = "docs/rust-api-guide.md";
+const REPORT_PATH: &str = "api/DOCUMENTATION-COVERAGE.md";
+const UPSTREAM_COMMIT: &str = "2aa70bb68513b39795da5d13c88f31b86e85a3ba";
+
+const PUBLIC_CRATES: [(&str, &str); 15] = [
+    ("libremetaverse", "crates/libremetaverse"),
+    ("libremetaverse-types", "crates/libremetaverse-types"),
+    (
+        "libremetaverse-structured-data",
+        "crates/libremetaverse-structured-data",
+    ),
+    ("libremetaverse-imaging", "crates/libremetaverse-imaging"),
+    (
+        "libremetaverse-imaging-skia",
+        "crates/libremetaverse-imaging-skia",
+    ),
+    (
+        "libremetaverse-prim-mesher",
+        "crates/libremetaverse-prim-mesher",
+    ),
+    (
+        "libremetaverse-rendering-simple",
+        "crates/libremetaverse-rendering-simple",
+    ),
+    (
+        "libremetaverse-rendering-mesh-foundry",
+        "crates/libremetaverse-rendering-mesh-foundry",
+    ),
+    (
+        "libremetaverse-lsl-tools",
+        "crates/libremetaverse-lsl-tools",
+    ),
+    ("libremetaverse-rlv", "crates/libremetaverse-rlv"),
+    (
+        "libremetaverse-utilities",
+        "crates/libremetaverse-utilities",
+    ),
+    (
+        "libremetaverse-voice-vivox",
+        "crates/libremetaverse-voice-vivox",
+    ),
+    (
+        "libremetaverse-voice-webrtc",
+        "crates/libremetaverse-voice-webrtc",
+    ),
+    ("libremetaverse-openjpeg", "crates/libremetaverse-openjpeg"),
+    ("libremetaverse-opus", "crates/libremetaverse-opus"),
+];
+
+const PROGRAMS: [&str; 9] = [
+    "osd-inspector",
+    "simple-bot",
+    "packet-dump",
+    "prim-inspector",
+    "inventory-explorer",
+    "irc-gateway",
+    "test-client",
+    "vivox-test",
+    "webrtc-test",
+];
+
+const REQUIRED_GUIDE_SECTIONS: [&str; 11] = [
+    "## Choosing crates and features",
+    "## Naming and overload migration",
+    "## Ownership and disposal",
+    "## Async work and cancellation",
+    "## Errors",
+    "## Events and subscriptions",
+    "## Threading and callbacks",
+    "## Security boundaries",
+    "## Native prerequisites",
+    "## Live `OpenSim` setup",
+    "## Programs and operational tools",
+];
+
+const REQUIRED_MAPPING_FIELDS: [&str; 11] = [
+    "csharp_id",
+    "csharp_signature",
+    "rust_crate",
+    "rust_item_path",
+    "rust_signature",
+    "ownership",
+    "asyncness",
+    "error_model",
+    "overload_decision",
+    "mapping_kind",
+    "status",
+];
+
+#[derive(Debug, Serialize)]
+struct DocumentationEvidence {
+    schema: u32,
+    upstream_commit: &'static str,
+    public_crates: usize,
+    mapped_public_types: usize,
+    documented_public_types: usize,
+    mapped_members: usize,
+    documented_members: usize,
+    tested_rust_snippets: usize,
+    linked_programs: usize,
+    checked_local_links: usize,
+    required_guide_sections: usize,
+    status: &'static str,
+}
+
+struct Snapshot {
+    mapped_public_types: usize,
+    documented_public_types: usize,
+    mapped_members: usize,
+    documented_members: usize,
+    rust_snippets: usize,
+    local_links: usize,
+}
+
+/// Regenerates the deterministic mapped API documentation coverage report.
+///
+/// # Errors
+///
+/// Returns an error if documentation inputs are incomplete or cannot be read.
+pub fn write_documentation_report(root: &Path) -> Result<()> {
+    let snapshot = inspect(root)?;
+    fs::write(root.join(REPORT_PATH), render_report(&snapshot))?;
+    Ok(())
+}
+
+/// Validates public crate, mapped item, guide, link, snippet, and program docs.
+///
+/// # Errors
+///
+/// Returns an error when any mapped public item lacks its exact C# concept ID,
+/// a public crate lacks publishable root documentation, a local link is broken,
+/// a required migration topic/program is absent, or the checked report is stale.
+pub fn audit_documentation(root: &Path, evidence: &Path) -> Result<()> {
+    if evidence.exists() {
+        return Err(MatrixError::new(format!(
+            "{} already exists; preserve or remove it before rerunning the audit",
+            evidence.display()
+        )));
+    }
+    let snapshot = inspect(root)?;
+    let expected_report = render_report(&snapshot);
+    let report = fs::read_to_string(root.join(REPORT_PATH))?;
+    if report != expected_report {
+        return Err(MatrixError::new(
+            "documentation coverage report is stale; run documentation-report",
+        ));
+    }
+    let record = DocumentationEvidence {
+        schema: 1,
+        upstream_commit: UPSTREAM_COMMIT,
+        public_crates: PUBLIC_CRATES.len(),
+        mapped_public_types: snapshot.mapped_public_types,
+        documented_public_types: snapshot.documented_public_types,
+        mapped_members: snapshot.mapped_members,
+        documented_members: snapshot.documented_members,
+        tested_rust_snippets: snapshot.rust_snippets,
+        linked_programs: PROGRAMS.len(),
+        checked_local_links: snapshot.local_links,
+        required_guide_sections: REQUIRED_GUIDE_SECTIONS.len(),
+        status: "ok",
+    };
+    if let Some(parent) = evidence.parent() {
+        fs::create_dir_all(parent)?;
+    }
+    let mut file = OpenOptions::new()
+        .write(true)
+        .create_new(true)
+        .open(evidence)?;
+    serde_json::to_writer_pretty(&mut file, &record)?;
+    file.write_all(b"\n")?;
+    file.sync_all()?;
+    Ok(())
+}
+
+fn inspect(root: &Path) -> Result<Snapshot> {
+    validate_crates(root)?;
+    let guide = fs::read_to_string(root.join(GUIDE_PATH))?;
+    validate_guide(&guide)?;
+    let generated_docs = generated_doc_lines(root)?;
+    let mapped_members = validate_member_mapping(root, &generated_docs)?;
+    let mapped_public_types = validate_type_mapping(root, &generated_docs)?;
+    let local_links = validate_markdown_links(root)?;
+    let rust_snippets = guide.matches("```rust").count();
+    if rust_snippets < 4 {
+        return Err(MatrixError::new(
+            "Rust API guide must contain at least four compiled snippets",
+        ));
+    }
+    Ok(Snapshot {
+        mapped_public_types,
+        documented_public_types: mapped_public_types,
+        mapped_members,
+        documented_members: mapped_members,
+        rust_snippets,
+        local_links,
+    })
+}
+
+fn validate_crates(root: &Path) -> Result<()> {
+    for (name, relative) in PUBLIC_CRATES {
+        let directory = root.join(relative);
+        let manifest = fs::read_to_string(directory.join("Cargo.toml"))?;
+        if !manifest.contains(&format!("name = \"{name}\""))
+            || !manifest.lines().any(|line| {
+                line.trim_start()
+                    .strip_prefix("description = ")
+                    .is_some_and(|description| description.len() >= 24)
+            })
+        {
+            return Err(MatrixError::new(format!(
+                "public crate {name} lacks package documentation metadata"
+            )));
+        }
+        let library = fs::read_to_string(directory.join("src/lib.rs"))?;
+        let crate_doc_bytes: usize = library
+            .lines()
+            .take_while(|line| line.starts_with("//!") || line.trim().is_empty())
+            .filter(|line| line.starts_with("//!"))
+            .map(str::len)
+            .sum();
+        if crate_doc_bytes < 120 {
+            return Err(MatrixError::new(format!(
+                "public crate {name} needs a useful crate-level overview"
+            )));
+        }
+    }
+    let core = fs::read_to_string(root.join("crates/libremetaverse/src/lib.rs"))?;
+    if !core.contains("include_str!(\"../../../docs/rust-api-guide.md\")") {
+        return Err(MatrixError::new(
+            "the tested Rust API guide is not included in libremetaverse rustdoc",
+        ));
+    }
+    Ok(())
+}
+
+fn validate_guide(guide: &str) -> Result<()> {
+    if !guide.contains("https://github.com/cinderblocks/libremetaverse")
+        || !guide.contains(UPSTREAM_COMMIT)
+    {
+        return Err(MatrixError::new(
+            "Rust API guide lacks the pinned C# source link",
+        ));
+    }
+    for section in REQUIRED_GUIDE_SECTIONS {
+        if !guide.contains(section) {
+            return Err(MatrixError::new(format!(
+                "Rust API guide lacks required section {section}"
+            )));
+        }
+    }
+    for program in PROGRAMS {
+        if !guide.contains(&format!("`{program}`")) {
+            return Err(MatrixError::new(format!(
+                "Rust API guide does not link program {program}"
+            )));
+        }
+    }
+    Ok(())
+}
+
+fn generated_doc_lines(root: &Path) -> Result<BTreeSet<String>> {
+    let mut lines = BTreeSet::new();
+    for (_, relative) in PUBLIC_CRATES {
+        let generated = root.join(relative).join("src/generated.rs");
+        if generated.exists() {
+            lines.extend(
+                fs::read_to_string(generated)?
+                    .lines()
+                    .map(str::trim)
+                    .filter(|line| line.starts_with("///"))
+                    .map(str::to_owned),
+            );
+        }
+    }
+    Ok(lines)
+}
+
+fn validate_member_mapping(root: &Path, docs: &BTreeSet<String>) -> Result<usize> {
+    let (header, rows) = read_tsv(&root.join(MAPPING_PATH))?;
+    let positions = positions(&header, &REQUIRED_MAPPING_FIELDS)?;
+    let id_position = positions["csharp_id"];
+    let mut ids = BTreeSet::new();
+    for row in &rows {
+        for field in REQUIRED_MAPPING_FIELDS {
+            if row[positions[field]].trim().is_empty() {
+                return Err(MatrixError::new(format!(
+                    "mapping row has empty documentation context field {field}"
+                )));
+            }
+        }
+        let id = &row[id_position];
+        if !ids.insert(id.clone()) {
+            return Err(MatrixError::new(format!("duplicate mapped member {id}")));
+        }
+        if !docs.contains(&format!("/// C# member: `{id}`.")) {
+            return Err(MatrixError::new(format!(
+                "mapped member {id} lacks its C# concept documentation marker"
+            )));
+        }
+        let signature = &row[positions["csharp_signature"]];
+        if !docs.contains(&format!("/// C# signature: `{signature}`.")) {
+            return Err(MatrixError::new(format!(
+                "mapped member {id} lacks its C# signature documentation"
+            )));
+        }
+        let ownership = &row[positions["ownership"]];
+        let asyncness = &row[positions["asyncness"]];
+        if !docs.contains(&format!(
+            "/// Mapping contract: ownership `{ownership}`; async `{asyncness}`;"
+        )) {
+            return Err(MatrixError::new(format!(
+                "mapped member {id} lacks ownership/async documentation"
+            )));
+        }
+        let error_model = &row[positions["error_model"]];
+        let overload = &row[positions["overload_decision"]];
+        let mapping_kind = &row[positions["mapping_kind"]];
+        if !docs.contains(&format!(
+            "/// errors `{error_model}`; overload `{overload}`; kind `{mapping_kind}`."
+        )) {
+            return Err(MatrixError::new(format!(
+                "mapped member {id} lacks error/overload/kind documentation"
+            )));
+        }
+    }
+    Ok(rows.len())
+}
+
+fn validate_type_mapping(root: &Path, docs: &BTreeSet<String>) -> Result<usize> {
+    let (header, rows) = read_tsv(&root.join(TYPES_PATH))?;
+    let required = [
+        "csharp_type_id",
+        "csharp_signature",
+        "source_kind",
+        "rust_crate",
+        "rust_path",
+        "mapping_decision",
+        "status",
+    ];
+    let positions = positions(&header, &required)?;
+    let mut count = 0;
+    let mut ids = BTreeSet::new();
+    for row in &rows {
+        for field in required {
+            if row[positions[field]].trim().is_empty() {
+                return Err(MatrixError::new(format!(
+                    "type row has empty documentation context field {field}"
+                )));
+            }
+        }
+        let id = &row[positions["csharp_type_id"]];
+        if !ids.insert(id.clone()) {
+            return Err(MatrixError::new(format!("duplicate mapped type {id}")));
+        }
+        if id.starts_with("T:LibreMetaverse")
+            && row[positions["source_kind"]] != "referenced_support_trait"
+        {
+            count += 1;
+            if !docs.contains(&format!("/// C# type: `{id}`.")) {
+                return Err(MatrixError::new(format!(
+                    "mapped public type {id} lacks its C# concept documentation marker"
+                )));
+            }
+            let signature = &row[positions["csharp_signature"]];
+            if !docs.contains(&format!(
+                "/// Native Rust mapping of C# `{signature}` using decision"
+            )) {
+                return Err(MatrixError::new(format!(
+                    "mapped public type {id} lacks useful C# signature documentation"
+                )));
+            }
+            let decision = &row[positions["mapping_decision"]];
+            if !docs.contains(&format!(
+                "/// `{decision}`. See the [pinned C# source](https://github.com/cinderblocks/libremetaverse/tree/{UPSTREAM_COMMIT})."
+            )) {
+                return Err(MatrixError::new(format!(
+                    "mapped public type {id} lacks its mapping decision/source link"
+                )));
+            }
+        }
+    }
+    Ok(count)
+}
+
+fn read_tsv(path: &Path) -> Result<(Vec<String>, Vec<Vec<String>>)> {
+    let text = fs::read_to_string(path)?;
+    let mut lines = text.lines();
+    let header = lines
+        .next()
+        .ok_or_else(|| MatrixError::new(format!("{} is empty", path.display())))?
+        .split('\t')
+        .map(str::to_owned)
+        .collect::<Vec<_>>();
+    let rows = lines
+        .filter(|line| !line.is_empty())
+        .map(|line| line.split('\t').map(str::to_owned).collect::<Vec<_>>())
+        .collect::<Vec<_>>();
+    if rows.iter().any(|row| row.len() != header.len()) {
+        return Err(MatrixError::new(format!(
+            "{} contains a malformed row",
+            path.display()
+        )));
+    }
+    Ok((header, rows))
+}
+
+fn positions<const N: usize>(
+    header: &[String],
+    required: &[&'static str; N],
+) -> Result<BTreeMap<&'static str, usize>> {
+    let mut result = BTreeMap::new();
+    for &field in required {
+        let position = header
+            .iter()
+            .position(|candidate| candidate == field)
+            .ok_or_else(|| MatrixError::new(format!("TSV lacks required column {field}")))?;
+        result.insert(field, position);
+    }
+    Ok(result)
+}
+
+fn markdown_files(root: &Path) -> Result<Vec<PathBuf>> {
+    let mut files = vec![root.join("README.md"), root.join("programs/README.md")];
+    for directory in [root.join("docs"), root.join("api"), root.join("crates")] {
+        collect_markdown(&directory, &mut files)?;
+    }
+    files.sort();
+    files.dedup();
+    files.retain(|path| path != &root.join(REPORT_PATH));
+    Ok(files)
+}
+
+fn collect_markdown(directory: &Path, files: &mut Vec<PathBuf>) -> Result<()> {
+    for entry in fs::read_dir(directory)? {
+        let path = entry?.path();
+        if path.is_dir() {
+            collect_markdown(&path, files)?;
+        } else if path.extension().and_then(|value| value.to_str()) == Some("md") {
+            files.push(path);
+        }
+    }
+    Ok(())
+}
+
+fn validate_markdown_links(root: &Path) -> Result<usize> {
+    let mut checked = 0;
+    for file in markdown_files(root)? {
+        let text = fs::read_to_string(&file)?;
+        let mut remainder = text.as_str();
+        while let Some(start) = remainder.find("](") {
+            remainder = &remainder[start + 2..];
+            let Some(end) = remainder.find(')') else {
+                return Err(MatrixError::new(format!(
+                    "{} contains an unterminated Markdown link",
+                    file.display()
+                )));
+            };
+            let raw = remainder[..end].trim().trim_matches(['<', '>']);
+            remainder = &remainder[end + 1..];
+            let target = raw.split_whitespace().next().unwrap_or_default();
+            if target.is_empty()
+                || target.starts_with("http://")
+                || target.starts_with("https://")
+                || target.starts_with("mailto:")
+            {
+                continue;
+            }
+            let (path, fragment) = target
+                .split_once('#')
+                .map_or((target, None), |(path, fragment)| (path, Some(fragment)));
+            let resolved = if path.is_empty() {
+                file.clone()
+            } else {
+                file.parent().unwrap_or(root).join(path)
+            };
+            if !resolved.exists() {
+                return Err(MatrixError::new(format!(
+                    "broken local Markdown link {target} in {}",
+                    file.strip_prefix(root).unwrap_or(&file).display()
+                )));
+            }
+            if let Some(fragment) = fragment.filter(|fragment| !fragment.is_empty())
+                && resolved.extension().and_then(|value| value.to_str()) == Some("md")
+            {
+                let destination = fs::read_to_string(&resolved)?;
+                if !markdown_anchors(&destination).contains(fragment) {
+                    return Err(MatrixError::new(format!(
+                        "broken local Markdown anchor {target} in {}",
+                        file.strip_prefix(root).unwrap_or(&file).display()
+                    )));
+                }
+            }
+            checked += 1;
+        }
+    }
+    Ok(checked)
+}
+
+fn markdown_anchors(markdown: &str) -> BTreeSet<String> {
+    let mut anchors = BTreeSet::new();
+    let mut occurrences = BTreeMap::<String, usize>::new();
+    let mut fenced = false;
+    for line in markdown.lines() {
+        if line.trim_start().starts_with("```") {
+            fenced = !fenced;
+            continue;
+        }
+        if fenced {
+            continue;
+        }
+        let heading = line.trim_start_matches('#').trim();
+        if heading.is_empty() || heading.len() == line.trim().len() {
+            continue;
+        }
+        let base = heading
+            .chars()
+            .filter_map(|character| {
+                if character.is_alphanumeric() || matches!(character, '-' | '_') {
+                    Some(character.to_ascii_lowercase())
+                } else if character.is_whitespace() {
+                    Some('-')
+                } else {
+                    None
+                }
+            })
+            .collect::<String>();
+        let occurrence = occurrences.entry(base.clone()).or_default();
+        let anchor = if *occurrence == 0 {
+            base.clone()
+        } else {
+            format!("{base}-{occurrence}")
+        };
+        *occurrence += 1;
+        anchors.insert(anchor);
+    }
+    anchors
+}
+
+fn render_report(snapshot: &Snapshot) -> String {
+    format!(
+        "# Documentation coverage\n\n\
+Generated by `metacrate-ci-matrix documentation-report`; do not edit by hand.\n\n\
+| Surface | Documented | Required | Coverage |\n\
+| --- | ---: | ---: | ---: |\n\
+| Publishable public crates | {crates} | {crates} | 100% |\n\
+| Pinned LibreMetaverse public types | {types_documented} | {types} | 100% |\n\
+| Mapped public members | {members_documented} | {members} | 100% |\n\
+| Compiled Rust guide snippets | {snippets} | 4 minimum | pass |\n\
+| Linked native programs | {programs} | {programs} | 100% |\n\
+| Checked local Markdown links | {links} | {links} | 100% |\n\n\
+Every mapped item is tied to its exact C# documentation ID and to the ownership, \
+asyncness, error, overload, mapping-kind, and Rust-signature decisions in \
+[`RUST-MAPPING.tsv`](RUST-MAPPING.tsv). Public types are tied to the corresponding \
+type mapping. The upstream source is pinned to \
+[`{commit}`](https://github.com/cinderblocks/libremetaverse/tree/{commit}).\n",
+        crates = PUBLIC_CRATES.len(),
+        types_documented = snapshot.documented_public_types,
+        types = snapshot.mapped_public_types,
+        members_documented = snapshot.documented_members,
+        members = snapshot.mapped_members,
+        snippets = snapshot.rust_snippets,
+        programs = PROGRAMS.len(),
+        links = snapshot.local_links,
+        commit = UPSTREAM_COMMIT,
+    )
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn markdown_anchor_generation_matches_linkable_headings() {
+        let anchors = markdown_anchors(
+            "# API guide\n\n## Live `OpenSim` setup\n## API guide\n```text\n# ignored\n```\n",
+        );
+        assert!(anchors.contains("api-guide"));
+        assert!(anchors.contains("live-opensim-setup"));
+        assert!(anchors.contains("api-guide-1"));
+        assert!(!anchors.contains("ignored"));
+    }
+
+    #[test]
+    fn report_contains_exact_documentation_totals() {
+        let root = Path::new(env!("CARGO_MANIFEST_DIR")).join("../..");
+        let snapshot = inspect(&root).unwrap();
+        assert_eq!(snapshot.mapped_members, 30_789);
+        assert_eq!(snapshot.documented_members, snapshot.mapped_members);
+        assert_eq!(snapshot.mapped_public_types, 3_066);
+        assert_eq!(
+            snapshot.documented_public_types,
+            snapshot.mapped_public_types
+        );
+        assert!(snapshot.rust_snippets >= 4);
+        assert!(snapshot.local_links > 0);
+    }
+}

tools/ci-matrix/src/lib.rs

@@ -12,8 +12,10 @@ use std::process::{Command, ExitStatus, Stdio};
 use std::time::{SystemTime, UNIX_EPOCH};
 
 mod dependency;
+mod documentation;
 
 pub use dependency::audit_dependencies;
+pub use documentation::{audit_documentation, write_documentation_report};
 
 pub const MATRIX_PATH: &str = "ci/release-matrix.json";
 const WORKFLOW_PATH: &str = ".gitea/workflows/release-matrix.yml";

tools/ci-matrix/src/main.rs

@@ -1,4 +1,7 @@
-use metacrate_ci_matrix::{audit, audit_dependencies, load, run, workspace_root};
+use metacrate_ci_matrix::{
+    audit, audit_dependencies, audit_documentation, load, run, workspace_root,
+    write_documentation_report,
+};
 use std::path::{Path, PathBuf};
 
 fn main() {
@@ -46,9 +49,27 @@ fn execute() -> Result<(), Box<dyn std::error::Error>> {
             audit_dependencies(&root, &evidence)?;
             println!("dependency policy: ok ({})", evidence.display());
         }
+        Some("documentation-report") if arguments.next().is_none() => {
+            write_documentation_report(&root)?;
+            println!("documentation coverage report: updated");
+        }
+        Some("documentation-audit") => {
+            let flag = arguments
+                .next()
+                .ok_or("documentation-audit requires --evidence FILE")?;
+            let evidence = arguments
+                .next()
+                .ok_or("documentation-audit requires --evidence FILE")?;
+            if flag != "--evidence" || arguments.next().is_some() {
+                return Err("usage: ci-matrix documentation-audit --evidence FILE".into());
+            }
+            let evidence = absolute_or_rooted(&root, &evidence);
+            audit_documentation(&root, &evidence)?;
+            println!("documentation coverage: ok ({})", evidence.display());
+        }
         _ => {
             return Err(
-                "usage: ci-matrix audit | run PROFILE --evidence FILE | dependency-audit --evidence FILE"
+                "usage: ci-matrix audit | run PROFILE --evidence FILE | dependency-audit --evidence FILE | documentation-report | documentation-audit --evidence FILE"
                     .into(),
             );
         }