Implement native inventory store and cache (#61)

.gitea/workflows/codegen.yml

@@ -15,8 +15,11 @@ on:
       - "tools/check_milestone_08.py"
       - "tools/check_milestone_09_issue_59.py"
       - "tools/check_milestone_09_issue_60.py"
+      - "tools/check_milestone_09_issue_61.py"
       - "crates/libremetaverse/src/agent_movement.rs"
+      - "crates/libremetaverse/src/inventory.rs"
       - "docs/agent-movement.md"
+      - "docs/inventory.md"
       - "crates/libremetaverse/Cargo.toml"
       - "crates/libremetaverse/src/generated.rs"
       - "crates/libremetaverse/src/packets.rs"
@@ -28,6 +31,8 @@ on:
       - "tests/compat/src/**"
       - "tests/compat/tests/wire_semantics.rs"
       - "tests/compat/tests/appearance_visual_semantics.rs"
+      - "tests/compat/tests/inventory_store_semantics.rs"
+      - "tests/compat/tests/misclassified_link_semantics.rs"
       - "tests/upstream-tests.json"
   pull_request:
     paths:
@@ -43,8 +48,11 @@ on:
       - "tools/check_milestone_08.py"
       - "tools/check_milestone_09_issue_59.py"
       - "tools/check_milestone_09_issue_60.py"
+      - "tools/check_milestone_09_issue_61.py"
       - "crates/libremetaverse/src/agent_movement.rs"
+      - "crates/libremetaverse/src/inventory.rs"
       - "docs/agent-movement.md"
+      - "docs/inventory.md"
       - "crates/libremetaverse/Cargo.toml"
       - "crates/libremetaverse/src/generated.rs"
       - "crates/libremetaverse/src/packets.rs"
@@ -56,6 +64,8 @@ on:
       - "tests/compat/src/**"
       - "tests/compat/tests/wire_semantics.rs"
       - "tests/compat/tests/appearance_visual_semantics.rs"
+      - "tests/compat/tests/inventory_store_semantics.rs"
+      - "tests/compat/tests/misclassified_link_semantics.rs"
       - "tests/upstream-tests.json"
   workflow_dispatch:
 
@@ -79,6 +89,7 @@ jobs:
           python3 tools/check_milestone_08.py
           python3 tools/check_milestone_09_issue_59.py
           python3 tools/check_milestone_09_issue_60.py
+          python3 tools/check_milestone_09_issue_61.py
           cargo check --manifest-path tests/api-compile/Cargo.toml --locked
       - name: Test generated wire codecs and translated parity cases
         run: |
@@ -86,6 +97,8 @@ jobs:
           cargo test -p libremetaverse --test packet_wire
           cargo test -p libremetaverse --test packet_osd
           cargo test -p libremetaverse-compat-tests --test wire_semantics
+          cargo test -p libremetaverse-compat-tests --test inventory_store_semantics
+          cargo test -p libremetaverse-compat-tests --test misclassified_link_semantics
       - name: Test generated visual and foliage catalogs
         run: |
           cargo test -p libremetaverse --test generated_catalogs

.gitea/workflows/rust-workspace.yml

@@ -25,9 +25,10 @@ jobs:
           components: rustfmt
       - name: Check formatting
         run: cargo fmt --all -- --check
-      - name: Audit native AgentManager boundaries
+      - name: Audit native milestone 09 boundaries
         run: |
           python3 tools/check_milestone_09_issue_59.py
           python3 tools/check_milestone_09_issue_60.py
+          python3 tools/check_milestone_09_issue_61.py
       - name: Compile every workspace target with bounded memory
         run: cargo check --workspace --all-targets --locked -j 1

Cargo.lock

@@ -844,6 +844,7 @@ version = "0.0.1"
 dependencies = [
  "bcdec_rs",
  "flate2",
+ "futures-channel",
  "futures-util",
  "libremetaverse-imaging",
  "libremetaverse-structured-data",

README.md

@@ -348,3 +348,12 @@ diagnosable crossing state machine with recovery to the old simulator. Paused
 time tests and a loopback fake grid verify exact wire fields and clean worker
 shutdown. The compatibility and lifecycle contract is documented in
 [`docs/agent-movement.md`](docs/agent-movement.md).
+
+Native inventory now preserves the complete item/folder model hierarchy,
+permissions, concrete subclasses, parent/descendant counts, link resolution,
+root/library semantics, system-folder lookup, and C# sort flags. Local mutation
+is transport-independent and observer callbacks run after locks are released.
+The versioned 64 MiB-bounded cache validates complete snapshots before install
+and uses same-directory temporary replacement with rollback. The compatibility
+and persistence contract is documented in
+[`docs/inventory.md`](docs/inventory.md).

api/SHIM-COVERAGE.md

@@ -4,7 +4,7 @@ Generated by `python3 tools/generate_api_shims.py`; do not edit by hand.
 
 | Assembly | Types | Members | Status |
 |---|---:|---:|---|
-| `LibreMetaverse` | 2,711 | 27,281 | native implementation: 128 types / 14,611 members; remaining surface is callable failure-only shims |
+| `LibreMetaverse` | 2,711 | 27,281 | native implementation: 153 types / 14,746 members; remaining surface is callable failure-only shims |
 | `LibreMetaverse.Imaging.Abstractions` | 3 | 20 | native implementation: 3 types / 20 members; no generated shims remain |
 | `LibreMetaverse.Imaging.Skia` | 1 | 3 | native implementation: 1 type / 3 members; no generated shims remain |
 | `LibreMetaverse.LslTools` | 164 | 768 | callable failure-only shim |

crates/libremetaverse/Cargo.toml

@@ -15,6 +15,7 @@ jpeg2000 = ["libremetaverse-imaging/jpeg2000"]
 [dependencies]
 bcdec_rs = { version = "0.2.0", optional = true }
 flate2 = "1.1.2"
+futures-channel = "0.3.31"
 futures-util = "0.3.31"
 libremetaverse-imaging = { path = "../libremetaverse-imaging" }
 libremetaverse-structured-data = { path = "../libremetaverse-structured-data" }

crates/libremetaverse/src/lib.rs

@@ -14,6 +14,7 @@ mod client_core;
 mod download_manager;
 mod event_queue;
 mod gesture;
+mod inventory;
 #[rustfmt::skip] // Deterministic machine output is formatted by the pinned generator.
 mod foliage_catalog;
 mod generated;

docs/inventory.md

@@ -0,0 +1,38 @@
+# Inventory models, hierarchy, and cache
+
+The native inventory layer mirrors the value and hierarchy behavior of
+LibreMetaverse's `InventoryBase`, `InventoryItem`, `InventoryFolder`,
+`InventoryNode`, and `Inventory` types without depending on a network
+transport. Concrete item subclasses remain concrete when inserted, queried,
+notified, or restored from disk; link detection is based on `AssetType`, not on
+the Rust wrapper type. This preserves the C# misclassified-link behavior.
+
+Store mutation is serialized by one inventory state lock. Parent/child and link
+indexes and recursive item counts are rebuilt as one transaction, with cycle
+and depth bounds. Missing parents become placeholder folders until their real
+folder record arrives. Self-loops and longer parent cycles remain queryable but
+are not linked into a cyclic ownership graph. Removing a folder removes its
+bounded descendant subtree. Root and library roots are tracked independently,
+and direct system folders can be found by `FolderType`. Sorting supports the
+`ByDate`, `FoldersByName`, and `SystemFoldersToTop` flags.
+
+Added, updated, and removed notifications clone their arguments and subscriber
+list before invoking observers. No inventory, node, or subscriber lock is held
+during a callback, so observers may safely query or mutate the store. A panic
+in one observer is isolated by the shared event registry.
+
+Cache files start with `INVCACHE`, a little-endian format version, owner and
+root identities, and tagged concrete records. Reads are limited to 64 MiB,
+one million records, one MiB per string, and a 512-level hierarchy. Invalid
+magic, versions, UTF-8, enums, duplicate IDs, zero IDs, truncated records, and
+trailing data are rejected before the live store is replaced. Synchronous
+restore returns `-1` for compatibility; asynchronous restore returns the typed
+error. Saves snapshot under the state lock, release it before I/O, write and
+sync a same-directory temporary file, then replace the destination. The
+replacement includes a rollback path on platforms that cannot rename over an
+existing file.
+
+The focused compatibility suites are
+`inventory_store_semantics` and `misclassified_link_semantics`; native unit
+tests additionally cover concrete cache round trips, permissions, corruption,
+unknown versions, system-folder sorting, and callback re-entry.

tests/api-compile/Cargo.lock

@@ -716,6 +716,7 @@ version = "0.0.1"
 dependencies = [
  "bcdec_rs",
  "flate2",
+ "futures-channel",
  "futures-util",
  "libremetaverse-imaging",
  "libremetaverse-structured-data",

tests/api-compile/src/lib.rs

@@ -39120,7 +39120,7 @@ fn member_06311(value: &mut libremetaverse::Inventory) {
 }
 
 // C# member: `M:LibreMetaverse.Inventory.GetValueOrDefault``1(LibreMetaverse.UUID)`.
-fn member_06312<T>(value: &mut libremetaverse::Inventory) {
+fn member_06312<T: Clone + 'static>(value: &mut libremetaverse::Inventory) {
     let _ = value.get_value_or_default_with_uuid_a7c63fbe::<T>(argument());
 }
 
@@ -39160,7 +39160,7 @@ fn member_06319(value: &mut libremetaverse::Inventory) {
 }
 
 // C# member: `M:LibreMetaverse.Inventory.TryGetValue``1(LibreMetaverse.UUID,``0@)`.
-fn member_06320<T>(value: &mut libremetaverse::Inventory) {
+fn member_06320<T: Clone + 'static>(value: &mut libremetaverse::Inventory) {
     let _ = value.try_get_value_with_uuid_t::<T>(argument(), argument());
 }
 

tests/compat/tests/wire_shims.rs

@@ -19,9 +19,11 @@ fn each_wire_data_domain_is_callable_and_failure_only() {
     let packet = UseCircuitCodePacket::new_with_constructor()
         .expect("generated packet constructor is native");
     assert_eq!(packet.circuit_code.code, 0);
-    assert_eq!(
-        member_id(AgentDropGroupMessage::new()),
-        "M:LibreMetaverse.Messages.Linden.AgentDropGroupMessage.#ctor"
+    assert!(
+        AgentDropGroupMessage::new()
+            .expect("native generated message constructor")
+            .agent_data_block
+            .is_empty()
     );
     assert_eq!(
         member_id(AssetTexture::new_with_constructor()),
@@ -47,13 +49,12 @@ fn each_wire_data_domain_is_callable_and_failure_only() {
         member_id(OarFile::package_archive(String::new(), String::new())),
         "M:LibreMetaverse.Assets.OarFile.PackageArchive(System.String,System.String)"
     );
+    let bit_pack = BitPack::new(Vec::new(), 0).expect("native bit pack constructor");
+    assert_eq!(bit_pack.byte_pos(), 0);
+    assert_eq!(bit_pack.bit_pos(), 0);
     assert_eq!(
-        member_id(BitPack::new(Vec::new(), 0)),
-        "M:LibreMetaverse.BitPack.#ctor(System.Byte[],System.Int32)"
-    );
-    assert_eq!(
-        member_id(Permissions::new(0, 0, 0, 0, 0)),
-        "M:LibreMetaverse.Permissions.#ctor(System.UInt32,System.UInt32,System.UInt32,System.UInt32,System.UInt32)"
+        Permissions::new(0, 0, 0, 0, 0).expect("native permissions constructor"),
+        Permissions::no_permissions()
     );
 }
 

tools/check_milestone_09_issue_61.py

@@ -0,0 +1,76 @@
+#!/usr/bin/env python3
+"""Audit issue 61's native inventory model, store, and cache boundary."""
+
+from __future__ import annotations
+
+from pathlib import Path
+import re
+
+import generate_api_shims
+
+
+ROOT = Path(__file__).resolve().parents[1]
+SOURCE = ROOT / "crates" / "libremetaverse" / "src" / "inventory.rs"
+DOC = ROOT / "docs" / "inventory.md"
+STORE_TESTS = ROOT / "tests" / "compat" / "tests" / "inventory_store_semantics.rs"
+LINK_TESTS = ROOT / "tests" / "compat" / "tests" / "misclassified_link_semantics.rs"
+STUB_RE = re.compile(r"\b(?:not_implemented|unimplemented_api)\b|\b(?:todo|unimplemented)!\s*\(")
+
+REQUIRED_TYPES = {
+    "T:LibreMetaverse.Inventory",
+    "T:LibreMetaverse.InventoryAnimation",
+    "T:LibreMetaverse.InventoryAttachment",
+    "T:LibreMetaverse.InventoryBase",
+    "T:LibreMetaverse.InventoryCallingCard",
+    "T:LibreMetaverse.InventoryCategory",
+    "T:LibreMetaverse.InventoryFolder",
+    "T:LibreMetaverse.InventoryGesture",
+    "T:LibreMetaverse.InventoryItem",
+    "T:LibreMetaverse.InventoryLSL",
+    "T:LibreMetaverse.InventoryLandmark",
+    "T:LibreMetaverse.InventoryMaterial",
+    "T:LibreMetaverse.InventoryNode",
+    "T:LibreMetaverse.InventoryNodeDictionary",
+    "T:LibreMetaverse.InventoryNotecard",
+    "T:LibreMetaverse.InventoryObject",
+    "T:LibreMetaverse.InventoryObjectAddedEventArgs",
+    "T:LibreMetaverse.InventoryObjectRemovedEventArgs",
+    "T:LibreMetaverse.InventoryObjectUpdatedEventArgs",
+    "T:LibreMetaverse.InventorySettings",
+    "T:LibreMetaverse.InventorySnapshot",
+    "T:LibreMetaverse.InventorySound",
+    "T:LibreMetaverse.InventoryTexture",
+    "T:LibreMetaverse.InventoryWearable",
+    "T:LibreMetaverse.Permissions",
+}
+
+
+def require_markers(path: Path, markers: tuple[str, ...]) -> None:
+    text = path.read_text()
+    missing = [marker for marker in markers if marker not in text]
+    if missing:
+        raise SystemExit(f"{path.name}: audit evidence missing: " + ", ".join(missing))
+
+
+def main() -> None:
+    missing = sorted(REQUIRED_TYPES - generate_api_shims.NATIVE_TYPES.keys())
+    if missing:
+        raise SystemExit("issue 61 native types missing: " + ", ".join(missing))
+    if STUB_RE.search(SOURCE.read_text()):
+        raise SystemExit("issue 61 owned Rust stubs remain in inventory.rs")
+    require_markers(SOURCE, (
+        "MAX_CACHE_BYTES", "MAX_CACHE_RECORDS", "MAX_HIERARCHY_DEPTH",
+        "rebuild_indexes_and_counts", "replace_file", "would_create_cycle",
+        "callbacks_run_after_store_lock_is_released",
+        "cache_rejects_corruption_and_unknown_versions",
+        "cache_round_trip_preserves_roots_concrete_type_and_permissions",
+        "system_folders_sort_first_and_are_discoverable",
+    ))
+    require_markers(DOC, ("misclassified-link", "64 MiB", "rollback", "No inventory"))
+    require_markers(STORE_TESTS, ("concurrent_add_move_remove_stress", "link_index_updates_on_asset_change"))
+    require_markers(LINK_TESTS, ("inventory_store_indexer_returns_wearable", "resolved_item_id_on_misclassified_texture_link"))
+    print("issue 61 audit: native models, concrete hierarchy storage, bounded cache, lock-free callbacks, and compatibility regressions are present")
+
+
+if __name__ == "__main__":
+    main()

tools/generate_api_shims.py

@@ -63,6 +63,31 @@ NATIVE_TYPES = {
     "T:LibreMetaverse.HttpCapsClient.ProgressReport": "crate::caps_http::HttpCapsClientProgressReport",
     "T:LibreMetaverse.InstantMessage": "crate::agent_manager::InstantMessage",
     "T:LibreMetaverse.InstantMessageEventArgs": "crate::agent_manager::InstantMessageEventArgs",
+    "T:LibreMetaverse.Inventory": "crate::inventory::Inventory",
+    "T:LibreMetaverse.InventoryAnimation": "crate::inventory::InventoryAnimation",
+    "T:LibreMetaverse.InventoryAttachment": "crate::inventory::InventoryAttachment",
+    "T:LibreMetaverse.InventoryBase": "crate::inventory::InventoryBase",
+    "T:LibreMetaverse.InventoryCallingCard": "crate::inventory::InventoryCallingCard",
+    "T:LibreMetaverse.InventoryCategory": "crate::inventory::InventoryCategory",
+    "T:LibreMetaverse.InventoryFolder": "crate::inventory::InventoryFolder",
+    "T:LibreMetaverse.InventoryGesture": "crate::inventory::InventoryGesture",
+    "T:LibreMetaverse.InventoryItem": "crate::inventory::InventoryItem",
+    "T:LibreMetaverse.InventoryLSL": "crate::inventory::InventoryLSL",
+    "T:LibreMetaverse.InventoryLandmark": "crate::inventory::InventoryLandmark",
+    "T:LibreMetaverse.InventoryMaterial": "crate::inventory::InventoryMaterial",
+    "T:LibreMetaverse.InventoryNode": "crate::inventory::InventoryNode",
+    "T:LibreMetaverse.InventoryNodeDictionary": "crate::inventory::InventoryNodeDictionary",
+    "T:LibreMetaverse.InventoryNotecard": "crate::inventory::InventoryNotecard",
+    "T:LibreMetaverse.InventoryObject": "crate::inventory::InventoryObject",
+    "T:LibreMetaverse.InventoryObjectAddedEventArgs": "crate::inventory::InventoryObjectAddedEventArgs",
+    "T:LibreMetaverse.InventoryObjectRemovedEventArgs": "crate::inventory::InventoryObjectRemovedEventArgs",
+    "T:LibreMetaverse.InventoryObjectUpdatedEventArgs": "crate::inventory::InventoryObjectUpdatedEventArgs",
+    "T:LibreMetaverse.InventorySettings": "crate::inventory::InventorySettings",
+    "T:LibreMetaverse.InventorySnapshot": "crate::inventory::InventorySnapshot",
+    "T:LibreMetaverse.InventorySound": "crate::inventory::InventorySound",
+    "T:LibreMetaverse.InventoryTexture": "crate::inventory::InventoryTexture",
+    "T:LibreMetaverse.InventoryWearable": "crate::inventory::InventoryWearable",
+    "T:LibreMetaverse.Permissions": "crate::inventory::Permissions",
     "T:LibreMetaverse.ChatEventArgs": "crate::agent_messages::ChatEventArgs",
     "T:LibreMetaverse.ChatSessionMember": "crate::agent_messages::ChatSessionMember",
     "T:LibreMetaverse.ChatSessionMemberAddedEventArgs": "crate::agent_messages::ChatSessionMemberAddedEventArgs",
@@ -1560,6 +1585,21 @@ VALUE_DERIVES = {
     "T:LibreMetaverse.Vector3d": "Clone, Copy, Debug",
     "T:LibreMetaverse.WorldSettings": "Clone, Debug, Eq, PartialEq",
 }
+
+# Native implementations occasionally need Rust-only bounds that have no
+# direct C# metadata equivalent. C#'s `where T : class` permits returning a
+# reference to the stored object; the Rust inventory store returns an owned
+# value, so its generic compatibility accessors clone the concrete item.
+NATIVE_MEMBER_BOUNDS = {
+    "M:LibreMetaverse.Inventory.GetValueOrDefault``1(LibreMetaverse.UUID)": {
+        "T": "T: Clone + 'static",
+    },
+    "M:LibreMetaverse.Inventory.TryGetValue``1(LibreMetaverse.UUID,``0@)": {
+        "T": "T: Clone + 'static",
+    },
+}
+
+
 def generic_names(item: dict) -> list[str]:
     return [parameter["name"] for parameter in item.get("generic_parameters", [])]
 
@@ -1649,6 +1689,11 @@ def render_compile_fixture(
                     *(owner_bounds.get(name, name) for name in owner_names),
                     *generic_declarations(member, mapper, owner_names),
                 ]
+                member_bounds = NATIVE_MEMBER_BOUNDS.get(member["doc_id"], {})
+                parameters = [
+                    member_bounds.get(parameter.split(":", 1)[0], parameter)
+                    for parameter in parameters
+                ]
                 if member["doc_id"] == "M:LibreMetaverse.Utils.EnumTryParse``1(System.String,``0@)":
                     parameters = [
                         *owner_names,

tools/generate_rust_mapping.py

@@ -993,6 +993,7 @@ def validate_generated_shims() -> None:
                     "crate::genepool_catalog::",
                     "crate::gesture::",
                     "crate::json_codec::",
+                    "crate::inventory::",
                     "crate::message_codec::",
                     "crate::message_decoder::",
                     "crate::notation::",