Harden concurrency and resource lifecycle (#101)

crates/libremetaverse/src/client_core.rs

@@ -242,20 +242,25 @@ impl ClientRuntime {
         }
 
         self.cancellation.cancel();
-        let mut services = self
-            .services
-            .lock()
-            .unwrap_or_else(std::sync::PoisonError::into_inner);
+        // Service shutdown is user-controlled code. Move registrations out
+        // before invoking it so re-entrant lifecycle calls cannot deadlock on
+        // the registry mutex.
+        let mut services = std::mem::take(
+            &mut *self
+                .services
+                .lock()
+                .unwrap_or_else(std::sync::PoisonError::into_inner),
+        );
         services.sort_by_key(|entry| (entry.service.shutdown_phase(), entry.insertion_order));
         let mut first_error = None;
-        for entry in services.iter() {
+        for entry in &services {
             if let Err(error) = entry.service.shutdown()
                 && first_error.is_none()
             {
                 first_error = Some(error);
             }
         }
-        services.clear();
+        drop(services);
         clear_cached(&self.agent_throttle_sender);
         clear_cached(&self.inventory_manager);
         clear_cached(&self.inventory_ais_client);
@@ -1623,6 +1628,41 @@ mod tests {
         }
     }
 
+    struct ReentrantService {
+        client: Mutex<Option<GridClient>>,
+        calls: AtomicUsize,
+    }
+
+    impl ClientService for ReentrantService {
+        fn name(&self) -> &'static str {
+            "reentrant"
+        }
+
+        fn shutdown(&self) -> Result<(), ClientCoreError> {
+            self.calls.fetch_add(1, Ordering::AcqRel);
+            let result = self
+                .client
+                .lock()
+                .unwrap_or_else(std::sync::PoisonError::into_inner)
+                .as_mut()
+                .expect("client installed")
+                .register_service(Arc::new(RecordingService {
+                    name: "late",
+                    phase: ShutdownPhase::Manager,
+                    order: Arc::new(Mutex::new(Vec::new())),
+                    calls: AtomicUsize::new(0),
+                }));
+            assert!(matches!(
+                result,
+                Err(ClientCoreError::InvalidLifecycle {
+                    state: ClientLifecycleState::ShuttingDown,
+                    ..
+                })
+            ));
+            Ok(())
+        }
+    }
+
     #[test]
     #[allow(clippy::float_cmp)] // Exact representable constants are the compatibility contract.
     fn defaults_match_reference_and_validate() {
@@ -1762,4 +1802,29 @@ mod tests {
         assert_eq!(network.calls.load(Ordering::Relaxed), 1);
         assert_eq!(manager.calls.load(Ordering::Relaxed), 1);
     }
+
+    #[test]
+    fn shutdown_does_not_hold_service_registry_lock_across_callbacks() {
+        let service = Arc::new(ReentrantService {
+            client: Mutex::new(None),
+            calls: AtomicUsize::new(0),
+        });
+        let client = GridClient::builder()
+            .with_service(service.clone())
+            .build()
+            .unwrap();
+        *service
+            .client
+            .lock()
+            .unwrap_or_else(std::sync::PoisonError::into_inner) = Some(client.clone());
+
+        client.shutdown().unwrap();
+        assert_eq!(client.lifecycle_state(), ClientLifecycleState::Disposed);
+        assert_eq!(service.calls.load(Ordering::Acquire), 1);
+        service
+            .client
+            .lock()
+            .unwrap_or_else(std::sync::PoisonError::into_inner)
+            .take();
+    }
 }