Implement Android TOTP cached discovery and search (#110)

crates/storage/src/mobile_authentication.rs

@@ -1,7 +1,7 @@
 //! Shared mobile authentication state; native frontends supply input and present results.
 
 use std::{
-    collections::BTreeMap,
+    collections::{BTreeMap, BTreeSet},
     error::Error,
     fmt,
     path::Path,
@@ -31,7 +31,7 @@ use crate::{
     },
     mobile_totp::{
         MobileOtpCodePresentation, MobileTotpDetail, MobileTotpError, MobileTotpOperation,
-        MobileTotpPage, MobileTotpService, otp_code_presentation,
+        MobileTotpPage, MobileTotpRefreshResult, MobileTotpService, otp_code_presentation,
     },
     mobile_watch::{
         MobileWatchSnapshotState, MobileWatchSnapshotStatus, WatchSnapshotError,
@@ -1899,11 +1899,37 @@ impl MobileAuthentication {
         Option<Vec<crate::mobile_passwords::navigation::NavigationRow>>,
         MobileAuthenticationError,
     > {
-        self.cached_totp_page()?
+        self.search_cached_totp_navigation("", &MobileTotpOperation::default())
+    }
+
+    pub fn search_cached_totp_navigation(
+        &self,
+        query: &str,
+        operation: &MobileTotpOperation,
+    ) -> Result<
+        Option<Vec<crate::mobile_passwords::navigation::NavigationRow>>,
+        MobileAuthenticationError,
+    > {
+        self.ensure_repository_idle()?;
+        if query.len() > 256 || query.chars().any(char::is_control) {
+            return Err(entry_detail(
+                "TOTP Search Is Unavailable",
+                "the search text is invalid",
+            ));
+        }
+        MobileTotpService::new(&self.repository, &self.keys)
+            .search_cached_page_cancellable(
+                &BTreeSet::new(),
+                &self.config.source().with_file_name("totp-catalog.toml"),
+                query,
+                operation,
+            )
+            .map_err(totp_error)?
             .map(|page| {
                 page.rows()
                     .iter()
                     .map(|row| {
+                        operation.check_cancelled().map_err(totp_error)?;
                         let path = EntryPath::parse(row.path()).map_err(entry_error)?;
                         crate::mobile_passwords::navigation::entry_navigation_row(
                             &self.repository,
@@ -1916,6 +1942,56 @@ impl MobileAuthentication {
             .transpose()
     }
 
+    /// Android refresh reuses a stable catalog while locked and authenticates only for discovery.
+    pub fn refresh_totp_catalog(
+        &self,
+        operation: &MobileTotpOperation,
+        passphrase: Option<SecretBytes>,
+    ) -> Result<MobileTotpRefreshResult, MobileAuthenticationError> {
+        self.ensure_repository_idle()?;
+        let _discovery = self.totp_discovery.lock().map_err(|_| {
+            entry_detail(
+                "TOTP Discovery Is Unavailable",
+                "the discovery lock is unavailable",
+            )
+        })?;
+        let cache_path = self.config.source().with_file_name("totp-catalog.toml");
+        match MobileTotpService::new(&self.repository, &self.keys)
+            .refresh_cached(&cache_path, operation)
+        {
+            Ok(page) => {
+                return Ok(MobileTotpRefreshResult {
+                    progress: operation.progress(),
+                    cache_notice: page.cache_notice().map(str::to_owned),
+                });
+            }
+            Err(MobileTotpError::AuthenticationRequired) => {}
+            Err(error) => return Err(totp_error(error)),
+        }
+        operation.check_cancelled().map_err(totp_error)?;
+        if self.ensure_active().is_err() {
+            self.unlock_totp(passphrase)?;
+        }
+        let (handle, key) = {
+            let status = self.status()?;
+            let active = status.active.as_ref().ok_or_else(locked_error)?;
+            (active.handle.clone(), active.key.clone())
+        };
+        let page = MobileTotpService::new(&self.repository, &self.keys)
+            .discover(
+                &BTreeSet::new(),
+                &mut KeyOnlyProvider::new(handle, &key),
+                &cache_path,
+                operation,
+            )
+            .map_err(totp_error)?;
+        self.ensure_active()?;
+        Ok(MobileTotpRefreshResult {
+            progress: operation.progress(),
+            cache_notice: page.cache_notice().map(str::to_owned),
+        })
+    }
+
     pub fn search_cached_totp_page(
         &self,
         query: &str,
@@ -3452,6 +3528,54 @@ mod tests {
         );
     }
 
+    #[test]
+    fn android_catalog_refresh_authenticates_only_for_uncached_ciphertext() {
+        use super::MobileAuthenticationErrorKind;
+        use crate::{mobile_totp::MobileTotpOperation, repository::SecretBytes};
+        let (_root, authentication, _) =
+            otp_fixture("otpauth://totp/RFC:alice?secret=GEZDGNBVGY3TQOJQGEZDGNBVGY3TQOJQ");
+        let cold = MobileTotpOperation::default();
+        assert_eq!(
+            authentication
+                .refresh_totp_catalog(&cold, None)
+                .unwrap_err()
+                .kind(),
+            MobileAuthenticationErrorKind::PassphraseRequired
+        );
+        assert_eq!(
+            cold.progress().phase(),
+            crate::mobile_totp::MobileTotpDiscoveryPhase::Preparing
+        );
+        let completed = authentication
+            .refresh_totp_catalog(
+                &MobileTotpOperation::default(),
+                Some(SecretBytes::new(b"fixture-alice-passphrase".to_vec())),
+            )
+            .unwrap();
+        assert_eq!(completed.progress.matches(), 1);
+        authentication.manual_lock().unwrap();
+        let warm = authentication
+            .refresh_totp_catalog(
+                &MobileTotpOperation::default(),
+                Some(SecretBytes::new(b"incorrect and must not be used".to_vec())),
+            )
+            .unwrap();
+        assert_eq!(warm.progress.cache_hits(), warm.progress.total());
+        assert!(!authentication.state().unwrap().unlocked());
+        let rows = authentication
+            .search_cached_totp_navigation(" OTP ", &MobileTotpOperation::default())
+            .unwrap()
+            .unwrap();
+        assert_eq!(rows.len(), 1);
+        let cancelled = MobileTotpOperation::default();
+        cancelled.cancel();
+        assert!(
+            authentication
+                .search_cached_totp_navigation("", &cancelled)
+                .is_err()
+        );
+    }
+
     #[test]
     fn otp_presentation_matches_rfc_vectors_and_boundaries_without_renewing_the_lease() {
         use crate::{otp::OtpCodeValidity, repository::SecretBytes};

crates/storage/src/mobile_totp.rs

@@ -50,6 +50,12 @@ pub struct MobileTotpDiscoveryProgress {
     unavailable: u32,
 }
 
+#[derive(Debug)]
+pub struct MobileTotpRefreshResult {
+    pub progress: MobileTotpDiscoveryProgress,
+    pub cache_notice: Option<String>,
+}
+
 impl MobileTotpDiscoveryProgress {
     pub fn phase(&self) -> MobileTotpDiscoveryPhase {
         self.phase
@@ -99,8 +105,12 @@ impl Default for MobileTotpOperation {
 
 impl MobileTotpOperation {
     pub fn cancel(&self) {
-        self.cancelled.store(true, Ordering::Relaxed);
-        self.update(|progress| progress.phase = MobileTotpDiscoveryPhase::Cancelled);
+        self.update(|progress| {
+            if progress.phase != MobileTotpDiscoveryPhase::Complete {
+                self.cancelled.store(true, Ordering::Relaxed);
+                progress.phase = MobileTotpDiscoveryPhase::Cancelled;
+            }
+        });
     }
 
     pub fn progress(&self) -> MobileTotpDiscoveryProgress {
@@ -110,7 +120,7 @@ impl MobileTotpOperation {
         )
     }
 
-    fn check_cancelled(&self) -> Result<(), MobileTotpError> {
+    pub(crate) fn check_cancelled(&self) -> Result<(), MobileTotpError> {
         if self.cancelled.load(Ordering::Relaxed) {
             Err(OtpError::Cancelled.into())
         } else {
@@ -119,11 +129,26 @@ impl MobileTotpOperation {
     }
 
     fn update(&self, update: impl FnOnce(&mut MobileTotpDiscoveryProgress)) {
-        match self.progress.lock() {
-            Ok(mut progress) => update(&mut progress),
-            Err(poisoned) => update(&mut poisoned.into_inner()),
+        let mut progress = self
+            .progress
+            .lock()
+            .unwrap_or_else(|poisoned| poisoned.into_inner());
+        if !self.cancelled.load(Ordering::Relaxed) {
+            update(&mut progress);
         }
     }
+
+    fn complete(&self, save: impl FnOnce()) -> Result<(), MobileTotpError> {
+        let mut progress = self
+            .progress
+            .lock()
+            .unwrap_or_else(|poisoned| poisoned.into_inner());
+        self.check_cancelled()?;
+        // Cancellation must not split final deletion pruning from successful completion.
+        save();
+        progress.phase = MobileTotpDiscoveryPhase::Complete;
+        Ok(())
+    }
 }
 
 #[derive(Clone, Debug, Eq, PartialEq)]
@@ -429,7 +454,7 @@ impl<'a> MobileTotpService<'a> {
         cache_path: &Path,
         operation: &MobileTotpOperation,
     ) -> Result<MobileTotpPage, MobileTotpError> {
-        self.discover_with_cache(shared, provider, cache_path, operation, true)
+        self.discover_with_cache(shared, provider, cache_path, operation, true, false)
     }
 
     pub fn rebuild(
@@ -439,7 +464,32 @@ impl<'a> MobileTotpService<'a> {
         cache_path: &Path,
         operation: &MobileTotpOperation,
     ) -> Result<MobileTotpPage, MobileTotpError> {
-        self.discover_with_cache(shared, provider, cache_path, operation, false)
+        self.discover_with_cache(shared, provider, cache_path, operation, false, false)
+    }
+
+    /// A complete unchanged catalog can be refreshed while locked without reading secrets.
+    pub fn refresh_cached(
+        &self,
+        cache_path: &Path,
+        operation: &MobileTotpOperation,
+    ) -> Result<MobileTotpPage, MobileTotpError> {
+        struct NoSecrets;
+        impl SecretProvider for NoSecrets {
+            fn secret_for(
+                &mut self,
+                _: &crate::crypto::KeyInfo,
+            ) -> Result<SecretBytes, crate::crypto::SecretProviderError> {
+                Err(crate::crypto::SecretProviderError::Missing)
+            }
+        }
+        self.discover_with_cache(
+            &BTreeSet::new(),
+            &mut NoSecrets,
+            cache_path,
+            operation,
+            true,
+            true,
+        )
     }
 
     fn discover_with_cache(
@@ -449,19 +499,33 @@ impl<'a> MobileTotpService<'a> {
         cache_path: &Path,
         operation: &MobileTotpOperation,
         reuse_cache: bool,
+        cache_only: bool,
     ) -> Result<MobileTotpPage, MobileTotpError> {
         operation.check_cancelled()?;
-        let initial_inventory = inventory(self.repository)?;
-        operation.update(|progress| {
-            progress.phase = MobileTotpDiscoveryPhase::Inspecting;
-            progress.total = u32::try_from(initial_inventory.len()).unwrap_or(u32::MAX);
-        });
+        let initial_inventory = inventory(self.repository, operation)?;
 
         let store = store_identity(self.repository.root_path(), cache_path);
         let (mut cached, mut cache_notice) = load_cache(cache_path, &store);
         if !reuse_cache {
             cached = None;
         }
+        if cache_only
+            && !cached.as_ref().is_some_and(|catalog| {
+                initial_inventory.iter().all(|(path, _, hash)| {
+                    catalog
+                        .entries
+                        .get(&path.to_string())
+                        .is_some_and(|record| record.ciphertext_hash == *hash)
+                })
+            })
+        {
+            return Err(MobileTotpError::AuthenticationRequired);
+        }
+        operation.check_cancelled()?;
+        operation.update(|progress| {
+            progress.phase = MobileTotpDiscoveryPhase::Inspecting;
+            progress.total = u32::try_from(initial_inventory.len()).unwrap_or(u32::MAX);
+        });
         let mut records = BTreeMap::new();
         let mut rows = Vec::new();
         let mut unavailable_entries = 0_u32;
@@ -507,7 +571,7 @@ impl<'a> MobileTotpService<'a> {
         }
 
         operation.check_cancelled()?;
-        if initial_inventory != inventory(self.repository)? {
+        if initial_inventory != inventory(self.repository, operation)? {
             return Err(MobileTotpError::ConcurrentModification);
         }
         operation.update(|progress| progress.phase = MobileTotpDiscoveryPhase::Saving);
@@ -516,13 +580,14 @@ impl<'a> MobileTotpService<'a> {
             store,
             entries: records,
         };
-        if let Err(error) = save_cache(cache_path, &catalog) {
-            cache_notice = Some(error.to_string());
-        }
+        operation.complete(|| {
+            if let Err(error) = save_cache(cache_path, &catalog) {
+                cache_notice = Some(error.to_string());
+            }
+        })?;
 
         sort_rows(&mut rows);
         let selected = rows.iter().filter(|row| row.shared_with_watch).count();
-        operation.update(|progress| progress.phase = MobileTotpDiscoveryPhase::Complete);
         Ok(MobileTotpPage {
             rows,
             unavailable_entries,
@@ -573,29 +638,55 @@ impl<'a> MobileTotpService<'a> {
         cache_path: &Path,
         query: &str,
     ) -> Option<MobileTotpPage> {
+        self.search_cached_page_cancellable(
+            shared,
+            cache_path,
+            query,
+            &MobileTotpOperation::default(),
+        )
+        .ok()
+        .flatten()
+    }
+
+    pub fn search_cached_page_cancellable(
+        &self,
+        shared: &BTreeSet<EntryPath>,
+        cache_path: &Path,
+        query: &str,
+        operation: &MobileTotpOperation,
+    ) -> Result<Option<MobileTotpPage>, MobileTotpError> {
+        operation.check_cancelled()?;
         let store = store_identity(self.repository.root_path(), cache_path);
         let (catalog, _) = load_cache(cache_path, &store);
+        let Some(catalog) = catalog else {
+            return Ok(None);
+        };
         let query = query.trim().to_lowercase();
-        let mut rows: Vec<_> = catalog?
-            .entries
-            .into_iter()
-            .filter(|(_, record)| record.is_totp)
-            .filter_map(|(path, record)| {
-                let path = EntryPath::parse(&path).ok()?;
-                let ciphertext = self.repository.read_entry(&path).ok()?;
-                (digest(ciphertext.as_bytes()) == record.ciphertext_hash).then_some(path)
-            })
-            .filter(|path| query.is_empty() || path.to_string().to_lowercase().contains(&query))
-            .map(|path| cached_row(&path, shared.contains(&path)))
-            .collect();
+        let mut rows = Vec::new();
+        for (path, record) in catalog.entries {
+            operation.check_cancelled()?;
+            if !record.is_totp || !path.to_lowercase().contains(&query) {
+                continue;
+            }
+            let Ok(path) = EntryPath::parse(&path) else {
+                continue;
+            };
+            let Ok(ciphertext) = self.repository.read_entry(&path) else {
+                continue;
+            };
+            if digest(ciphertext.as_bytes()) == record.ciphertext_hash {
+                rows.push(cached_row(&path, shared.contains(&path)));
+            }
+        }
         sort_rows(&mut rows);
         let selected = rows.iter().filter(|row| row.shared_with_watch).count();
-        Some(MobileTotpPage {
+        operation.check_cancelled()?;
+        Ok(Some(MobileTotpPage {
             rows,
             unavailable_entries: 0,
             cache_notice: None,
             watch: snapshot_status(selected),
-        })
+        }))
     }
 
     pub fn reconcile_document(
@@ -726,11 +817,13 @@ struct CachedCatalog {
 
 fn inventory(
     repository: &Repository,
-) -> Result<Vec<(EntryPath, EncryptedEntry, String)>, RepositoryError> {
+    operation: &MobileTotpOperation,
+) -> Result<Vec<(EntryPath, EncryptedEntry, String)>, MobileTotpError> {
     repository
         .snapshot()?
         .entries()
         .map(|entry| {
+            operation.check_cancelled()?;
             let encrypted = repository.read_entry(entry.path())?;
             let ciphertext_hash = digest(encrypted.as_bytes());
             Ok((entry.path().clone(), encrypted, ciphertext_hash))
@@ -1013,6 +1106,7 @@ pub enum MobileTotpError {
     Repository(RepositoryError),
     Otp(OtpError),
     ConcurrentModification,
+    AuthenticationRequired,
 }
 
 impl fmt::Display for MobileTotpError {
@@ -1022,6 +1116,9 @@ impl fmt::Display for MobileTotpError {
             Self::Otp(error) => error.fmt(formatter),
             Self::ConcurrentModification => formatter
                 .write_str("the password store changed while TOTP discovery was in progress"),
+            Self::AuthenticationRequired => {
+                formatter.write_str("authenticate to inspect new or changed entries")
+            }
         }
     }
 }
@@ -1039,3 +1136,31 @@ impl From<OtpError> for MobileTotpError {
         Self::Otp(error)
     }
 }
+
+#[cfg(test)]
+mod operation_tests {
+    use super::*;
+
+    #[test]
+    fn cancellation_cannot_split_final_save_from_completion() {
+        let cancelled = MobileTotpOperation::default();
+        cancelled.update(|progress| progress.phase = MobileTotpDiscoveryPhase::Saving);
+        cancelled.cancel();
+        let mut saved = false;
+        assert!(cancelled.complete(|| saved = true).is_err());
+        assert!(!saved);
+        assert_eq!(
+            cancelled.progress().phase(),
+            MobileTotpDiscoveryPhase::Cancelled
+        );
+
+        let completed = MobileTotpOperation::default();
+        completed.complete(|| saved = true).unwrap();
+        completed.cancel();
+        assert!(saved);
+        assert_eq!(
+            completed.progress().phase(),
+            MobileTotpDiscoveryPhase::Complete
+        );
+    }
+}

crates/storage/tests/mobile_totp.rs

@@ -228,6 +228,14 @@ fn totp_cache_reuses_ciphertext_hashes_and_removes_deleted_entries() -> TestResu
     assert_eq!(warm_secrets.requests, 0);
     assert_eq!(warm.progress().cache_hits(), warm.progress().total());
     assert_eq!(warm_page.rows(), page.rows());
+    let locked = MobileTotpOperation::default();
+    assert_eq!(service.refresh_cached(&cache, &locked)?.rows(), page.rows());
+    assert_eq!(locked.progress().cache_hits(), locked.progress().total());
+    locked.cancel();
+    assert_eq!(
+        locked.progress().phase(),
+        MobileTotpDiscoveryPhase::Complete
+    );
     assert_eq!(service.cache_status(&cache).0 as usize, page.rows().len());
     let rebuild = MobileTotpOperation::default();
     let mut rebuild_secrets = CountingSecrets::all(&fixture);
@@ -269,6 +277,17 @@ fn totp_cache_reuses_ciphertext_hashes_and_removes_deleted_entries() -> TestResu
         "ordinary",
         b"otpauth://totp/New:new@example.com?secret=GEZDGNBVGY3TQOJQGEZDGNBVGY3TQOJQ&issuer=New\n",
     )?;
+    let preserved = fs::read(&cache)?;
+    let requires_authentication = MobileTotpOperation::default();
+    assert!(matches!(
+        service.refresh_cached(&cache, &requires_authentication),
+        Err(MobileTotpError::AuthenticationRequired)
+    ));
+    assert_eq!(
+        requires_authentication.progress().phase(),
+        MobileTotpDiscoveryPhase::Preparing
+    );
+    assert_eq!(fs::read(&cache)?, preserved);
     let changed = MobileTotpOperation::default();
     let mut changed_secrets = CountingSecrets::all(&fixture);
     let changed_page =
@@ -327,6 +346,10 @@ fn totp_cache_reuses_ciphertext_hashes_and_removes_deleted_entries() -> TestResu
 
     let cancelled = MobileTotpOperation::default();
     cancelled.cancel();
+    assert!(matches!(
+        service.search_cached_page_cancellable(&BTreeSet::new(), &cache, "team", &cancelled),
+        Err(MobileTotpError::Otp(OtpError::Cancelled))
+    ));
     let mut cancelled_secrets = CountingSecrets::all(&fixture);
     assert!(matches!(
         service.discover(&BTreeSet::new(), &mut cancelled_secrets, &cache, &cancelled,),
@@ -490,6 +513,99 @@ fn cancelled_discovery_checkpoints_completed_entries() -> TestResult {
     service.discover(&BTreeSet::new(), &mut resumed_secrets, &cache, &resumed)?;
     assert!(resumed.progress().cache_hits() >= 1);
     assert!(resumed_secrets.requests < resumed.progress().total() as usize);
+    let deleted = repository
+        .snapshot()?
+        .entries()
+        .next()
+        .unwrap()
+        .path()
+        .clone();
+    repository.remove_entry(&deleted)?;
+    write_plaintext(
+        &repository,
+        &keys,
+        "new-checkpoint",
+        b"otpauth://totp/Test?secret=GEZDGNBVGY3TQOJQGEZDGNBVGY3TQOJQ\n",
+    )?;
+    let interrupted = MobileTotpOperation::default();
+    let mut secrets = CancellingSecrets {
+        inner: FixtureSecrets::all(&fixture),
+        operation: &interrupted,
+        requests: 0,
+    };
+    assert!(matches!(
+        service.discover(&BTreeSet::new(), &mut secrets, &cache, &interrupted),
+        Err(MobileTotpError::Otp(OtpError::Cancelled))
+    ));
+    let checkpoint: toml::Value = toml::from_str(&fs::read_to_string(&cache)?)?;
+    assert!(checkpoint["entries"].get(deleted.to_string()).is_some());
+    service.discover(
+        &BTreeSet::new(),
+        &mut FixtureSecrets::all(&fixture),
+        &cache,
+        &MobileTotpOperation::default(),
+    )?;
+    let completed: toml::Value = toml::from_str(&fs::read_to_string(&cache)?)?;
+    assert!(completed["entries"].get(deleted.to_string()).is_none());
+    Ok(())
+}
+
+#[test]
+fn large_cached_search_filters_paths_without_reading_secrets() -> TestResult {
+    let fixture = FixtureSet::load()?;
+    let store = fixture.materialize_store("basic")?;
+    let repository = Repository::open(store.path())?;
+    let keys = KeyStore::load(fixture.path("keys"))?;
+    write_plaintext(
+        &repository,
+        &keys,
+        "seed",
+        b"otpauth://totp/Test?secret=GEZDGNBVGY3TQOJQGEZDGNBVGY3TQOJQ\n",
+    )?;
+    let directory = tempfile::tempdir()?;
+    let cache = directory.path().join("totp-catalog.toml");
+    let service = MobileTotpService::new(&repository, &keys);
+    service.discover(
+        &BTreeSet::new(),
+        &mut FixtureSecrets::all(&fixture),
+        &cache,
+        &MobileTotpOperation::default(),
+    )?;
+    let ciphertext = repository.read_entry(&EntryPath::parse("seed")?)?;
+    let mut catalog: toml::Value = toml::from_str(&fs::read_to_string(&cache)?)?;
+    let entries = catalog["entries"].as_table_mut().unwrap();
+    let record = entries["seed"].clone();
+    for index in 0..1_000 {
+        let path = format!("large/entry-{index:04}");
+        repository.write_entry(&EntryPath::parse(&path)?, &ciphertext)?;
+        entries.insert(path, record.clone());
+    }
+    fs::write(&cache, toml::to_string(&catalog)?)?;
+    let result = service
+        .search_cached_page_cancellable(
+            &BTreeSet::new(),
+            &cache,
+            " ENTRY-0999 ",
+            &MobileTotpOperation::default(),
+        )?
+        .unwrap();
+    assert_eq!(
+        result
+            .rows()
+            .iter()
+            .map(|row| row.path())
+            .collect::<Vec<_>>(),
+        ["large/entry-0999"]
+    );
+    let all = service
+        .search_cached_page_cancellable(
+            &BTreeSet::new(),
+            &cache,
+            "large/",
+            &MobileTotpOperation::default(),
+        )?
+        .unwrap();
+    assert_eq!(all.rows().len(), 1_000);
     Ok(())
 }