Cache TOTP discovery and report progress (#74)

crates/storage/src/mobile_totp.rs

@@ -1,13 +1,126 @@
 //! Storage-owned TOTP catalog and detail state for native mobile frontends.
 
-use std::{collections::BTreeSet, error::Error, fmt};
+use std::{
+    collections::{BTreeMap, BTreeSet},
+    error::Error,
+    fmt, fs,
+    io::Write as _,
+    path::Path,
+    sync::{
+        Mutex,
+        atomic::{AtomicBool, Ordering},
+    },
+};
+
+use cap_std::{ambient_authority, fs::Dir};
+use cap_tempfile::TempFile;
+use data_encoding::HEXLOWER;
+use serde::{Deserialize, Serialize};
+use sha2::{Digest as _, Sha256};
 
 use crate::{
     crypto::{KeyStore, SecretProvider},
     otp::{OtpError, OtpKind, OtpService},
-    repository::{EntryPath, Repository, RepositoryError, SecretBytes},
+    repository::{EncryptedEntry, EntryPath, Repository, RepositoryError, SecretBytes},
 };
 
+const CACHE_VERSION: u32 = 1;
+
+#[derive(Clone, Copy, Debug, Eq, PartialEq)]
+pub enum MobileTotpDiscoveryPhase {
+    Preparing,
+    Inspecting,
+    Saving,
+    Complete,
+    Cancelled,
+}
+
+#[derive(Clone, Debug, Eq, PartialEq)]
+pub struct MobileTotpDiscoveryProgress {
+    phase: MobileTotpDiscoveryPhase,
+    total: u32,
+    inspected: u32,
+    cache_hits: u32,
+    matches: u32,
+    unavailable: u32,
+}
+
+impl MobileTotpDiscoveryProgress {
+    pub fn phase(&self) -> MobileTotpDiscoveryPhase {
+        self.phase
+    }
+
+    pub fn total(&self) -> u32 {
+        self.total
+    }
+
+    pub fn inspected(&self) -> u32 {
+        self.inspected
+    }
+
+    pub fn cache_hits(&self) -> u32 {
+        self.cache_hits
+    }
+
+    pub fn matches(&self) -> u32 {
+        self.matches
+    }
+
+    pub fn unavailable(&self) -> u32 {
+        self.unavailable
+    }
+}
+
+pub struct MobileTotpOperation {
+    cancelled: AtomicBool,
+    progress: Mutex<MobileTotpDiscoveryProgress>,
+}
+
+impl Default for MobileTotpOperation {
+    fn default() -> Self {
+        Self {
+            cancelled: AtomicBool::new(false),
+            progress: Mutex::new(MobileTotpDiscoveryProgress {
+                phase: MobileTotpDiscoveryPhase::Preparing,
+                total: 0,
+                inspected: 0,
+                cache_hits: 0,
+                matches: 0,
+                unavailable: 0,
+            }),
+        }
+    }
+}
+
+impl MobileTotpOperation {
+    pub fn cancel(&self) {
+        self.cancelled.store(true, Ordering::Relaxed);
+        self.update(|progress| progress.phase = MobileTotpDiscoveryPhase::Cancelled);
+    }
+
+    pub fn progress(&self) -> MobileTotpDiscoveryProgress {
+        self.progress.lock().map_or_else(
+            |poisoned| poisoned.into_inner().clone(),
+            |value| value.clone(),
+        )
+    }
+
+    fn check_cancelled(&self) -> Result<(), MobileTotpError> {
+        if self.cancelled.load(Ordering::Relaxed) {
+            Err(OtpError::Cancelled.into())
+        } else {
+            Ok(())
+        }
+    }
+
+    fn update(&self, update: impl FnOnce(&mut MobileTotpDiscoveryProgress)) {
+        match self.progress.lock() {
+            Ok(mut progress) => update(&mut progress),
+            Err(poisoned) => update(&mut poisoned.into_inner()),
+        }
+    }
+}
+
 #[derive(Clone, Copy, Debug, Eq, PartialEq)]
 pub enum MobileWatchSnapshotState {
     Unavailable,
@@ -70,6 +183,7 @@ impl MobileTotpRow {
 pub struct MobileTotpPage {
     rows: Vec<MobileTotpRow>,
     unavailable_entries: u32,
+    cache_notice: Option<String>,
     watch: MobileWatchSnapshotStatus,
 }
 
@@ -82,6 +196,10 @@ impl MobileTotpPage {
         self.unavailable_entries
     }
 
+    pub fn cache_notice(&self) -> Option<&str> {
+        self.cache_notice.as_deref()
+    }
+
     pub fn watch(&self) -> &MobileWatchSnapshotStatus {
         &self.watch
     }
@@ -184,21 +302,129 @@ impl<'a> MobileTotpService<'a> {
                 Err(_) => {}
             }
         }
-        rows.sort_by(|left, right| {
-            left.title
-                .to_lowercase()
-                .cmp(&right.title.to_lowercase())
-                .then_with(|| {
-                    left.account
-                        .to_lowercase()
-                        .cmp(&right.account.to_lowercase())
-                })
-                .then_with(|| left.path.cmp(&right.path))
-        });
+        sort_rows(&mut rows);
         let selected = rows.iter().filter(|row| row.shared_with_watch).count();
         Ok(MobileTotpPage {
             rows,
             unavailable_entries,
+            cache_notice: None,
+            watch: snapshot_status(selected),
+        })
+    }
+
+    pub fn discover(
+        &self,
+        shared: &BTreeSet<EntryPath>,
+        provider: &mut impl SecretProvider,
+        cache_path: &Path,
+        operation: &MobileTotpOperation,
+    ) -> Result<MobileTotpPage, MobileTotpError> {
+        operation.check_cancelled()?;
+        let initial_inventory = inventory(self.repository)?;
+        operation.update(|progress| {
+            progress.phase = MobileTotpDiscoveryPhase::Inspecting;
+            progress.total = u32::try_from(initial_inventory.len()).unwrap_or(u32::MAX);
+        });
+
+        let store = store_identity(self.repository.root_path());
+        let (cached, mut cache_notice) = load_cache(cache_path, &store);
+        let mut checkpoint = CachedCatalog {
+            version: CACHE_VERSION,
+            store: store.clone(),
+            entries: cached
+                .as_ref()
+                .map_or_else(BTreeMap::new, |catalog| catalog.entries.clone()),
+        };
+        let mut records = BTreeMap::new();
+        let mut rows = Vec::new();
+        let mut unavailable_entries = 0_u32;
+        for (path, encrypted, ciphertext_hash) in &initial_inventory {
+            operation.check_cancelled()?;
+            let path_text = path.to_string();
+            let cached_record = cached
+                .as_ref()
+                .and_then(|catalog| catalog.entries.get(&path_text))
+                .filter(|record| record.ciphertext_hash == *ciphertext_hash);
+            let (record, changed) = if let Some(record) = cached_record {
+                operation
+                    .update(|progress| progress.cache_hits = progress.cache_hits.saturating_add(1));
+                (record.clone(), false)
+            } else {
+                match inspect_entry(self.repository, self.keys, path, encrypted, provider) {
+                    Ok(record) => (record.with_ciphertext_hash(ciphertext_hash.clone()), true),
+                    Err(OtpError::Crypto(_)) => {
+                        unavailable_entries = unavailable_entries.saturating_add(1);
+                        operation.update(|progress| {
+                            progress.unavailable = progress.unavailable.saturating_add(1)
+                        });
+                        operation.update(|progress| {
+                            progress.inspected = progress.inspected.saturating_add(1)
+                        });
+                        continue;
+                    }
+                    Err(OtpError::Repository(error)) => return Err(error.into()),
+                    Err(error) => return Err(error.into()),
+                }
+            };
+            if record.is_totp {
+                rows.push(cached_row(path, shared.contains(path)));
+                operation.update(|progress| progress.matches = progress.matches.saturating_add(1));
+            }
+            if changed {
+                checkpoint.entries.insert(path_text.clone(), record.clone());
+                if let Err(error) = save_cache(cache_path, &checkpoint) {
+                    cache_notice = Some(error.to_string());
+                }
+            }
+            records.insert(path_text, record);
+            operation.update(|progress| progress.inspected = progress.inspected.saturating_add(1));
+        }
+
+        operation.check_cancelled()?;
+        if initial_inventory != inventory(self.repository)? {
+            return Err(MobileTotpError::ConcurrentModification);
+        }
+        operation.update(|progress| progress.phase = MobileTotpDiscoveryPhase::Saving);
+        let catalog = CachedCatalog {
+            version: CACHE_VERSION,
+            store,
+            entries: records,
+        };
+        if let Err(error) = save_cache(cache_path, &catalog) {
+            cache_notice = Some(error.to_string());
+        }
+
+        sort_rows(&mut rows);
+        let selected = rows.iter().filter(|row| row.shared_with_watch).count();
+        operation.update(|progress| progress.phase = MobileTotpDiscoveryPhase::Complete);
+        Ok(MobileTotpPage {
+            rows,
+            unavailable_entries,
+            cache_notice,
+            watch: snapshot_status(selected),
+        })
+    }
+
+    pub fn cached_page(
+        &self,
+        shared: &BTreeSet<EntryPath>,
+        cache_path: &Path,
+    ) -> Option<MobileTotpPage> {
+        let store = store_identity(self.repository.root_path());
+        let (catalog, _) = load_cache(cache_path, &store);
+        let mut rows: Vec<_> = catalog?
+            .entries
+            .into_iter()
+            .filter(|(_, record)| record.is_totp)
+            .filter_map(|(path, _)| EntryPath::parse(&path).ok())
+            .map(|path| cached_row(&path, shared.contains(&path)))
+            .collect();
+        sort_rows(&mut rows);
+        let selected = rows.iter().filter(|row| row.shared_with_watch).count();
+        Some(MobileTotpPage {
+            rows,
+            unavailable_entries: 0,
+            cache_notice: None,
             watch: snapshot_status(selected),
         })
     }
@@ -234,6 +460,150 @@ impl<'a> MobileTotpService<'a> {
     }
 }
 
+#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)]
+struct CachedRecord {
+    ciphertext_hash: String,
+    is_totp: bool,
+}
+
+impl CachedRecord {
+    fn with_ciphertext_hash(mut self, ciphertext_hash: String) -> Self {
+        self.ciphertext_hash = ciphertext_hash;
+        self
+    }
+}
+
+#[derive(Debug, Deserialize, Eq, PartialEq, Serialize)]
+struct CachedCatalog {
+    version: u32,
+    store: String,
+    entries: BTreeMap<String, CachedRecord>,
+}
+
+fn inventory(
+    repository: &Repository,
+) -> Result<Vec<(EntryPath, EncryptedEntry, String)>, RepositoryError> {
+    repository
+        .snapshot()?
+        .entries()
+        .map(|entry| {
+            let encrypted = repository.read_entry(entry.path())?;
+            let ciphertext_hash = digest(encrypted.as_bytes());
+            Ok((entry.path().clone(), encrypted, ciphertext_hash))
+        })
+        .collect()
+}
+
+fn inspect_entry(
+    repository: &Repository,
+    keys: &KeyStore,
+    path: &EntryPath,
+    encrypted: &EncryptedEntry,
+    provider: &mut impl SecretProvider,
+) -> Result<CachedRecord, OtpError> {
+    let is_totp = match OtpService::new(repository, keys).uri_encrypted(path, encrypted, provider) {
+        Ok(uri) => uri.kind() == OtpKind::Totp,
+        Err(OtpError::MissingUri { .. } | OtpError::AmbiguousUri { .. }) => false,
+        Err(OtpError::Crypto(error)) => return Err(OtpError::Crypto(error)),
+        Err(OtpError::Repository(error)) => return Err(OtpError::Repository(error)),
+        Err(_) => false,
+    };
+    Ok(CachedRecord {
+        ciphertext_hash: String::new(),
+        is_totp,
+    })
+}
+
+fn load_cache(path: &Path, store: &str) -> (Option<CachedCatalog>, Option<String>) {
+    if let Ok(metadata) = fs::symlink_metadata(path)
+        && (metadata.file_type().is_symlink() || !metadata.is_file())
+    {
+        return (
+            None,
+            Some("The TOTP cache was not a regular file and was rebuilt.".to_owned()),
+        );
+    }
+    let contents = match fs::read_to_string(path) {
+        Ok(contents) => contents,
+        Err(error) if error.kind() == std::io::ErrorKind::NotFound => return (None, None),
+        Err(_) => {
+            return (
+                None,
+                Some("The TOTP cache could not be read and was rebuilt.".to_owned()),
+            );
+        }
+    };
+    let catalog = toml::from_str::<CachedCatalog>(&contents)
+        .ok()
+        .filter(|catalog| catalog.version == CACHE_VERSION && catalog.store == store);
+    match catalog {
+        Some(catalog) => (Some(catalog), None),
+        None => (
+            None,
+            Some("The TOTP cache was outdated or damaged and was rebuilt.".to_owned()),
+        ),
+    }
+}
+
+fn save_cache(path: &Path, catalog: &CachedCatalog) -> Result<(), MobileTotpCacheError> {
+    let serialized = toml::to_string(catalog).map_err(|_| MobileTotpCacheError::Encode)?;
+    let parent = path.parent().ok_or(MobileTotpCacheError::Write)?;
+    let name = path.file_name().ok_or(MobileTotpCacheError::Write)?;
+    let directory = Dir::open_ambient_dir(parent, ambient_authority())
+        .map_err(|_| MobileTotpCacheError::Write)?;
+    if let Ok(metadata) = directory.symlink_metadata(name)
+        && (metadata.file_type().is_symlink() || !metadata.is_file())
+    {
+        return Err(MobileTotpCacheError::Write);
+    }
+    let mut temporary = TempFile::new(&directory).map_err(|_| MobileTotpCacheError::Write)?;
+    set_private_permissions(&temporary)?;
+    temporary
+        .write_all(serialized.as_bytes())
+        .and_then(|()| temporary.as_file().sync_all())
+        .and_then(|()| temporary.replace(name))
+        .and_then(|()| directory.open(".").and_then(|file| file.sync_all()))
+        .map_err(|_| MobileTotpCacheError::Write)
+}
+
+#[cfg(unix)]
+fn set_private_permissions(temporary: &TempFile<'_>) -> Result<(), MobileTotpCacheError> {
+    use cap_std::fs::{Permissions, PermissionsExt as _};
+
+    temporary
+        .as_file()
+        .set_permissions(Permissions::from_mode(0o600))
+        .map_err(|_| MobileTotpCacheError::Write)
+}
+
+#[cfg(not(unix))]
+fn set_private_permissions(_temporary: &TempFile<'_>) -> Result<(), MobileTotpCacheError> {
+    Ok(())
+}
+
+fn store_identity(root: &Path) -> String {
+    digest(root.to_string_lossy().as_bytes())
+}
+
+fn digest(bytes: &[u8]) -> String {
+    HEXLOWER.encode(&Sha256::digest(bytes))
+}
+
+#[derive(Debug)]
+enum MobileTotpCacheError {
+    Encode,
+    Write,
+}
+
+impl fmt::Display for MobileTotpCacheError {
+    fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
+        match self {
+            Self::Encode => formatter.write_str("The TOTP cache could not be encoded."),
+            Self::Write => formatter.write_str("The TOTP cache could not be saved."),
+        }
+    }
+}
+
 fn row(
     path: &EntryPath,
     issuer: Option<&str>,
@@ -250,6 +620,33 @@ fn row(
     }
 }
 
+fn cached_row(path: &EntryPath, shared_with_watch: bool) -> MobileTotpRow {
+    let path = path.to_string();
+    let title = path.rsplit('/').next().unwrap_or(&path).to_owned();
+    MobileTotpRow {
+        path: path.clone(),
+        issuer: None,
+        account: title.clone(),
+        title,
+        detail: path,
+        shared_with_watch,
+    }
+}
+
+fn sort_rows(rows: &mut [MobileTotpRow]) {
+    rows.sort_by(|left, right| {
+        left.title
+            .to_lowercase()
+            .cmp(&right.title.to_lowercase())
+            .then_with(|| {
+                left.account
+                    .to_lowercase()
+                    .cmp(&right.account.to_lowercase())
+            })
+            .then_with(|| left.path.cmp(&right.path))
+    });
+}
+
 fn snapshot_status(selected: usize) -> MobileWatchSnapshotStatus {
     if selected == 0 {
         MobileWatchSnapshotStatus {
@@ -275,6 +672,7 @@ fn snapshot_status(selected: usize) -> MobileWatchSnapshotStatus {
 pub enum MobileTotpError {
     Repository(RepositoryError),
     Otp(OtpError),
+    ConcurrentModification,
 }
 
 impl fmt::Display for MobileTotpError {
@@ -282,6 +680,8 @@ impl fmt::Display for MobileTotpError {
         match self {
             Self::Repository(error) => error.fmt(formatter),
             Self::Otp(error) => error.fmt(formatter),
+            Self::ConcurrentModification => formatter
+                .write_str("the password store changed while TOTP discovery was in progress"),
         }
     }
 }