Harden Android accessibility, localization and startup (#105)

Implement independent hardening and regression coverage. Keep #105 open for final feature and environment coverage after #106–#110; document the completion boundary and resume order.

.gitea/scripts/android_ci.py

@@ -3,6 +3,7 @@
 import argparse
 import hashlib
 import json
+import math
 import os
 from pathlib import Path
 import platform
@@ -99,6 +100,7 @@ def verification_phases():
         (":app:assembleDebug", ":app:assembleRelease", ":app:bundleRelease"),
         *((f":{module}:assembleDebugAndroidTest",) for module in ("app", "core", "rust-bridge", "platform")),
         *((f":{module}:ktlintCheck", f":{module}:lintDebug") for module in MODULES),
+        (":app:assembleBenchmark", ":app:assembleProfile", ":performance:assembleBenchmark", ":performance:ktlintCheck"),
         ("androidVerification",),
     )
 
@@ -166,7 +168,7 @@ def clean_ci(root=ROOT):
     if os.environ.get("CI") != "true":
         raise RuntimeError("Build cleanup is restricted to CI checkouts")
     paths = [root / "target", root / "android/build"]
-    paths.extend(root / "android" / module / "build" for module in MODULES)
+    paths.extend(root / "android" / module / "build" for module in (*MODULES, "performance"))
     if any(path.resolve() != path or (path.exists() and not path.is_dir()) for path in paths):
         raise RuntimeError("CI build directories must not contain symlinks or unexpected files")
     for path in paths:
@@ -244,6 +246,30 @@ def build():
     checksum_artifacts(output)
 
 
+PERFORMANCE_BUDGETS = {
+    "editor_100_updates_ms": 500,
+    "editor_update_p95_us": 32000,
+    "cache_rebuild_ms": 15000,
+    "qr_analysis_ms": 5000,
+    "tree_1000_navigation_ms": 3000,
+    "tree_1000_search_ms": 5000,
+}
+
+
+def performance_metrics(result):
+    values = {}
+    for name, number in re.findall(r"^INSTRUMENTATION_STATUS: ironstorage_([a-z0-9_]+)=([0-9]{1,12})$", result, re.MULTILINE):
+        if name in PERFORMANCE_BUDGETS:
+            if name in values:
+                raise RuntimeError("Duplicate performance measurement")
+            values[name] = int(number)
+    if values.keys() != PERFORMANCE_BUDGETS.keys():
+        raise RuntimeError("Missing required journey measurements")
+    if any(values[name] >= bound for name, bound in PERFORMANCE_BUDGETS.items()):
+        raise RuntimeError("Native journey exceeded its performance budget")
+    return values
+
+
 def emulator(serial):
     adb = SDK / "platform-tools/adb"
     def command(*args):
@@ -271,6 +297,7 @@ def emulator(serial):
               ("core", "de.rfc1437.ironstorage.core.test", None),
               ("rust-bridge", "de.rfc1437.ironstorage.rustbridge.test", None)]
     native_timings = []
+    journey_timings = {}
     for module, package, _ in suites:
         test_apk = next((ROOT / "android" / module / "build/outputs/apk/androidTest/debug").glob("*.apk"))
         command("install", "-r", test_apk)
@@ -291,6 +318,8 @@ def emulator(serial):
             raw.write_text(result)
             raw.chmod(0o600)
             raise RuntimeError("ARM emulator instrumentation failed (raw output retained locally only)")
+        if module == "core":
+            journey_timings = performance_metrics(result)
         # Release instrumentation-owned activities and native owners before real app starts.
         command("shell", "am", "force-stop", package)
     # No activity state or application data is deleted between process recreation runs.
@@ -304,18 +333,105 @@ def emulator(serial):
         timings.append(int(values["TotalTime"].strip()))
     output = ROOT / "android/build/ci-artifacts"
     output.mkdir(parents=True, exist_ok=True)
-    (output / f"emulator-api{api}-{pages}.json").write_text(json.dumps({"api": api, "abi": abi, "page_size": pages, "cold_start_ms": timings, "native_load_ns": native_timings, "instrumentation": "passed"}, indent=2) + "\n")
+    (output / f"emulator-api{api}-{pages}.json").write_text(json.dumps({"api": api, "abi": abi, "page_size": pages, "cold_start_ms": timings, "native_load_ns": native_timings, "journey_timings": journey_timings, "instrumentation": "passed"}, indent=2) + "\n")
     checksum_artifacts(output)
     print(f"ARM API {api}, {pages}-byte pages: instrumentation and five cold starts passed")
 
 
+def startup_metrics(data):
+    values = {}
+    for row in data["benchmarks"]:
+        mode = {"coldStart": "cold", "warmStart": "warm"}.get(row["name"])
+        if mode is None:
+            continue
+        if mode in values or row["repeatIterations"] != 5:
+            raise RuntimeError("Startup measurements must have five isolated iterations")
+        samples = {}
+        for metric, key in [("timeToInitialDisplayMs", "initial_display_ms"),
+                            ("timeToFullDisplayMs", "full_display_ms")]:
+            runs = row["metrics"][metric]["runs"]
+            budget = 5000 if mode == "cold" else 3000
+            if len(runs) != 5 or any(type(value) not in (int, float) or
+                                     not math.isfinite(value) or value <= 0 or value >= budget for value in runs):
+                raise RuntimeError("Incomplete or over-budget startup measurement")
+            samples[key] = runs
+        values[mode] = samples
+    if values.keys() != {"cold", "warm"}:
+        raise RuntimeError("Both cold and warm startup measurements are required")
+    return values
+
+
+def performance(serial, profiles=False):
+    adb = SDK / "platform-tools/adb"
+    def command(*args):
+        return run(adb, "-s", serial, *args, capture=True).stdout.strip()
+    abi = command("shell", "getprop", "ro.product.cpu.abi")
+    api = int(command("shell", "getprop", "ro.build.version.sdk"))
+    if abi not in ABIS or command("shell", "getprop", "ro.kernel.qemu") != "1":
+        raise RuntimeError("Performance validation requires an ARM emulator")
+    if profiles and api < 34:
+        raise RuntimeError("Data-preserving baseline generation requires API 34 or newer")
+    variant = "profile" if profiles else "benchmark"
+    gradle(":app:assemble" + variant.capitalize(), ":performance:assembleBenchmark")
+    command("install", "-r", ROOT / f"android/app/build/outputs/apk/{variant}/app-{variant}.apk")
+    harness = "de.rfc1437.ironstorage.performance"
+    command("install", "-r", "-t", ROOT / "android/performance/build/outputs/apk/benchmark/performance-benchmark.apk")
+    directory = "/sdcard/Android/media/" + harness + "/additional_test_output"
+    command("shell", "mkdir", "-p", directory)
+    test = "BaselineProfile" if profiles else "StartupBenchmark"
+    result = command("shell", "am", "instrument", "-w", "-r", "-e", "class", harness + "." + test,
+                     "-e", "additionalTestOutputDir", directory, "-e", "androidx.benchmark.suppressErrors", "EMULATOR",
+                     harness + "/androidx.test.runner.AndroidJUnitRunner")
+    local = ROOT / "android/build/local-validation" / ("performance-" + serial)
+    local.mkdir(parents=True, exist_ok=True)
+    local.chmod(0o700)
+    (local / (test + ".txt")).write_text(result)
+    (local / (test + ".txt")).chmod(0o600)
+    command("shell", "am", "force-stop", harness)
+    if "FAILURES!!!" in result or "INSTRUMENTATION_FAILED" in result or "OK (" not in result:
+        raise RuntimeError("Performance instrumentation failed; raw output stays local")
+    command("pull", directory, local)
+    if not profiles:
+        data = json.loads((local / "additional_test_output" / (harness + "-benchmarkData.json")).read_text())
+        metrics = startup_metrics(data)
+        output = ROOT / "android/build/ci-artifacts"
+        output.mkdir(parents=True, exist_ok=True)
+        (output / f"startup-api{api}-{abi}.json").write_text(json.dumps({"api": api, "abi": abi, "startup": metrics}, allow_nan=False, indent=2) + "\n")
+        checksum_artifacts(output)
+    print(f"ARM API {api}: {test} passed; inspect local artifacts before promoting numeric metrics or generated profiles")
+
+
 class SelfTest(unittest.TestCase):
+    def test_startup_requires_complete_bounded_numeric_samples(self):
+        def data():
+            return {"benchmarks": [{"name": name, "repeatIterations": 5, "metrics": {
+                metric: {"runs": [1, 2, 3, 4, 5]} for metric in ("timeToInitialDisplayMs", "timeToFullDisplayMs")
+            }} for name in ("coldStart", "warmStart")]}
+        self.assertEqual(set(startup_metrics(data())), {"cold", "warm"})
+        for samples in ([1, 2], [1, 2, 3, 4, float("nan")], [1, 2, 3, 4, 5000], [1, 2, 3, 4, "secret"], [1, 2, 3, 4, True]):
+            invalid = data()
+            invalid["benchmarks"][0]["metrics"]["timeToFullDisplayMs"]["runs"] = samples
+            with self.assertRaises(RuntimeError):
+                startup_metrics(invalid)
+        invalid = data()
+        invalid["benchmarks"].append(invalid["benchmarks"][0])
+        with self.assertRaises(RuntimeError):
+            startup_metrics(invalid)
+
+    def test_performance_metrics_retain_only_reviewed_numbers_and_enforce_budgets(self):
+        valid = "\n".join(f"INSTRUMENTATION_STATUS: ironstorage_{name}=1" for name in PERFORMANCE_BUDGETS)
+        self.assertEqual(performance_metrics(valid + "\nINSTRUMENTATION_STATUS: ironstorage_secret=999"), dict.fromkeys(PERFORMANCE_BUDGETS, 1))
+        for invalid in (valid.replace("editor_100_updates_ms=1", "editor_100_updates_ms=500"), valid.replace("qr_analysis_ms=1", "qr_analysis_ms=NaN"), valid + "\nINSTRUMENTATION_STATUS: ironstorage_qr_analysis_ms=1"):
+            with self.assertRaises(RuntimeError):
+                performance_metrics(invalid)
+
+
     def test_isolated_phases_preserve_all_verification_tasks(self):
         phases = verification_phases()
         expected = {f":{module}:{task}" for module in MODULES
                     for task in ("testDebugUnitTest", "ktlintCheck", "lintDebug")}
         expected.update(f":{module}:assembleDebugAndroidTest" for module in ("app", "core", "rust-bridge", "platform"))
-        expected.update((":app:assembleDebug", ":app:assembleRelease", ":app:bundleRelease", "androidVerification"))
+        expected.update((":app:assembleDebug", ":app:assembleRelease", ":app:bundleRelease", ":app:assembleBenchmark", ":app:assembleProfile", ":performance:assembleBenchmark", ":performance:ktlintCheck", "androidVerification"))
         self.assertEqual({task for phase in phases for task in phase}, expected)
         self.assertEqual(phases[-1], ("androidVerification",))
         self.assertTrue(all(len(phase) <= len(MODULES) for phase in phases))
@@ -423,7 +539,7 @@ class SelfTest(unittest.TestCase):
 
 if __name__ == "__main__":
     parser = argparse.ArgumentParser()
-    parser.add_argument("command", choices=["prepare-linux", "build", "emulator", "test", "clean-ci"])
+    parser.add_argument("command", choices=["prepare-linux", "build", "emulator", "performance", "profiles", "test", "clean-ci"])
     parser.add_argument("serial", nargs="?")
     args = parser.parse_args()
     if args.command not in ("test", "clean-ci") and not (os.environ.get("ANDROID_HOME") or os.environ.get("ANDROID_SDK_ROOT")):
@@ -437,6 +553,9 @@ if __name__ == "__main__":
     elif args.command == "build":
         build()
     elif args.serial:
-        emulator(args.serial)
+        if args.command in ("performance", "profiles"):
+            performance(args.serial, profiles=args.command == "profiles")
+        else:
+            emulator(args.serial)
     else:
-        parser.error("emulator requires an ADB serial")
+        parser.error("emulator, performance and profiles require an ADB serial")