Establish native Linux ARM64 Android CI and verification gates (#94)

.gitea/scripts/android_ci.py

@@ -0,0 +1,392 @@
+#!/usr/bin/env python3
+"""Build-only orchestration; never shipped or invoked by the Android application."""
+import argparse
+import hashlib
+import json
+import os
+from pathlib import Path
+import platform
+import re
+import runpy
+import shutil
+import tarfile
+import urllib.request
+import zipfile
+import xml.etree.ElementTree as ET
+import subprocess
+import tempfile
+import unittest
+
+ROOT = Path(__file__).resolve().parents[2]
+ABIS = ("arm64-v8a", "armeabi-v7a")
+SDK = Path(os.environ.get("ANDROID_HOME", os.environ.get("ANDROID_SDK_ROOT", "")))
+CARGO = os.environ.get("CARGO", "cargo")
+
+
+def run(*args, capture=False, **kwargs):
+    return subprocess.run([str(a) for a in args], check=True, text=True,
+                          stdout=subprocess.PIPE if capture else None, **kwargs)
+
+
+def tool(*args):
+    return run(CARGO, "+1.97.1", "run", "--locked", "-p", "ironstorage-android-build", "--", *args)
+
+
+def download(spec, path):
+    if not path.exists():
+        with urllib.request.urlopen(spec["url"], timeout=60) as response, path.open("wb") as output:
+            shutil.copyfileobj(response, output)
+    if hashlib.sha256(path.read_bytes()).hexdigest() != spec["sha256"]:
+        raise RuntimeError("CI toolchain checksum mismatch")
+
+
+def prepare_linux():
+    if platform.system() != "Linux" or platform.machine() not in ("aarch64", "arm64"):
+        raise RuntimeError("Native CI toolchains require Linux ARM64")
+    SDK.mkdir(parents=True, exist_ok=True)
+    policy = json.loads((ROOT / "android/ci-toolchains.json").read_text())
+    with tempfile.TemporaryDirectory(prefix="ironstorage-tools-") as directory:
+        directory = Path(directory)
+        for key, spec in policy.items():
+            download(spec, directory / key)
+        cli = SDK / "cmdline-tools/latest"
+        if not cli.exists():
+            destination = SDK / "cmdline-tools"
+            with zipfile.ZipFile(directory / "sdk_cli") as archive:
+                archive.extractall(destination)
+            (destination / "cmdline-tools").rename(cli)
+            for path in (cli / "bin").iterdir():
+                path.chmod(0o755)
+        ndk = SDK / "ndk/30.0.16248370"
+        if not ndk.exists():
+            destination = SDK / "ndk"
+            destination.mkdir(parents=True, exist_ok=True)
+            with tarfile.open(directory / "ndk") as archive:
+                archive.extractall(destination, filter="data")
+            (destination / "android-ndk-r30").rename(ndk)
+        prebuilt = ndk / "toolchains/llvm/prebuilt"
+        alias = prebuilt / "linux-x86_64"
+        # cargo-ndk 4.1.2 hardcodes this directory name on Linux; binaries remain native ARM64.
+        if not alias.exists():
+            alias.symlink_to("linux-arm64", target_is_directory=True)
+        native = SDK / "native-arm64"
+        if not native.exists():
+            with zipfile.ZipFile(directory / "sdk_native") as archive:
+                archive.extractall(native)
+            for path in native.rglob("*"):
+                if path.is_file():
+                    path.chmod(0o755)
+    run(cli / "bin/sdkmanager", "--licenses", input="y\n" * 100)
+    run(cli / "bin/sdkmanager", "platforms;android-37.0", "build-tools;36.0.0")
+    run(prebuilt / "linux-arm64/bin/clang", "--version")
+    run(prebuilt / "linux-arm64/bin/ld.lld", "--version")
+    run(native / "build-tools/aapt2", "version")
+
+
+def gradle(*args):
+    options = [ROOT / "android/gradlew", "-p", ROOT / "android", "--no-configuration-cache", "--console=plain"]
+    if platform.system() == "Linux" and platform.machine() in ("aarch64", "arm64"):
+        options.append("-Pandroid.aapt2FromMavenOverride=" + str(ROOT / ".gitea/scripts/native-arm64/aapt2"))
+    return run(*options, *args, cwd=ROOT)
+
+
+def select_symbols(symbols, output):
+    allowed = {f"{abi}/{lib}.dbg" for abi in ABIS for lib in ("libironstorage_android.so", "libjnidispatch.so")}
+    seen = set()
+    with zipfile.ZipFile(symbols) as source, zipfile.ZipFile(output, "w", compression=zipfile.ZIP_DEFLATED) as destination:
+        for name in source.namelist():
+            if name.split("/", 1)[0] in ABIS:
+                if name not in allowed or name in seen:
+                    raise RuntimeError("Unreviewed or duplicate native debug symbol path")
+                seen.add(name)
+                with source.open(name) as original, destination.open(name, "w", force_zip64=True) as target:
+                    shutil.copyfileobj(original, target)
+    if seen != allowed:
+        raise RuntimeError("Supported ARM debug symbols missing")
+
+
+def checksum_artifacts(output):
+    checksums = {path.name: hashlib.sha256(path.read_bytes()).hexdigest() for path in sorted(output.iterdir()) if path.is_file() and path.name != "sha256.json"}
+    (output / "sha256.json").write_text(json.dumps(checksums, indent=2) + "\n")
+
+
+def backup_references(manifest, resource_table):
+    application = ET.fromstring(manifest).find("application")
+    if application is None:
+        raise RuntimeError("Packaged application missing")
+    ids = []
+    for attribute, name in (("fullBackupContent", "backup_rules"), ("dataExtractionRules", "data_extraction_rules")):
+        values = re.findall(r"(?m)^\s*resource (0x[0-9a-f]{8}) xml/" + name + r"(?:\s|$)", resource_table)
+        if len(values) != 1:
+            raise RuntimeError("Backup resource symbol missing or ambiguous")
+        if application.get("{http://schemas.android.com/apk/res/android}" + attribute) != "@ref/" + values[0]:
+            raise RuntimeError("Packaged manifest selects an unreviewed backup resource")
+        ids.extend(values)
+    if len(set(ids)) != 2:
+        raise RuntimeError("Backup resource symbols overlap")
+
+
+def clear_native_intermediates(target):
+    paths = [target / triple for triple in ("aarch64-linux-android", "armv7-linux-androideabi")]
+    if target.resolve() != target.absolute() or any(path.is_symlink() or (path.exists() and not path.is_dir()) for path in paths):
+        raise RuntimeError("Native scratch path must not contain symlinks or unexpected files")
+    for path in paths:
+        if path.exists():
+            shutil.rmtree(path)
+
+
+def clean_ci(root=ROOT):
+    if os.environ.get("CI") != "true":
+        raise RuntimeError("Build cleanup is restricted to CI checkouts")
+    paths = [root / "target", root / "android/build"]
+    paths.extend(root / "android" / module / "build" for module in ("app", "core", "features", "platform", "rust-bridge"))
+    if any(path.resolve() != path or (path.exists() and not path.is_dir()) for path in paths):
+        raise RuntimeError("CI build directories must not contain symlinks or unexpected files")
+    for path in paths:
+        if path.exists():
+            shutil.rmtree(path)
+
+
+def build():
+    ci = os.environ.get("CI") == "true"
+    scratch = ROOT / "android/build/native-cargo"
+    if ci:
+        if scratch.resolve() != scratch:
+            raise RuntimeError("CI Cargo scratch directory must not contain symlinks")
+        os.environ["CARGO_TARGET_DIR"] = str(scratch)
+    # Resolve/check the distribution before spending time compiling the host tool.
+    gradle("--version")
+    # UniFFI generation is memory-heavy; compile host tools before Gradle build JVMs.
+    run(CARGO, "+1.97.1", "build", "--locked", "-p", "ironstorage-android-build", cwd=ROOT)
+    if ci:
+        for variant in ("Debug", "Release"):
+            gradle(f":rust-bridge:prepare{variant}RustBindings")
+            clear_native_intermediates(scratch)
+    gradle("androidVerification")
+    output = ROOT / "android/build/ci-artifacts"
+    output.mkdir(parents=True, exist_ok=True)
+    apk = ROOT / "android/app/build/outputs/apk/release/app-release-unsigned.apk"
+    symbols = ROOT / "android/app/build/outputs/native-debug-symbols/release/native-debug-symbols.zip"
+    selected_symbols = output / "native-debug-symbols.zip"
+    select_symbols(symbols, selected_symbols)
+    tool("audit-release", apk, selected_symbols)
+    analyzer = SDK / "cmdline-tools/latest/bin/apkanalyzer"
+    aapt2 = ROOT / ".gitea/scripts/native-arm64/aapt2" if platform.system() == "Linux" else SDK / "build-tools/36.0.0/aapt2"
+    for variant, path in [("debug", ROOT / "android/app/build/outputs/apk/debug/app-debug.apk"), ("release", apk)]:
+        manifest = output / (variant + "-manifest.xml")
+        manifest.write_text(run(analyzer, "manifest", "print", path, capture=True).stdout)
+        backup_references(manifest.read_text(), run(aapt2, "dump", "resources", path, capture=True).stdout)
+        rules = []
+        for name in ("backup_rules", "data_extraction_rules"):
+            resource = run(analyzer, "resources", "value", "--type", "xml", "--config", "default", "--name", name, path, capture=True).stdout.strip()
+            decoded = output / f"{variant}-{name}.xml"
+            decoded.write_text(run(analyzer, "resources", "xml", "--file", resource, path, capture=True).stdout)
+            rules.append(decoded)
+        tool("audit-manifest", manifest, *rules)
+    trees = []
+    for target in ("aarch64-linux-android", "armv7-linux-androideabi"):
+        trees.append(run(CARGO, "+1.97.1", "tree", "--locked", "-p", "ironstorage-android", "--target", target, "--edges", "normal", "--prefix", "none", "--format", "{p}|{l}|{r}", capture=True, cwd=ROOT).stdout)
+    tree = ROOT / "android/build/rust-dependencies.txt"
+    tree.write_text("\n".join(trees))
+    inventory = ROOT / "android/build/reports/android-dependencies.json"
+    rows = []
+    for module in ("app", "core", "features", "platform", "rust-bridge"):
+        rows.extend(json.loads((ROOT / "android" / module / "build/reports/android-dependencies.json").read_text()))
+    inventory.parent.mkdir(parents=True, exist_ok=True)
+    inventory.write_text(json.dumps(rows, sort_keys=True) + "\n")
+    tool("sbom", inventory, tree, ROOT / "Cargo.lock", ROOT / "android/dependency-policy.json", output / "sbom.cdx.json")
+    for module in ("app", "core", "features", "platform", "rust-bridge"):
+        reports = ROOT / "android" / module / "build/test-results"
+        if module in ("app", "core") or list(reports.rglob("TEST-*.xml")):
+            tool("safe-reports", reports, output / (module + "-unit-tests.json"))
+    files = [ROOT / "android/app/build/outputs/apk/debug/app-debug.apk", apk,
+             ROOT / "android/app/build/outputs/bundle/release/app-release.aab",
+             ROOT / "android/app/build/outputs/mapping/release/mapping.txt"]
+    for path in files:
+        shutil.copy2(path, output / path.name)
+    checksum_artifacts(output)
+
+
+def emulator(serial):
+    adb = SDK / "platform-tools/adb"
+    def command(*args):
+        return run(adb, "-s", serial, *args, capture=True).stdout.strip()
+    abi = command("shell", "getprop", "ro.product.cpu.abi")
+    if abi not in ABIS:
+        raise RuntimeError("Only ARM emulators are in scope")
+    if command("shell", "getprop", "ro.kernel.qemu") != "1":
+        raise RuntimeError("Physical devices are outside validation scope")
+    api = int(command("shell", "getprop", "ro.build.version.sdk"))
+    try:
+        pages = int(command("shell", "getconf", "PAGE_SIZE"))
+    except subprocess.CalledProcessError:
+        sizes = {int(line.split()[1]) * 1024 for line in command("shell", "cat", "/proc/self/smaps").splitlines() if line.startswith("KernelPageSize:")}
+        if len(sizes) != 1:
+            raise RuntimeError("Cannot determine emulator kernel page size")
+        pages = sizes.pop()
+    if pages == 16384:
+        command("shell", "setprop", "bionic.linker.16kb.app_compat.enabled", "false")
+        command("shell", "setprop", "pm.16kb.app_compat.disabled", "true")
+    apk = ROOT / "android/app/build/outputs/apk/debug/app-debug.apk"
+    command("install", "-r", apk)
+    # Android's test runner and Rust fixtures cover typed errors, callbacks and cancellation.
+    suites = [("app", "de.rfc1437.ironstorage.test", None),
+              ("rust-bridge", "de.rfc1437.ironstorage.rustbridge.test", None)]
+    native_timings = []
+    for module, package, _ in suites:
+        test_apk = next((ROOT / "android" / module / "build/outputs/apk/androidTest/debug").glob("*.apk"))
+        command("install", "-r", test_apk)
+        if module == "rust-bridge":
+            for _ in range(5):
+                command("shell", "am", "force-stop", package)
+                measured = command("shell", "am", "instrument", "-w", "-r", "-e", "class", "de.rfc1437.ironstorage.bridge.NativeLoadPerformanceTest", package + "/androidx.test.runner.AndroidJUnitRunner")
+                samples = [int(line.split("=", 1)[1]) for line in measured.splitlines() if line.startswith("INSTRUMENTATION_STATUS: ironstorage_native_load_ns=")]
+                if len(samples) != 1 or "OK (1 test)" not in measured:
+                    raise RuntimeError("Native cold-load measurement failed")
+                native_timings.extend(samples)
+        result = command("shell", "am", "instrument", "-w", "-r", package + "/androidx.test.runner.AndroidJUnitRunner")
+        if "FAILURES!!!" in result or "INSTRUMENTATION_FAILED" in result or "OK (" not in result:
+            local = ROOT / "android/build/local-validation"
+            local.mkdir(parents=True, exist_ok=True)
+            local.chmod(0o700)
+            raw = local / f"{serial}-{module}-failure.txt"
+            raw.write_text(result)
+            raw.chmod(0o600)
+            raise RuntimeError("ARM emulator instrumentation failed (raw output retained locally only)")
+    # No activity state or application data is deleted between process recreation runs.
+    timings = []
+    for _ in range(5):
+        command("shell", "am", "force-stop", "de.rfc1437.ironstorage")
+        result = command("shell", "am", "start", "-W", "-n", "de.rfc1437.ironstorage/.MainActivity")
+        values = dict(line.split(":", 1) for line in result.splitlines() if ":" in line)
+        if values.get("Status", "").strip() != "ok":
+            raise RuntimeError("Cold start failed")
+        timings.append(int(values["TotalTime"].strip()))
+    output = ROOT / "android/build/ci-artifacts"
+    output.mkdir(parents=True, exist_ok=True)
+    (output / f"emulator-api{api}-{pages}.json").write_text(json.dumps({"api": api, "abi": abi, "page_size": pages, "cold_start_ms": timings, "native_load_ns": native_timings, "instrumentation": "passed"}, indent=2) + "\n")
+    checksum_artifacts(output)
+    print(f"ARM API {api}, {pages}-byte pages: instrumentation and five cold starts passed")
+
+
+class SelfTest(unittest.TestCase):
+    def test_ci_cleanup_refuses_local_or_redirected_build_directories(self):
+        previous = os.environ.get("CI")
+        try:
+            with tempfile.TemporaryDirectory() as directory:
+                root = Path(directory).resolve()
+                (root / "target").mkdir()
+                (root / "target/public-fixture").write_text("artifact")
+                os.environ.pop("CI", None)
+                with self.assertRaisesRegex(RuntimeError, "restricted to CI"):
+                    clean_ci(root)
+                self.assertTrue((root / "target/public-fixture").exists())
+                os.environ["CI"] = "true"
+                (root / "android/app").mkdir(parents=True)
+                (root / "android/app/build").symlink_to(root / "target", target_is_directory=True)
+                with self.assertRaisesRegex(RuntimeError, "symlinks"):
+                    clean_ci(root)
+                self.assertTrue((root / "target/public-fixture").exists())
+                (root / "android/app/build").unlink()
+                (root / "android/app/build").mkdir()
+                (root / "android/app/src").mkdir()
+                clean_ci(root)
+                self.assertFalse((root / "target").exists())
+                self.assertFalse((root / "android/app/build").exists())
+                self.assertTrue((root / "android/app/src").is_dir())
+        finally:
+            if previous is None:
+                os.environ.pop("CI", None)
+            else:
+                os.environ["CI"] = previous
+
+    def test_native_scratch_cleanup_keeps_host_tools_and_rejects_symlinks(self):
+        with tempfile.TemporaryDirectory() as directory:
+            target = Path(directory).resolve()
+            triples = ("aarch64-linux-android", "armv7-linux-androideabi")
+            for name in (*triples, "debug"):
+                (target / name).mkdir()
+                (target / name / "public-fixture").write_text("artifact")
+            clear_native_intermediates(target)
+            self.assertTrue((target / "debug/public-fixture").is_file())
+            self.assertTrue(all(not (target / name).exists() for name in triples))
+            (target / triples[0]).symlink_to(target / "debug", target_is_directory=True)
+            with self.assertRaisesRegex(RuntimeError, "symlinks"):
+                clear_native_intermediates(target)
+            self.assertTrue((target / "debug/public-fixture").is_file())
+
+    def test_manifest_must_select_the_audited_backup_resources(self):
+        manifest = '<manifest xmlns:android="http://schemas.android.com/apk/res/android"><application android:fullBackupContent="@ref/0x7f110000" android:dataExtractionRules="@ref/0x7f110001"/></manifest>'
+        symbols = "    resource 0x7f110000 xml/backup_rules\n    resource 0x7f110001 xml/data_extraction_rules\n"
+        backup_references(manifest, symbols)
+        for wrong in (manifest.replace("0x7f110000", "0x7f110002"), manifest.replace("@ref/0x7f110001", "false")):
+            with self.assertRaisesRegex(RuntimeError, "unreviewed backup resource"):
+                backup_references(wrong, symbols)
+        for wrong in (symbols.replace("backup_rules", "other_rules"), symbols + symbols, symbols.replace("0x7f110001", "0x7f110000")):
+            with self.assertRaises(RuntimeError):
+                backup_references(manifest, wrong)
+        with self.assertRaisesRegex(RuntimeError, "symbols overlap"):
+            backup_references(manifest.replace("0x7f110001", "0x7f110000"), symbols.replace("0x7f110001", "0x7f110000"))
+
+    def test_aapt2_adapter_preserves_paths_and_other_options(self):
+        adapter = runpy.run_path(ROOT / ".gitea/scripts/native-arm64/aapt2")
+        self.assertEqual(adapter["arguments"](["optimize", "input.apk", "--resource-path-shortening-map=/tmp/map with spaces=ok", "--shorten-resource-paths", "-o", "output.apk"]),
+                         ["optimize", "input.apk", "--resource-path-shortening-map", "/tmp/map with spaces=ok", "--shorten-resource-paths", "-o", "output.apk"])
+        self.assertEqual(adapter["arguments"](["daemon"]), ["daemon"])
+
+    def test_toolchain_checksum_rejects_changed_archive(self):
+        with tempfile.TemporaryDirectory() as directory:
+            path = Path(directory) / "fixture"
+            path.write_bytes(b"public archive fixture")
+            spec = {"sha256": hashlib.sha256(path.read_bytes()).hexdigest()}
+            download(spec, path)
+            path.write_bytes(b"changed")
+            with self.assertRaisesRegex(RuntimeError, "checksum mismatch"):
+                download(spec, path)
+
+    def test_symbol_selection_and_repeatable_checksum_manifest(self):
+        with tempfile.TemporaryDirectory() as directory:
+            directory = Path(directory)
+            source = directory / "source.zip"
+            output = directory / "output.zip"
+            names = [f"{abi}/{lib}.dbg" for abi in ABIS for lib in ("libironstorage_android.so", "libjnidispatch.so")]
+            with zipfile.ZipFile(source, "w") as archive:
+                for name in names + ["unused/libjnidispatch.so.dbg"]:
+                    archive.writestr(name, b"public symbols")
+            select_symbols(source, output)
+            with zipfile.ZipFile(output) as archive:
+                self.assertEqual(set(archive.namelist()), set(names))
+            checksum_artifacts(directory)
+            first = (directory / "sha256.json").read_bytes()
+            checksum_artifacts(directory)
+            self.assertEqual(first, (directory / "sha256.json").read_bytes())
+            self.assertNotIn("sha256.json", json.loads(first))
+            with zipfile.ZipFile(source, "w") as archive:
+                archive.writestr(names[0], b"public symbols")
+            with self.assertRaisesRegex(RuntimeError, "missing"):
+                select_symbols(source, output)
+            with zipfile.ZipFile(source, "w") as archive:
+                archive.writestr("arm64-v8a/unreviewed.dbg", b"public symbols")
+            with self.assertRaisesRegex(RuntimeError, "Unreviewed"):
+                select_symbols(source, output)
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+    parser.add_argument("command", choices=["prepare-linux", "build", "emulator", "test", "clean-ci"])
+    parser.add_argument("serial", nargs="?")
+    args = parser.parse_args()
+    if args.command not in ("test", "clean-ci") and not (os.environ.get("ANDROID_HOME") or os.environ.get("ANDROID_SDK_ROOT")):
+        parser.error("ANDROID_HOME or ANDROID_SDK_ROOT is required")
+    if args.command == "test":
+        unittest.main(argv=[__file__])
+    elif args.command == "clean-ci":
+        clean_ci()
+    elif args.command == "prepare-linux":
+        prepare_linux()
+    elif args.command == "build":
+        build()
+    elif args.serial:
+        emulator(args.serial)
+    else:
+        parser.error("emulator requires an ADB serial")