#!/usr/bin/env python3 """Build-only orchestration; never shipped or invoked by the Android application.""" import argparse import hashlib import json import os from pathlib import Path import platform import re import runpy import shutil import tarfile import urllib.request import zipfile import xml.etree.ElementTree as ET import subprocess import tempfile import unittest ROOT = Path(__file__).resolve().parents[2] ABIS = ("arm64-v8a", "armeabi-v7a") MODULES = ("app", "core", "features", "platform", "rust-bridge") SDK = Path(os.environ.get("ANDROID_HOME", os.environ.get("ANDROID_SDK_ROOT", ""))) CARGO = os.environ.get("CARGO", "cargo") def run(*args, capture=False, **kwargs): return subprocess.run([str(a) for a in args], check=True, text=True, stdout=subprocess.PIPE if capture else None, **kwargs) def tool(*args): return run(CARGO, "+1.97.1", "run", "--locked", "-p", "ironstorage-android-build", "--", *args) def download(spec, path): if not path.exists(): with urllib.request.urlopen(spec["url"], timeout=60) as response, path.open("wb") as output: shutil.copyfileobj(response, output) if hashlib.sha256(path.read_bytes()).hexdigest() != spec["sha256"]: raise RuntimeError("CI toolchain checksum mismatch") def prepare_linux(): if platform.system() != "Linux" or platform.machine() not in ("aarch64", "arm64"): raise RuntimeError("Native CI toolchains require Linux ARM64") SDK.mkdir(parents=True, exist_ok=True) policy = json.loads((ROOT / "android/ci-toolchains.json").read_text()) with tempfile.TemporaryDirectory(prefix="ironstorage-tools-") as directory: directory = Path(directory) for key, spec in policy.items(): download(spec, directory / key) cli = SDK / "cmdline-tools/latest" if not cli.exists(): destination = SDK / "cmdline-tools" with zipfile.ZipFile(directory / "sdk_cli") as archive: archive.extractall(destination) (destination / "cmdline-tools").rename(cli) for path in (cli / "bin").iterdir(): path.chmod(0o755) ndk = SDK / "ndk/30.0.16248370" if not ndk.exists(): destination = SDK / "ndk" destination.mkdir(parents=True, exist_ok=True) with tarfile.open(directory / "ndk") as archive: archive.extractall(destination, filter="data") (destination / "android-ndk-r30").rename(ndk) prebuilt = ndk / "toolchains/llvm/prebuilt" alias = prebuilt / "linux-x86_64" # cargo-ndk 4.1.2 hardcodes this directory name on Linux; binaries remain native ARM64. if not alias.exists(): alias.symlink_to("linux-arm64", target_is_directory=True) native = SDK / "native-arm64" if not native.exists(): with zipfile.ZipFile(directory / "sdk_native") as archive: archive.extractall(native) for path in native.rglob("*"): if path.is_file(): path.chmod(0o755) run(cli / "bin/sdkmanager", "--licenses", input="y\n" * 100) run(cli / "bin/sdkmanager", "platforms;android-37.0", "build-tools;36.0.0") run(prebuilt / "linux-arm64/bin/clang", "--version") run(prebuilt / "linux-arm64/bin/ld.lld", "--version") run(native / "build-tools/aapt2", "version") def gradle(*args): options = [ROOT / "android/gradlew", "-p", ROOT / "android", "--no-configuration-cache", "--console=plain"] if os.environ.get("CI") == "true": options.extend(("--no-daemon", "-Pkotlin.compiler.execution.strategy=in-process")) if platform.system() == "Linux" and platform.machine() in ("aarch64", "arm64"): options.append("-Pandroid.aapt2FromMavenOverride=" + str(ROOT / ".gitea/scripts/native-arm64/aapt2")) return run(*options, *args, cwd=ROOT) def verification_phases(): return ( tuple(f":{module}:testDebugUnitTest" for module in MODULES), (":app:assembleDebug", ":app:assembleRelease", ":app:bundleRelease"), *((f":{module}:assembleDebugAndroidTest",) for module in ("app", "rust-bridge", "platform")), *((f":{module}:ktlintCheck", f":{module}:lintDebug") for module in MODULES), ("androidVerification",), ) def resources(): facts = {"available_disk_bytes": shutil.disk_usage(ROOT).free} memory = Path("/proc/meminfo") if memory.exists(): facts["memory_kib"] = {key: int(value.split()[0]) for line in memory.read_text().splitlines() for key, value in [line.split(":", 1)] if key in ("MemTotal", "MemAvailable", "SwapTotal", "SwapFree")} for name in ("memory.current", "memory.max", "memory.swap.max", "memory.events"): path = Path("/sys/fs/cgroup") / name if path.exists(): facts[name] = path.read_text().strip() print("CI resource facts " + json.dumps(facts, sort_keys=True), flush=True) def select_symbols(symbols, output): allowed = {f"{abi}/{lib}.dbg" for abi in ABIS for lib in ("libironstorage_android.so", "libjnidispatch.so")} seen = set() with zipfile.ZipFile(symbols) as source, zipfile.ZipFile(output, "w", compression=zipfile.ZIP_DEFLATED) as destination: for name in source.namelist(): if name.split("/", 1)[0] in ABIS: if name not in allowed or name in seen: raise RuntimeError("Unreviewed or duplicate native debug symbol path") seen.add(name) with source.open(name) as original, destination.open(name, "w", force_zip64=True) as target: shutil.copyfileobj(original, target) if seen != allowed: raise RuntimeError("Supported ARM debug symbols missing") def checksum_artifacts(output): checksums = {path.name: hashlib.sha256(path.read_bytes()).hexdigest() for path in sorted(output.iterdir()) if path.is_file() and path.name != "sha256.json"} (output / "sha256.json").write_text(json.dumps(checksums, indent=2) + "\n") def backup_references(manifest, resource_table): application = ET.fromstring(manifest).find("application") if application is None: raise RuntimeError("Packaged application missing") ids = [] for attribute, name in (("fullBackupContent", "backup_rules"), ("dataExtractionRules", "data_extraction_rules")): values = re.findall(r"(?m)^\s*resource (0x[0-9a-f]{8}) xml/" + name + r"(?:\s|$)", resource_table) if len(values) != 1: raise RuntimeError("Backup resource symbol missing or ambiguous") if application.get("{http://schemas.android.com/apk/res/android}" + attribute) != "@ref/" + values[0]: raise RuntimeError("Packaged manifest selects an unreviewed backup resource") ids.extend(values) if len(set(ids)) != 2: raise RuntimeError("Backup resource symbols overlap") def clear_native_intermediates(target): paths = [target / triple for triple in ("aarch64-linux-android", "armv7-linux-androideabi")] if target.resolve() != target.absolute() or any(path.is_symlink() or (path.exists() and not path.is_dir()) for path in paths): raise RuntimeError("Native scratch path must not contain symlinks or unexpected files") for path in paths: if path.exists(): shutil.rmtree(path) def clean_ci(root=ROOT): if os.environ.get("CI") != "true": raise RuntimeError("Build cleanup is restricted to CI checkouts") paths = [root / "target", root / "android/build"] paths.extend(root / "android" / module / "build" for module in MODULES) if any(path.resolve() != path or (path.exists() and not path.is_dir()) for path in paths): raise RuntimeError("CI build directories must not contain symlinks or unexpected files") for path in paths: if path.exists(): shutil.rmtree(path) def build(): ci = os.environ.get("CI") == "true" scratch = ROOT / "android/build/native-cargo" if ci: resources() if scratch.resolve() != scratch: raise RuntimeError("CI Cargo scratch directory must not contain symlinks") os.environ["CARGO_TARGET_DIR"] = str(scratch) # Resolve/check the distribution before spending time compiling the host tool. gradle("--version") # UniFFI generation is memory-heavy; compile host tools before Gradle build JVMs. run(CARGO, "+1.97.1", "build", "--locked", "-p", "ironstorage-android-build", cwd=ROOT) if ci: for variant in ("Debug", "Release"): gradle(f":rust-bridge:prepare{variant}RustBindings") clear_native_intermediates(scratch) if ci: # Release retained JVM/compiler state between memory-heavy build phases. for phase in verification_phases(): print("CI Gradle phase " + " ".join(phase), flush=True) try: gradle(*phase) finally: resources() else: gradle("androidVerification") output = ROOT / "android/build/ci-artifacts" output.mkdir(parents=True, exist_ok=True) apk = ROOT / "android/app/build/outputs/apk/release/app-release-unsigned.apk" symbols = ROOT / "android/app/build/outputs/native-debug-symbols/release/native-debug-symbols.zip" selected_symbols = output / "native-debug-symbols.zip" select_symbols(symbols, selected_symbols) tool("audit-release", apk, selected_symbols) analyzer = SDK / "cmdline-tools/latest/bin/apkanalyzer" aapt2 = ROOT / ".gitea/scripts/native-arm64/aapt2" if platform.system() == "Linux" else SDK / "build-tools/36.0.0/aapt2" for variant, path in [("debug", ROOT / "android/app/build/outputs/apk/debug/app-debug.apk"), ("release", apk)]: manifest = output / (variant + "-manifest.xml") manifest.write_text(run(analyzer, "manifest", "print", path, capture=True).stdout) backup_references(manifest.read_text(), run(aapt2, "dump", "resources", path, capture=True).stdout) rules = [] for name in ("backup_rules", "data_extraction_rules"): resource = run(analyzer, "resources", "value", "--type", "xml", "--config", "default", "--name", name, path, capture=True).stdout.strip() decoded = output / f"{variant}-{name}.xml" decoded.write_text(run(analyzer, "resources", "xml", "--file", resource, path, capture=True).stdout) rules.append(decoded) tool("audit-manifest", manifest, *rules) trees = [] for target in ("aarch64-linux-android", "armv7-linux-androideabi"): trees.append(run(CARGO, "+1.97.1", "tree", "--locked", "-p", "ironstorage-android", "--target", target, "--edges", "normal", "--prefix", "none", "--format", "{p}|{l}|{r}", capture=True, cwd=ROOT).stdout) tree = ROOT / "android/build/rust-dependencies.txt" tree.write_text("\n".join(trees)) inventory = ROOT / "android/build/reports/android-dependencies.json" rows = [] for module in MODULES: rows.extend(json.loads((ROOT / "android" / module / "build/reports/android-dependencies.json").read_text())) inventory.parent.mkdir(parents=True, exist_ok=True) inventory.write_text(json.dumps(rows, sort_keys=True) + "\n") tool("sbom", inventory, tree, ROOT / "Cargo.lock", ROOT / "android/dependency-policy.json", output / "sbom.cdx.json") for module in MODULES: reports = ROOT / "android" / module / "build/test-results" if module in ("app", "core") or list(reports.rglob("TEST-*.xml")): tool("safe-reports", reports, output / (module + "-unit-tests.json")) files = [ROOT / "android/app/build/outputs/apk/debug/app-debug.apk", apk, ROOT / "android/app/build/outputs/bundle/release/app-release.aab", ROOT / "android/app/build/outputs/mapping/release/mapping.txt"] for path in files: shutil.copy2(path, output / path.name) checksum_artifacts(output) def emulator(serial): adb = SDK / "platform-tools/adb" def command(*args): return run(adb, "-s", serial, *args, capture=True).stdout.strip() abi = command("shell", "getprop", "ro.product.cpu.abi") if abi not in ABIS: raise RuntimeError("Only ARM emulators are in scope") if command("shell", "getprop", "ro.kernel.qemu") != "1": raise RuntimeError("Physical devices are outside validation scope") api = int(command("shell", "getprop", "ro.build.version.sdk")) try: pages = int(command("shell", "getconf", "PAGE_SIZE")) except subprocess.CalledProcessError: sizes = {int(line.split()[1]) * 1024 for line in command("shell", "cat", "/proc/self/smaps").splitlines() if line.startswith("KernelPageSize:")} if len(sizes) != 1: raise RuntimeError("Cannot determine emulator kernel page size") pages = sizes.pop() if pages == 16384: command("shell", "setprop", "bionic.linker.16kb.app_compat.enabled", "false") command("shell", "setprop", "pm.16kb.app_compat.disabled", "true") apk = ROOT / "android/app/build/outputs/apk/debug/app-debug.apk" command("install", "-r", apk) # Android's test runner and Rust fixtures cover typed errors, callbacks and cancellation. suites = [("app", "de.rfc1437.ironstorage.test", None), ("rust-bridge", "de.rfc1437.ironstorage.rustbridge.test", None)] native_timings = [] for module, package, _ in suites: test_apk = next((ROOT / "android" / module / "build/outputs/apk/androidTest/debug").glob("*.apk")) command("install", "-r", test_apk) if module == "rust-bridge": for _ in range(5): command("shell", "am", "force-stop", package) measured = command("shell", "am", "instrument", "-w", "-r", "-e", "class", "de.rfc1437.ironstorage.bridge.NativeLoadPerformanceTest", package + "/androidx.test.runner.AndroidJUnitRunner") samples = [int(line.split("=", 1)[1]) for line in measured.splitlines() if line.startswith("INSTRUMENTATION_STATUS: ironstorage_native_load_ns=")] if len(samples) != 1 or "OK (1 test)" not in measured: raise RuntimeError("Native cold-load measurement failed") native_timings.extend(samples) result = command("shell", "am", "instrument", "-w", "-r", package + "/androidx.test.runner.AndroidJUnitRunner") if "FAILURES!!!" in result or "INSTRUMENTATION_FAILED" in result or "OK (" not in result: local = ROOT / "android/build/local-validation" local.mkdir(parents=True, exist_ok=True) local.chmod(0o700) raw = local / f"{serial}-{module}-failure.txt" raw.write_text(result) raw.chmod(0o600) raise RuntimeError("ARM emulator instrumentation failed (raw output retained locally only)") # No activity state or application data is deleted between process recreation runs. timings = [] for _ in range(5): command("shell", "am", "force-stop", "de.rfc1437.ironstorage") result = command("shell", "am", "start", "-W", "-n", "de.rfc1437.ironstorage/.MainActivity") values = dict(line.split(":", 1) for line in result.splitlines() if ":" in line) if values.get("Status", "").strip() != "ok": raise RuntimeError("Cold start failed") timings.append(int(values["TotalTime"].strip())) output = ROOT / "android/build/ci-artifacts" output.mkdir(parents=True, exist_ok=True) (output / f"emulator-api{api}-{pages}.json").write_text(json.dumps({"api": api, "abi": abi, "page_size": pages, "cold_start_ms": timings, "native_load_ns": native_timings, "instrumentation": "passed"}, indent=2) + "\n") checksum_artifacts(output) print(f"ARM API {api}, {pages}-byte pages: instrumentation and five cold starts passed") class SelfTest(unittest.TestCase): def test_isolated_phases_preserve_all_verification_tasks(self): phases = verification_phases() expected = {f":{module}:{task}" for module in MODULES for task in ("testDebugUnitTest", "ktlintCheck", "lintDebug")} expected.update(f":{module}:assembleDebugAndroidTest" for module in ("app", "rust-bridge", "platform")) expected.update((":app:assembleDebug", ":app:assembleRelease", ":app:bundleRelease", "androidVerification")) self.assertEqual({task for phase in phases for task in phase}, expected) self.assertEqual(phases[-1], ("androidVerification",)) self.assertTrue(all(len(phase) <= len(MODULES) for phase in phases)) def test_ci_cleanup_refuses_local_or_redirected_build_directories(self): previous = os.environ.get("CI") try: with tempfile.TemporaryDirectory() as directory: root = Path(directory).resolve() (root / "target").mkdir() (root / "target/public-fixture").write_text("artifact") os.environ.pop("CI", None) with self.assertRaisesRegex(RuntimeError, "restricted to CI"): clean_ci(root) self.assertTrue((root / "target/public-fixture").exists()) os.environ["CI"] = "true" (root / "android/app").mkdir(parents=True) (root / "android/app/build").symlink_to(root / "target", target_is_directory=True) with self.assertRaisesRegex(RuntimeError, "symlinks"): clean_ci(root) self.assertTrue((root / "target/public-fixture").exists()) (root / "android/app/build").unlink() (root / "android/app/build").mkdir() (root / "android/app/src").mkdir() clean_ci(root) self.assertFalse((root / "target").exists()) self.assertFalse((root / "android/app/build").exists()) self.assertTrue((root / "android/app/src").is_dir()) finally: if previous is None: os.environ.pop("CI", None) else: os.environ["CI"] = previous def test_native_scratch_cleanup_keeps_host_tools_and_rejects_symlinks(self): with tempfile.TemporaryDirectory() as directory: target = Path(directory).resolve() triples = ("aarch64-linux-android", "armv7-linux-androideabi") for name in (*triples, "debug"): (target / name).mkdir() (target / name / "public-fixture").write_text("artifact") clear_native_intermediates(target) self.assertTrue((target / "debug/public-fixture").is_file()) self.assertTrue(all(not (target / name).exists() for name in triples)) (target / triples[0]).symlink_to(target / "debug", target_is_directory=True) with self.assertRaisesRegex(RuntimeError, "symlinks"): clear_native_intermediates(target) self.assertTrue((target / "debug/public-fixture").is_file()) def test_manifest_must_select_the_audited_backup_resources(self): manifest = '' symbols = " resource 0x7f110000 xml/backup_rules\n resource 0x7f110001 xml/data_extraction_rules\n" backup_references(manifest, symbols) for wrong in (manifest.replace("0x7f110000", "0x7f110002"), manifest.replace("@ref/0x7f110001", "false")): with self.assertRaisesRegex(RuntimeError, "unreviewed backup resource"): backup_references(wrong, symbols) for wrong in (symbols.replace("backup_rules", "other_rules"), symbols + symbols, symbols.replace("0x7f110001", "0x7f110000")): with self.assertRaises(RuntimeError): backup_references(manifest, wrong) with self.assertRaisesRegex(RuntimeError, "symbols overlap"): backup_references(manifest.replace("0x7f110001", "0x7f110000"), symbols.replace("0x7f110001", "0x7f110000")) def test_aapt2_adapter_preserves_paths_and_other_options(self): adapter = runpy.run_path(ROOT / ".gitea/scripts/native-arm64/aapt2") self.assertEqual(adapter["arguments"](["optimize", "input.apk", "--resource-path-shortening-map=/tmp/map with spaces=ok", "--shorten-resource-paths", "-o", "output.apk"]), ["optimize", "input.apk", "--resource-path-shortening-map", "/tmp/map with spaces=ok", "--shorten-resource-paths", "-o", "output.apk"]) self.assertEqual(adapter["arguments"](["daemon"]), ["daemon"]) def test_toolchain_checksum_rejects_changed_archive(self): with tempfile.TemporaryDirectory() as directory: path = Path(directory) / "fixture" path.write_bytes(b"public archive fixture") spec = {"sha256": hashlib.sha256(path.read_bytes()).hexdigest()} download(spec, path) path.write_bytes(b"changed") with self.assertRaisesRegex(RuntimeError, "checksum mismatch"): download(spec, path) def test_symbol_selection_and_repeatable_checksum_manifest(self): with tempfile.TemporaryDirectory() as directory: directory = Path(directory) source = directory / "source.zip" output = directory / "output.zip" names = [f"{abi}/{lib}.dbg" for abi in ABIS for lib in ("libironstorage_android.so", "libjnidispatch.so")] with zipfile.ZipFile(source, "w") as archive: for name in names + ["unused/libjnidispatch.so.dbg"]: archive.writestr(name, b"public symbols") select_symbols(source, output) with zipfile.ZipFile(output) as archive: self.assertEqual(set(archive.namelist()), set(names)) checksum_artifacts(directory) first = (directory / "sha256.json").read_bytes() checksum_artifacts(directory) self.assertEqual(first, (directory / "sha256.json").read_bytes()) self.assertNotIn("sha256.json", json.loads(first)) with zipfile.ZipFile(source, "w") as archive: archive.writestr(names[0], b"public symbols") with self.assertRaisesRegex(RuntimeError, "missing"): select_symbols(source, output) with zipfile.ZipFile(source, "w") as archive: archive.writestr("arm64-v8a/unreviewed.dbg", b"public symbols") with self.assertRaisesRegex(RuntimeError, "Unreviewed"): select_symbols(source, output) if __name__ == "__main__": parser = argparse.ArgumentParser() parser.add_argument("command", choices=["prepare-linux", "build", "emulator", "test", "clean-ci"]) parser.add_argument("serial", nargs="?") args = parser.parse_args() if args.command not in ("test", "clean-ci") and not (os.environ.get("ANDROID_HOME") or os.environ.get("ANDROID_SDK_ROOT")): parser.error("ANDROID_HOME or ANDROID_SDK_ROOT is required") if args.command == "test": unittest.main(argv=[__file__]) elif args.command == "clean-ci": clean_ci() elif args.command == "prepare-linux": prepare_linux() elif args.command == "build": build() elif args.serial: emulator(args.serial) else: parser.error("emulator requires an ADB serial")