272 lines
14 KiB
Python
272 lines
14 KiB
Python
#!/usr/bin/env python3
|
|
"""Build-only orchestration; never shipped or invoked by the Android application."""
|
|
import argparse
|
|
import hashlib
|
|
import json
|
|
import os
|
|
from pathlib import Path
|
|
import platform
|
|
import runpy
|
|
import shutil
|
|
import tarfile
|
|
import urllib.request
|
|
import zipfile
|
|
import subprocess
|
|
import tempfile
|
|
import unittest
|
|
|
|
ROOT = Path(__file__).resolve().parents[2]
|
|
ABIS = ("arm64-v8a", "armeabi-v7a")
|
|
SDK = Path(os.environ.get("ANDROID_HOME", os.environ.get("ANDROID_SDK_ROOT", "")))
|
|
CARGO = os.environ.get("CARGO", "cargo")
|
|
|
|
|
|
def run(*args, capture=False, **kwargs):
|
|
return subprocess.run([str(a) for a in args], check=True, text=True,
|
|
stdout=subprocess.PIPE if capture else None, **kwargs)
|
|
|
|
|
|
def tool(*args):
|
|
return run(CARGO, "+1.97.1", "run", "--locked", "-p", "ironstorage-android-build", "--", *args)
|
|
|
|
|
|
def download(spec, path):
|
|
if not path.exists():
|
|
with urllib.request.urlopen(spec["url"], timeout=60) as response, path.open("wb") as output:
|
|
shutil.copyfileobj(response, output)
|
|
if hashlib.sha256(path.read_bytes()).hexdigest() != spec["sha256"]:
|
|
raise RuntimeError("CI toolchain checksum mismatch")
|
|
|
|
|
|
def prepare_linux():
|
|
if platform.system() != "Linux" or platform.machine() not in ("aarch64", "arm64"):
|
|
raise RuntimeError("Native CI toolchains require Linux ARM64")
|
|
SDK.mkdir(parents=True, exist_ok=True)
|
|
policy = json.loads((ROOT / "android/ci-toolchains.json").read_text())
|
|
with tempfile.TemporaryDirectory(prefix="ironstorage-tools-") as directory:
|
|
directory = Path(directory)
|
|
for key, spec in policy.items():
|
|
download(spec, directory / key)
|
|
cli = SDK / "cmdline-tools/latest"
|
|
if not cli.exists():
|
|
destination = SDK / "cmdline-tools"
|
|
with zipfile.ZipFile(directory / "sdk_cli") as archive:
|
|
archive.extractall(destination)
|
|
(destination / "cmdline-tools").rename(cli)
|
|
for path in (cli / "bin").iterdir():
|
|
path.chmod(0o755)
|
|
ndk = SDK / "ndk/30.0.16248370"
|
|
if not ndk.exists():
|
|
destination = SDK / "ndk"
|
|
destination.mkdir(parents=True, exist_ok=True)
|
|
with tarfile.open(directory / "ndk") as archive:
|
|
archive.extractall(destination, filter="data")
|
|
(destination / "android-ndk-r30").rename(ndk)
|
|
prebuilt = ndk / "toolchains/llvm/prebuilt"
|
|
alias = prebuilt / "linux-x86_64"
|
|
# cargo-ndk 4.1.2 hardcodes this directory name on Linux; binaries remain native ARM64.
|
|
if not alias.exists():
|
|
alias.symlink_to("linux-arm64", target_is_directory=True)
|
|
native = SDK / "native-arm64"
|
|
if not native.exists():
|
|
with zipfile.ZipFile(directory / "sdk_native") as archive:
|
|
archive.extractall(native)
|
|
for path in native.rglob("*"):
|
|
if path.is_file():
|
|
path.chmod(0o755)
|
|
run(cli / "bin/sdkmanager", "--licenses", input="y\n" * 100)
|
|
run(cli / "bin/sdkmanager", "platforms;android-37.0", "build-tools;36.0.0")
|
|
run(prebuilt / "linux-arm64/bin/clang", "--version")
|
|
run(prebuilt / "linux-arm64/bin/ld.lld", "--version")
|
|
run(native / "build-tools/aapt2", "version")
|
|
|
|
|
|
def gradle(*args):
|
|
options = [ROOT / "android/gradlew", "-p", ROOT / "android", "--no-configuration-cache", "--console=plain"]
|
|
if platform.system() == "Linux" and platform.machine() in ("aarch64", "arm64"):
|
|
options.append("-Pandroid.aapt2FromMavenOverride=" + str(ROOT / ".gitea/scripts/native-arm64/aapt2"))
|
|
return run(*options, *args, cwd=ROOT)
|
|
|
|
|
|
def select_symbols(symbols, output):
|
|
allowed = {f"{abi}/{lib}.dbg" for abi in ABIS for lib in ("libironstorage_android.so", "libjnidispatch.so")}
|
|
seen = set()
|
|
with zipfile.ZipFile(symbols) as source, zipfile.ZipFile(output, "w", compression=zipfile.ZIP_DEFLATED) as destination:
|
|
for name in source.namelist():
|
|
if name.split("/", 1)[0] in ABIS:
|
|
if name not in allowed or name in seen:
|
|
raise RuntimeError("Unreviewed or duplicate native debug symbol path")
|
|
seen.add(name)
|
|
with source.open(name) as original, destination.open(name, "w", force_zip64=True) as target:
|
|
shutil.copyfileobj(original, target)
|
|
if seen != allowed:
|
|
raise RuntimeError("Supported ARM debug symbols missing")
|
|
|
|
|
|
def checksum_artifacts(output):
|
|
checksums = {path.name: hashlib.sha256(path.read_bytes()).hexdigest() for path in sorted(output.iterdir()) if path.is_file() and path.name != "sha256.json"}
|
|
(output / "sha256.json").write_text(json.dumps(checksums, indent=2) + "\n")
|
|
|
|
|
|
def build():
|
|
gradle("androidVerification")
|
|
output = ROOT / "android/build/ci-artifacts"
|
|
output.mkdir(parents=True, exist_ok=True)
|
|
apk = ROOT / "android/app/build/outputs/apk/release/app-release-unsigned.apk"
|
|
symbols = ROOT / "android/app/build/outputs/native-debug-symbols/release/native-debug-symbols.zip"
|
|
selected_symbols = output / "native-debug-symbols.zip"
|
|
select_symbols(symbols, selected_symbols)
|
|
tool("audit-release", apk, selected_symbols)
|
|
analyzer = SDK / "cmdline-tools/latest/bin/apkanalyzer"
|
|
for variant, path in [("debug", ROOT / "android/app/build/outputs/apk/debug/app-debug.apk"), ("release", apk)]:
|
|
manifest = output / (variant + "-manifest.xml")
|
|
manifest.write_text(run(analyzer, "manifest", "print", path, capture=True).stdout)
|
|
tool("audit-manifest", manifest, ROOT / "android/app/src/main/res/xml/backup_rules.xml", ROOT / "android/app/src/main/res/xml/data_extraction_rules.xml")
|
|
trees = []
|
|
for target in ("aarch64-linux-android", "armv7-linux-androideabi"):
|
|
trees.append(run(CARGO, "+1.97.1", "tree", "--locked", "-p", "ironstorage-android", "--target", target, "--edges", "normal", "--prefix", "none", "--format", "{p}|{l}|{r}", capture=True, cwd=ROOT).stdout)
|
|
tree = ROOT / "android/build/rust-dependencies.txt"
|
|
tree.write_text("\n".join(trees))
|
|
inventory = ROOT / "android/build/reports/android-dependencies.json"
|
|
rows = []
|
|
for module in ("app", "core", "features", "platform", "rust-bridge"):
|
|
rows.extend(json.loads((ROOT / "android" / module / "build/reports/android-dependencies.json").read_text()))
|
|
inventory.parent.mkdir(parents=True, exist_ok=True)
|
|
inventory.write_text(json.dumps(rows, sort_keys=True) + "\n")
|
|
tool("sbom", inventory, tree, ROOT / "Cargo.lock", ROOT / "android/dependency-policy.json", output / "sbom.cdx.json")
|
|
for module in ("app", "core", "features", "platform", "rust-bridge"):
|
|
reports = ROOT / "android" / module / "build/test-results"
|
|
if module in ("app", "core") or list(reports.rglob("TEST-*.xml")):
|
|
tool("safe-reports", reports, output / (module + "-unit-tests.json"))
|
|
files = [ROOT / "android/app/build/outputs/apk/debug/app-debug.apk", apk,
|
|
ROOT / "android/app/build/outputs/bundle/release/app-release.aab",
|
|
ROOT / "android/app/build/outputs/mapping/release/mapping.txt"]
|
|
for path in files:
|
|
shutil.copy2(path, output / path.name)
|
|
checksum_artifacts(output)
|
|
|
|
|
|
def emulator(serial):
|
|
adb = SDK / "platform-tools/adb"
|
|
def command(*args):
|
|
return run(adb, "-s", serial, *args, capture=True).stdout.strip()
|
|
abi = command("shell", "getprop", "ro.product.cpu.abi")
|
|
if abi not in ABIS:
|
|
raise RuntimeError("Only ARM emulators are in scope")
|
|
if command("shell", "getprop", "ro.kernel.qemu") != "1":
|
|
raise RuntimeError("Physical devices are outside validation scope")
|
|
api = int(command("shell", "getprop", "ro.build.version.sdk"))
|
|
try:
|
|
pages = int(command("shell", "getconf", "PAGE_SIZE"))
|
|
except subprocess.CalledProcessError:
|
|
sizes = {int(line.split()[1]) * 1024 for line in command("shell", "cat", "/proc/self/smaps").splitlines() if line.startswith("KernelPageSize:")}
|
|
if len(sizes) != 1:
|
|
raise RuntimeError("Cannot determine emulator kernel page size")
|
|
pages = sizes.pop()
|
|
if pages == 16384:
|
|
command("shell", "setprop", "bionic.linker.16kb.app_compat.enabled", "false")
|
|
command("shell", "setprop", "pm.16kb.app_compat.disabled", "true")
|
|
apk = ROOT / "android/app/build/outputs/apk/debug/app-debug.apk"
|
|
command("install", "-r", apk)
|
|
# Android's test runner and Rust fixtures cover typed errors, callbacks and cancellation.
|
|
suites = [("app", "de.rfc1437.ironstorage.test", None),
|
|
("rust-bridge", "de.rfc1437.ironstorage.rustbridge.test", None)]
|
|
native_timings = []
|
|
for module, package, _ in suites:
|
|
test_apk = next((ROOT / "android" / module / "build/outputs/apk/androidTest/debug").glob("*.apk"))
|
|
command("install", "-r", test_apk)
|
|
if module == "rust-bridge":
|
|
for _ in range(5):
|
|
command("shell", "am", "force-stop", package)
|
|
measured = command("shell", "am", "instrument", "-w", "-r", "-e", "class", "de.rfc1437.ironstorage.bridge.NativeLoadPerformanceTest", package + "/androidx.test.runner.AndroidJUnitRunner")
|
|
samples = [int(line.split("=", 1)[1]) for line in measured.splitlines() if line.startswith("INSTRUMENTATION_STATUS: ironstorage_native_load_ns=")]
|
|
if len(samples) != 1 or "OK (1 test)" not in measured:
|
|
raise RuntimeError("Native cold-load measurement failed")
|
|
native_timings.extend(samples)
|
|
result = command("shell", "am", "instrument", "-w", "-r", package + "/androidx.test.runner.AndroidJUnitRunner")
|
|
if "FAILURES!!!" in result or "INSTRUMENTATION_FAILED" in result or "OK (" not in result:
|
|
local = ROOT / "android/build/local-validation"
|
|
local.mkdir(parents=True, exist_ok=True)
|
|
local.chmod(0o700)
|
|
raw = local / f"{serial}-{module}-failure.txt"
|
|
raw.write_text(result)
|
|
raw.chmod(0o600)
|
|
raise RuntimeError("ARM emulator instrumentation failed (raw output retained locally only)")
|
|
# No activity state or application data is deleted between process recreation runs.
|
|
timings = []
|
|
for _ in range(5):
|
|
command("shell", "am", "force-stop", "de.rfc1437.ironstorage")
|
|
result = command("shell", "am", "start", "-W", "-n", "de.rfc1437.ironstorage/.MainActivity")
|
|
values = dict(line.split(":", 1) for line in result.splitlines() if ":" in line)
|
|
if values.get("Status", "").strip() != "ok":
|
|
raise RuntimeError("Cold start failed")
|
|
timings.append(int(values["TotalTime"].strip()))
|
|
output = ROOT / "android/build/ci-artifacts"
|
|
output.mkdir(parents=True, exist_ok=True)
|
|
(output / f"emulator-api{api}-{pages}.json").write_text(json.dumps({"api": api, "abi": abi, "page_size": pages, "cold_start_ms": timings, "native_load_ns": native_timings, "instrumentation": "passed"}, indent=2) + "\n")
|
|
checksum_artifacts(output)
|
|
print(f"ARM API {api}, {pages}-byte pages: instrumentation and five cold starts passed")
|
|
|
|
|
|
class SelfTest(unittest.TestCase):
|
|
def test_aapt2_adapter_preserves_paths_and_other_options(self):
|
|
adapter = runpy.run_path(ROOT / ".gitea/scripts/native-arm64/aapt2")
|
|
self.assertEqual(adapter["arguments"](["optimize", "input.apk", "--resource-path-shortening-map=/tmp/map with spaces=ok", "--shorten-resource-paths", "-o", "output.apk"]),
|
|
["optimize", "input.apk", "--resource-path-shortening-map", "/tmp/map with spaces=ok", "--shorten-resource-paths", "-o", "output.apk"])
|
|
self.assertEqual(adapter["arguments"](["daemon"]), ["daemon"])
|
|
|
|
def test_toolchain_checksum_rejects_changed_archive(self):
|
|
with tempfile.TemporaryDirectory() as directory:
|
|
path = Path(directory) / "fixture"
|
|
path.write_bytes(b"public archive fixture")
|
|
spec = {"sha256": hashlib.sha256(path.read_bytes()).hexdigest()}
|
|
download(spec, path)
|
|
path.write_bytes(b"changed")
|
|
with self.assertRaisesRegex(RuntimeError, "checksum mismatch"):
|
|
download(spec, path)
|
|
|
|
def test_symbol_selection_and_repeatable_checksum_manifest(self):
|
|
with tempfile.TemporaryDirectory() as directory:
|
|
directory = Path(directory)
|
|
source = directory / "source.zip"
|
|
output = directory / "output.zip"
|
|
names = [f"{abi}/{lib}.dbg" for abi in ABIS for lib in ("libironstorage_android.so", "libjnidispatch.so")]
|
|
with zipfile.ZipFile(source, "w") as archive:
|
|
for name in names + ["unused/libjnidispatch.so.dbg"]:
|
|
archive.writestr(name, b"public symbols")
|
|
select_symbols(source, output)
|
|
with zipfile.ZipFile(output) as archive:
|
|
self.assertEqual(set(archive.namelist()), set(names))
|
|
checksum_artifacts(directory)
|
|
first = (directory / "sha256.json").read_bytes()
|
|
checksum_artifacts(directory)
|
|
self.assertEqual(first, (directory / "sha256.json").read_bytes())
|
|
self.assertNotIn("sha256.json", json.loads(first))
|
|
with zipfile.ZipFile(source, "w") as archive:
|
|
archive.writestr(names[0], b"public symbols")
|
|
with self.assertRaisesRegex(RuntimeError, "missing"):
|
|
select_symbols(source, output)
|
|
with zipfile.ZipFile(source, "w") as archive:
|
|
archive.writestr("arm64-v8a/unreviewed.dbg", b"public symbols")
|
|
with self.assertRaisesRegex(RuntimeError, "Unreviewed"):
|
|
select_symbols(source, output)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
parser = argparse.ArgumentParser()
|
|
parser.add_argument("command", choices=["prepare-linux", "build", "emulator", "test"])
|
|
parser.add_argument("serial", nargs="?")
|
|
args = parser.parse_args()
|
|
if args.command != "test" and not (os.environ.get("ANDROID_HOME") or os.environ.get("ANDROID_SDK_ROOT")):
|
|
parser.error("ANDROID_HOME or ANDROID_SDK_ROOT is required")
|
|
if args.command == "test":
|
|
unittest.main(argv=[__file__])
|
|
elif args.command == "prepare-linux":
|
|
prepare_linux()
|
|
elif args.command == "build":
|
|
build()
|
|
elif args.serial:
|
|
emulator(args.serial)
|
|
else:
|
|
parser.error("emulator requires an ADB serial")
|