Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
503 changes: 503 additions & 0 deletions .github/workflows/android-compat.yml

Large diffs are not rendered by default.

125 changes: 125 additions & 0 deletions .github/workflows/native-deps-cache.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,125 @@
name: Warm the native dependency cache

# Dawn is ~16 minutes per ABI and the pins move a few times a year, so it should
# be built about as often as the pins move. It was in fact being built on every
# single release, and on every branch that needed it, because of where the cache
# was being written from.
#
# A GitHub cache is readable from the ref that created it, that ref's descendants,
# and the default branch. The release pipeline runs on a TAG — so the 937 MB
# android and 401 MB ios caches it saved (v0.36.0's and v0.37.0's, both still
# there) are readable by nothing at all. Not by a branch, not by a PR, and not by
# the next release either, since every new tag is a new scope. Each release paid
# for a cold Dawn build and then wrote its result somewhere it could never be read
# from again.
#
# Writing it from the default branch instead is the whole fix: a default-branch
# cache is in scope for every run in the repository.
#
# This is restore-first and builds only on a miss, which is what makes the weekly
# trigger cheap. GitHub evicts a cache untouched for 7 days, and a restore counts
# as touching it — so on the normal week this job downloads the cache, finds it
# complete, and stops. It rebuilds only when the pins actually moved.

on:
push:
branches: [master]
paths:
# The only thing that invalidates the cache is the pins themselves.
- 'toolchain.manifest.json'
# And this file, so that merging it warms the cache once instead of leaving
# it cold until the pins next move or the weekly trigger comes round. A
# workflow_dispatch cannot stand in for that: dispatch only resolves against
# the default branch, so there is no way to run this before it is merged.
- '.github/workflows/native-deps-cache.yml'
schedule:
# Weekly, purely to keep the 7-day eviction clock from running out.
- cron: '0 3 * * 1'
workflow_dispatch:
inputs:
force-rebuild:
description: 'Build even if the cache already has this pin (for a corrupt entry).'
required: false
default: false
type: boolean

permissions:
contents: read

jobs:
warm:
name: Dawn for ${{ matrix.target }}
runs-on: ${{ matrix.os }}
timeout-minutes: 150
strategy:
fail-fast: false
matrix:
include:
- target: android
os: ubuntu-latest
- target: ios
os: macos-latest
steps:
# No submodules: Dawn and QuickJS are fetched into build/native-deps by
# --fetch-deps, and third_party is the HOST's dependency, which this never
# builds.
- uses: actions/checkout@v7

- uses: ./.github/actions/setup

- name: Read the native dependency pins
id: pins
run: |
node -e "const p=require('./toolchain.manifest.json').native;\
console.log('dawn='+p.dawn.commit);console.log('quickjs='+p.quickjs.commit)" >> "$GITHUB_OUTPUT"

# restore, not the combined action: a hit must end this job rather than
# trigger a save of what it just downloaded.
- name: Is this pin already built
id: restore
if: ${{ !inputs.force-rebuild }}
uses: actions/cache/restore@v4
with:
path: build/native-deps
key: native-deps-${{ matrix.target }}-${{ steps.pins.outputs.dawn }}-${{ steps.pins.outputs.quickjs }}

- name: Report what the restore found
run: |
if [ '${{ steps.restore.outputs.cache-hit }}' = 'true' ]; then
echo "cache is warm for these pins — the restore refreshed its eviction clock, nothing to build"
else
echo "no cache for these pins — building Dawn"
fi

- name: Set up the Android NDK
if: matrix.target == 'android' && steps.restore.outputs.cache-hit != 'true'
uses: nttld/setup-ndk@v1
id: ndk
with:
ndk-version: r28

# Dawn builds with Ninja and a macOS runner has neither it nor CMake; the
# Android SDK carries its own pair.
- name: Set up CMake + Ninja
if: matrix.target == 'ios' && steps.restore.outputs.cache-hit != 'true'
run: brew install cmake ninja

- name: Fetch Dawn + QuickJS at their pinned commits
if: steps.restore.outputs.cache-hit != 'true'
run: node build-tools/cli.js native --fetch-deps

# Every ABI (android) and every sysroot slice (ios) the consumers ask for.
# Warming one and leaving the other cold would look like a hit and behave
# like a miss.
- name: Build Dawn
if: steps.restore.outputs.cache-hit != 'true'
env:
ANDROID_NDK_HOME: ${{ steps.ndk.outputs.ndk-path }}
run: node build-tools/cli.js native --target ${{ matrix.target }} --build-deps

- name: Save it for every ref in the repository
if: steps.restore.outputs.cache-hit != 'true'
uses: actions/cache/save@v4
with:
path: build/native-deps
key: native-deps-${{ matrix.target }}-${{ steps.pins.outputs.dawn }}-${{ steps.pins.outputs.quickjs }}
3 changes: 2 additions & 1 deletion build-tools/cli.js
Original file line number Diff line number Diff line change
Expand Up @@ -162,11 +162,12 @@ program
.description('Build the native (embedded-Dawn) host for Android or iOS arm64')
.option('--target <target>', 'android or ios', 'android')
.option('--fetch-deps', 'Check out Dawn + QuickJS-ng at the pinned commits, then stop', false)
.option('--build-deps', 'Build Dawn for every ABI/slice the target needs, then stop (for warming a cache)', false)
.option('--dawn <dir>', 'Dawn source dir (default: the pinned checkout; or ESTELLA_DAWN_DIR)')
.option('--dawn-build <dir>', 'Dawn build dir for this target (default: <dawn>/out-<target>, built if absent)')
.option('--quickjs <dir>', 'QuickJS-ng source dir (default: the pinned checkout; or ESTELLA_QUICKJS_DIR)')
.option('--abi <abi>', 'Android ABI', 'arm64-v8a')
.option('--platform <platform>', 'Android platform', 'android-33')
.option('--platform <platform>', 'Android platform', 'android-29')
.option('--ios-min <version>', 'iOS deployment target', '17.0')
.option('--simulator', 'iOS: build the simulator slice (needs a simulator Dawn)', false)
.option('--package', 'Assemble the app around --content from the installed runtime template: Android a signed APK, iOS an Xcode project', false)
Expand Down
46 changes: 44 additions & 2 deletions build-tools/tasks/native.js
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@ import { requireSdk, requireNdk, sdkCmake } from '../utils/android.js';
import { emitNativeTemplate, writeTemplateIndex, readEngineVersion } from './nativeTemplateEmit.js';
import { fetchNativeDeps, pinnedDep, ensureDawnBuild, dawnLibrary, DAWN_TARGETS } from './nativeDeps.js';
import {
BYTECODE_FILE, findTemplate, iosTemplateSources, templateStoreDir,
ANDROID_ABIS, BYTECODE_FILE, findTemplate, iosTemplateSources, templateStoreDir,
} from '../utils/nativeTemplate.js';
import { readAppConfig, fillTemplate, iosInterfaceOrientations } from '../utils/nativeApp.js';
import { emitIosXcodeProject } from '../utils/iosProject.js';
Expand Down Expand Up @@ -312,7 +312,14 @@ function quickjsDir(options) {
const fwd = (p) => p.replace(/\\/g, '/');

async function buildAndroidHost(options) {
const { abi = 'arm64-v8a', platform = 'android-33' } = options;
// The platform MUST equal the manifest's minSdkVersion. The NDK emits a weak
// reference for an API newer than the target and a strong one otherwise, and
// `__builtin_available` is compiled out in the second case — so building above
// the declared floor turns every guard into dead code and every guarded symbol
// into a load-time requirement. At android-33 that shipped a host which could
// not dlopen below API 31: `cannot locate symbol APerformanceHint_getManager`,
// on Android 10 and 11, before a line of our code ran.
const { abi = 'arm64-v8a', platform = 'android-29' } = options;
const rootDir = config.paths.root;

const sdk = requireSdk();
Expand Down Expand Up @@ -644,8 +651,43 @@ async function emitFromExistingBuild(target, options) {
});
}

/**
* Build the pinned dependencies for a target and stop.
*
* For a job that exists only to populate the dependency cache. Dawn is the whole
* cost — the host beside it is minutes — so building the host too would only add
* ways for a cache warm-up to fail for reasons that have nothing to do with what
* it is caching.
*
* EVERY build tree the consumers look for, not just the default one: Dawn's is
* per ABI on Android and per sysroot on iOS, so a warm-up that did one would
* leave the release cold in exactly the half it did not do.
*/
async function buildNativeDeps(options) {
const target = (options.target || 'android').toLowerCase();
if (target === 'android') {
const sdk = requireSdk();
const ndk = requireNdk(sdk);
const { cmake, ninja } = sdkCmake(sdk);
for (const abi of ANDROID_ABIS) {
logger.step(`Dawn for android/${abi}...`);
await dawnPaths({ ...options, abi }, 'android', { ndk, cmake, ninja });
}
logger.success(`Dawn built for ${ANDROID_ABIS.join(', ')}`);
return;
}
const developerDir = await iosDeveloperDir();
const env = developerDir ? { DEVELOPER_DIR: developerDir } : undefined;
for (const slice of ['ios', 'ios-sim']) {
logger.step(`Dawn for ${slice}...`);
await dawnPaths(options, slice, { env });
}
logger.success('Dawn built for ios, ios-sim');
}

export async function buildNative(options = {}) {
if (options.fetchDeps) return fetchNativeDeps(options);
if (options.buildDeps) return buildNativeDeps(options);
if (options.templateIndex) {
return writeTemplateIndex(path.isAbsolute(options.templateIndex)
? options.templateIndex : path.join(config.paths.root, options.templateIndex));
Expand Down
2 changes: 1 addition & 1 deletion build-tools/tasks/nativeDeps.js
Original file line number Diff line number Diff line change
Expand Up @@ -145,7 +145,7 @@ export async function ensureDawnBuild(options) {
? [
`-DCMAKE_TOOLCHAIN_FILE=${path.join(options.ndk, 'build', 'cmake', 'android.toolchain.cmake')}`,
`-DANDROID_ABI=${options.abi || 'arm64-v8a'}`,
`-DANDROID_PLATFORM=${options.androidPlatform || 'android-33'}`,
`-DANDROID_PLATFORM=${options.androidPlatform || 'android-29'}`,
'-DANDROID_STL=c++_shared',
'-DDAWN_ENABLE_VULKAN=ON', '-DDAWN_ENABLE_METAL=OFF',
// Shared on Android (the APK ships the .so); static on iOS (an app
Expand Down
2 changes: 1 addition & 1 deletion build-tools/tasks/nativeTemplateEmit.js
Original file line number Diff line number Diff line change
Expand Up @@ -200,7 +200,7 @@ export async function emitNativeTemplate(options) {
spineVersion: options.spineVersion || '4.2',
...(platform === 'ios' ? { deploymentTarget: options.deploymentTarget || '17.0' } : {}),
...(platform === 'android'
? { androidPlatform: options.androidPlatform || 'android-33', abis: templateAbis(dir) }
? { androidPlatform: options.androidPlatform || 'android-29', abis: templateAbis(dir) }
: {}),
});

Expand Down
2 changes: 1 addition & 1 deletion build-tools/utils/apk.js
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,7 @@ import { fillTemplate, androidScreenOrientation } from './nativeApp.js';
* smaller page size also divides. */
const PAGE_ALIGNMENT = 16384;

/** APK Signature Scheme v2. minSdk 26 is well past the API 24 that introduced it,
/** APK Signature Scheme v2. minSdk 29 is well past the API 24 that introduced it,
* so v1 (JAR signing) would be dead weight — and its PKCS#7 is the only part of
* APK signing that is genuinely hard to write. */
const V2_BLOCK_ID = 0x7109871a;
Expand Down
2 changes: 1 addition & 1 deletion build-tools/utils/gradleProject.js
Original file line number Diff line number Diff line change
Expand Up @@ -54,7 +54,7 @@ export function gradleManifest(templateXml, app) {
HAS_CODE: 'true',
});

const minSdk = Number(/android:minSdkVersion="(\d+)"/.exec(filled)?.[1] ?? 26);
const minSdk = Number(/android:minSdkVersion="(\d+)"/.exec(filled)?.[1] ?? 29);
const targetSdk = Number(/android:targetSdkVersion="(\d+)"/.exec(filled)?.[1] ?? 33);

const xml = filled
Expand Down
4 changes: 2 additions & 2 deletions desktop/tests/android-project.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -110,7 +110,7 @@ describe('the Android Studio project an export writes', () => {
expect(gradle).toContain('versionCode = 7');
expect(gradle).toContain('versionName = "1.2"');
// Taken from the manifest template rather than restated, so the two agree.
expect(gradle).toContain('minSdk = 26');
expect(gradle).toContain('minSdk = 29');
expect(gradle).toContain('targetSdk = 33');
});

Expand Down Expand Up @@ -163,7 +163,7 @@ describe('the Android Studio project an export writes', () => {
describe('gradleManifest', () => {
it('reads the SDK levels it strips, so the build script can state them', () => {
const { minSdk, targetSdk } = gradleManifest(readFileSync(MANIFEST_TEMPLATE, 'utf8'), APP);
expect(minSdk).toBe(26);
expect(minSdk).toBe(29);
expect(targetSdk).toBe(33);
});
});
12 changes: 11 additions & 1 deletion native/android/host/AndroidManifest.xml.in
Original file line number Diff line number Diff line change
Expand Up @@ -8,12 +8,22 @@
hard requirement, and the config changes the host handles rather than being
restarted for. `hasCode` follows whether the Java shim compiled — the game is
native, and the only Java in the package is the IME's side of a text field.

minSdkVersion 29, and the NDK MUST build against the same number (`--platform`,
build-tools/tasks/native.js). The host calls AFontMatcher_create unguarded and
that is API 29, so nothing below it could ever have run; 26 promised three
releases the code never supported. The equality is what makes the availability
guards work: the NDK emits a weak reference only for an API newer than the
build target, so building above the declared floor compiles every
`__builtin_available` out and turns its symbol into a load-time requirement.
Built at android-33, the released host could not dlopen below API 31 —
"cannot locate symbol APerformanceHint_getManager" on Android 10 and 11.
-->
<manifest xmlns:android="http://schemas.android.com/apk/res/android"
package="@APP_ID@"
android:versionCode="@VERSION_CODE@"
android:versionName="@VERSION_NAME@">
<uses-sdk android:minSdkVersion="26" android:targetSdkVersion="33" />
<uses-sdk android:minSdkVersion="29" android:targetSdkVersion="33" />
<uses-feature android:name="android.hardware.vulkan.version" android:version="0x400003" android:required="true" />
<application android:label="@APP_NAME@" android:icon="@mipmap/ic_launcher"
android:hasCode="@HAS_CODE@" android:extractNativeLibs="false">
Expand Down
63 changes: 49 additions & 14 deletions native/host/platform/android.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@
* Licensed under the Apache License, Version 2.0.
*/
#include <android_native_app_glue.h>
#include <dlfcn.h>
#include <android/log.h>
#include <sys/system_properties.h>
#include <android/input.h>
Expand All @@ -22,7 +23,6 @@
#include <android/font_matcher.h>
#include <android/native_window.h>
#include <android/choreographer.h>
#include <android/performance_hint.h>
#include <jni.h>
#include <unistd.h>

Expand Down Expand Up @@ -499,21 +499,59 @@ void onAppCmd(android_app* app, int32_t cmd) {
// The loop then reports what each frame actually cost, and the system ramps on
// that instead of on a guess. Busy-waiting to fake demand is the alternative the
// platform documentation explicitly warns against.
//
// Resolved by hand, through dlsym, rather than called directly.
//
// ADPF is API 33 and this host builds against the manifest's floor, so the NDK
// marks those four symbols `unavailable`: a hard compile error, and no
// `__builtin_available` guard changes that — the annotation is not "call me
// under a check", it is "this build cannot see me". Raising the build target is
// what makes them callable, and that is precisely the mistake being fixed here:
// at android-33 every availability guard in this file became dead code and every
// guarded symbol became a load-time requirement, so the released host could not
// dlopen on Android 10 or 11 at all.
//
// The NDK's own answer is ANDROID_WEAK_API_DEFS, which turns such symbols into
// weak references. Looking them up here instead keeps the decision in the code
// that depends on it, where it is visible, rather than in a toolchain flag whose
// absence would silently restore the same class of failure.
struct PerformanceHints {
APerformanceHintSession* session = nullptr;
void* session = nullptr;

// One display refresh. The Choreographer is what calls the frame, so the
// deadline is the panel's cadence, whatever it happens to be.
static constexpr int64_t kFrameBudgetNanos = 16'666'667;

// Opaque on purpose: the header's typedefs come with the same availability
// annotations, and nothing here needs to know what a session is.
using GetManagerFn = void* (*)();
using CreateSessionFn = void* (*)(void*, const int32_t*, size_t, int64_t);
using ReportFn = int (*)(void*, int64_t);
using CloseFn = void (*)(void*);

ReportFn reportFn = nullptr;
CloseFn closeFn = nullptr;

void open() {
if (__builtin_available(android 31, *)) {
APerformanceHintManager* manager = APerformanceHint_getManager();
if (!manager) return; // no session on a device (or emulator) without one
// The frame loop's own thread, and it lives as long as the app does —
// the session wants long-lived tids, not ones that come and go.
const int32_t tid = static_cast<int32_t>(gettid());
session = APerformanceHint_createSession(manager, &tid, 1, kFrameBudgetNanos);
// RTLD_DEFAULT: libandroid is already loaded — this asks whether THIS
// platform version exports the symbols, which is the actual question.
const auto getManager = reinterpret_cast<GetManagerFn>(
dlsym(RTLD_DEFAULT, "APerformanceHint_getManager"));
const auto createSession = reinterpret_cast<CreateSessionFn>(
dlsym(RTLD_DEFAULT, "APerformanceHint_createSession"));
reportFn = reinterpret_cast<ReportFn>(
dlsym(RTLD_DEFAULT, "APerformanceHint_reportActualWorkDuration"));
closeFn = reinterpret_cast<CloseFn>(
dlsym(RTLD_DEFAULT, "APerformanceHint_closeSession"));

if (getManager && createSession && reportFn && closeFn) {
if (void* manager = getManager()) {
// The frame loop's own thread, and it lives as long as the app
// does — the session wants long-lived tids, not ones that come
// and go.
const int32_t tid = static_cast<int32_t>(gettid());
session = createSession(manager, &tid, 1, kFrameBudgetNanos);
}
}
__android_log_print(session ? ANDROID_LOG_INFO : ANDROID_LOG_WARN, LOG_TAG,
"perf hints: %s", session ? "session up (ADPF)" : "unavailable");
Expand All @@ -523,14 +561,11 @@ struct PerformanceHints {
if (!session) return;
const auto nanos = std::chrono::duration_cast<std::chrono::nanoseconds>(frame).count();
if (nanos <= 0) return; // the API rejects a non-positive duration
if (__builtin_available(android 31, *)) {
APerformanceHint_reportActualWorkDuration(session, nanos);
}
reportFn(session, nanos);
}

~PerformanceHints() {
if (!session) return;
if (__builtin_available(android 31, *)) APerformanceHint_closeSession(session);
if (session) closeFn(session);
}
};

Expand Down
Loading
Loading