Skip to content

Commit 940c071

Browse files
matthargettmatthargett
committed
cache the NDKs instead of re-downloading and re-unpacking each time. for that matter, use cccache so that incremental updates from upstream bun-sh/webkit don't trigger the 90+ minute (time three, for each NDK) build.
1 parent d7dbeb7 commit 940c071

File tree

8 files changed

+486
-30
lines changed

8 files changed

+486
-30
lines changed

‎.github/workflows/build_and_test.yml‎

Lines changed: 179 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,20 @@
11
name: Build jsc-android and test
22

33
on:
4-
workflow_dispatch: {}
4+
workflow_dispatch:
5+
inputs:
6+
publish:
7+
description: 'Publish npm packages after build/test succeed'
8+
type: boolean
9+
default: false
10+
npm-tag:
11+
description: 'npm dist-tag'
12+
required: false
13+
default: latest
14+
dry-run:
15+
description: 'Run npm publish in dry-run mode'
16+
type: boolean
17+
default: true
518
push:
619
branches: [main]
720
pull_request:
@@ -12,6 +25,9 @@ jobs:
1225
env:
1326
ORG_GRADLE_PROJECT_signingKey: ${{ secrets.GPG_SIGNING_KEY }}
1427
ORG_GRADLE_PROJECT_signingPassword: ${{ secrets.GPG_SIGNING_PASSWORD }}
28+
NDK_VERSION_27: '27.1.12297006'
29+
NDK_VERSION_28: '28.2.13676358'
30+
BUILD_CACHE_VERSION: v1
1531

1632
steps:
1733
- uses: actions/checkout@v4
@@ -30,39 +46,126 @@ jobs:
3046
with:
3147
node-version: 22
3248

49+
- name: Compute build cache key
50+
id: build-hash
51+
run: |
52+
HASH=$(node scripts/build-hash.js)
53+
echo "hash=$HASH" >> "$GITHUB_OUTPUT"
54+
55+
- name: Restore JSC build artifacts
56+
id: cache-dist
57+
uses: actions/cache@v4
58+
with:
59+
path: |
60+
dist-ndk27
61+
dist-ndk27.unstripped
62+
dist-ndk28
63+
dist-ndk28.unstripped
64+
dist-ndk29
65+
dist-ndk29.unstripped
66+
key: jsc-dist-${{ env.BUILD_CACHE_VERSION }}-${{ steps.build-hash.outputs.hash }}
67+
68+
- name: Restore WebKit sources
69+
id: cache-download
70+
uses: actions/cache@v4
71+
with:
72+
path: build/download
73+
key: jsc-download-${{ env.BUILD_CACHE_VERSION }}-${{ steps.build-hash.outputs.hash }}
74+
restore-keys: |
75+
jsc-download-${{ env.BUILD_CACHE_VERSION }}-
76+
3377
- name: Install packages
3478
run: |
3579
sudo apt-get update
36-
sudo apt-get install coreutils curl git wget python3 ruby gperf -y
80+
sudo apt-get install coreutils curl git wget python3 ruby gperf ccache -y
81+
shell: bash
82+
83+
- name: Restore ccache
84+
id: cache-ccache
85+
uses: actions/cache@v4
86+
with:
87+
path: ~/.cache/ccache
88+
key: ccache-${{ env.BUILD_CACHE_VERSION }}-${{ runner.os }}-${{ env.NDK_VERSION_27 }}-${{ steps.build-hash.outputs.hash }}
89+
restore-keys: |
90+
ccache-${{ env.BUILD_CACHE_VERSION }}-${{ runner.os }}-${{ env.NDK_VERSION_27 }}-
91+
92+
- name: Configure ccache
93+
run: |
94+
mkdir -p ~/.cache/ccache
95+
if command -v ccache >/dev/null 2>&1; then
96+
ccache --max-size=5G
97+
ccache --zero-stats || true
98+
fi
3799
shell: bash
38100

101+
- name: Cache Android NDK r27
102+
id: cache-ndk-27
103+
uses: actions/cache@v4
104+
with:
105+
path: ${{ env.ANDROID_HOME }}/ndk/${{ env.NDK_VERSION_27 }}
106+
key: android-ndk-${{ runner.os }}-${{ env.NDK_VERSION_27 }}
107+
39108
- name: Install Android packages
40109
run: |
41110
export PATH=$PATH:$ANDROID_HOME/cmdline-tools/latest/bin:$ANDROID_HOME/platform-tools
42111
yes | sdkmanager --licenses || true
43-
sdkmanager \
44-
"cmake;3.22.1" \
45-
"ndk;27.1.12297006"
46-
# move out builtin icu headers from ndk and prevent icu build errors
47-
mv "${ANDROID_HOME}/ndk/27.1.12297006/toolchains/llvm/prebuilt/linux-x86_64/sysroot/usr/include/unicode" "${ANDROID_HOME}/ndk/27.1.12297006/toolchains/llvm/prebuilt/linux-x86_64/sysroot/usr/include/unicode2"
48-
49-
echo "ANDROID_NDK=$ANDROID_HOME/ndk/27.1.12297006" >> $GITHUB_ENV
50-
echo "PATH=$PATH:$ANDROID_HOME/cmdline-tools/latest/bin:$ANDROID_HOME/platform-tools" >> $GITHUB_ENV
112+
sdkmanager "cmake;3.22.1"
113+
if [[ ! -d "${ANDROID_HOME}/ndk/${NDK_VERSION_27}" ]]; then
114+
sdkmanager "ndk;${NDK_VERSION_27}"
115+
fi
116+
UNICODE_DIR="${ANDROID_HOME}/ndk/${NDK_VERSION_27}/toolchains/llvm/prebuilt/linux-x86_64/sysroot/usr/include/unicode"
117+
if [[ -d "${UNICODE_DIR}" ]]; then
118+
mv "${UNICODE_DIR}" "${UNICODE_DIR}2"
119+
fi
120+
echo "ANDROID_NDK=$ANDROID_HOME/ndk/${NDK_VERSION_27}" >> $GITHUB_ENV
121+
echo "PATH=$PATH" >> $GITHUB_ENV
122+
shell: bash
123+
124+
- name: Install dependencies
125+
run: yarn install --frozen-lockfile
126+
shell: bash
127+
128+
- name: Clean previous build outputs
129+
if: steps.cache-dist.outputs.cache-hit != 'true'
130+
run: |
131+
rm -rf dist dist.unstripped dist-ndk* build/target* build/compiled* build/cppruntime*
132+
shell: bash
133+
134+
- name: Download sources
135+
if: steps.cache-download.outputs.cache-hit != 'true'
136+
run: yarn download
51137
shell: bash
52138

53139
- name: Build
140+
if: steps.cache-dist.outputs.cache-hit != 'true'
141+
run: yarn start
142+
shell: bash
143+
144+
- name: Show ccache stats
145+
if: steps.cache-dist.outputs.cache-hit != 'true'
54146
run: |
55-
yarn install --frozen-lockfile
56-
yarn clean
57-
yarn download
58-
yarn start
147+
if command -v ccache >/dev/null 2>&1; then
148+
ccache --show-stats
149+
fi
59150
shell: bash
60151

61152
- name: Archive
62153
run: |
154+
rm -rf archive
63155
mkdir -p archive
64-
mv dist archive/
65-
mv dist.unstripped archive/
156+
shopt -s nullglob
157+
found=0
158+
for dir in dist-ndk*; do
159+
if [[ -d "$dir" ]]; then
160+
cp -R "$dir" archive/
161+
found=1
162+
fi
163+
done
164+
shopt -u nullglob
165+
if [[ $found -eq 0 ]]; then
166+
echo "No distribution directories were produced." >&2
167+
exit 1
168+
fi
66169
shell: bash
67170

68171
- uses: actions/upload-artifact@v4
@@ -99,8 +202,12 @@ jobs:
99202

100203
- name: Extract archive
101204
run: |
102-
mv archive/dist dist
103-
mv archive/dist.unstripped dist.unstripped
205+
shopt -s nullglob
206+
for dir in archive/dist-ndk*; do
207+
dest=$(basename "$dir")
208+
mv "$dir" "$dest"
209+
done
210+
shopt -u nullglob
104211
rmdir archive
105212
shell: bash
106213

@@ -128,6 +235,7 @@ jobs:
128235
target: google_apis
129236
working-directory: test
130237
script: |
238+
export JSC_GRADLE_DIST_PATH=../../dist-ndk27
131239
npx expo run:android --variant release --no-bundler
132240
adb logcat -c
133241
set +e
@@ -145,3 +253,56 @@ jobs:
145253
$HOME/.maestro/tests/**/*
146254
test/android/app/build/outputs/apk/release/app-release.apk
147255
test/adb.log
256+
257+
publish:
258+
if: github.event_name == 'workflow_dispatch' && github.event.inputs.publish == 'true'
259+
needs:
260+
- build
261+
- test
262+
runs-on: ubuntu-latest
263+
environment:
264+
name: npm-publish
265+
url: https://www.npmjs.com/package/jsc-android
266+
permissions:
267+
contents: read
268+
steps:
269+
- uses: actions/checkout@v4
270+
271+
- name: ⬢ Setup Node
272+
uses: actions/setup-node@v4
273+
with:
274+
node-version: 22
275+
276+
- uses: actions/download-artifact@v4
277+
with:
278+
name: archive
279+
path: archive
280+
281+
- name: Install dependencies
282+
run: yarn install --frozen-lockfile
283+
shell: bash
284+
285+
- name: Verify npm token availability
286+
if: github.event.inputs.dry-run != 'true'
287+
run: |
288+
if [[ -z "${NPM_TOKEN:-}" ]]; then
289+
echo "NPM_TOKEN secret is required for publishing." >&2
290+
exit 1
291+
fi
292+
env:
293+
NPM_TOKEN: ${{ secrets.NPM_TOKEN }}
294+
shell: bash
295+
296+
- name: Publish packages
297+
env:
298+
NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }}
299+
NPM_CONFIG_PROVENANCE: 'true'
300+
run: |
301+
TAG="${{ github.event.inputs.npm-tag }}"
302+
DRY_RUN="${{ github.event.inputs.dry-run }}"
303+
PUBLISH_ARGS=("-T" "$TAG")
304+
if [[ "$DRY_RUN" == 'true' ]]; then
305+
PUBLISH_ARGS+=("--dry-run")
306+
fi
307+
node scripts/publish.js "${PUBLISH_ARGS[@]}" archive
308+
shell: bash

‎package.json‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,7 @@
2020
"/dist"
2121
],
2222
"scripts": {
23-
"clean": "rm -rf dist; rm -rf build",
23+
"clean": "rm -rf dist dist.unstripped dist-ndk* dist-ndk*.unstripped; rm -rf build",
2424
"info": "./scripts/info.sh",
2525
"download": "./scripts/download.sh",
2626
"start": "./scripts/start.sh"

‎scripts/build-hash.js‎

Lines changed: 75 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,75 @@
1+
#!/usr/bin/env node
2+
3+
const crypto = require('crypto');
4+
const fs = require('fs');
5+
const path = require('path');
6+
7+
const rootDir = process.cwd();
8+
9+
const includePaths = [
10+
'package.json',
11+
'yarn.lock',
12+
'patches',
13+
'scripts',
14+
'lib',
15+
];
16+
17+
const excludePatterns = [/^scripts\/publish\.js$/];
18+
19+
const hash = crypto.createHash('sha256');
20+
21+
function shouldInclude(relPath) {
22+
if (relPath.includes('node_modules')) {
23+
return false;
24+
}
25+
if (relPath.startsWith('.git')) {
26+
return false;
27+
}
28+
return true;
29+
}
30+
31+
function shouldExclude(relPath) {
32+
return excludePatterns.some((pattern) => pattern.test(relPath));
33+
}
34+
35+
function addFile(filePath, relPath) {
36+
const content = fs.readFileSync(filePath);
37+
hash.update(relPath);
38+
hash.update('\0');
39+
hash.update(content);
40+
}
41+
42+
function walk(currentPath, basePath) {
43+
const entries = fs.readdirSync(currentPath).sort();
44+
entries.forEach((entry) => {
45+
const absPath = path.join(currentPath, entry);
46+
const relPath = path.relative(basePath, absPath).replace(/\\/g, '/');
47+
48+
if (!shouldInclude(relPath) || shouldExclude(relPath)) {
49+
return;
50+
}
51+
52+
const stat = fs.statSync(absPath);
53+
if (stat.isDirectory()) {
54+
walk(absPath, basePath);
55+
} else if (stat.isFile()) {
56+
addFile(absPath, relPath);
57+
}
58+
});
59+
}
60+
61+
includePaths.forEach((relativePath) => {
62+
const relative = relativePath.replace(/\\/g, '/');
63+
const abs = path.join(rootDir, relative);
64+
if (!fs.existsSync(abs)) {
65+
return;
66+
}
67+
const stat = fs.statSync(abs);
68+
if (stat.isDirectory()) {
69+
walk(abs, rootDir);
70+
} else if (stat.isFile()) {
71+
addFile(abs, path.relative(rootDir, abs).replace(/\\/g, '/'));
72+
}
73+
});
74+
75+
process.stdout.write(hash.digest('hex'));

‎scripts/compile/icu.sh‎

Lines changed: 9 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,13 @@ else
3535
CONFIGURE_PREFIX=()
3636
fi
3737

38+
CC_BIN=$CROSS_COMPILE_PLATFORM_CC-clang
39+
CXX_BIN=$CROSS_COMPILE_PLATFORM_CC-clang++
40+
if [[ -n "$JSC_CCACHE_BIN" ]]; then
41+
CC_BIN="$JSC_CCACHE_BIN $CC_BIN"
42+
CXX_BIN="$JSC_CCACHE_BIN $CXX_BIN"
43+
fi
44+
3845
"${CONFIGURE_PREFIX[@]}" $TARGETDIR/icu/source/configure --prefix=${INSTALL_DIR} \
3946
$BUILD_TYPE_CONFIG \
4047
--host=$CROSS_COMPILE_PLATFORM \
@@ -52,8 +59,8 @@ fi
5259
CFLAGS="$ICU_CFLAGS" \
5360
CXXFLAGS="$ICU_CXXFLAGS" \
5461
LDFLAGS="$ICU_LDFLAGS" \
55-
CC=$CROSS_COMPILE_PLATFORM_CC-clang \
56-
CXX=$CROSS_COMPILE_PLATFORM_CC-clang++ \
62+
CC="$CC_BIN" \
63+
CXX="$CXX_BIN" \
5764
AR=$TOOLCHAIN_DIR/bin/llvm-ar \
5865
LD=$TOOLCHAIN_DIR/bin/ld \
5966
RANLIB=$TOOLCHAIN_DIR/bin/llvm-ranlib \

‎scripts/compile/jsc.sh‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,11 @@ source $SCRIPT_DIR/common.sh
66
CMAKE_FOLDER=$(cd $ANDROID_HOME/cmake && ls -1 | sort -r | head -1)
77
PATH=$TOOLCHAIN_DIR/bin:$ANDROID_HOME/cmake/$CMAKE_FOLDER/bin/:$PATH
88

9+
CCACHE_CMAKE_ARGS=""
10+
if [[ -n "$JSC_CCACHE_BIN" ]]; then
11+
CCACHE_CMAKE_ARGS="-DCMAKE_C_COMPILER_LAUNCHER=${JSC_CCACHE_BIN} -DCMAKE_CXX_COMPILER_LAUNCHER=${JSC_CCACHE_BIN}"
12+
fi
13+
914
rm -rf $TARGETDIR/webkit/$CROSS_COMPILE_PLATFORM-${FLAVOR}
1015
rm -rf $TARGETDIR/webkit/WebKitBuild
1116
cd $TARGETDIR/webkit/Tools/Scripts
@@ -75,6 +80,7 @@ $TARGETDIR/webkit/Tools/Scripts/build-webkit \
7580
-DCMAKE_CXX_STANDARD_REQUIRED=ON \
7681
-DCMAKE_CXX_EXTENSIONS=OFF \
7782
-DCMAKE_VERBOSE_MAKEFILE=on \
83+
$CCACHE_CMAKE_ARGS \
7884
-DENABLE_API_TESTS=OFF \
7985
-DENABLE_SAMPLING_PROFILER=OFF \
8086
-DENABLE_DFG_JIT=OFF \

0 commit comments

Comments
 (0)