From 6768b0a5b217a40e9129a5971700e93ab72bc276 Mon Sep 17 00:00:00 2001 From: Shihaam Abdul Rahman Date: Sat, 22 Aug 2026 16:14:56 +0500 Subject: [PATCH] build works now --- .gitea/workflows/build.yml | 5 +- .gitignore | 6 +- Dockerfile | 4 + build.sh | 50 ++++++- overlay/server/htk-entry.ts | 50 +++++-- .../server/0002-bun-compat-tmp-import.patch | 33 +++++ ...003-bun-compat-adbkit-default-import.patch | 33 +++++ ...04-bun-compat-drop-oclif-self-update.patch | 24 ++++ release.sh | 67 ++++++++++ scripts/build.sh | 8 ++ scripts/patch-node-modules.sh | 123 ++++++++++++++++++ scripts/patch.sh | 50 +++++++ scripts/smoke.sh | 19 +++ scripts/ws-check.js | 56 ++++++++ 14 files changed, 506 insertions(+), 22 deletions(-) create mode 100644 patches/server/0002-bun-compat-tmp-import.patch create mode 100644 patches/server/0003-bun-compat-adbkit-default-import.patch create mode 100644 patches/server/0004-bun-compat-drop-oclif-self-update.patch create mode 100755 release.sh create mode 100755 scripts/patch-node-modules.sh create mode 100644 scripts/ws-check.js diff --git a/.gitea/workflows/build.yml b/.gitea/workflows/build.yml index 1158d05..bbed60c 100644 --- a/.gitea/workflows/build.yml +++ b/.gitea/workflows/build.yml @@ -20,12 +20,11 @@ jobs: fetch-depth: 0 - name: Build Linux binary (docker) - working-directory: .build/prod - run: docker compose run --build --rm prod + run: docker compose run --build --rm release-httptoolkit - name: Create Gitea release & upload assets env: GITEA_SERVER_URL: ${{ gitea.server_url }} GITEA_REPOSITORY: ${{ gitea.repository }} PAT_GITEA: ${{ secrets.PAT_GITEA }} - run: bash .build/prod/release.sh + run: bash release.sh diff --git a/.gitignore b/.gitignore index 711f01a..a4fb632 100644 --- a/.gitignore +++ b/.gitignore @@ -1,11 +1,15 @@ # Build outputs /dist/ -/out/ +/builds/ # Upstream sources are cloned fresh at build time, not tracked here /httptoolkit-ui/ /httptoolkit-server/ +# build.sh parks the server tree here while smoke-testing that the binary is +# genuinely self-contained. Only survives a build that died mid-test. +/.htk-server-hidden/ + # Local tooling / editor /.claude/ *.log diff --git a/Dockerfile b/Dockerfile index d379b24..f074f53 100644 --- a/Dockerfile +++ b/Dockerfile @@ -15,6 +15,10 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ jq \ && rm -rf /var/lib/apt/lists/* +# Install Bun (used for the single-file `--compile` step) +RUN curl -fsSL https://bun.sh/install | bash +ENV PATH="/root/.bun/bin:${PATH}" + WORKDIR /source CMD /source/build.sh diff --git a/build.sh b/build.sh index 0ef7478..6d66b26 100755 --- a/build.sh +++ b/build.sh @@ -1,16 +1,54 @@ #!/usr/bin/env bash - +# +# Runs INSIDE the builder container (see compose.yml / Dockerfile): +# +# docker compose run --rm release-httptoolkit +# +# Clones upstream, applies our patches + overlay, builds the UI, embeds it into +# the server and compiles a single self-contained binary -> builds/httptoolkit. +# set -euo pipefail cd /source +# The repo is bind-mounted from the host (owned by a different uid), so git +# refuses to operate on it until we mark it safe: +git config --global --add safe.directory '*' OUT_DIR="/source/builds" mkdir -p "$OUT_DIR" -git clone https://github.com/httptoolkit/httptoolkit-ui -git clone https://github.com/httptoolkit/httptoolkit-server +# Clone upstream -> scripts/patch.sh -> build UI -> embed -> bun compile. +# Produces dist/httptoolkit: +bash scripts/build.sh + +# Smoke-test the binary before we consider it releasable: it must serve the UI +# and bring up the management API, i.e. all native modules actually load. +# +# Critically, we hide the build tree first. Paths like +# /source/httptoolkit-server/node_modules/... are baked into the bundle, so any +# native module that bun failed to embed still *works* here while those files sit +# on disk - and then dies on a user's machine with "Cannot find module". Moving +# the tree aside forces the binary to prove it is genuinely self-contained. +HIDDEN=/source/.htk-server-hidden +rm -rf "$HIDDEN" +mv httptoolkit-server "$HIDDEN" +trap 'mv "$HIDDEN" /source/httptoolkit-server 2>/dev/null || true' EXIT + +echo +echo "### Smoke 1/2: self-containment (build tree hidden)" +bash scripts/smoke.sh dist/httptoolkit + +mv "$HIDDEN" /source/httptoolkit-server +trap - EXIT + +# Second pass with the tree back, because the WebSocket check needs node and the +# server's node_modules to run a real mockttp admin client against the binary. +# That path (admin session + subscription stream) is invisible to HTTP probing and +# is exactly where bun's built-in `ws` shim used to crash the server. +echo +echo "### Smoke 2/2: admin WebSocket + subscription stream" +WS_CHECK=1 bash scripts/smoke.sh dist/httptoolkit -## -## add the patch codes here -## cp -v dist/httptoolkit "$OUT_DIR/httptoolkit" +echo "==> Artifacts:" +ls -lh "$OUT_DIR" diff --git a/overlay/server/htk-entry.ts b/overlay/server/htk-entry.ts index 8f2ea24..3bd1259 100644 --- a/overlay/server/htk-entry.ts +++ b/overlay/server/htk-entry.ts @@ -1,9 +1,13 @@ /* * Single-file binary entry point (compiled with `bun build --compile`). * - * Not part of upstream. Copied to the server submodule root by scripts/apply.sh. - * The built UI is embedded into the binary as `ui.tar.gz` (created by - * scripts/build.sh) and extracted to a temp dir on first run. + * Not part of upstream. Copied to the server clone root by scripts/patch.sh. + * Two archives are embedded into the binary by scripts/build.sh and extracted to + * a temp dir on first run: + * - ui.tar.gz the built web UI, served over HTTP + * - overrides.tar.gz the interception overrides tree (terminal shims, the JVM + * agent, frida, the browser webextension) that upstream + * expects to find on disk at APP_ROOT/overrides * * Usage: httptoolkit [--port 8080] [--server-port 45457] [--mockttp-port 45456] [--no-open] */ @@ -17,11 +21,16 @@ import * as fs from 'fs'; import * as fsp from 'fs/promises'; import { x as extractTar } from 'tar'; -import { startHtkApp } from './src/htk-app'; -import { SERVER_VERSION } from './src/constants'; - -// Embedded UI bundle. Resolves to a path Bun can read from the compiled binary: +// Embedded archives. These resolve to paths Bun can read from the compiled binary: import uiArchive from './ui.tar.gz' with { type: 'file' }; +import overridesArchive from './overrides.tar.gz' with { type: 'file' }; + +// Deliberately NOT imported from './src/constants': that module computes APP_ROOT +// at load time, and we have to set HTK_APP_ROOT before it is ever evaluated (see +// main()). package.json carries the version without pulling in any of that. +import pkg from './package.json'; + +const SERVER_VERSION: string = pkg.version; function printHelp() { console.log(`HTTP Toolkit (self-hosted single binary) v${SERVER_VERSION} @@ -64,16 +73,18 @@ function parseArgs(argv: string[]) { return opts; } -async function extractUi(): Promise { - const dir = path.join(os.tmpdir(), `httptoolkit-ui-${SERVER_VERSION}`); +// Extract one embedded tarball into `dir`, unless it's already there for this +// version. Modes are preserved, which matters: overrides/path holds executable +// shims that only work if they stay +x. +async function extractArchive(archivePath: string, dir: string): Promise { const marker = path.join(dir, '.extracted'); if (fs.existsSync(marker)) return dir; // already extracted for this version await fsp.rm(dir, { recursive: true, force: true }); await fsp.mkdir(dir, { recursive: true }); - const bytes = new Uint8Array(await Bun.file(uiArchive).arrayBuffer()); - const tmpArchive = path.join(dir, 'ui.tar.gz'); + const bytes = new Uint8Array(await Bun.file(archivePath).arrayBuffer()); + const tmpArchive = path.join(dir, 'archive.tar.gz'); await fsp.writeFile(tmpArchive, bytes); await extractTar({ file: tmpArchive, cwd: dir }); @@ -84,7 +95,22 @@ async function extractUi(): Promise { async function main() { const opts = parseArgs(process.argv.slice(2)); - const uiDir = await extractUi(); + + // One app root per version, laid out the way upstream expects to find it: + // /httptoolkit-/ui + // /httptoolkit-/overrides + const appRoot = path.join(os.tmpdir(), `httptoolkit-${SERVER_VERSION}`); + const uiDir = path.join(appRoot, 'ui'); + + await extractArchive(uiArchive, uiDir); + await extractArchive(overridesArchive, path.join(appRoot, 'overrides')); + + // Must be set before ./src/constants is evaluated, since APP_ROOT (and hence + // OVERRIDES_DIR) is computed once at module load. That's why the app is pulled + // in with a dynamic import here rather than a static one at the top of the file. + process.env.HTK_APP_ROOT = appRoot; + + const { startHtkApp } = await import('./src/htk-app'); await startHtkApp({ ...opts, uiDir }); } diff --git a/patches/server/0002-bun-compat-tmp-import.patch b/patches/server/0002-bun-compat-tmp-import.patch new file mode 100644 index 0000000..219821b --- /dev/null +++ b/patches/server/0002-bun-compat-tmp-import.patch @@ -0,0 +1,33 @@ +Bun's bundler mis-compiles `import * as tmp from 'tmp'`. + +tmp's entrypoint ends with `Object.defineProperty(module.exports, 'tmpdir', {...})`. +When bun builds an ESM namespace object for that CJS module it emits a reference +to `exports_tmp` without ever declaring the binding, so the compiled binary dies +on startup with `ReferenceError: exports_tmp is not defined`. + +Importing it as CJS instead keeps bun on its `__commonJS` path, which handles the +`defineProperty` pattern correctly. Only the `tmp.file` value and the `Options` +type are used here, so the switch is behaviour-preserving. + +diff --git a/src/util/fs.ts b/src/util/fs.ts +index 21c99ef..0d2224f 100644 +--- a/src/util/fs.ts ++++ b/src/util/fs.ts +@@ -1,6 +1,7 @@ + import * as fs from 'fs'; + import * as path from 'path'; +-import * as tmp from 'tmp'; ++import type { Options as TmpOptions } from 'tmp'; ++const tmp: typeof import('tmp') = require('tmp'); + import { lookpath } from 'lookpath'; + import { isErrorLike } from '@httptoolkit/util'; + +@@ -62,7 +63,7 @@ export const resolveCommandPath = (path: string): Promise => + export const commandExists = (path: string): Promise => + resolveCommandPath(path).then((result) => result !== undefined); + +-export const createTmp = (options: tmp.Options = {}) => new Promise<{ ++export const createTmp = (options: TmpOptions = {}) => new Promise<{ + path: string, + fd: number, + cleanupCallback: () => void diff --git a/patches/server/0003-bun-compat-adbkit-default-import.patch b/patches/server/0003-bun-compat-adbkit-default-import.patch new file mode 100644 index 0000000..ff6a471 --- /dev/null +++ b/patches/server/0003-bun-compat-adbkit-default-import.patch @@ -0,0 +1,33 @@ +Pin the CJS default-import interop for adbkit, which bun and tsc disagree on. + +adbkit is a tsc-compiled CJS package: it sets `__esModule` and puts its real +default export at `exports.default`. Upstream builds with tsc + esModuleInterop, +where `import adb from '@devicefarmer/adbkit'` therefore resolves to +`exports.default` - the Adb class. + +Bun instead applies native ESM/Node semantics, where the default of a CJS module +is `module.exports` *itself*. It emits `__toESM(require(...), 1)` and, because +that `isNodeMode` flag is set, unconditionally does `default = mod` regardless of +`__esModule`. So `adb` ends up as the whole exports object and the Adb class is +one level deeper, giving `Adb.default.createClient is not a function` at startup +(AndroidAdbInterceptor is constructed eagerly by buildInterceptors, so this kills +the whole server, not just Android interception). + +Requiring the module explicitly and reaching for `.default` ourselves makes the +interop unambiguous under either toolchain. `Adb` is only ever used in type +positions here, so it becomes a type-only import. + +diff --git a/src/interceptors/android/adb-commands.ts b/src/interceptors/android/adb-commands.ts +index 26307ce..2f74c4d 100644 +--- a/src/interceptors/android/adb-commands.ts ++++ b/src/interceptors/android/adb-commands.ts +@@ -1,7 +1,8 @@ + import * as stream from 'stream'; + import * as path from 'path'; + +-import adb, * as Adb from '@devicefarmer/adbkit'; ++import type * as Adb from '@devicefarmer/adbkit'; ++const adb = (require('@devicefarmer/adbkit') as typeof import('@devicefarmer/adbkit')).default; + import { delay, isErrorLike } from '@httptoolkit/util'; + + import { logError } from '../../error-tracking'; diff --git a/patches/server/0004-bun-compat-drop-oclif-self-update.patch b/patches/server/0004-bun-compat-drop-oclif-self-update.patch new file mode 100644 index 0000000..4ea295c --- /dev/null +++ b/patches/server/0004-bun-compat-drop-oclif-self-update.patch @@ -0,0 +1,24 @@ +diff --git a/src/index.ts b/src/index.ts +index a13dcab..f2f3942 100644 +--- a/src/index.ts ++++ b/src/index.ts +@@ -15,7 +15,18 @@ import { + MockRTCAdminPlugin + } from 'mockrtc'; + +-import updateCommand from '@oclif/plugin-update/lib/commands/update'; ++// Self-update is meaningless in a single self-contained binary: oclif's updater ++// rewrites files inside an oclif install directory that doesn't exist here. Worse, ++// importing it drags @oclif/plugin-update into the bundle, and that reads ++// @oclif/command/package.json from disk when it loads - a path that only exists on ++// the build machine, so the compiled binary dies at startup everywhere else. ++// Back off for 6 hours like the EEXIT branch below, so the UI stops re-asking. ++const updateCommand = { ++ run: (_channel: string[]): Promise => { ++ console.log('Self-update is not supported in this build - download a new binary to update.'); ++ return delay(1000 * 60 * 60 * 6, { unref: true }); ++ } ++}; + + import { HttpToolkitServerApi } from './api/api-server'; + import { checkBrowserConfig } from './browsers'; diff --git a/release.sh b/release.sh new file mode 100755 index 0000000..ac97336 --- /dev/null +++ b/release.sh @@ -0,0 +1,67 @@ +#!/usr/bin/env bash +# +# Publish a NEW Gitea release for this build and upload everything in builds/. +# Never deletes existing releases/tags - each build gets its own tag. +# +# Config via env - in CI these come from the workflow's secrets: +# GITEA_SERVER_URL, GITEA_REPOSITORY, PAT_GITEA +# For a manual run, pass them inline rather than keeping a token on disk: +# GITEA_SERVER_URL=... GITEA_REPOSITORY=... PAT_GITEA=... ./release.sh +# Tag: $1 if given, else derived as v- +# (a build suffix is appended if that tag already exists). +# +set -euo pipefail +cd "$(dirname "$0")" + +: "${GITEA_SERVER_URL:?}" +: "${GITEA_REPOSITORY:?}" +: "${PAT_GITEA:?}" + +ROOT="$(git rev-parse --show-toplevel)" +SHA="$(git rev-parse HEAD)" +SHORT="$(git rev-parse --short HEAD)" +OUT_DIR="$(pwd)/builds" +REPO_API="${GITEA_SERVER_URL}/api/v1/repos/${GITEA_REPOSITORY}" +AUTH=(-H "Authorization: token ${PAT_GITEA}") + +# Resolve a unique release tag: +TAG="${1:-}" +if [ -z "$TAG" ]; then + VER="$(jq -r .version "$ROOT/httptoolkit-server/package.json")" + TAG="v${VER}-${SHORT}" + # If a release for this commit already exists (e.g. a scheduled rebuild of an + # unchanged commit), keep the old one and make this tag unique instead. + if curl -sf "${AUTH[@]}" "${REPO_API}/releases/tags/${TAG}" > /dev/null 2>&1; then + TAG="${TAG}-${GITHUB_RUN_NUMBER:-$(date -u +%Y%m%d%H%M%S)}" + fi +fi + +echo "==> Creating release $TAG @ $SHORT" +RESP=$(curl -s -X POST "${REPO_API}/releases" "${AUTH[@]}" \ + -H "Content-Type: application/json" \ + -d "{ + \"tag_name\": \"${TAG}\", + \"target_commitish\": \"${SHA}\", + \"name\": \"HTTP Toolkit ${TAG}\", + \"body\": \"Self-hosted single-file build from ${SHORT}.\", + \"draft\": false, + \"prerelease\": false + }") + +RID=$(echo "$RESP" | jq -r '.id // empty') +if [ -z "$RID" ]; then + echo "!! Failed to create release:"; echo "$RESP"; exit 1 +fi +echo "==> Release id $RID" + +# Upload artifacts. +shopt -s nullglob +for f in "$OUT_DIR"/*; do + [ -f "$f" ] || continue + name=$(basename "$f") + echo "==> Uploading $name" + curl -s -X POST "${REPO_API}/releases/${RID}/assets?name=${name}" "${AUTH[@]}" \ + -H "Content-Type: application/octet-stream" \ + --data-binary "@${f}" > /dev/null +done +echo "==> Release complete." diff --git a/scripts/build.sh b/scripts/build.sh index 6836bc7..11c6838 100755 --- a/scripts/build.sh +++ b/scripts/build.sh @@ -22,12 +22,20 @@ echo "==> Building UI" ( cd httptoolkit-ui && npm ci && npm run build ) echo "==> Preparing server + embedding UI" +# overrides/ is the interception support tree (terminal shims, JVM agent, frida, +# webextension). Upstream reads it off disk at APP_ROOT/overrides, so it has to be +# embedded too - the binary extracts both archives on startup. -p keeps the +x bits +# on the shims in overrides/path. ( cd httptoolkit-server \ && npm ci \ && npm install --no-save tar@^7 \ && tar -czf ui.tar.gz -C ../httptoolkit-ui/dist . \ + && tar -czpf overrides.tar.gz -C overrides . \ && rm -rf ui && cp -r ../httptoolkit-ui/dist ui ) +echo "==> Patching installed deps for bun" +bash scripts/patch-node-modules.sh + echo "==> Compiling single-file binary -> dist/httptoolkit" mkdir -p dist ( cd httptoolkit-server && bun build ./htk-entry.ts --compile --outfile "$ROOT/dist/httptoolkit" ) diff --git a/scripts/patch-node-modules.sh b/scripts/patch-node-modules.sh new file mode 100755 index 0000000..7e51d8e --- /dev/null +++ b/scripts/patch-node-modules.sh @@ -0,0 +1,123 @@ +#!/usr/bin/env bash +# +# Bun-compatibility fixups applied to installed dependencies. Runs AFTER `npm ci` +# (it edits node_modules) and BEFORE `bun build --compile`. +# +set -euo pipefail +ROOT="$(cd "$(dirname "$0")/.." && pwd)" +cd "$ROOT/httptoolkit-server" + +# TypeScript's UMD emit wraps the module body in a factory that takes `require` +# as a *parameter*: +# +# })(function (require, exports) { +# var bplist_creator_1 = __importDefault(require("bplist-creator")); +# +# That parameter shadows the module-scope `require`, so bun's bundler can't +# statically link the call and leaves it as a runtime require. In a --compile'd +# binary there is no node_modules to fall back on, so the import fails at startup +# ("Cannot find package 'bplist-creator'"). Note that merely forcing the package +# into the bundle does NOT help: bun's runtime require only resolves call sites +# it linked at build time. +# +# Renaming the parameter un-shadows `require`, so the calls bind to the +# module-scope require that bun does link. The CJS branch still passes the real +# require in as the (now unused) first argument, so behaviour is unchanged. +deshadow_umd_require() { + local dir="$1" found=0 already=0 + for file in "$dir"/*.js; do + [ -e "$file" ] || continue + if grep -q '})(function (require, exports) {' "$file"; then + sed -i 's/})(function (require, exports) {/})(function (__umdRequire, exports) {/' "$file" + found=$((found + 1)) + elif grep -q '__umdRequire' "$file"; then + already=$((already + 1)) + fi + done + echo " - $dir: patched $found, already patched $already" + # Idempotent: re-running over an existing tree is fine. Only a tree with + # neither form means the dependency changed shape under us. + [ $((found + already)) -gt 0 ] \ + || { echo "!! no UMD wrappers found in $dir (dependency changed?)"; exit 1; } +} + +echo "==> [node_modules] de-shadowing UMD require() calls for bun" +# Reached via @httptoolkit/browser-launcher -> simple-plist (darwin browser +# detection). Bun bundles every platform branch regardless of target, so this +# has to resolve even in a Linux build. +deshadow_umd_require node_modules/simple-plist/dist + +# Native .node addons only get embedded into the compiled binary if bun can +# resolve the require statically. node-datachannel (via mockrtc) instead builds a +# require with module.createRequire(): +# +# const require$1 = module$1.createRequire(pathToFileURL(__filename).href); +# const nodeDataChannel = require$1('../build/Release/node_datachannel.node'); +# +# bun can't see through that, so the addon is left out and the binary resolves it +# relative to the *build-time* __filename at runtime - working on the build +# machine while failing everywhere else with "Cannot find module +# '../build/Release/node_datachannel.node'". Using a plain literal require makes +# bun embed the addon as an asset instead. +replace_literal() { + local file="$1" from="$2" to="$3" + [ -f "$file" ] || { echo "!! expected $file to exist (dependency changed?)"; exit 1; } + if ! grep -qF "$from" "$file"; then + # Idempotent: already-applied is fine, a missing target is not. + if grep -qF "$to" "$file"; then + echo " - $file: already patched" + return 0 + fi + echo "!! pattern not found in $file: $from"; exit 1 + fi + python3 - "$file" "$from" "$to" <<'PY' +import sys +path, old, new = sys.argv[1], sys.argv[2], sys.argv[3] +with open(path) as f: content = f.read() +with open(path, 'w') as f: f.write(content.replace(old, new)) +PY + echo " - $file: $from -> $to" +} + +echo "==> [node_modules] making native addon requires statically resolvable" +replace_literal node_modules/node-datachannel/lib/index.cjs \ + "require\$1('../build/Release/node_datachannel.node')" \ + "require('../build/Release/node_datachannel.node')" + +# bun ships its own built-in `ws` shim and resolves the bare specifier "ws" to it, +# shadowing the real npm package. That shim has no createWebSocketStream (and no +# Receiver): it throws "Not supported yet in Bun". mockttp's admin server calls +# Ws.createWebSocketStream() for every admin client connection, so the server +# process died the moment the UI opened a session - after the UI had already +# loaded and looked fine. +# +# Requiring a subpath instead of the bare name escapes bun's shim and gets the +# real ws@8, which implements all of this. ws' exports map doesn't publish +# ./index.js, so add it before pointing anything at it. +redirect_ws_to_real_package() { + local dir="$1" count=0 + # Publish the subpath we're about to require: + if [ "$(jq -r '.exports["./index.js"] // "none"' node_modules/ws/package.json)" = "none" ]; then + jq '.exports["./index.js"] = "./index.js"' node_modules/ws/package.json > node_modules/ws/package.json.tmp + mv node_modules/ws/package.json.tmp node_modules/ws/package.json + echo " - ws/package.json: exposed ./index.js subpath" + else + echo " - ws/package.json: subpath already exposed" + fi + + while IFS= read -r file; do + sed -i 's|require("ws")|require("ws/index.js")|g' "$file" + echo " - ${file#node_modules/}: -> ws/index.js" + count=$((count + 1)) + done < <(grep -rl 'require("ws")' "$dir" --include=*.js 2>/dev/null || true) + + local already + already=$(grep -rl 'require("ws/index.js")' "$dir" --include=*.js 2>/dev/null | wc -l) + [ "$already" -gt 0 ] \ + || { echo "!! no bare ws requires found in $dir (dependency changed?)"; exit 1; } +} + +echo "==> [node_modules] redirecting ws requires past bun's built-in shim" +redirect_ws_to_real_package node_modules/mockttp/dist + +echo "==> patch-node-modules.sh complete." diff --git a/scripts/patch.sh b/scripts/patch.sh index 054bc6c..8e1c431 100755 --- a/scripts/patch.sh +++ b/scripts/patch.sh @@ -21,6 +21,56 @@ apply_patches() { apply_patches ui httptoolkit-ui apply_patches server httptoolkit-server +# `src/config.d.ts` is a types-only module: tsc erases `import { HtkConfig } from +# './config'` entirely, but bun's bundler resolves imports for real and fails +# ("Could not resolve: ./config") because a .d.ts isn't a bundleable source file. +# Promote it to a real .ts module so bun can resolve it - it holds only exported +# interfaces, so it compiles down to nothing. Derived from upstream rather than +# vendored, so it can't drift out of sync with the real type. +promote_dts_to_ts() { + local dts="$1" ts="${1%.d.ts}.ts" + [ -f "$dts" ] || { echo "!! expected $dts to exist (upstream moved?)"; exit 1; } + # Anything other than exported type declarations would need real runtime code: + if grep -qE '^\s*(declare|import|export\s+(const|let|var|function|class|default))' "$dts"; then + echo "!! $dts is no longer types-only - it needs a real runtime module now" + exit 1 + fi + echo " - $dts -> $ts" + mv "$dts" "$ts" +} + +echo "==> [server] promoting types-only modules to real modules (for bun)" +promote_dts_to_ts httptoolkit-server/src/config.d.ts + +# Upstream targets es2020 and never sets useDefineForClassFields, so tsc defaults +# it to false: class field initialisers are emitted *inside* the constructor, +# after parameter properties are assigned. Bun ignores that inferred default and +# emits native ES class fields, which run BEFORE the constructor body - so any +# class doing `constructor(private config: X)` plus a field initialiser that +# reads `this.config` (e.g. ElectronInterceptor.certData) crashes with +# "undefined is not an object". Set the flag explicitly to restore tsc semantics. +# APP_ROOT is derived from __dirname, which in a compiled binary is a build-machine +# path that doesn't exist for users. Everything hanging off it then breaks at +# runtime - OVERRIDES_DIR most visibly ("Unable to access jarfile +# .../overrides/java-agent.jar", "lstat .../overrides/webextension"), which takes +# out JVM/terminal interception and the WebRTC webextension. Make it overridable so +# htk-entry.ts can point it at the directory it extracts the embedded overrides to. +echo "==> [server] making APP_ROOT overridable via HTK_APP_ROOT" +APP_ROOT_LINE="export const APP_ROOT = path.join(__dirname, '..');" +if grep -qF "$APP_ROOT_LINE" httptoolkit-server/src/constants.ts; then + sed -i "s|$APP_ROOT_LINE|export const APP_ROOT = process.env.HTK_APP_ROOT \|\| path.join(__dirname, '..');|" \ + httptoolkit-server/src/constants.ts +elif grep -q 'HTK_APP_ROOT' httptoolkit-server/src/constants.ts; then + echo " - already applied" +else + echo "!! APP_ROOT definition not found in constants.ts (upstream moved?)"; exit 1 +fi + +echo "==> [server] pinning useDefineForClassFields=false (tsc semantics for bun)" +jq '.compilerOptions.useDefineForClassFields = false' \ + httptoolkit-server/tsconfig.json > httptoolkit-server/tsconfig.json.tmp +mv httptoolkit-server/tsconfig.json.tmp httptoolkit-server/tsconfig.json + echo "==> Copying overlay into httptoolkit-server" cp -v overlay/server/htk-entry.ts httptoolkit-server/htk-entry.ts cp -v overlay/server/src/htk-app.ts httptoolkit-server/src/htk-app.ts diff --git a/scripts/smoke.sh b/scripts/smoke.sh index 80d49a9..ecd5a91 100755 --- a/scripts/smoke.sh +++ b/scripts/smoke.sh @@ -37,6 +37,25 @@ echo "==> UI responded; checking it looks like the HTK app" grep -qi 'httptoolkit\|
/dev/null; then + echo "==> checking admin WebSocket + subscription stream on :$MOCKTTP_PORT" + NODE_PATH="$NM" node "$ROOT/scripts/ws-check.js" \ + "http://127.0.0.1:${MOCKTTP_PORT}" "http://localhost:${PORT}" \ + || { echo "!! admin WebSocket check failed"; exit 1; } + # That check is only meaningful if it didn't take the server down with it: + kill -0 "$PID" 2>/dev/null || { echo "!! binary died during WebSocket check"; exit 1; } + else + echo "==> SKIP WebSocket check (needs node + httptoolkit-server/node_modules)" + fi +fi + echo "==> checking management API is listening on :$SERVER_PORT" # corsGate rejects requests without an allowed Origin with a JSON 403 - that's a # *successful* connection (the server is up), which is all we need to confirm. diff --git a/scripts/ws-check.js b/scripts/ws-check.js new file mode 100644 index 0000000..bce3cda --- /dev/null +++ b/scripts/ws-check.js @@ -0,0 +1,56 @@ +/* + * Exercises the admin-server WebSocket path that the UI uses, which the plain + * HTTP smoke test never touches. + * + * mockttp's admin server wraps each admin client's connection with + * Ws.createWebSocketStream(). Under bun's built-in `ws` shim that throws + * "Not supported yet in Bun" and kills the whole server the moment the UI opens a + * session - so a binary can pass every HTTP check and still die on first real use. + * + * Run with node (this is a *client*; the binary under test is the server). + * Usage: node ws-check.js [adminServerUrl] + */ +const mockttp = require('mockttp'); + +const adminServerUrl = process.argv[2] || 'http://127.0.0.1:45456'; +// The admin server runs corsOptions.strict, so it requires an Origin from +// MOCKTTP_ALLOWED_ORIGINS - the same one the browser UI sends. +const origin = process.argv[3] || 'http://localhost:8099'; + +(async () => { + const server = mockttp.getRemote({ + adminServerUrl, + client: { headers: { origin } } + }); + + // start() opens the admin WebSocket + subscription stream. This is the call + // that used to crash the server process. + await server.start(); + console.log(` admin session started, mock server on port ${server.port}`); + + await server.forGet('/ws-check').thenReply(200, 'ws-check-ok'); + + const res = await fetch(`http://127.0.0.1:${server.port}/ws-check`); + const body = await res.text(); + if (body !== 'ws-check-ok') { + throw new Error(`unexpected proxied response: ${JSON.stringify(body)}`); + } + console.log(' proxied request through the mock server OK'); + + // Confirm the subscription stream is live, not just connected. Subscribing is + // async on a remote client, so settle before generating traffic or we race it. + const seen = new Promise((resolve) => server.on('request', (r) => resolve(r.path))); + await new Promise((r) => setTimeout(r, 1000)); + await fetch(`http://127.0.0.1:${server.port}/ws-check`); + const path = await Promise.race([ + seen, + new Promise((_, rej) => setTimeout(() => rej(new Error('no event over subscription stream')), 10000)) + ]); + console.log(` received '${path}' event over the subscription stream`); + + await server.stop(); + console.log(' admin session stopped cleanly'); +})().catch((e) => { + console.error('!! ws-check FAILED:', e.message); + process.exit(1); +});