Compare commits

..

11 Commits

Author SHA1 Message Date
Michael Telatynski
9f00594ae3
Attempt to kill off hak
Signed-off-by: Michael Telatynski <7t3chguy@gmail.com>
2025-02-28 14:13:58 +00:00
Michael Telatynski
02c23e87b5
Update imports
Signed-off-by: Michael Telatynski <7t3chguy@gmail.com>
2025-02-28 13:28:27 +00:00
Michael Telatynski
3e7b10894a
Merge remote-tracking branch 'origin/t3chguy/keytar-forked' into t3chguy/keytar-forked 2025-02-28 13:23:44 +00:00
Michael Telatynski
be9214fe0b
Merge branch 't3chguy/keytar-no-hak' of https://github.com/vector-im/element-desktop into t3chguy/keytar-forked
Signed-off-by: Michael Telatynski <7t3chguy@gmail.com>

# Conflicts:
#	hak/keytar-forked/build.ts
#	hak/keytar-forked/check.ts
#	hak/keytar-forked/hak.json
#	package.json
#	src/@types/keytar.d.ts
#	src/keytar.ts
2025-02-28 13:23:38 +00:00
Michael Telatynski
2219ca9ca3
Merge branch 'develop' into t3chguy/keytar-no-hak 2025-02-28 13:20:13 +00:00
Michael Telatynski
e4dac2a210
Iterate
Signed-off-by: Michael Telatynski <7t3chguy@gmail.com>
2025-02-28 12:33:47 +00:00
Michael Telatynski
36446c108b
Merge branch 'develop' into t3chguy/keytar-forked 2025-02-28 12:30:46 +00:00
Michael Telatynski
5ee122cdae
Add logging
Signed-off-by: Michael Telatynski <7t3chguy@gmail.com>
2025-02-28 12:14:10 +00:00
Michael Telatynski
fde278d5b5
Leverage electron-builder to build keytar rather than hak
Signed-off-by: Michael Telatynski <7t3chguy@gmail.com>
2025-02-28 11:15:42 +00:00
Michael Telatynski
08b61a7e3a
Iterate
Signed-off-by: Michael Telatynski <7t3chguy@gmail.com>
2025-02-28 11:02:02 +00:00
Michael Telatynski
6ae09710c6
Test keytar-forked
Signed-off-by: Michael Telatynski <7t3chguy@gmail.com>
2025-02-28 10:59:16 +00:00
29 changed files with 393 additions and 1446 deletions

View File

@ -1,23 +0,0 @@
on: [push, workflow_dispatch]
jobs:
build-element:
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@v3
- name: Download Cringe Studios configuration
run: "mkdir cringe && wget -O cringe/config.json https://chat.cringe-studios.com/config.json"
- name: Docker setup
run: "docker build --platform linux/amd64 -t element-desktop-dockerbuild -f dockerbuild/Dockerfile ."
- name: Docker build
run: >
ls -la && PWD=$(pwd) scripts/in-docker.sh bash -c 'ls -la &&
yarn add matrix-seshat &&
yarn &&
yarn run fetch --noverify --cfgdir cringe &&
SQLCIPHER_BUNDLED=1 yarn run docker:build:native &&
yarn run docker:build --linux rpm --linux deb --linux tar.xz --linux AppImage --win squirrel'

View File

@ -92,6 +92,7 @@ jobs:
uses: ./.github/workflows/build_linux.yaml
with:
arch: ${{ matrix.arch }}
config: ${{ needs.prepare.outputs.config }}
sqlcipher: ${{ matrix.sqlcipher }}
version: ${{ needs.prepare.outputs.nightly-version }}

View File

@ -35,6 +35,7 @@ jobs:
sqlcipher: [system, static]
arch: [amd64, arm64]
with:
config: ${{ github.event.pull_request.base.ref == 'develop' && 'element.io/nightly' || 'element.io/release' }}
sqlcipher: ${{ matrix.sqlcipher }}
arch: ${{ matrix.arch }}

View File

@ -8,6 +8,10 @@ on:
type: string
required: true
description: "The architecture to build for, one of 'amd64' | 'arm64'"
config:
type: string
required: true
description: "The config directory to use"
version:
type: string
required: false
@ -62,14 +66,6 @@ jobs:
with:
name: webapp
- name: Cache .hak
id: cache
uses: actions/cache@v4
with:
key: ${{ runner.os }}-${{ github.ref_name }}-${{ inputs.sqlcipher }}-${{ inputs.arch }}-${{ hashFiles('hakHash', 'electronVersion', 'dockerbuild/*') }}
path: |
./.hak
- uses: actions/setup-node@v4
with:
node-version-file: .node-version
@ -91,7 +87,7 @@ jobs:
# This allows contributors to test changes to the dockerbuild image within a pull request
- name: Build docker image
uses: docker/build-push-action@471d1dc4e07e5cdedd4c2171150001c434f0b7a4 # v6
uses: docker/build-push-action@ca877d9245402d1537745e0e356eab47c3520991 # v6
if: steps.changed_files.outputs.any_modified == 'true'
with:
file: dockerbuild/Dockerfile
@ -99,23 +95,11 @@ jobs:
platforms: linux/${{ inputs.arch }}
tags: ${{ env.HAK_DOCKER_IMAGE }}
- name: Build Natives
if: steps.cache.outputs.cache-hit != 'true'
run: |
docker run \
-v ${{ github.workspace }}:/work -w /work \
-e SQLCIPHER_BUNDLED \
$HAK_DOCKER_IMAGE \
yarn build:native
- name: Fix permissions on .hak
run: sudo chown -R $USER:$USER .hak
- name: Check native libraries in hak dependencies
run: |
shopt -s globstar
for filename in ./.hak/hakModules/**/*.node; do
for filename in ./node_modules/**/*.node; do
./scripts/glibc-check.sh $filename
done

View File

@ -39,14 +39,6 @@ jobs:
with:
name: webapp
- name: Cache .hak
id: cache
uses: actions/cache@v4
with:
key: ${{ runner.os }}-${{ hashFiles('hakHash', 'electronVersion') }}
path: |
./.hak
- name: Install Rust
if: steps.cache.outputs.cache-hit != 'true'
run: |
@ -68,13 +60,11 @@ jobs:
- name: Install Deps
run: "yarn install --frozen-lockfile"
# Python 3.12 drops distutils which keytar relies on
- name: Install setuptools
run: pip3 install setuptools
- name: Build Natives
if: steps.cache.outputs.cache-hit != 'true'
run: yarn build:native:universal
run: |
# Python 3.12 drops distutils which keytar relies on
pip3 install setuptools
# We split these because electron-builder gets upset if we set CSC_LINK even to an empty string
- name: "[Signed] Build App"

View File

@ -34,9 +34,12 @@ on:
packages-dir:
description: "The directory non-deb packages for this run should live in within packages.element.io"
value: ${{ inputs.nightly && 'nightly' || 'desktop' }}
# This is just a simple pass-through of the input to simplify reuse of complex inline conditions
# These are just simple pass-throughs of the input to simplify reuse of complex inline conditions
config:
description: "The relative path to the config file for this run"
value: ${{ inputs.config }}
deploy:
description: "Whether the build should be deployed to production"
description: "The relative path to the config file for this run"
value: ${{ inputs.deploy }}
permissions: {}
jobs:

View File

@ -56,8 +56,7 @@ jobs:
"ia32": {
"target": "i686-pc-windows-msvc",
"build-args": "--ia32",
"arch": "x86",
"extra_config": "{\"user_notice\": {\"title\": \"Your desktop support ends soon\",\"description\": \"Support for 32-bit Windows installations will end soon, this impacts you. Transition to the web or mobile app for continued access.\"}}"
"arch": "x86"
}
}
@ -67,14 +66,6 @@ jobs:
with:
name: webapp
- name: Cache .hak
id: cache
uses: actions/cache@v4
with:
key: ${{ runner.os }}-${{ inputs.arch }}-${{ hashFiles('hakHash', 'electronVersion') }}
path: |
./.hak
# ActiveTCL package on choco is from 2015,
# this one is newer but includes more than we need
- name: Choco install tclsh
@ -106,20 +97,6 @@ jobs:
- name: Install Deps
run: "yarn install --frozen-lockfile"
- name: Insert config snippet
if: steps.config.outputs.extra_config != ''
shell: bash
run: |
mkdir config-edit
yarn asar extract webapp.asar config-edit
cd config-edit
mv config.json old-config.json
echo '${{ steps.config.outputs.extra_config }}' | jq -s '.[0] * .[1]' old-config.json - > config.json
rm old-config.json
cd ..
rm webapp.asar
yarn asar pack config-edit/ webapp.asar
- name: Set up sqlcipher macros
if: steps.cache.outputs.cache-hit != 'true' && contains(inputs.arch, 'arm')
shell: pwsh
@ -132,12 +109,6 @@ jobs:
with:
arch: ${{ steps.config.outputs.arch || inputs.arch }}
- name: Build Natives
if: steps.cache.outputs.cache-hit != 'true'
run: |
refreshenv
yarn build:native --target ${{ steps.config.outputs.target }}
- name: Install and configure eSigner CKA
run: |
Set-StrictMode -Version 'Latest'

View File

@ -22,15 +22,15 @@ jobs:
- uses: actions/checkout@v4
- name: Set up QEMU
uses: docker/setup-qemu-action@29109295f81e9208d7d86ff1c6c12d2833863392 # v3
uses: docker/setup-qemu-action@4574d27a4764455b42196d70a065bc6853246a25 # v3
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@b5ca514318bd6ebac0fb2aedd5d36ec1b5c232a2 # v3
uses: docker/setup-buildx-action@f7ce87c1d6bead3e36075b2ce75da1f6cc28aaca # v3
with:
install: true
- name: Build test image
uses: docker/build-push-action@471d1dc4e07e5cdedd4c2171150001c434f0b7a4 # v6
uses: docker/build-push-action@ca877d9245402d1537745e0e356eab47c3520991 # v6
with:
file: dockerbuild/Dockerfile
push: false
@ -52,7 +52,7 @@ jobs:
- name: Extract metadata for Docker
id: meta
if: github.event_name != 'pull_request'
uses: docker/metadata-action@902fa8ec7d6ecbf8d84d538b9b233a880e428804 # v5
uses: docker/metadata-action@369eb591f429131d6889c46b94e711f089e6ca96 # v5
with:
images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
tags: |
@ -61,7 +61,7 @@ jobs:
- name: Build and push Docker image
if: github.event_name != 'pull_request'
uses: docker/build-push-action@471d1dc4e07e5cdedd4c2171150001c434f0b7a4 # v6
uses: docker/build-push-action@ca877d9245402d1537745e0e356eab47c3520991 # v6
with:
file: dockerbuild/Dockerfile
push: true

View File

@ -14,7 +14,6 @@ RUN apt-get -qq update && apt-get -y -qq dist-upgrade && \
# Used by seshat (when not SQLCIPHER_STATIC) \
libsqlcipher-dev && \
apt-get purge -y --auto-remove && rm -rf /var/lib/apt/lists/*
RUN dpkg --add-architecture i386 && apt-get update && apt-get install -y wine mono-devel rpm
RUN ln -s /usr/bin/python3 /usr/bin/python & ln -s /usr/bin/pip3 /usr/bin/pip
ENV DEBUG_COLORS=true

View File

@ -93,14 +93,7 @@ const config: Omit<Writable<Configuration>, "electronFuses"> & {
afterPack: async (context: AfterPackContext) => {
await injectAsarIntegrity(context);
},
files: [
"package.json",
{
from: ".hak/hakModules",
to: "node_modules",
},
"lib/**",
],
files: ["package.json", "lib/**"],
extraResources: [
{
from: "res/img",

View File

@ -1,34 +0,0 @@
/*
Copyright 2024 New Vector Ltd.
Copyright 2020, 2021 The Matrix.org Foundation C.I.C.
SPDX-License-Identifier: AGPL-3.0-only OR GPL-3.0-only OR LicenseRef-Element-Commercial
Please see LICENSE files in the repository root for full details.
*/
import type HakEnv from "../../scripts/hak/hakEnv.js";
import type { DependencyInfo } from "../../scripts/hak/dep.js";
export default async function (hakEnv: HakEnv, moduleInfo: DependencyInfo): Promise<void> {
const env = hakEnv.makeGypEnv();
if (!hakEnv.isHost()) {
env.CARGO_BUILD_TARGET = hakEnv.getTargetId();
}
console.log("Running yarn install");
await hakEnv.spawn("yarn", ["install"], {
cwd: moduleInfo.moduleBuildDir,
env,
shell: true,
});
const buildTarget = hakEnv.wantsStaticSqlCipher() ? "build-bundled" : "build";
console.log("Running yarn build");
await hakEnv.spawn("yarn", ["run", buildTarget], {
cwd: moduleInfo.moduleBuildDir,
env,
shell: true,
});
}

View File

@ -1,55 +0,0 @@
/*
Copyright 2024 New Vector Ltd.
Copyright 2020, 2021 The Matrix.org Foundation C.I.C.
SPDX-License-Identifier: AGPL-3.0-only OR GPL-3.0-only OR LicenseRef-Element-Commercial
Please see LICENSE files in the repository root for full details.
*/
import childProcess from "node:child_process";
import fsProm from "node:fs/promises";
import type HakEnv from "../../scripts/hak/hakEnv.js";
import type { Tool } from "../../scripts/hak/hakEnv.js";
import type { DependencyInfo } from "../../scripts/hak/dep.js";
export default async function (hakEnv: HakEnv, moduleInfo: DependencyInfo): Promise<void> {
const tools: Tool[] = [
["rustc", "--version"],
["python", "--version"], // node-gyp uses python for reasons beyond comprehension
];
if (hakEnv.isWin()) {
tools.push(["perl", "--version"]); // for openssl configure
tools.push(["nasm", "-v"]); // for openssl building
tools.push(["patch", "--version"]); // to patch sqlcipher Makefile.msc
tools.push(["nmake", "/?"]);
} else {
tools.push(["make", "--version"]);
}
await hakEnv.checkTools(tools);
// Ensure Rust target exists (nb. we avoid depending on rustup)
await new Promise((resolve, reject) => {
const rustc = childProcess.execFile(
"rustc",
["--target", hakEnv.getTargetId(), "--emit=obj", "-o", "tmp", "-"],
(err, out) => {
if (err) {
reject(
"rustc can't build for target " +
hakEnv.getTargetId() +
": ensure target is installed via `rustup target add " +
hakEnv.getTargetId() +
"` " +
"or your package manager if not using `rustup`",
);
}
fsProm.unlink("tmp").then(resolve);
},
);
rustc.stdin!.write("fn main() {}");
rustc.stdout!.pipe(process.stdout);
rustc.stderr!.pipe(process.stderr);
rustc.stdin!.end();
});
}

View File

@ -1,7 +0,0 @@
{
"scripts": {
"check": "check.ts",
"build": "build.ts"
},
"copy": "index.node"
}

View File

@ -1,11 +0,0 @@
{
"compilerOptions": {
"moduleResolution": "node",
"esModuleInterop": true,
"target": "es2022",
"sourceMap": false,
"strict": true,
"lib": ["es2022"]
},
"include": ["../scripts/@types/*.d.ts", "./**/*.ts"]
}

View File

@ -5,6 +5,7 @@ export default {
project: ["**/*.{js,ts}"],
ignoreDependencies: [
// Brought in via hak scripts
"keytar-forked",
"matrix-seshat",
// Required for `action-validator`
"@action-validator/*",

View File

@ -27,17 +27,14 @@
"asar-webapp": "asar p webapp webapp.asar",
"start": "yarn run build:ts && yarn run build:res && electron .",
"lint": "yarn lint:types && yarn lint:js && yarn lint:workflows",
"lint:js": "eslint --max-warnings 0 src hak playwright scripts && prettier --check .",
"lint:js-fix": "eslint --fix --max-warnings 0 src hak playwright scripts && prettier --log-level=warn --write .",
"lint:types": "yarn lint:types:src && yarn lint:types:test && yarn lint:types:scripts && yarn lint:types:hak",
"lint:js": "eslint --max-warnings 0 src playwright scripts && prettier --check .",
"lint:js-fix": "eslint --fix --max-warnings 0 src playwright scripts && prettier --log-level=warn --write .",
"lint:types": "yarn lint:types:src && yarn lint:types:test && yarn lint:types:scripts",
"lint:types:src": "tsc --noEmit",
"lint:types:test": "tsc --noEmit -p playwright/tsconfig.json",
"lint:types:scripts": "tsc --noEmit -p scripts/tsconfig.json",
"lint:types:hak": "tsc --noEmit -p hak/tsconfig.json",
"lint:workflows": "find .github/workflows -type f \\( -iname '*.yaml' -o -iname '*.yml' \\) | xargs -I {} sh -c 'echo \"Linting {}\"; action-validator \"{}\"'",
"lint:knip": "knip",
"build:native": "yarn run hak",
"build:native:universal": "yarn run hak --target x86_64-apple-darwin fetchandbuild && yarn run hak --target aarch64-apple-darwin fetchandbuild && yarn run hak --target x86_64-apple-darwin --target aarch64-apple-darwin copyandlink",
"build:32": "yarn run build:ts && yarn run build:res && electron-builder --ia32",
"build:64": "yarn run build:ts && yarn run build:res && electron-builder --x64",
"build:universal": "yarn run build:ts && yarn run build:res && electron-builder --universal",
@ -45,7 +42,6 @@
"build:ts": "tsc",
"build:res": "tsx scripts/copy-res.ts",
"docker:setup": "docker build --platform linux/amd64 -t element-desktop-dockerbuild -f dockerbuild/Dockerfile .",
"docker:build:native": "scripts/in-docker.sh yarn run hak",
"docker:build": "scripts/in-docker.sh yarn run build",
"docker:install": "scripts/in-docker.sh yarn install",
"clean": "rimraf webapp.asar dist packages deploys lib",
@ -57,13 +53,14 @@
"postinstall": "electron-builder install-app-deps"
},
"dependencies": {
"@sentry/electron": "^6.0.0",
"@sentry/electron": "^5.0.0",
"@standardnotes/electron-clear-data": "^1.0.5",
"auto-launch": "^5.0.5",
"counterpart": "^0.18.6",
"electron-store": "^10.0.0",
"electron-window-state": "^5.0.3",
"keytar-forked": "7.10.0",
"matrix-seshat": "https://gitpkg.vercel.app/matrix-org/seshat/seshat-node?t3chguy/binding-gyp",
"minimist": "^1.2.6",
"png-to-ico": "^2.1.1",
"uuid": "^11.0.0"
@ -80,16 +77,16 @@
"@types/auto-launch": "^5.0.1",
"@types/counterpart": "^0.18.1",
"@types/minimist": "^1.2.1",
"@types/node": "18.19.79",
"@types/node": "18.19.76",
"@types/pacote": "^11.1.1",
"@typescript-eslint/eslint-plugin": "^8.0.0",
"@typescript-eslint/parser": "^8.0.0",
"app-builder-lib": "26.0.10",
"app-builder-lib": "26.0.8",
"chokidar": "^4.0.0",
"detect-libc": "^2.0.0",
"electron": "34.3.0",
"electron-builder": "26.0.10",
"electron-builder-squirrel-windows": "26.0.10",
"electron": "34.2.0",
"electron-builder": "26.0.8",
"electron-builder-squirrel-windows": "26.0.8",
"electron-devtools-installer": "^4.0.0",
"eslint": "^8.26.0",
"eslint-config-google": "^0.14.0",
@ -110,13 +107,10 @@
"rimraf": "^6.0.0",
"tar": "^7.0.0",
"tsx": "^4.19.2",
"typescript": "5.8.2"
},
"hakDependencies": {
"matrix-seshat": "^4.0.1"
"typescript": "5.7.3"
},
"resolutions": {
"@types/node": "18.19.79",
"@types/node": "18.19.76",
"config-file-ts": "0.2.8-rc1"
}
}

View File

@ -1,81 +0,0 @@
# hak
This tool builds native dependencies for element-desktop. Here follows some very minimal
documentation for it.
Goals:
- Must build compiled native node modules in a shippable state
(ie. only dynamically linked against libraries that will be on the
target system, all unnecessary files removed).
- Must be able to build any native module, no matter what build system
it uses (electron-rebuild is supposed to do this job but only works
for modules that use gyp).
It's also loosely designed to be a general tool and agnostic to what it's
actually building. It's used here to build modules for the electron app
but should work equally well for building modules for normal node.
# Running
Hak is invoked with a command and a dependency, eg. `yarn run hak fetch matrix-seshat`.
If no dependencies are given, hak runs the command on all dependencies.
# Files
There are a lot of files involved:
- scripts/hak/... - The tool itself
- hak/[dependency] - Files provided by the app that tell hak how to build each of its native dependencies.
Contains a hak.json file and also some script files, each of which must be referenced in hak.json.
- .hak/ - Files generated by hak in the course of doing its job. Includes the dependency module itself and
any of the native dependency's native dependencies.
- .hak/[dependency]/build - An extracted copy of the dependency's node module used to build it.
- .hak/[dependency]/out - Another extracted copy of the dependency, this one contains only what will be shipped.
# Workings
Hak works around native node modules that try to fetch or build their native component in
the npm 'install' phase - modules that do this will typically end up with native components
targeted to the build platform and the node that npm/yarn is using, which is no good for an
electron app.
It does this by installing it with `--ignore-scripts` and then using `yarn link` to keep the
dependency module separate so yarn doesn't try to run its install / postinstall script
at other points (eg. whenever you `yarn add` a random other dependency).
This also means that the dependencies cannot be listed in `dependencies` or
`devDependencies` in the project, since this would cause npm / yarn to install them and
try to fetch their native parts. Instead, they are listed in `hakDependencies` which
hak reads to install them for you.
Hak will _not_ install dependencies for the copy of the module it links into your
project, so if your native module has javascript dependencies that are actually needed at
runtime (and not just to fetch / build the native parts), it won't work.
Hak will generate a `.yarnrc` in the project directory to set the link directory to its
own in the .hak directory (unless one already exists, in which case this is your problem).
# Lifecycle
Hak is divided into lifecycle stages, in order:
- fetch - Download and extract the source of the dependency
- link - Link the copy of the dependency into your node_modules directory
- build - The Good Stuff. Configure and build any native dependencies, then the module itself.
- copy - Copy the built artifact from the module build directory to the module output directory.
# hak.json
The scripts section contains scripts used for lifecycle stages that need them (fetch, build).
It also contains 'prune' and 'copy' which are globs of files to delete from the output module directory
and copy over from the module build directory to the output module directory, respectively.
# Shortcomings
Hak doesn't know about dependencies between lifecycle stages, ie. it doesn't know that you need to
'fetch' before you can 'build', etc. You get to run each individually, and remember
the right order.
There is also a _lot_ of duplication in the command execution: we should abstract away
some of the boilerplate required to run commands & so forth.

View File

@ -1,14 +0,0 @@
/*
Copyright 2024 New Vector Ltd.
Copyright 2020 The Matrix.org Foundation C.I.C.
SPDX-License-Identifier: AGPL-3.0-only OR GPL-3.0-only OR LicenseRef-Element-Commercial
Please see LICENSE files in the repository root for full details.
*/
import type { DependencyInfo } from "./dep.js";
import type HakEnv from "./hakEnv.js";
export default async function build(hakEnv: HakEnv, moduleInfo: DependencyInfo): Promise<void> {
await moduleInfo.scripts.build(hakEnv, moduleInfo);
}

View File

@ -1,14 +0,0 @@
/*
Copyright 2024 New Vector Ltd.
Copyright 2020 The Matrix.org Foundation C.I.C.
SPDX-License-Identifier: AGPL-3.0-only OR GPL-3.0-only OR LicenseRef-Element-Commercial
Please see LICENSE files in the repository root for full details.
*/
import type { DependencyInfo } from "./dep.js";
import type HakEnv from "./hakEnv.js";
export default async function check(hakEnv: HakEnv, moduleInfo: DependencyInfo): Promise<void> {
await moduleInfo.scripts.check?.(hakEnv, moduleInfo);
}

View File

@ -1,19 +0,0 @@
/*
Copyright 2024 New Vector Ltd.
Copyright 2020 The Matrix.org Foundation C.I.C.
SPDX-License-Identifier: AGPL-3.0-only OR GPL-3.0-only OR LicenseRef-Element-Commercial
Please see LICENSE files in the repository root for full details.
*/
import path from "node:path";
import { rimraf } from "rimraf";
import type { DependencyInfo } from "./dep.js";
import type HakEnv from "./hakEnv.js";
export default async function clean(hakEnv: HakEnv, moduleInfo: DependencyInfo): Promise<void> {
await rimraf(moduleInfo.moduleDotHakDir);
await rimraf(path.join(hakEnv.dotHakDir, "links", moduleInfo.name));
await rimraf(path.join(hakEnv.projectRoot, "node_modules", moduleInfo.name));
}

View File

@ -1,62 +0,0 @@
/*
Copyright 2024 New Vector Ltd.
Copyright 2020, 2021 The Matrix.org Foundation C.I.C.
SPDX-License-Identifier: AGPL-3.0-only OR GPL-3.0-only OR LicenseRef-Element-Commercial
Please see LICENSE files in the repository root for full details.
*/
import path from "node:path";
import fsProm from "node:fs/promises";
import childProcess from "node:child_process";
import { glob } from "glob";
import { mkdirp } from "mkdirp";
import type HakEnv from "./hakEnv.js";
import type { DependencyInfo } from "./dep.js";
export default async function copy(hakEnv: HakEnv, moduleInfo: DependencyInfo): Promise<void> {
if (moduleInfo.cfg.copy) {
// If there are multiple moduleBuildDirs, singular moduleBuildDir
// is the same as moduleBuildDirs[0], so we're just listing the contents
// of the first one.
const files = await glob(moduleInfo.cfg.copy, {
cwd: moduleInfo.moduleBuildDir,
});
if (moduleInfo.moduleBuildDirs.length > 1) {
if (!hakEnv.isMac()) {
console.error(
"You asked me to copy multiple targets but I've only been taught " + "how to do that on macOS.",
);
throw new Error("Can't copy multiple targets on this platform");
}
for (const f of files) {
const components = moduleInfo.moduleBuildDirs.map((dir) => path.join(dir, f));
const dst = path.join(moduleInfo.moduleOutDir, f);
await mkdirp(path.dirname(dst));
await new Promise<void>((resolve, reject) => {
childProcess.execFile("lipo", ["-create", "-output", dst, ...components], (err) => {
if (err) {
reject(err);
} else {
resolve();
}
});
});
}
} else {
console.log("Copying files from " + moduleInfo.moduleBuildDir + " to " + moduleInfo.moduleOutDir);
for (const f of files) {
console.log("\t" + f);
const src = path.join(moduleInfo.moduleBuildDir, f);
const dst = path.join(moduleInfo.moduleOutDir, f);
await mkdirp(path.dirname(dst));
await fsProm.copyFile(src, dst);
}
}
}
}

View File

@ -1,24 +0,0 @@
/*
Copyright 2024 New Vector Ltd.
Copyright 2021 The Matrix.org Foundation C.I.C.
SPDX-License-Identifier: AGPL-3.0-only OR GPL-3.0-only OR LicenseRef-Element-Commercial
Please see LICENSE files in the repository root for full details.
*/
import type HakEnv from "./hakEnv.js";
export interface DependencyInfo {
name: string;
version: string;
cfg: Record<string, any>;
moduleHakDir: string;
moduleDotHakDir: string;
moduleTargetDotHakDir: string;
moduleBuildDir: string;
moduleBuildDirs: string[];
moduleOutDir: string;
nodeModuleBinDir: string;
depPrefix: string;
scripts: Record<string, (hakEnv: HakEnv, moduleInfo: DependencyInfo) => Promise<void>>;
}

View File

@ -1,51 +0,0 @@
/*
Copyright 2024 New Vector Ltd.
Copyright 2020 The Matrix.org Foundation C.I.C.
SPDX-License-Identifier: AGPL-3.0-only OR GPL-3.0-only OR LicenseRef-Element-Commercial
Please see LICENSE files in the repository root for full details.
*/
import fsProm from "node:fs/promises";
import pacote from "pacote";
import type HakEnv from "./hakEnv.js";
import type { DependencyInfo } from "./dep.js";
export default async function fetch(hakEnv: HakEnv, moduleInfo: DependencyInfo): Promise<void> {
let haveModuleBuildDir;
try {
const stats = await fsProm.stat(moduleInfo.moduleBuildDir);
haveModuleBuildDir = stats.isDirectory();
} catch {
haveModuleBuildDir = false;
}
if (haveModuleBuildDir) return;
console.log("Fetching " + moduleInfo.name + "@" + moduleInfo.version);
const packumentCache = new Map();
await pacote.extract(`${moduleInfo.name}@${moduleInfo.version}`, moduleInfo.moduleBuildDir, {
packumentCache,
});
console.log("Running yarn install in " + moduleInfo.moduleBuildDir);
await hakEnv.spawn("yarn", ["install", "--ignore-scripts"], {
cwd: moduleInfo.moduleBuildDir,
});
// also extract another copy to the output directory at this point
// nb. we do not yarn install in the output copy: we could install in
// production mode to get only runtime dependencies and not devDependencies,
// but usually native modules come with dependencies that are needed for
// building/fetching the native modules (eg. node-pre-gyp) rather than
// actually used at runtime: we do not want to bundle these into our app.
// We therefore just install no dependencies at all, and accept that any
// actual runtime dependencies will have to be added to the main app's
// dependencies. We can't tell what dependencies are real runtime deps
// and which are just used for native module building.
await pacote.extract(`${moduleInfo.name}@${moduleInfo.version}`, moduleInfo.moduleOutDir, {
packumentCache,
});
}

View File

@ -1,130 +0,0 @@
/*
Copyright 2024 New Vector Ltd.
Copyright 2020, 2021 The Matrix.org Foundation C.I.C.
SPDX-License-Identifier: AGPL-3.0-only OR GPL-3.0-only OR LicenseRef-Element-Commercial
Please see LICENSE files in the repository root for full details.
*/
import path from "node:path";
import os from "node:os";
import { getElectronVersionFromInstalled } from "app-builder-lib/out/electron/electronVersion.js";
import childProcess, { type SpawnOptions } from "node:child_process";
import { type Arch, type Target, TARGETS, getHost, isHostId, type TargetId } from "./target.js";
async function getRuntimeVersion(projectRoot: string): Promise<string> {
const electronVersion = await getElectronVersionFromInstalled(projectRoot);
if (!electronVersion) {
throw new Error("Can't determine Electron version");
}
return electronVersion;
}
export type Tool = [cmd: string, ...args: string[]];
export default class HakEnv {
public readonly target: Target;
public runtime: string = "electron";
public runtimeVersion?: string;
public dotHakDir: string;
public constructor(
public readonly projectRoot: string,
targetId: TargetId | null,
) {
const target = targetId ? TARGETS[targetId] : getHost();
if (!target) {
throw new Error(`Unknown target ${targetId}!`);
}
this.target = target;
this.dotHakDir = path.join(this.projectRoot, ".hak");
}
public async init(): Promise<void> {
this.runtimeVersion = await getRuntimeVersion(this.projectRoot);
}
public getTargetId(): TargetId {
return this.target.id;
}
public isWin(): boolean {
return this.target.platform === "win32";
}
public isMac(): boolean {
return this.target.platform === "darwin";
}
public isLinux(): boolean {
return this.target.platform === "linux";
}
public isFreeBSD(): boolean {
return this.target.platform === "freebsd";
}
public getTargetArch(): Arch {
return this.target.arch;
}
public isHost(): boolean {
return isHostId(this.target.id);
}
public makeGypEnv(): Record<string, string | undefined> {
return {
...process.env,
npm_config_arch: this.target.arch,
npm_config_target_arch: this.target.arch,
npm_config_disturl: "https://electronjs.org/headers",
npm_config_runtime: this.runtime,
npm_config_target: this.runtimeVersion,
npm_config_build_from_source: "true",
npm_config_devdir: path.join(os.homedir(), ".electron-gyp"),
};
}
public wantsStaticSqlCipher(): boolean {
return !(this.isLinux() || this.isFreeBSD()) || process.env.SQLCIPHER_BUNDLED == "1";
}
public spawn(
cmd: string,
args: string[],
{ ignoreWinCmdlet, ...options }: SpawnOptions & { ignoreWinCmdlet?: boolean } = {},
): Promise<void> {
return new Promise((resolve, reject) => {
const proc = childProcess.spawn(cmd + (!ignoreWinCmdlet && this.isWin() ? ".cmd" : ""), args, {
stdio: "inherit",
// We need shell mode on Windows to be able to launch `.cmd` executables
// See https://nodejs.org/en/blog/vulnerability/april-2024-security-releases-2
shell: this.isWin(),
...options,
});
proc.on("exit", (code) => {
if (code) {
reject(code);
} else {
resolve();
}
});
});
}
public async checkTools(tools: Tool[]): Promise<void> {
for (const [tool, ...args] of tools) {
try {
await this.spawn(tool, args, {
ignoreWinCmdlet: true,
stdio: ["ignore"],
shell: false,
});
} catch {
throw new Error(`Can't find ${tool}`);
}
}
}
}

View File

@ -1,147 +0,0 @@
/*
Copyright 2024 New Vector Ltd.
Copyright 2020, 2021 The Matrix.org Foundation C.I.C.
SPDX-License-Identifier: AGPL-3.0-only OR GPL-3.0-only OR LicenseRef-Element-Commercial
Please see LICENSE files in the repository root for full details.
*/
import path, { dirname } from "node:path";
import { fileURLToPath } from "node:url";
import HakEnv from "./hakEnv.js";
import type { TargetId } from "./target.js";
import type { DependencyInfo } from "./dep.js";
import { loadJsonFile } from "../../src/utils.js";
import packageJson from "../../package.json";
// These can only be run on specific modules
const MODULECOMMANDS = ["check", "fetch", "link", "build", "copy", "clean"];
// Shortcuts for multiple commands at once (useful for building universal binaries
// because you can run the fetch/build for each arch and then copy/link once)
const METACOMMANDS: Record<string, string[]> = {
fetchandbuild: ["check", "fetch", "build"],
copyandlink: ["copy", "link"],
};
// Scripts valid in a hak.json 'scripts' section
const HAKSCRIPTS = ["check", "fetch", "build"];
const __dirname = dirname(fileURLToPath(import.meta.url));
async function main(): Promise<void> {
const prefix = path.join(__dirname, "..", "..");
const targetIds: TargetId[] = [];
// Apply `--target <target>` option if specified
// Can be specified multiple times for the copy command to bundle
// multiple arches into a single universal output module)
for (;;) {
// eslint-disable-line no-constant-condition
const targetIndex = process.argv.indexOf("--target");
if (targetIndex === -1) break;
if (targetIndex + 1 >= process.argv.length) {
console.error("--target option specified without a target");
process.exit(1);
}
// Extract target ID and remove from args
targetIds.push(process.argv.splice(targetIndex, 2)[1] as TargetId);
}
const hakEnvs = targetIds.map((tid) => new HakEnv(prefix, tid));
if (hakEnvs.length == 0) hakEnvs.push(new HakEnv(prefix, null));
for (const h of hakEnvs) {
await h.init();
}
const hakEnv = hakEnvs[0];
const deps: Record<string, DependencyInfo> = {};
const hakDepsCfg = packageJson.hakDependencies || {};
for (const dep in hakDepsCfg) {
const hakJsonPath = path.join(prefix, "hak", dep, "hak.json");
let hakJson: Record<string, any>;
try {
hakJson = loadJsonFile(hakJsonPath);
} catch {
console.error("No hak.json found for " + dep + ".");
console.log("Expecting " + hakJsonPath);
process.exit(1);
}
deps[dep] = {
name: dep,
version: hakDepsCfg[dep as keyof typeof hakDepsCfg],
cfg: hakJson,
moduleHakDir: path.join(prefix, "hak", dep),
moduleDotHakDir: path.join(hakEnv.dotHakDir, dep),
moduleTargetDotHakDir: path.join(hakEnv.dotHakDir, dep, hakEnv.getTargetId()),
moduleBuildDir: path.join(hakEnv.dotHakDir, dep, hakEnv.getTargetId(), "build"),
moduleBuildDirs: hakEnvs.map((h) => path.join(h.dotHakDir, dep, h.getTargetId(), "build")),
moduleOutDir: path.join(hakEnv.dotHakDir, "hakModules", dep),
nodeModuleBinDir: path.join(hakEnv.dotHakDir, dep, hakEnv.getTargetId(), "build", "node_modules", ".bin"),
depPrefix: path.join(hakEnv.dotHakDir, dep, hakEnv.getTargetId(), "opt"),
scripts: {},
};
for (const s of HAKSCRIPTS) {
if (hakJson.scripts?.[s]) {
// Shockingly, using path.join and backslashes here doesn't work on Windows
const scriptModule = await import(`../../hak/${dep}/${hakJson.scripts[s]}`);
if (scriptModule.default) {
deps[dep].scripts[s] = scriptModule.default;
} else {
deps[dep].scripts[s] = scriptModule;
}
}
}
}
let cmds: string[];
if (process.argv.length < 3) {
cmds = ["check", "fetch", "build", "copy", "link"];
} else if (METACOMMANDS[process.argv[2]]) {
cmds = METACOMMANDS[process.argv[2]];
} else {
cmds = [process.argv[2]];
}
if (hakEnvs.length > 1 && cmds.some((c) => !["copy", "link"].includes(c))) {
// We allow link here too for convenience because it's completely arch independent
console.error("Multiple targets only supported with the copy command");
return;
}
let modules = process.argv.slice(3);
if (modules.length === 0) modules = Object.keys(deps);
for (const cmd of cmds) {
if (!MODULECOMMANDS.includes(cmd)) {
console.error("Unknown command: " + cmd);
console.log("Commands I know about:");
for (const cmd of MODULECOMMANDS) {
console.log("\t" + cmd);
}
process.exit(1);
}
const cmdFunc = (await import("./" + cmd)).default;
for (const mod of modules) {
const depInfo = deps[mod];
if (depInfo === undefined) {
console.log("Module " + mod + " not found - is it in hakDependencies " + "in your package.json?");
process.exit(1);
}
console.log("hak " + cmd + ": " + mod);
await cmdFunc(hakEnv, depInfo);
}
}
}
main().catch((err) => {
console.error(err);
process.exit(1);
});

View File

@ -1,47 +0,0 @@
/*
Copyright 2024 New Vector Ltd.
Copyright 2020 The Matrix.org Foundation C.I.C.
SPDX-License-Identifier: AGPL-3.0-only OR GPL-3.0-only OR LicenseRef-Element-Commercial
Please see LICENSE files in the repository root for full details.
*/
import path from "node:path";
import os from "node:os";
import fsProm from "node:fs/promises";
import type HakEnv from "./hakEnv.js";
import { type DependencyInfo } from "./dep.js";
export default async function link(hakEnv: HakEnv, moduleInfo: DependencyInfo): Promise<void> {
const yarnrc = path.join(hakEnv.projectRoot, ".yarnrc");
// this is fairly terrible but it's reasonably clunky to either parse a yarnrc
// properly or get yarn to do it, so this will probably suffice for now.
// We just check to see if there is a local .yarnrc at all, and assume that
// if you've put one there yourself, you probably know what you're doing and
// we won't meddle with it.
// Also we do this for each module which is unnecessary, but meh.
try {
await fsProm.stat(yarnrc);
} catch {
await fsProm.writeFile(
yarnrc,
// XXX: 1. This must be absolute, as yarn will resolve link directories
// relative to the closest project root, which means when we run it
// in the dependency project, it will put the link directory in its
// own project folder rather than the main project.
// 2. The parser gets very confused by strings with colons in them
// (ie. Windows absolute paths) but strings in quotes get parsed as
// JSON so need to be valid JSON encoded strings (ie. have the
// backslashes escaped). JSON.stringify will add quotes and escape.
"--link-folder " + JSON.stringify(path.join(hakEnv.dotHakDir, "links")) + os.EOL,
);
}
await hakEnv.spawn("yarn", ["link"], {
cwd: moduleInfo.moduleOutDir,
});
await hakEnv.spawn("yarn", ["link", moduleInfo.name], {
cwd: hakEnv.projectRoot,
});
}

View File

@ -1,217 +0,0 @@
/*
Copyright 2024 New Vector Ltd.
Copyright 2021 The Matrix.org Foundation C.I.C.
SPDX-License-Identifier: AGPL-3.0-only OR GPL-3.0-only OR LicenseRef-Element-Commercial
Please see LICENSE files in the repository root for full details.
*/
import { GLIBC, MUSL, familySync as processLibC } from "detect-libc";
// We borrow Rust's target naming scheme as a way of expressing all target
// details in a single string.
// See https://doc.rust-lang.org/rustc/platform-support.html.
export type TargetId =
| "aarch64-apple-darwin"
| "x86_64-apple-darwin"
| "universal-apple-darwin"
| "i686-pc-windows-msvc"
| "x86_64-pc-windows-msvc"
| "aarch64-pc-windows-msvc"
| "i686-unknown-freebsd"
| "x86_64-unknown-freebsd"
| "aarch64-unknown-freebsd"
| "i686-unknown-linux-musl"
| "i686-unknown-linux-gnu"
| "x86_64-unknown-linux-musl"
| "x86_64-unknown-linux-gnu"
| "aarch64-unknown-linux-musl"
| "aarch64-unknown-linux-gnu"
| "powerpc64le-unknown-linux-musl"
| "powerpc64le-unknown-linux-gnu";
// Values are expected to match those used in `process.platform`.
export type Platform = "darwin" | "freebsd" | "linux" | "win32";
// Values are expected to match those used in `process.arch`.
export type Arch = "arm64" | "ia32" | "x64" | "ppc64" | "universal";
// Values are expected to match those used by Visual Studio's `vcvarsall.bat`.
// See https://docs.microsoft.com/cpp/build/building-on-the-command-line?view=msvc-160#vcvarsall-syntax
export type VcVarsArch = "amd64" | "arm64" | "x86";
export type Target = {
id: TargetId;
platform: Platform;
arch: Arch;
};
export type WindowsTarget = Target & {
platform: "win32";
vcVarsArch: VcVarsArch;
};
export type LinuxTarget = Target & {
platform: "linux";
libC: typeof GLIBC | typeof MUSL;
};
export type UniversalTarget = Target & {
arch: "universal";
subtargets: Target[];
};
const aarch64AppleDarwin: Target = {
id: "aarch64-apple-darwin",
platform: "darwin",
arch: "arm64",
};
const x8664AppleDarwin: Target = {
id: "x86_64-apple-darwin",
platform: "darwin",
arch: "x64",
};
const universalAppleDarwin: UniversalTarget = {
id: "universal-apple-darwin",
platform: "darwin",
arch: "universal",
subtargets: [aarch64AppleDarwin, x8664AppleDarwin],
};
const i686PcWindowsMsvc: WindowsTarget = {
id: "i686-pc-windows-msvc",
platform: "win32",
arch: "ia32",
vcVarsArch: "x86",
};
const x8664PcWindowsMsvc: WindowsTarget = {
id: "x86_64-pc-windows-msvc",
platform: "win32",
arch: "x64",
vcVarsArch: "amd64",
};
const aarch64WindowsMsvc: WindowsTarget = {
id: "aarch64-pc-windows-msvc",
platform: "win32",
arch: "arm64",
vcVarsArch: "arm64",
};
const i686UnknownFreebsd: Target = {
id: "i686-unknown-freebsd",
platform: "freebsd",
arch: "ia32",
};
const x8664UnknownFreebsd: Target = {
id: "x86_64-unknown-freebsd",
platform: "freebsd",
arch: "x64",
};
const aarch64UnknownFreebsd: Target = {
id: "aarch64-unknown-freebsd",
platform: "freebsd",
arch: "arm64",
};
const x8664UnknownLinuxGnu: LinuxTarget = {
id: "x86_64-unknown-linux-gnu",
platform: "linux",
arch: "x64",
libC: GLIBC,
};
const x8664UnknownLinuxMusl: LinuxTarget = {
id: "x86_64-unknown-linux-musl",
platform: "linux",
arch: "x64",
libC: MUSL,
};
const i686UnknownLinuxGnu: LinuxTarget = {
id: "i686-unknown-linux-gnu",
platform: "linux",
arch: "ia32",
libC: GLIBC,
};
const i686UnknownLinuxMusl: LinuxTarget = {
id: "i686-unknown-linux-musl",
platform: "linux",
arch: "ia32",
libC: MUSL,
};
const aarch64UnknownLinuxGnu: LinuxTarget = {
id: "aarch64-unknown-linux-gnu",
platform: "linux",
arch: "arm64",
libC: GLIBC,
};
const aarch64UnknownLinuxMusl: LinuxTarget = {
id: "aarch64-unknown-linux-musl",
platform: "linux",
arch: "arm64",
libC: MUSL,
};
const powerpc64leUnknownLinuxGnu: LinuxTarget = {
id: "powerpc64le-unknown-linux-gnu",
platform: "linux",
arch: "ppc64",
libC: GLIBC,
};
const powerpc64leUnknownLinuxMusl: LinuxTarget = {
id: "powerpc64le-unknown-linux-musl",
platform: "linux",
arch: "ppc64",
libC: MUSL,
};
export const TARGETS: Record<TargetId, Target> = {
// macOS
"aarch64-apple-darwin": aarch64AppleDarwin,
"x86_64-apple-darwin": x8664AppleDarwin,
"universal-apple-darwin": universalAppleDarwin,
// Windows
"i686-pc-windows-msvc": i686PcWindowsMsvc,
"x86_64-pc-windows-msvc": x8664PcWindowsMsvc,
"aarch64-pc-windows-msvc": aarch64WindowsMsvc,
// FreeBSD
"i686-unknown-freebsd": i686UnknownFreebsd,
"x86_64-unknown-freebsd": x8664UnknownFreebsd,
"aarch64-unknown-freebsd": aarch64UnknownFreebsd,
// Linux
"i686-unknown-linux-musl": i686UnknownLinuxMusl,
"i686-unknown-linux-gnu": i686UnknownLinuxGnu,
"x86_64-unknown-linux-musl": x8664UnknownLinuxMusl,
"x86_64-unknown-linux-gnu": x8664UnknownLinuxGnu,
"aarch64-unknown-linux-musl": aarch64UnknownLinuxMusl,
"aarch64-unknown-linux-gnu": aarch64UnknownLinuxGnu,
"powerpc64le-unknown-linux-musl": powerpc64leUnknownLinuxMusl,
"powerpc64le-unknown-linux-gnu": powerpc64leUnknownLinuxGnu,
};
export function getHost(): Target | undefined {
return Object.values(TARGETS).find(
(target) =>
target.platform === process.platform &&
target.arch === process.arch &&
(process.platform !== "linux" || (target as LinuxTarget).libC === processLibC()),
);
}
export function isHostId(id: TargetId): boolean {
return getHost()?.id === id;
}
export function isHost(target: Target): boolean {
return getHost()?.id === target.id;
}

View File

@ -8,21 +8,16 @@ if [ $? != 0 ]; then
exit 1
fi
echo "${PWD}"
echo $(ls "${PWD}")
# Taken from https://www.electron.build/multi-platform-build#docker
# Pass through any vars prefixed with INDOCKER_, removing the prefix
docker run --rm \
docker run --rm -ti \
--platform linux/amd64 \
--env-file <(env | grep -E '^INDOCKER_' | sed -e 's/^INDOCKER_//') \
--env ELECTRON_CACHE="/root/.cache/electron" \
--env ELECTRON_BUILDER_CACHE="/root/.cache/electron-builder" \
-v ${PWD}:/project \
-v ${PWD}/docker/node_modules:/project/node_modules \
-v ${PWD}/docker/.hak:/project/.hak \
-v ${PWD}/docker/.gnupg:/root/.gnupg \
-v ~/.cache/electron:/root/.cache/electron \
-v ~/.cache/electron-builder:/root/.cache/electron-builder \
--workdir "/project" \
"$IMAGE" "$@"

761
yarn.lock

File diff suppressed because it is too large Load Diff