From 19c496f7a3530fb5beec1d158d3401d85ee8742a Mon Sep 17 00:00:00 2001 From: Johannes Faltermeier Date: Mon, 3 Mar 2025 12:10:37 +0100 Subject: [PATCH] Mac Builds for 'x64' + 'arm64' * since we only have x64 node in Eclipse CI we will build for x64 and arm64 on Github runners * Zipped dist directories will be made available to the Jenkins build via a Github pre-release * Zips will be produced by verification Job for now * adjusts the Jenkinsbuild to get the zips from the pre-release, unpack the dmg, sign the files, repack, and continue build as before * add a new location on download.eclipse.org for mac arm builds (the latest file is called the same for both mac archs, so we need two locations for updates) * enable tests for arm --- .github/workflows/build.yml | 33 ++- Jenkinsfile | 204 ++++++++++++++---- PUBLISHING.md | 8 +- applications/browser/package.json | 4 +- applications/electron/package.json | 9 +- .../electron/scripts/sign-directory.ts | 178 +++++++++++++++ applications/electron/scripts/sign.sh | 82 +++++++ applications/electron/test/app.spec.js | 25 ++- lerna.json | 2 +- package.json | 2 +- theia-extensions/launcher/package.json | 2 +- theia-extensions/product/package.json | 2 +- theia-extensions/updater/package.json | 2 +- 13 files changed, 496 insertions(+), 57 deletions(-) create mode 100644 applications/electron/scripts/sign-directory.ts diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 1214d3f2d..07646a361 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -19,7 +19,7 @@ jobs: strategy: fail-fast: false matrix: - os: [windows-2019, ubuntu-22.04, macos-13] + os: [windows-2019, ubuntu-22.04, macos-13, macos-14] # macOS-13 is for x64, macOS-14 is for arm64 node: ['20.x'] runs-on: ${{ matrix.os }} @@ -42,7 +42,8 @@ jobs: with: python-version: '3.11' - - name: Build and package + - name: Build and package (Windows, Linux) + if: runner.os == 'Windows' || runner.os == 'Linux' shell: bash run: | yarn --skip-integrity-check --network-timeout 100000 @@ -53,6 +54,34 @@ jobs: NODE_OPTIONS: --max_old_space_size=4096 GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} # https://github.com/microsoft/vscode-ripgrep/issues/9 + - name: Update electron-builder.yml for macOS-14 + if: matrix.os == 'macos-14' + run: | + sed -i '' 's|https://download.eclipse.org/theia/ide/latest/macos|https://download.eclipse.org/theia/ide/latest/macos-arm|g' applications/electron/electron-builder.yml + + - name: Build and package (Mac) + if: runner.os == 'macOS' + shell: bash + run: | + yarn --skip-integrity-check --network-timeout 100000 + yarn build + yarn download:plugins + yarn package:applications + env: + NODE_OPTIONS: --max_old_space_size=4096 + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} # https://github.com/microsoft/vscode-ripgrep/issues/9 + + - name: Upload Mac Dist Files + if: runner.os == 'macOS' + uses: actions/upload-artifact@4cec3d8aa04e39d1a68397de0c4cd6fb9dce8ec1 #4.6.1 + with: + name: ${{ matrix.os == 'macos-13' && 'mac-x64' || matrix.os == 'macos-14' && 'mac-arm64'}} + path: | + applications/electron/dist/** + !applications/electron/dist/mac/** + !applications/electron/dist/mac-arm64/** + retention-days: 1 + - name: Test (Linux) if: matrix.tests != 'skip' && runner.os == 'Linux' uses: GabrielBB/xvfb-action@86d97bde4a65fe9b290c0b3fb92c2c4ed0e5302d #1.6 diff --git a/Jenkinsfile b/Jenkinsfile index 8b29c2db5..dc0dc245a 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -121,14 +121,17 @@ spec: } } stage('Create Mac Installer') { + options { + skipDefaultCheckout true + } agent { label 'macos' } steps { script { - buildInstaller(60) + createMacInstaller() } - stash includes: "${toStash}", name: 'mac' + stash includes: "${toStashDist}", name: 'mac' } post { failure { @@ -248,12 +251,14 @@ spec: container('theia-dev') { withCredentials([string(credentialsId: "github-bot-token", variable: 'GITHUB_TOKEN')]) { script { - signInstaller('dmg', 'mac') - notarizeInstaller('dmg') + signInstaller('dmg', 'mac', 'mac-x64') + notarizeInstaller('dmg', 'mac-x64') + signInstaller('dmg', 'mac', 'mac-arm64') + notarizeInstaller('dmg', 'mac-arm64') } } } - stash includes: "${toStash}", name: 'mac2' + stash includes: "${toStashDist}", name: 'mac2' } } stage('Recreate Zip with Ditto for correct file permissions') { @@ -266,34 +271,46 @@ spec: def packageJSON = readJSON file: "package.json" String version = "${packageJSON.version}" - def notarizedDmg = "${distFolder}/TheiaIDE.dmg" + def architectures = ['mac-x64', 'mac-arm64'] + architectures.each { arch -> + + String targetFolder = "${distFolder}/${arch}" + def notarizedDmg = "${targetFolder}/TheiaIDE.dmg" + + // We'll mount and then copy the .app out of the DMG + def mountPoint = "${targetFolder}/TheiaIDE-mount" + def extractedFolder = "${targetFolder}/TheiaIDE-extracted" + def rezippedFile = "${targetFolder}/TheiaIDE-rezipped.zip" + def archSuffix = arch == 'mac-arm64' ? '-arm64' : '' + def finalZip = "${targetFolder}/TheiaIDE-${version}${archSuffix}-mac.zip" + sh "rm -rf \"${extractedFolder}\" \"${mountPoint}\"" + sh "mkdir -p \"${extractedFolder}\" \"${mountPoint}\"" + sh "hdiutil attach \"${notarizedDmg}\" -mountpoint \"${mountPoint}\"" - // We'll mount and then copy the .app out of the DMG - def mountPoint = "${distFolder}/TheiaIDE-mount" - def extractedFolder = "${distFolder}/TheiaIDE-extracted" - def rezippedFile = "${distFolder}/TheiaIDE-rezipped.zip" - def finalZip = "${distFolder}/TheiaIDE-${version}-mac.zip" - sh "rm -rf \"${extractedFolder}\" \"${mountPoint}\"" - sh "mkdir -p \"${extractedFolder}\" \"${mountPoint}\"" - sh "hdiutil attach \"${notarizedDmg}\" -mountpoint \"${mountPoint}\"" + sleep 5 + sh "ls -al ${mountPoint}" + sh "ls -al ${mountPoint}/TheiaIDE.app" - // Copy the .app from the DMG to a folder we can zip - sh "ditto \"${mountPoint}/TheiaIDE.app\" \"${extractedFolder}/TheiaIDE.app\"" + // Copy the .app from the DMG to a folder we can zip + sh "ditto \"${mountPoint}/TheiaIDE.app\" \"${extractedFolder}/TheiaIDE.app\"" - // Unmount the DMG - sh "hdiutil detach \"${mountPoint}\"" + // Unmount the DMG + sh "hdiutil detach \"${mountPoint}\"" - // Zip with ditto - sh "ditto -c -k \"${extractedFolder}\" \"${rezippedFile}\"" + // Zip with ditto + sh "ditto -c -k \"${extractedFolder}\" \"${rezippedFile}\"" + + // Replace the old zip with the newly created one + sh "rm -f \"${finalZip}\"" + sh "mv \"${rezippedFile}\" \"${finalZip}\"" + + // Cleanup + sh "rm -rf \"${extractedFolder}\" \"${mountPoint}\"" + } - // Replace the old zip with the newly created one - sh "rm -f \"${finalZip}\"" - sh "mv \"${rezippedFile}\" \"${finalZip}\"" - // Cleanup - sh "rm -rf \"${extractedFolder}\" \"${mountPoint}\"" } - stash includes: "${toStash}", name: 'mac3' + stash includes: "${toStashDist}", name: 'mac3' } } stage('Update Metadata and Upload Mac') { @@ -352,14 +369,17 @@ spec: script { def packageJSON = readJSON file: "package.json" String version = "${packageJSON.version}" - updateMetadata('TheiaIDE-' + version + '-mac.zip', 'latest-mac.yml', 'macos', false, '.zip', 1200) - updateMetadata('TheiaIDE.dmg', 'latest-mac.yml', 'macos', false, '.dmg', 1200) + updateMetadata('mac-x64/TheiaIDE-' + version + '-mac.zip', 'mac-x64/latest-mac.yml', 'macos', false, '.zip', 1200) + updateMetadata('mac-x64/TheiaIDE.dmg', 'mac-x64/latest-mac.yml', 'macos', false, '.dmg', 1200) + updateMetadata('mac-arm64/TheiaIDE-' + version + '-arm64-mac.zip', 'mac-arm64/latest-mac.yml', 'macos-arm', false, '.zip', 1200) + updateMetadata('mac-arm64/TheiaIDE.dmg', 'mac-arm64/latest-mac.yml', 'macos-arm', false, '.dmg', 1200) } } } container('jnlp') { script { - uploadInstaller('macos') + uploadInstaller('macos', 'mac-x64') + uploadInstaller('macos-arm', 'mac-arm64') } } } @@ -441,6 +461,104 @@ spec: } } +def detachVolume(String mountpoint) { + try { + sh "hdiutil detach \"${mountpoint}\" -force" + } catch (Exception ex) { + echo "Failed to detach ${mountpoint}: ${ex}" + } +} + +def createMacInstaller() { + // Step 0: Clean up any previously mounted DMG files and only then checkout scm + def pathToMacArm64 = "/${pwd()}/applications/electron/dist/mac-arm64/TheiaIDE-dmg-mounted" + def pathToMacX64 = "/${pwd()}/applications/electron/dist/mac-x64/TheiaIDE-dmg-mounted" + def path2ToMacArm64 = "/${pwd()}/applications/electron/dist/mac-arm64/TheiaIDE-mount" + def path2ToMacX64 = "/${pwd()}/applications/electron/dist/mac-x64/TheiaIDE-mount" + detachVolume(pathToMacArm64) + detachVolume(pathToMacX64) + detachVolume(path2ToMacArm64) + detachVolume(path2ToMacX64) + checkout scm + + // Step 1: Ensure the directory exists + sh 'mkdir -p applications/electron/dist' + + // Step 2: Download the zip files + sh 'curl -L -o applications/electron/dist/mac-arm64.zip https://github.com/eclipse-theia/theia-ide/releases/download/pre-release/mac-arm64.zip' + sh 'curl -L -o applications/electron/dist/mac-x64.zip https://github.com/eclipse-theia/theia-ide/releases/download/pre-release/mac-x64.zip' + + // Step 3: Extract the zip files + sh 'unzip applications/electron/dist/mac-arm64.zip -d applications/electron/dist/mac-arm64' + sh 'unzip applications/electron/dist/mac-x64.zip -d applications/electron/dist/mac-x64' + + // Step 4: Delete the zip files + sh 'rm applications/electron/dist/mac-arm64.zip applications/electron/dist/mac-x64.zip' + + // Step 5: List contents to verify + sh 'ls -al applications/electron/dist/mac-arm64 applications/electron/dist/mac-x64' + + // Step 6: Unpack DMG files for signing + sh 'rm -rf applications/electron/dist/mac-arm64/TheiaIDE-dmg-mounted applications/electron/dist/mac-x64/TheiaIDE-dmg-mounted' + sh 'mkdir -p applications/electron/dist/mac-arm64/TheiaIDE-dmg-mounted applications/electron/dist/mac-x64/TheiaIDE-dmg-mounted' + sh 'hdiutil attach applications/electron/dist/mac-arm64/TheiaIDE.dmg -mountpoint applications/electron/dist/mac-arm64/TheiaIDE-dmg-mounted' + sh 'hdiutil attach applications/electron/dist/mac-x64/TheiaIDE.dmg -mountpoint applications/electron/dist/mac-x64/TheiaIDE-dmg-mounted' + sh 'mkdir -p applications/electron/dist/mac-arm64/TheiaIDE-dmg-layout/.background' + sh 'mkdir -p applications/electron/dist/mac-x64/TheiaIDE-dmg-layout/.background' + sh ''' + if [ -f applications/electron/dist/mac-arm64/TheiaIDE-dmg-mounted/.DS_Store ]; then + cp applications/electron/dist/mac-arm64/TheiaIDE-dmg-mounted/.DS_Store applications/electron/dist/mac-arm64/TheiaIDE-dmg-layout/ + fi + ''' + sh ''' + if [ -f applications/electron/dist/mac-x64/TheiaIDE-dmg-mounted/.DS_Store ]; then + cp applications/electron/dist/mac-x64/TheiaIDE-dmg-mounted/.DS_Store applications/electron/dist/mac-x64/TheiaIDE-dmg-layout/ + fi + ''' + sh 'cp -R applications/electron/dist/mac-arm64/TheiaIDE-dmg-mounted/TheiaIDE.app applications/electron/dist/mac-arm64/TheiaIDE-dmg-layout/TheiaIDE.app' + sh 'cp -R applications/electron/dist/mac-x64/TheiaIDE-dmg-mounted/TheiaIDE.app applications/electron/dist/mac-x64/TheiaIDE-dmg-layout/TheiaIDE.app' + sh 'ln -s /Applications applications/electron/dist/mac-arm64/TheiaIDE-dmg-layout/Applications' + sh 'ln -s /Applications applications/electron/dist/mac-x64/TheiaIDE-dmg-layout/Applications' + sh 'hdiutil detach applications/electron/dist/mac-arm64/TheiaIDE-dmg-mounted' + sh 'hdiutil detach applications/electron/dist/mac-x64/TheiaIDE-dmg-mounted' + sh 'ls -al applications/electron/dist/mac-arm64/TheiaIDE-dmg-layout applications/electron/dist/mac-x64/TheiaIDE-dmg-layout' + sh 'ls -al applications/electron/dist/mac-arm64/TheiaIDE-dmg-layout/TheiaIDE.app applications/electron/dist/mac-x64/TheiaIDE-dmg-layout/TheiaIDE.app' + + // Step 7: Remove quarantine bits from all files + sh 'xattr -d -r com.apple.quarantine applications/electron/dist/mac-arm64/TheiaIDE-dmg-layout || true' + sh 'xattr -d -r com.apple.quarantine applications/electron/dist/mac-x64/TheiaIDE-dmg-layout || true' + + // Step 8: Sign binaries - only when it's a release + if (isRelease()) { + sh 'yarn --frozen-lockfile --force' + sshagent(['projects-storage.eclipse.org-bot-ssh']) { + def appPathArm64 = "/${pwd()}/applications/electron/dist/mac-arm64/TheiaIDE-dmg-layout/TheiaIDE.app" + def appPathX64 = "/${pwd()}/applications/electron/dist/mac-x64/TheiaIDE-dmg-layout/TheiaIDE.app" + sh "yarn electron sign:directory -d \"${appPathArm64}\"" + sh "yarn electron sign:directory -d \"${appPathX64}\"" + } + } else { + echo "This is not a release, so skipping binary signing for branch ${env.BRANCH_NAME}" + } + sh 'ls -al applications/electron/dist/mac-arm64/TheiaIDE-dmg-layout applications/electron/dist/mac-x64/TheiaIDE-dmg-layout' + sh 'ls -al applications/electron/dist/mac-arm64/TheiaIDE-dmg-layout/TheiaIDE.app applications/electron/dist/mac-x64/TheiaIDE-dmg-layout/TheiaIDE.app' + + // Step 9: Remove existing DMG files + sh 'rm -f applications/electron/dist/mac-arm64/TheiaIDE.dmg applications/electron/dist/mac-x64/TheiaIDE.dmg' + + // Step 10: Create the final DMG + sh 'hdiutil create -volname TheiaIDE -srcfolder applications/electron/dist/mac-arm64/TheiaIDE-dmg-layout -fs HFS+ -format UDZO applications/electron/dist/mac-arm64/TheiaIDE.dmg' + sh 'hdiutil create -volname TheiaIDE -srcfolder applications/electron/dist/mac-x64/TheiaIDE-dmg-layout -fs HFS+ -format UDZO applications/electron/dist/mac-x64/TheiaIDE.dmg' + + // Step 11: Cleanup TheiaIDE-dmg-layout + sh 'rm -rf applications/electron/dist/mac-arm64/TheiaIDE-dmg-layout applications/electron/dist/mac-x64/TheiaIDE-dmg-layout' + + // Step 12: Cleanup files we don't require + sh 'find applications/electron/dist/mac-arm64 -type f ! -name "TheiaIDE.dmg" ! -name "latest-mac.yml" -delete' + sh 'find applications/electron/dist/mac-x64 -type f ! -name "TheiaIDE.dmg" ! -name "latest-mac.yml" -delete' + sh 'ls -al applications/electron/dist/mac-arm64 applications/electron/dist/mac-x64' +} + def buildInstaller(int sleepBetweenRetries) { int maxRetry = 1 String buildPackageCmd @@ -479,13 +597,15 @@ def buildInstaller(int sleepBetweenRetries) { } } -def signInstaller(String ext, String os) { +def signInstaller(String ext, String os, String arch = '') { if (!isRelease()) { echo "This is not a release, so skipping installer signing for branch ${env.BRANCH_NAME}" return } - List installers = findFiles(glob: "${distFolder}/*.${ext}") + // Adjust the dist folder to include architecture if supplied + String targetFolder = arch ? "${distFolder}/${arch}" : distFolder + List installers = findFiles(glob: "${targetFolder}/*.${ext}") // https://wiki.eclipse.org/IT_Infrastructure_Doc#Web_service if (os == 'mac') { @@ -497,22 +617,25 @@ def signInstaller(String ext, String os) { } if (installers.size() == 1) { - sh "curl -o ${distFolder}/signed-${installers[0].name} -F file=@${installers[0].path} ${url}" + sh "curl -o ${targetFolder}/signed-${installers[0].name} -F file=@${installers[0].path} ${url}" sh "rm ${installers[0].path}" - sh "mv ${distFolder}/signed-${installers[0].name} ${installers[0].path}" + sh "mv ${targetFolder}/signed-${installers[0].name} ${installers[0].path}" } else { error("Error during signing: installer not found or multiple installers exist: ${installers.size()}") } } -def notarizeInstaller(String ext) { +def notarizeInstaller(String ext, String arch = '') { if (!isRelease()) { echo "This is not a release, so skipping installer notarizing for branch ${env.BRANCH_NAME}" return } String service = 'https://cbi.eclipse.org/macos/xcrun' - List installers = findFiles(glob: "${distFolder}/*.${ext}") + + // Adjust the dist folder to include architecture if supplied + String targetFolder = arch ? "${distFolder}/${arch}" : distFolder + List installers = findFiles(glob: "${targetFolder}/*.${ext}") if (installers.size() == 1) { String response = sh(script: "curl -X POST -F file=@${installers[0].path} -F \'options={\"primaryBundleId\": \"eclipse.theia\", \"staple\": true};type=application/json\' ${service}/notarize", returnStdout: true) @@ -531,9 +654,9 @@ def notarizeInstaller(String ext) { error("Failed to notarize ${installers[0].name}: ${response}") } - sh "curl -o ${distFolder}/stapled-${installers[0].name} ${service}/${uuid}/download" + sh "curl -o ${targetFolder}/stapled-${installers[0].name} ${service}/${uuid}/download" sh "rm ${installers[0].path}" - sh "mv ${distFolder}/stapled-${installers[0].name} ${installers[0].path}" + sh "mv ${targetFolder}/stapled-${installers[0].name} ${installers[0].path}" } else { error("Error during notarization: installer not found or multiple installers exist: ${installers.size()}") } @@ -562,17 +685,18 @@ def updateMetadata(String executable, String yaml, String platform, Boolean upda } } -def uploadInstaller(String platform) { +def uploadInstaller(String platform, String arch = '') { if (isReleaseBranch()) { + String targetFolder = arch ? "${distFolder}/${arch}" : distFolder def packageJSON = readJSON file: "package.json" String version = "${packageJSON.version}" sshagent(['projects-storage.eclipse.org-bot-ssh']) { sh "ssh genie.theia@projects-storage.eclipse.org rm -rf /home/data/httpd/download.eclipse.org/theia/ide-preview/${version}/${platform}" sh "ssh genie.theia@projects-storage.eclipse.org mkdir -p /home/data/httpd/download.eclipse.org/theia/ide-preview/${version}/${platform}" - sh "scp ${distFolder}/*.* genie.theia@projects-storage.eclipse.org:/home/data/httpd/download.eclipse.org/theia/ide-preview/${version}/${platform}" + sh "scp ${targetFolder}/*.* genie.theia@projects-storage.eclipse.org:/home/data/httpd/download.eclipse.org/theia/ide-preview/${version}/${platform}" sh "ssh genie.theia@projects-storage.eclipse.org rm -rf /home/data/httpd/download.eclipse.org/theia/ide-preview/latest/${platform}" sh "ssh genie.theia@projects-storage.eclipse.org mkdir -p /home/data/httpd/download.eclipse.org/theia/ide-preview/latest/${platform}" - sh "scp ${distFolder}/*.* genie.theia@projects-storage.eclipse.org:/home/data/httpd/download.eclipse.org/theia/ide-preview/latest/${platform}" + sh "scp ${targetFolder}/*.* genie.theia@projects-storage.eclipse.org:/home/data/httpd/download.eclipse.org/theia/ide-preview/latest/${platform}" } } else { echo "Skipped upload for branch ${env.BRANCH_NAME}" diff --git a/PUBLISHING.md b/PUBLISHING.md index 7aa9a630a..796c4e59e 100644 --- a/PUBLISHING.md +++ b/PUBLISHING.md @@ -50,7 +50,13 @@ Follow these steps to update dependencies and package versions: 6. If there was a Theia release, review breaking changes, new built-ins, and sample applications, and update code as necessary. -After completing these steps, open a PR with your changes. Merging the PR automatically triggers a preview release. +7. After completing these steps, open a PR with your changes. + +8. The PR will trigger a verification build that generates two zip files with mac artifacts. +Download these zips and replace them in this pre-release: . +These unsigned dmgs will be used as input for the Jenkins build. + +9. Merging the PR automatically triggers a preview release, so make sure step 8 is fully completed before merging. ## 3. Preview, Testing, and Release Process diff --git a/applications/browser/package.json b/applications/browser/package.json index 2c3d609d1..da33d6a59 100644 --- a/applications/browser/package.json +++ b/applications/browser/package.json @@ -3,7 +3,7 @@ "name": "theia-ide-browser-app", "description": "Eclipse Theia IDE browser product", "productName": "Theia IDE", - "version": "1.59.1", + "version": "1.59.2", "license": "MIT", "author": "Eclipse Theia ", "homepage": "https://github.com/eclipse-theia/theia-ide#readme", @@ -104,7 +104,7 @@ "@theia/vsx-registry": "1.59.0", "@theia/workspace": "1.59.0", "fs-extra": "^9.0.1", - "theia-ide-product-ext": "1.59.1" + "theia-ide-product-ext": "1.59.2" }, "devDependencies": { "@theia/cli": "1.59.0", diff --git a/applications/electron/package.json b/applications/electron/package.json index ca60fee78..014b67975 100644 --- a/applications/electron/package.json +++ b/applications/electron/package.json @@ -3,7 +3,7 @@ "name": "theia-ide-electron-app", "description": "Eclipse Theia IDE product", "productName": "Theia IDE", - "version": "1.59.1", + "version": "1.59.2", "main": "scripts/theia-electron-main.js", "license": "MIT", "author": "Eclipse Theia ", @@ -112,9 +112,9 @@ "@theia/vsx-registry": "1.59.0", "@theia/workspace": "1.59.0", "fs-extra": "^9.0.1", - "theia-ide-launcher-ext": "1.59.1", - "theia-ide-product-ext": "1.59.1", - "theia-ide-updater-ext": "1.59.1" + "theia-ide-launcher-ext": "1.59.2", + "theia-ide-product-ext": "1.59.2", + "theia-ide-updater-ext": "1.59.2" }, "devDependencies": { "@theia/cli": "1.59.0", @@ -158,6 +158,7 @@ "update:blockmap": "ts-node scripts/update-blockmap.ts", "update:theia": "ts-node ../../scripts/update-theia-version.ts", "update:next": "ts-node ../../scripts/update-theia-version.ts next", + "sign:directory": "ts-node scripts/sign-directory.ts", "test": "mocha --timeout 60000 \"./test/*.spec.js\"", "lint": "eslint --ext js,jsx,ts,tsx scripts && eslint --ext js,jsx,ts,tsx test", "lint:fix": "eslint --ext js,jsx,ts,tsx scripts --fix && eslint --ext js,jsx,ts,tsx test -fix" diff --git a/applications/electron/scripts/sign-directory.ts b/applications/electron/scripts/sign-directory.ts new file mode 100644 index 000000000..09cfc35bc --- /dev/null +++ b/applications/electron/scripts/sign-directory.ts @@ -0,0 +1,178 @@ +/******************************************************************************** + * Copyright (C) 2025 EclipseSource and others. + * + * This program and the accompanying materials are made available under the + * terms of the MIT License, which is available in the project root. + * + * SPDX-License-Identifier: MIT + ********************************************************************************/ + +import { hideBin } from 'yargs/helpers'; +import yargs from 'yargs/yargs'; +import path from 'path'; +import fs from 'fs'; +import child_process from 'child_process'; + +const signCommand = path.join(__dirname, 'sign.sh'); +const notarizeCommand = path.join(__dirname, 'notarize.sh'); +const entitlements = path.resolve(__dirname, '..', 'entitlements.plist'); + +// File extensions and patterns that need code signing on macOS +const BINARY_EXTENSIONS = ['.dylib', '.so', '.node', '.framework']; +const BINARY_PATTERNS = [ + /^MacOS\//, // Executable files in MacOS directory + /^Contents\/MacOS\//, // Executable files in Contents/MacOS directory +]; +const EXECUTABLE_NAMES = [ + 'node', 'electron', 'rg', 'macos-trash', 'chrome-sandbox' +]; + +// Function to check if a file is likely a binary that needs signing +function isBinaryFile(filePath: string): boolean { + const extension = path.extname(filePath); + const fileName = path.basename(filePath); + const relativePath = filePath.replace(/^.*?\.app\//, ''); // Get path relative to .app bundle + + // Check by extension + if (BINARY_EXTENSIONS.includes(extension)) { + return true; + } + + // Check by executable name + if (EXECUTABLE_NAMES.includes(fileName)) { + return true; + } + + // Check by pattern + for (const pattern of BINARY_PATTERNS) { + if (pattern.test(relativePath)) { + return true; + } + } + + // Check if file is executable (Unix-only check) + try { + const stat = fs.statSync(filePath); + if ((stat.mode & 0o111) !== 0) { // Check if execute bit is set + // Further verify it's a binary with 'file' command if available + try { + const fileType = child_process.execSync(`file "${filePath}"`).toString(); + return fileType.includes('Mach-O') || + fileType.includes('executable') || + fileType.includes('shared library') || + fileType.includes('dynamically linked'); + } catch (e) { + // If 'file' command fails, fall back to assuming it's a binary if it has execute permission + return true; + } + } + } catch (e) { + // If stat fails, skip this check + } + + return false; +} + +// Function to recursively find binaries in a directory +function findBinariesToSign(dirPath: string): string[] { + const result: string[] = []; + + function scanDirectory(currentPath: string): void { + const entries = fs.readdirSync(currentPath, { withFileTypes: true }); + + for (const entry of entries) { + const fullPath = path.join(currentPath, entry.name); + + // Skip node_modules and .git directories + if (entry.isDirectory() && + entry.name !== 'node_modules' && + entry.name !== '.git') { + scanDirectory(fullPath); + } else if (entry.isFile() && isBinaryFile(fullPath)) { + result.push(fullPath); + } + } + } + + scanDirectory(dirPath); + + // Sort by path depth (deepest first) to ensure nested binaries are signed first + return result.sort((a, b) => { + const aDepth = a.split(path.sep).length; + const bDepth = b.split(path.sep).length; + return bDepth - aDepth; + }); +} + +const signFile = (file: string) => { + const stat = fs.lstatSync(file); + const mode = stat.isFile() ? stat.mode : undefined; + + // Get SHA hash of file before signing - only for actual files, not directories + let shaBeforeSigning: string | undefined; + if (stat.isFile()) { + shaBeforeSigning = child_process.execSync(`shasum -a 256 "${file}"`).toString().trim(); + } + + console.log(`Signing ${file}...`); + child_process.spawnSync(signCommand, [ + path.basename(file), + entitlements + ], { + cwd: path.dirname(file), + maxBuffer: 1024 * 10000, + env: process.env, + stdio: 'inherit', + encoding: 'utf-8' + }); + + // Get SHA hash of file after signing - only for actual files, not directories + if (stat.isFile()) { + const shaAfterSigning = child_process.execSync(`shasum -a 256 "${file}"`).toString().trim(); + // Log a warning if the SHA hash hasn't changed after signing + if (shaBeforeSigning === shaAfterSigning) { + console.warn(`WARNING: SHA hash did not change after signing for ${file}. This might indicate the file was not properly signed.`); + } + } + + if (mode) { + console.log(`Setting attributes of ${file}...`); + fs.chmodSync(file, mode); + } +}; + +const argv = yargs(hideBin(process.argv)) + .option('directory', { alias: 'd', type: 'string', default: 'dist', description: 'The directory which contains the application to be signed' }) + .version(false) + .wrap(120) + .parseSync(); + +execute(); + +async function execute(): Promise { + console.log(`signCommand: ${signCommand}; notarizeCommand: ${notarizeCommand}; entitlements: ${entitlements}; directory: ${argv.directory}`); + + // First sign all individual binaries inside the app bundle + const binariesToSign = findBinariesToSign(argv.directory); + + for (const binaryPath of binariesToSign) { + signFile(binaryPath); + } + + // Then sign the main app bundle + console.log('Signing main application bundle...'); + signFile(argv.directory); + + // Notarize app + console.log('Notarizing application...'); + child_process.spawnSync(notarizeCommand, [ + path.basename(argv.directory), + 'eclipse.theia' + ], { + cwd: path.dirname(argv.directory), + maxBuffer: 1024 * 10000, + env: process.env, + stdio: 'inherit', + encoding: 'utf-8' + }); +} diff --git a/applications/electron/scripts/sign.sh b/applications/electron/scripts/sign.sh index c69083318..a5808f730 100755 --- a/applications/electron/scripts/sign.sh +++ b/applications/electron/scripts/sign.sh @@ -1,11 +1,17 @@ #!/bin/bash -x +# Enable debug output +set -x + INPUT=$1 ENTITLEMENTS=$2 NEEDS_UNZIP=false +echo "=== DEBUG: Starting signing process for $INPUT ===" + # if folder, zip it if [ -d "${INPUT}" ]; then + echo "=== DEBUG: Input is a directory, zipping it ===" NEEDS_UNZIP=true zip -r -q -y unsigned.zip "${INPUT}" rm -rf "${INPUT}" @@ -13,11 +19,25 @@ if [ -d "${INPUT}" ]; then fi # copy file to storage server +echo "=== DEBUG: Copying $INPUT to storage server ===" scp -p "${INPUT}" genie.theia@projects-storage.eclipse.org:./ +if [ $? -eq 0 ]; then + echo "=== DEBUG: Successfully copied $INPUT to storage server ===" +else + echo "=== ERROR: Failed to copy $INPUT to storage server ===" + exit 1 +fi rm -f "${INPUT}" # copy entitlements to storage server +echo "=== DEBUG: Copying entitlements file to storage server ===" scp -p "${ENTITLEMENTS}" genie.theia@projects-storage.eclipse.org:./entitlements.plist +if [ $? -eq 0 ]; then + echo "=== DEBUG: Successfully copied entitlements to storage server ===" +else + echo "=== ERROR: Failed to copy entitlements to storage server ===" + exit 1 +fi # name to use on server REMOTE_NAME=${INPUT##*/} @@ -25,23 +45,85 @@ REMOTE_NAME=${INPUT##*/} # sign over ssh # https://wiki.eclipse.org/IT_Infrastructure_Doc#Web_service ssh -q genie.theia@projects-storage.eclipse.org curl -f -o "\"signed-${REMOTE_NAME}\"" -F file=@"\"${REMOTE_NAME}\"" -F entitlements=@entitlements.plist https://cbi.eclipse.org/macos/codesign/sign +if [ $? -eq 0 ]; then + echo "=== DEBUG: Remote signing completed successfully ===" +else + echo "=== ERROR: Remote signing failed ===" + # Try to get error information + ssh -q genie.theia@projects-storage.eclipse.org "cat \"signed-${REMOTE_NAME}\" || echo 'No output file found'" + exit 1 +fi # copy signed file back from server +echo "=== DEBUG: Copying signed file back from storage server ===" scp -T -p genie.theia@projects-storage.eclipse.org:"\"./signed-${REMOTE_NAME}\"" "${INPUT}" +if [ $? -eq 0 ]; then + echo "=== DEBUG: Successfully retrieved signed file ===" +else + echo "=== ERROR: Failed to retrieve signed file ===" + exit 1 +fi + +# Check if the file was actually signed +echo "=== DEBUG: Verifying if file was signed properly ===" +if [ -f "${INPUT}" ]; then + # Get file size to verify it's not empty + FILE_SIZE=$(stat -f%z "${INPUT}" 2>/dev/null || stat -c%s "${INPUT}" 2>/dev/null) + echo "=== DEBUG: Signed file size: $FILE_SIZE bytes ===" + + # On macOS, we can verify code signature + if [[ "$OSTYPE" == "darwin"* ]]; then + echo "=== DEBUG: Checking code signature with codesign -vv ===" + codesign -vv "${INPUT}" || echo "=== WARNING: codesign verification failed ===" + fi +else + echo "=== ERROR: Signed file not found ===" + exit 1 +fi # ensure storage server is clean +echo "=== DEBUG: Cleaning up remote files ===" ssh -q genie.theia@projects-storage.eclipse.org rm -f "\"${REMOTE_NAME}\"" "\"signed-${REMOTE_NAME}\"" entitlements.plist +echo "=== DEBUG: Remote cleanup completed ===" # if unzip needed if [ "$NEEDS_UNZIP" = true ]; then + echo "=== DEBUG: Unzipping signed archive ===" unzip -qq "${INPUT}" if [ $? -ne 0 ]; then # echo contents if unzip failed + echo "=== ERROR: Unzip failed, showing file contents ===" output=$(cat $INPUT) echo "$output" exit 1 fi + echo "=== DEBUG: Unzip successful, removing zip file ===" rm -f "${INPUT}" + + # Perform deep codesign check on the directory if running on macOS + if [[ "$OSTYPE" == "darwin"* ]]; then + echo "=== DEBUG: Performing deep codesign verification on directory ===" + # Check if spctl is available (macOS security assessment tool) + if command -v spctl &> /dev/null; then + # Check if the directory is an app bundle + if [[ -d "$1" && "$1" == *.app ]]; then + echo "=== DEBUG: Verifying app bundle with spctl --assess --verbose ===" + spctl --assess --verbose "$1" || echo "=== WARNING: App bundle verification failed, may not pass notarization ===" + fi + fi + + # Find all binary files and check their signatures + echo "=== DEBUG: Checking individual binary signatures in $1 ===" + find "$1" -type f -exec file {} \; | grep -E "Mach-O|dylib" | cut -d: -f1 | while read binary; do + echo "Checking signature for $binary" + codesign --verify --deep --strict --verbose=2 "$binary" || echo "=== WARNING: Binary $binary has signature issues, may not pass notarization ===" + + # Check for hardened runtime + codesign -d --verbose=4 "$binary" 2>&1 | grep -q 'Runtime Version=10.0.0' || echo "=== WARNING: Binary $binary may not have hardened runtime enabled ===" + done + fi fi + +echo "=== DEBUG: Signing process completed for $1 ===" \ No newline at end of file diff --git a/applications/electron/test/app.spec.js b/applications/electron/test/app.spec.js index 3f10f54d0..7cfe846a9 100644 --- a/applications/electron/test/app.spec.js +++ b/applications/electron/test/app.spec.js @@ -1,11 +1,26 @@ const os = require('os'); const path = require('path'); const fs = require('fs'); +const { execSync } = require('child_process'); const { remote } = require('webdriverio'); const { expect } = require('chai'); const THEIA_LOAD_TIMEOUT = 15000; // 15 seconds +function isMacArm() { + if (os.platform() !== 'darwin'){ + return false; + } + try { + // Check the architecture using uname -m + const arch = execSync('uname -m').toString().trim(); + return arch === 'arm64'; + } catch (error) { + // Fall back to node's arch property if uname fails + return os.arch() === 'arm64'; + } +} + function getElectronMainJS() { const distFolder = path.join(__dirname, '..', 'dist'); switch (os.platform()) { @@ -30,9 +45,10 @@ function getElectronMainJS() { 'electron-main.js' ); case 'darwin': + const macFolder = isMacArm() ? 'mac-arm64' : 'mac'; return path.join( distFolder, - 'mac', + macFolder, 'TheiaIDE.app', 'Contents', 'Resources', @@ -94,14 +110,17 @@ function getBinaryPath() { 'TheiaIDE.exe' ); case 'darwin': - return path.join( + const macFolder = isMacArm() ? 'mac-arm64' : 'mac'; + const binaryPath = path.join( distFolder, - 'mac', + macFolder, 'TheiaIDE.app', 'Contents', 'MacOS', 'TheiaIDE' ); + console.log(`Using binary path for Mac ${isMacArm() ? 'ARM64' : 'Intel'}: ${binaryPath}`); + return binaryPath; default: return undefined; } diff --git a/lerna.json b/lerna.json index 26bd71966..9086f0cf5 100644 --- a/lerna.json +++ b/lerna.json @@ -1,6 +1,6 @@ { "lerna": "4.0.0", - "version": "1.59.1", + "version": "1.59.2", "useWorkspaces": true, "npmClient": "yarn", "command": { diff --git a/package.json b/package.json index 63e9ddd74..f0421b921 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "private": true, - "version": "1.59.1", + "version": "1.59.2", "license": "MIT", "author": "Rob Moran ", "homepage": "https://github.com/eclipse-theia/theia-ide#readme", diff --git a/theia-extensions/launcher/package.json b/theia-extensions/launcher/package.json index 95b8824c2..e1897009c 100644 --- a/theia-extensions/launcher/package.json +++ b/theia-extensions/launcher/package.json @@ -1,6 +1,6 @@ { "name": "theia-ide-launcher-ext", - "version": "1.59.1", + "version": "1.59.2", "keywords": [ "theia-extension" ], diff --git a/theia-extensions/product/package.json b/theia-extensions/product/package.json index a5d0f4d6b..122664f47 100644 --- a/theia-extensions/product/package.json +++ b/theia-extensions/product/package.json @@ -1,7 +1,7 @@ { "private": true, "name": "theia-ide-product-ext", - "version": "1.59.1", + "version": "1.59.2", "description": "Eclipse Theia IDE Product Branding", "dependencies": { "@theia/core": "1.59.0", diff --git a/theia-extensions/updater/package.json b/theia-extensions/updater/package.json index c204af535..5b0086dcc 100644 --- a/theia-extensions/updater/package.json +++ b/theia-extensions/updater/package.json @@ -1,7 +1,7 @@ { "private": true, "name": "theia-ide-updater-ext", - "version": "1.59.1", + "version": "1.59.2", "description": "Eclipse Theia IDE Updater", "dependencies": { "@theia/core": "1.59.0",