Skip to content

Commit

Permalink
feat: query compiler proof of concept (#5024)
Browse files Browse the repository at this point in the history
Separate query planning and execution and lower the query graph into a format executable by the client.
  • Loading branch information
aqrln authored Jan 21, 2025
1 parent bc5b021 commit eb5bc54
Show file tree
Hide file tree
Showing 93 changed files with 2,831 additions and 375 deletions.
8 changes: 8 additions & 0 deletions .envrc
Original file line number Diff line number Diff line change
Expand Up @@ -53,3 +53,11 @@ if command -v nix &> /dev/null && [ -z ${DISABLE_NIX+x} ]
then
use nix
fi

if [[ "$OSTYPE" == "linux-gnu"* ]] && command -v lld &> /dev/null && [ ! -f .cargo/config.toml ]; then
mkdir -p .cargo
cat << EOF > .cargo/config.toml
[target.$(uname -m)-unknown-linux-gnu]
rustflags = ["-C", "link-arg=-fuse-ld=lld"]
EOF
fi
96 changes: 48 additions & 48 deletions .github/workflows/build-engines.yml
Original file line number Diff line number Diff line change
Expand Up @@ -6,30 +6,30 @@ on:
push:
branches:
- main
- '*.*.x'
- 'integration/*'
- "*.*.x"
- "integration/*"
paths-ignore:
- '!.github/workflows/build-engines*'
- '.github/**'
- '.buildkite/**'
- '*.md'
- 'LICENSE'
- 'CODEOWNERS'
- 'renovate.json'
- "!.github/workflows/build-engines*"
- ".github/**"
- ".buildkite/**"
- "*.md"
- "LICENSE"
- "CODEOWNERS"
- "renovate.json"
workflow_dispatch:
pull_request:
paths-ignore:
- '!.github/workflows/build-engines*'
- '.github/**'
- '.buildkite/**'
- '*.md'
- 'LICENSE'
- 'CODEOWNERS'
- 'renovate.json'
- "!.github/workflows/build-engines*"
- ".github/**"
- ".buildkite/**"
- "*.md"
- "LICENSE"
- "CODEOWNERS"
- "renovate.json"

jobs:
is-release-necessary:
name: 'Decide if a release of the engines artifacts is necessary'
name: "Decide if a release of the engines artifacts is necessary"
runs-on: ubuntu-22.04
outputs:
release: ${{ steps.decision.outputs.release }}
Expand Down Expand Up @@ -59,7 +59,7 @@ jobs:
END_OF_COMMIT_MESSAGE
echo "Commit message contains [integration]: ${{ contains(steps.commit-msg.outputs.commit-msg, '[integration]') }}"
- name: 'Check if commit message conatains `[integration]` and the PR author has permissions to trigger the workflow'
- name: "Check if commit message conatains `[integration]` and the PR author has permissions to trigger the workflow"
id: check-commit-message
# See https://docs.github.com/en/graphql/reference/enums
# https://michaelheap.com/github-actions-check-permission/
Expand All @@ -68,16 +68,16 @@ jobs:
# - the PR author has permissions to trigger the workflow (must be part of the org or a collaborator)
if: |
github.event_name == 'pull_request' &&
contains(steps.commit-msg.outputs.commit-msg, '[integration]') &&
contains(steps.commit-msg.outputs.commit-msg, '[integration]') &&
(
github.event.pull_request.author_association == 'OWNER' ||
github.event.pull_request.author_association == 'MEMBER' ||
github.event.pull_request.author_association == 'CONTRIBUTOR' ||
github.event.pull_request.author_association == 'COLLABORATOR'
)
run: |
echo "Commit message contains [integration] and PR author has permissions"
# set value to GitHub output
echo "Commit message contains [integration] and PR author has permissions"
# set value to GitHub output
echo "release=true" >> $GITHUB_OUTPUT
#
Expand Down Expand Up @@ -118,9 +118,9 @@ jobs:
# https://github.com/peter-evans/find-comment/tree/v3/?tab=readme-ov-file#outputs
# Tip: Empty strings evaluate to zero in GitHub Actions expressions. e.g. If comment-id is an empty string steps.fc.outputs.comment-id == 0 evaluates to true.
if: |
github.event_name == 'workflow_dispatch' ||
github.event_name == 'push' ||
steps.check-commit-message.outputs.release == 'true' ||
github.event_name == 'workflow_dispatch' ||
github.event_name == 'push' ||
steps.check-commit-message.outputs.release == 'true' ||
steps.check-branch.outputs.release == 'true'
id: decision
Expand All @@ -140,7 +140,7 @@ jobs:
build-linux:
name: Build Engines for Linux
needs:
needs:
- is-release-necessary
if: ${{ needs.is-release-necessary.outputs.release == 'true' }}
uses: ./.github/workflows/build-engines-linux-template.yml
Expand All @@ -149,7 +149,7 @@ jobs:

build-macos-intel:
name: Build Engines for Apple Intel
needs:
needs:
- is-release-necessary
if: ${{ needs.is-release-necessary.outputs.release == 'true' }}
uses: ./.github/workflows/build-engines-apple-intel-template.yml
Expand All @@ -158,7 +158,7 @@ jobs:

build-macos-silicon:
name: Build Engines for Apple Silicon
needs:
needs:
- is-release-necessary
if: ${{ needs.is-release-necessary.outputs.release == 'true' }}
uses: ./.github/workflows/build-engines-apple-silicon-template.yml
Expand All @@ -167,7 +167,7 @@ jobs:

build-react-native:
name: Build Engines for React native
needs:
needs:
- is-release-necessary
if: ${{ needs.is-release-necessary.outputs.release == 'true' }}
uses: ./.github/workflows/build-engines-react-native-template.yml
Expand All @@ -176,15 +176,15 @@ jobs:

build-windows:
name: Build Engines for Windows
needs:
needs:
- is-release-necessary
if: ${{ needs.is-release-necessary.outputs.release == 'true' }}
uses: ./.github/workflows/build-engines-windows-template.yml
with:
commit: ${{ github.sha }}

release-artifacts:
name: 'Release artifacts from branch ${{ github.head_ref || github.ref_name }} for commit ${{ github.sha }}'
name: "Release artifacts from branch ${{ github.head_ref || github.ref_name }} for commit ${{ github.sha }}"
runs-on: ubuntu-22.04
concurrency:
group: ${{ github.sha }}
Expand All @@ -195,9 +195,9 @@ jobs:
- build-react-native
- build-windows
env:
BUCKET_NAME: 'prisma-builds'
BUCKET_NAME: "prisma-builds"
PRISMA_ENGINES_COMMIT_SHA: ${{ github.sha }}
DESTINATION_TARGET_PATH: 's3://prisma-builds/all_commits/${{ github.sha }}'
DESTINATION_TARGET_PATH: "s3://prisma-builds/all_commits/${{ github.sha }}"

steps:
# Because we need the scripts
Expand All @@ -215,22 +215,22 @@ jobs:
# run-id: 9526334324
# github-token: ${{ secrets.GITHUB_TOKEN }}

- name: 'R2: Check if artifacts were already built and uploaded before via `.finished` file'
- name: "R2: Check if artifacts were already built and uploaded before via `.finished` file"
env:
FILE_PATH: 'all_commits/${{ github.sha }}/.finished'
FILE_PATH_LEGACY: 'all_commits/${{ github.sha }}/rhel-openssl-1.1.x/.finished'
AWS_DEFAULT_REGION: 'auto'
FILE_PATH: "all_commits/${{ github.sha }}/.finished"
FILE_PATH_LEGACY: "all_commits/${{ github.sha }}/rhel-openssl-1.1.x/.finished"
AWS_DEFAULT_REGION: "auto"
AWS_ACCESS_KEY_ID: ${{ vars.R2_ACCESS_KEY_ID }}
AWS_SECRET_ACCESS_KEY: ${{ secrets.R2_SECRET_ACCESS_KEY }}
AWS_ENDPOINT_URL_S3: ${{ vars.R2_ENDPOINT }}
working-directory: .github/workflows/utils
run: bash checkFinishedMarker.sh

- name: 'S3: Check if artifacts were already built and uploaded before via `.finished` file'
- name: "S3: Check if artifacts were already built and uploaded before via `.finished` file"
env:
FILE_PATH: 'all_commits/${{ github.sha }}/.finished'
FILE_PATH_LEGACY: 'all_commits/${{ github.sha }}/rhel-openssl-1.1.x/.finished'
AWS_DEFAULT_REGION: 'eu-west-1'
FILE_PATH: "all_commits/${{ github.sha }}/.finished"
FILE_PATH_LEGACY: "all_commits/${{ github.sha }}/rhel-openssl-1.1.x/.finished"
AWS_DEFAULT_REGION: "eu-west-1"
AWS_ACCESS_KEY_ID: ${{ vars.AWS_ACCESS_KEY_ID }}
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
working-directory: .github/workflows/utils
Expand All @@ -254,7 +254,7 @@ jobs:
zip -r react-native/binaries.zip ios android
rm -rf ios android
- name: 'Create compressed engine files (.gz)'
- name: "Create compressed engine files (.gz)"
working-directory: engines-artifacts
run: |
set -eu
Expand All @@ -266,13 +266,13 @@ jobs:
ls -Rl .
- name: 'Create SHA256 checksum files (.sha256).'
- name: "Create SHA256 checksum files (.sha256)."
working-directory: engines-artifacts
run: |
set -eu
find . -type f | while read filename; do
sha256sum "$filename" > "$filename.sha256"
sha256sum "$filename" > "$filename.sha256"
echo "$filename.sha256 file created."
done
Expand All @@ -292,7 +292,7 @@ jobs:
run: gpg -K

# next to each file (excluding .sha256 files)
- name: 'Create a GPG detached signature (.sig)'
- name: "Create a GPG detached signature (.sig)"
working-directory: engines-artifacts
run: |
set -eu
Expand All @@ -303,18 +303,18 @@ jobs:
ls -Rl .
- name: 'Cloudflare R2: Upload to bucket and verify uploaded files then create `.finished` file'
- name: "Cloudflare R2: Upload to bucket and verify uploaded files then create `.finished` file"
# https://docs.aws.amazon.com/cli/v1/userguide/cli-configure-envvars.html
env:
AWS_DEFAULT_REGION: 'auto'
AWS_DEFAULT_REGION: "auto"
AWS_ACCESS_KEY_ID: ${{ vars.R2_ACCESS_KEY_ID }}
AWS_SECRET_ACCESS_KEY: ${{ secrets.R2_SECRET_ACCESS_KEY }}
AWS_ENDPOINT_URL_S3: ${{ vars.R2_ENDPOINT }}
run: bash .github/workflows/utils/uploadAndVerify.sh engines-artifacts-for-r2

- name: 'AWS S3: Upload to bucket and verify uploaded files then create `.finished` file'
- name: "AWS S3: Upload to bucket and verify uploaded files then create `.finished` file"
env:
AWS_DEFAULT_REGION: 'eu-west-1'
AWS_DEFAULT_REGION: "eu-west-1"
AWS_ACCESS_KEY_ID: ${{ vars.AWS_ACCESS_KEY_ID }}
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
run: bash .github/workflows/utils/uploadAndVerify.sh engines-artifacts-for-s3
Expand Down
24 changes: 3 additions & 21 deletions .github/workflows/formatting.yml
Original file line number Diff line number Diff line change
Expand Up @@ -18,34 +18,16 @@ concurrency:
cancel-in-progress: true

jobs:
clippy:
name: clippy linting
checks:
name: run lints and formatting checks
runs-on: ubuntu-latest
env:
RUSTFLAGS: "-Dwarnings"
steps:
- uses: actions/checkout@v4
- uses: actions-rust-lang/setup-rust-toolchain@v1
with:
components: clippy
targets: wasm32-unknown-unknown
# Check the whole workspace with clippy for the native compilation
# target, and query-engine-wasm and dependencies for wasm32-unknown-unknown.
# Note that `--all-targets` is unrelated to `--target` as in target platform,
# it is a shortcut for `--lib --bins --tests --benches --examples`.
- run: |
cargo clippy --workspace --all-features --all-targets
cargo clippy --all-features --all-targets -p query-engine-wasm -p prisma-schema-build --target wasm32-unknown-unknown
rustfmt:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- uses: actions-rust-lang/setup-rust-toolchain@v1
with:
components: rustfmt
- name: Check formatting
run: cargo fmt -- --check
- run: make pedantic

shellcheck:
runs-on: ubuntu-latest
Expand Down
65 changes: 65 additions & 0 deletions .github/workflows/publish-query-compiler-wasm.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,65 @@
name: Build and publish @prisma/query-compiler-wasm
run-name: npm - release @prisma/query-compiler-wasm@${{ github.event.inputs.packageVersion }} from ${{ github.event.inputs.enginesHash }} on ${{ github.event.inputs.npmDistTag }}

concurrency: publish-query-compiler-wasm

on:
# usually triggered via GH Actions Workflow in prisma/engines-wrapper repo
workflow_dispatch:
inputs:
packageVersion:
required: true
description: "New @prisma/query-compiler-wasm package version"
enginesHash:
required: true
description: "query-compiler commit to build"
npmDistTag:
required: true
default: "latest"
description: "npm dist-tag (e.g. latest or integration)"

jobs:
build:
name: Build and publish @prisma/query-compiler-wasm
runs-on: ubuntu-latest
steps:
- name: Print input
run: echo "${{ toJson(github.event.inputs) }}"

- uses: actions/checkout@v4
with:
ref: ${{ github.event.inputs.enginesHash }}

- uses: ./.github/workflows/include/rust-wasm-setup

- name: Build @prisma/query-compiler-wasm
run: make build-qc-wasm
env:
QE_WASM_VERSION: ${{ github.event.inputs.packageVersion }}

- name: Install Node.js
uses: actions/setup-node@v4
with:
node-version: "20.x"

- name: Set up NPM token for publishing
run: echo "//registry.npmjs.org/:_authToken=${{ secrets.NPM_TOKEN }}" > ~/.npmrc

- name: Publish @prisma/query-compiler-wasm
run: npm publish --access public --tag ${{ github.event.inputs.npmDistTag }}
working-directory: query-compiler/query-compiler-wasm/pkg

#
# Failure handlers
#
- name: Set current job url in SLACK_FOOTER env var
if: ${{ failure() }}
run: echo "SLACK_FOOTER=<$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID|Click here to go to the job logs>" >> $GITHUB_ENV
- name: Slack Notification on Failure
if: ${{ failure() }}
uses: rtCamp/[email protected]
env:
SLACK_TITLE: "Building and publishing @prisma/query-compiler-wasm failed :x:"
SLACK_COLOR: "#FF0000"
SLACK_CHANNEL: feed-prisma-query-compiler-wasm-publish-failures
SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK_WASM_FAILING }}
Loading

0 comments on commit eb5bc54

Please sign in to comment.