From 86378d27e684a622d17dd9c02d2bcb5832905803 Mon Sep 17 00:00:00 2001 From: Jeroen van Straten Date: Tue, 31 May 2022 16:39:17 +0200 Subject: [PATCH 1/3] chore: fix up release pipelines --- .github/workflows/python.yml | 7 ++++ .github/workflows/rust.yml | 32 ++++++++++++++++ ci/crates-io-wait.py | 73 ++++++++++++++++++++++++++++++++++++ 3 files changed, 112 insertions(+) create mode 100644 ci/crates-io-wait.py diff --git a/.github/workflows/python.yml b/.github/workflows/python.yml index 76ebea1e..e3bd5423 100644 --- a/.github/workflows/python.yml +++ b/.github/workflows/python.yml @@ -4,6 +4,7 @@ on: pull_request: push: branches: [ main ] + tags: [ 'v*.*.*' ] jobs: build: @@ -96,6 +97,12 @@ jobs: - uses: actions/download-artifact@v2 with: name: wheels + - name: Publish to GitHub release page + uses: softprops/action-gh-release@v1 + with: + files: | + *.whl + *.tar.gz - name: Publish to PyPI uses: messense/maturin-action@v1 env: diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index aa5edbeb..f773621e 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -4,6 +4,7 @@ on: pull_request: push: branches: [ main ] + tags: [ 'v*.*.*' ] jobs: check: @@ -101,3 +102,34 @@ jobs: - uses: Swatinem/rust-cache@v1 - name: Doc run: RUSTDOCFLAGS="-Dwarnings" cargo doc --workspace --all-features + + release: + name: Release + runs-on: ubuntu-latest + if: "startsWith(github.ref, 'refs/tags/')" + needs: [ check, test ] + steps: + - uses: actions/checkout@v2 + with: + submodules: recursive + - name: Fetch Substrait submodule tags + working-directory: substrait + run: | + git fetch --recurse-submodules=no origin +refs/tags/*:refs/tags/* + git describe --dirty --tags + - name: Local build to populate resource files + working-directory: rs + run: cargo build + - name: Publish substrait-validator-derive + working-directory: derive + env: + CARGO_REGISTRY_TOKEN: ${{ secrets.CARGO_REGISTRY_TOKEN }} + run: cargo publish --allow-dirty + - name: Wait for substrait-validator-derive to appear in the index + continue-on-error: true + run: python3 ci/crates-io-wait.py substrait-validator-derive `cat ci/version` 1800 + - name: Publish substrait-validator + working-directory: rs + env: + CARGO_REGISTRY_TOKEN: ${{ secrets.CARGO_REGISTRY_TOKEN }} + run: cargo publish --allow-dirty diff --git a/ci/crates-io-wait.py b/ci/crates-io-wait.py new file mode 100644 index 00000000..59c89b2b --- /dev/null +++ b/ci/crates-io-wait.py @@ -0,0 +1,73 @@ +#!/usr/bin/python3 +# SPDX-License-Identifier: Apache-2.0 + +import urllib.request +import json +import sys +import time + + +def crate_version_exists(crate, version): + """Returns whether the given version of the given crate exists on the + crates.io index.""" + + # Fetch the version info for the crate. + with urllib.request.urlopen( + "https://raw.githubusercontent.com/rust-lang/crates.io-index/" + f"master/{crate[0:2]}/{crate[2:4]}/{crate}" + ) as f: + data = f.read() + + # Parse version info. + versions = list(map(json.loads, data.decode("utf-8").strip().split("\n"))) + + # Check whether the requested version exists. + for version_data in versions: + if version_data.get("vers", None) == version: + return True + return False + + +def check(crate, version): + """Exits with code 0 if the given crate + version is found.""" + + try: + if crate_version_exists(crate, version): + print("Crate found!") + sys.exit(0) + except Exception as e: + print(f"{type(e).__name__}: {e}") + + +if __name__ == "__main__": + + if len(sys.argv) == 4: + _, crate, version, wait = sys.argv + crate = crate.strip() + version = version.strip() + wait_remain = int(wait.strip()) + + print("Checking...") + check(crate, version) + + period = 10 + while wait_remain > 0: + print(f"Waiting {period}s...") + sys.stdout.flush() + time.sleep(period) + wait_remain -= period + period = int(period * 1.25) + + print("Checking...") + sys.stdout.flush() + check(crate, version) + + print("Timeout expired!") + sys.exit(1) + + # Arguments invalid, print usage. + me = sys.argv[0] if sys.argv else "version.py" + print("Waits for a crate to appear on crates.io, since cargo publish doesn't.") + print("See https://github.com/rust-lang/cargo/issues/9507") + print(f"Usage: {me} ") + sys.exit(2) From 3d4b4b40d0e4c871c78ddd5b943243d988b49285 Mon Sep 17 00:00:00 2001 From: Jeroen van Straten Date: Tue, 7 Jun 2022 19:52:58 +0200 Subject: [PATCH 2/3] chore: use a release environment to better secure API tokens for releasing --- .github/workflows/python.yml | 1 + .github/workflows/rust.yml | 1 + 2 files changed, 2 insertions(+) diff --git a/.github/workflows/python.yml b/.github/workflows/python.yml index e3bd5423..4a7877ef 100644 --- a/.github/workflows/python.yml +++ b/.github/workflows/python.yml @@ -93,6 +93,7 @@ jobs: runs-on: ubuntu-latest if: "startsWith(github.ref, 'refs/tags/')" needs: [ build ] + environment: VALIDATOR_RELEASE steps: - uses: actions/download-artifact@v2 with: diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index f773621e..dbf64f0e 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -108,6 +108,7 @@ jobs: runs-on: ubuntu-latest if: "startsWith(github.ref, 'refs/tags/')" needs: [ check, test ] + environment: VALIDATOR_RELEASE steps: - uses: actions/checkout@v2 with: From 305a401ed64bac1a5f653ed46bd3e6f9b13f900d Mon Sep 17 00:00:00 2001 From: Jeroen van Straten Date: Thu, 23 Jun 2022 19:08:05 +0200 Subject: [PATCH 3/3] chore: fix issues from review --- .github/workflows/rust.yml | 5 ++++ ci/crates-io-wait.py | 59 +++++++++++++++++++------------------- 2 files changed, 34 insertions(+), 30 deletions(-) diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index dbf64f0e..32050567 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -113,6 +113,11 @@ jobs: - uses: actions/checkout@v2 with: submodules: recursive + - uses: actions-rs/toolchain@v1 + with: + profile: minimal + toolchain: stable + override: true - name: Fetch Substrait submodule tags working-directory: substrait run: | diff --git a/ci/crates-io-wait.py b/ci/crates-io-wait.py index 59c89b2b..a37fd965 100644 --- a/ci/crates-io-wait.py +++ b/ci/crates-io-wait.py @@ -1,10 +1,17 @@ #!/usr/bin/python3 # SPDX-License-Identifier: Apache-2.0 +""" +Waits for a crate to appear on crates.io, since cargo publish doesn't. +See https://github.com/rust-lang/cargo/issues/9507 +""" + + import urllib.request import json import sys import time +import argparse def crate_version_exists(crate, version): @@ -41,33 +48,25 @@ def check(crate, version): if __name__ == "__main__": - if len(sys.argv) == 4: - _, crate, version, wait = sys.argv - crate = crate.strip() - version = version.strip() - wait_remain = int(wait.strip()) - - print("Checking...") - check(crate, version) - - period = 10 - while wait_remain > 0: - print(f"Waiting {period}s...") - sys.stdout.flush() - time.sleep(period) - wait_remain -= period - period = int(period * 1.25) - - print("Checking...") - sys.stdout.flush() - check(crate, version) - - print("Timeout expired!") - sys.exit(1) - - # Arguments invalid, print usage. - me = sys.argv[0] if sys.argv else "version.py" - print("Waits for a crate to appear on crates.io, since cargo publish doesn't.") - print("See https://github.com/rust-lang/cargo/issues/9507") - print(f"Usage: {me} ") - sys.exit(2) + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("crate", type=str, help="the crate to wait for") + parser.add_argument("version", type=str, help="the version to wait for") + parser.add_argument("timeout", type=int, help="number of seconds to wait") + args = parser.parse_args() + + print("Checking...") + check(args.crate, args.version) + + wait_remain = args.timeout + period = 10 + while wait_remain > 0: + print(f"Waiting {period}s...", flush=True) + time.sleep(period) + wait_remain -= period + period = int(period * 1.25) + + print("Checking...", flush=True) + check(args.crate, args.version) + + print("Timeout expired!") + sys.exit(1)