Update from master

.github/scripts/common/lib.sh

@@ -307,3 +307,40 @@ function increment_rc_tag() {
   ((suffix++))
   echo $suffix
 }
+
+function relative_parent() {
+    echo "$1" | sed -E 's/(.*)\/(.*)\/\.\./\1/g'
+}
+
+# Find all the runtimes, it returns the result as JSON object, compatible to be
+# used as Github Workflow Matrix. This call is exposed by the `scan` command and can be used as:
+# podman run --rm -it -v /.../fellowship-runtimes:/build docker.io/chevdor/srtool:1.70.0-0.11.1 scan
+function find_runtimes() {
+    libs=($(git grep -I -r --cached --max-depth 20 --files-with-matches 'construct_runtime!' -- '*lib.rs'))
+    re=".*-runtime$"
+    JSON=$(jq --null-input '{ "include": [] }')
+
+    # EXCLUDED_RUNTIMES is a space separated list of runtime names (without the -runtime postfix)
+    # EXCLUDED_RUNTIMES=${EXCLUDED_RUNTIMES:-"substrate-test"}
+    IFS=' ' read -r -a exclusions <<< "$EXCLUDED_RUNTIMES"
+
+    for lib in "${libs[@]}"; do
+        crate_dir=$(dirname "$lib")
+        cargo_toml="$crate_dir/../Cargo.toml"
+
+        name=$(toml get -r $cargo_toml 'package.name')
+        chain=${name//-runtime/}
+
+        if [[ "$name" =~ $re ]] && ! [[ ${exclusions[@]} =~ $chain ]]; then
+            lib_dir=$(dirname "$lib")
+            runtime_dir=$(relative_parent "$lib_dir/..")
+            ITEM=$(jq --null-input \
+                --arg chain "$chain" \
+                --arg name "$name" \
+                --arg runtime_dir "$runtime_dir" \
+                '{ "chain": $chain, "crate": $name, "runtime_dir": $runtime_dir }')
+            JSON=$(echo $JSON | jq ".include += [$ITEM]")
+        fi
+    done
+    echo $JSON
+}

.github/workflows/check-licenses.yml

@@ -16,7 +16,7 @@ jobs:
     steps:
       - name: Checkout sources
         uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
-      - uses: actions/[email protected].0
+      - uses: actions/[email protected].1
         with:
           node-version: "18.x"
           registry-url: "https://npm.pkg.github.com"

.github/workflows/check-markdown.yml

@@ -16,7 +16,7 @@ jobs:
       - name: Checkout sources
         uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
 
-      - uses: actions/[email protected].0
+      - uses: actions/[email protected].1
         with:
           node-version: "18.x"
           registry-url: "https://npm.pkg.github.com"

.github/workflows/srtool.yml

@@ -0,0 +1,135 @@
+name: Srtool build
+
+env:
+  SUBWASM_VERSION: 0.20.0
+  TOML_CLI_VERSION: 0.2.4
+
+on:
+  push:
+    tags:
+      - "*"
+    branches:
+      - release-v[0-9]+.[0-9]+.[0-9]+*
+      - release-cumulus-v[0-9]+*
+      - release-polkadot-v[0-9]+*
+
+  schedule:
+    - cron: "00 02 * * 1" # 2AM weekly on monday
+
+  workflow_dispatch:
+
+jobs:
+  find-runtimes:
+      name: Scan repo paritytech/polkadot-sdk
+      outputs:
+        runtime: ${{ steps.get_runtimes_list.outputs.runtime }}
+      runs-on: ubuntu-latest
+      steps:
+        - name: Checkout
+          uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac # v4.0.0
+          with:
+            fetch-depth: 0
+
+        - name: Install tooling
+          run: |
+            URL=https://github.com/chevdor/toml-cli/releases/download/v${{ env.TOML_CLI_VERSION }}/toml_linux_amd64_v${{ env.TOML_CLI_VERSION }}.deb
+            curl -L $URL --output toml.deb
+            sudo dpkg -i toml.deb
+            toml --version; jq --version
+
+        - name: Scan runtimes
+          env:
+            EXCLUDED_RUNTIMES: "substrate-test"
+          run: |
+            . ./.github/scripts/common/lib.sh
+
+            echo "Github workspace: ${{ github.workspace }}"
+            echo "Current folder: $(pwd)"; ls -al
+            ls -al
+
+            MATRIX=$(find_runtimes | tee runtimes_list.json)
+            echo $MATRIX
+
+        - name: Get runtimes list
+          id: get_runtimes_list
+          run: |
+            ls -al
+            MATRIX=$(cat runtimes_list.json)
+            echo $MATRIX
+            echo "runtime=$MATRIX" >> $GITHUB_OUTPUT
+
+  srtool:
+    runs-on: ubuntu-latest
+    needs:
+      - find-runtimes
+    strategy:
+      fail-fast: false
+      matrix: ${{ fromJSON(needs.find-runtimes.outputs.runtime) }}
+
+    steps:
+      - uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac # v4.0.0
+        with:
+          fetch-depth: 0
+
+      - name: Srtool build
+        id: srtool_build
+        uses: chevdor/[email protected]
+        with:
+          chain: ${{ matrix.chain }}
+          runtime_dir: ${{ matrix.runtime_dir }}
+
+      - name: Summary
+        run: |
+          echo '${{ steps.srtool_build.outputs.json }}' | jq > ${{ matrix.chain }}-srtool-digest.json
+          cat ${{ matrix.chain }}-srtool-digest.json
+          echo "Compact Runtime: ${{ steps.srtool_build.outputs.wasm }}"
+          echo "Compressed Runtime: ${{ steps.srtool_build.outputs.wasm_compressed }}"
+
+      # it takes a while to build the runtime, so let's save the artifact as soon as we have it
+      - name: Archive Artifacts for ${{ matrix.chain }}
+        uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce # v3.1.2
+        with:
+          name: ${{ matrix.chain }}-runtime
+          path: |
+            ${{ steps.srtool_build.outputs.wasm }}
+            ${{ steps.srtool_build.outputs.wasm_compressed }}
+            ${{ matrix.chain }}-srtool-digest.json
+
+      # We now get extra information thanks to subwasm
+      - name: Install subwasm
+        run: |
+          wget https://github.com/chevdor/subwasm/releases/download/v${{ env.SUBWASM_VERSION }}/subwasm_linux_amd64_v${{ env.SUBWASM_VERSION }}.deb
+          sudo dpkg -i subwasm_linux_amd64_v${{ env.SUBWASM_VERSION }}.deb
+          subwasm --version
+
+      - name: Show Runtime information
+        shell: bash
+        run: |
+          subwasm info ${{ steps.srtool_build.outputs.wasm }}
+          subwasm info ${{ steps.srtool_build.outputs.wasm_compressed }}
+          subwasm --json info ${{ steps.srtool_build.outputs.wasm }} > ${{ matrix.chain }}-info.json
+          subwasm --json info ${{ steps.srtool_build.outputs.wasm_compressed }} > ${{ matrix.chain }}-compressed-info.json
+
+      - name: Extract the metadata
+        shell: bash
+        run: |
+          subwasm meta ${{ steps.srtool_build.outputs.wasm }}
+          subwasm --json meta ${{ steps.srtool_build.outputs.wasm }} > ${{ matrix.chain }}-metadata.json
+
+      - name: Check the metadata diff
+        shell: bash
+        # the following subwasm call will error for chains that are not known and/or live, that includes shell for instance
+        run: |
+          subwasm diff ${{ steps.srtool_build.outputs.wasm }} --chain-b ${{ matrix.chain }} || \
+            echo "Subwasm call failed, check the logs. This is likely because ${{ matrix.chain }} is not known by subwasm" | \
+            tee ${{ matrix.chain }}-diff.txt
+
+      - name: Archive Subwasm results
+        uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce # v3.1.2
+        with:
+          name: ${{ matrix.chain }}-runtime
+          path: |
+            ${{ matrix.chain }}-info.json
+            ${{ matrix.chain }}-compressed-info.json
+            ${{ matrix.chain }}-metadata.json
+            ${{ matrix.chain }}-diff.txt

.gitlab/pipeline/build.yml

@@ -220,6 +220,7 @@ build-test-parachain:
 
 # DAG: build-runtime-assets -> build-runtime-collectives -> build-runtime-bridge-hubs
 # DAG: build-runtime-assets -> build-runtime-collectives -> build-runtime-contracts
+# DAG: build-runtime-assets -> build-runtime-coretime
 # DAG: build-runtime-assets -> build-runtime-starters -> build-runtime-testing
 build-runtime-assets:
   <<: *build-runtime-template
@@ -235,6 +236,15 @@ build-runtime-collectives:
     - job: build-runtime-assets
       artifacts: false
 
+build-runtime-coretime:
+  <<: *build-runtime-template
+  variables:
+    RUNTIME_PATH: "cumulus/parachains/runtimes/coretime"
+  # this is an artificial job dependency, for pipeline optimization using GitLab's DAGs
+  needs:
+    - job: build-runtime-assets
+      artifacts: false
+
 build-runtime-bridge-hubs:
   <<: *build-runtime-template
   variables:

.gitlab/pipeline/check.yml

@@ -8,6 +8,7 @@ cargo-clippy:
     RUSTFLAGS: "-D warnings"
   script:
     - SKIP_WASM_BUILD=1 cargo clippy --all-targets --locked --workspace
+    - SKIP_WASM_BUILD=1 cargo clippy --all-targets --all-features --locked --workspace
 
 check-try-runtime:
   stage: check
@@ -223,6 +224,19 @@ check-runtime-migration-collectives-westend:
     URI: "wss://westend-collectives-rpc.polkadot.io:443"
     COMMAND_EXTRA_ARGS: "--disable-spec-name-check"
 
+# Check runtime migrations for Parity managed coretime chain
+check-runtime-migration-coretime-rococo:
+  stage: check
+  extends:
+    - .docker-env
+    - .test-pr-refs
+    - .check-runtime-migration
+  variables:
+    NETWORK: "coretime-rococo"
+    PACKAGE: "coretime-rococo-runtime"
+    WASM: "coretime_rococo_runtime.compact.compressed.wasm"
+    URI: "wss://rococo-coretime-rpc.polkadot.io:443"
+
 find-fail-ci-phrase:
   stage: check
   variables:

.gitlab/pipeline/short-benchmarks.yml

@@ -74,6 +74,16 @@ short-benchmark-collectives-westend:
   variables:
     RUNTIME_CHAIN: collectives-westend-dev
 
+short-benchmark-coretime-rococo:
+  <<: *short-bench-cumulus
+  variables:
+    RUNTIME_CHAIN: coretime-rococo-dev
+
+short-benchmark-coretime-westend:
+  <<: *short-bench-cumulus
+  variables:
+    RUNTIME_CHAIN: coretime-westend-dev
+
 short-benchmark-glutton-westend:
   <<: *short-bench-cumulus
   variables:

.gitlab/pipeline/test.yml

@@ -303,8 +303,15 @@ node-bench-regression-guard:
       artifacts: true
   variables:
     CI_IMAGE: "paritytech/node-bench-regression-guard:latest"
+    # current git limit is 20, set to 100 to avoid failures (gitlab removes old artifacts)
+    GIT_DEPTH: 100
+    GIT_STRATEGY: fetch
   before_script: [""]
   script:
+    - if [ $(ls -la artifacts/benches/ | grep master | wc -l) == 0 ]; then
+      echo "Couldn't find master artifacts, consider increasing GIT_LIMIT variable";
+      exit 1;
+      fi
     - echo "------- IMPORTANT -------"
     - echo "node-bench-regression-guard depends on the results of a cargo-check-benches job"
     - echo "In case of this job failure, check your pipeline's cargo-check-benches"
@@ -520,6 +527,6 @@ test-syscalls:
     - ./list-syscalls.rb ../../../target/x86_64-unknown-linux-musl/production/polkadot-prepare-worker --only-used-syscalls | diff -u prepare-worker-syscalls -
   after_script:
     - if [[ "$CI_JOB_STATUS" == "failed" ]]; then
-        printf "The x86_64 syscalls used by the worker binaries have changed. Please review if this is expected and update polkadot/scripts/list-syscalls/*-worker-syscalls as needed.\n";
+      printf "The x86_64 syscalls used by the worker binaries have changed. Please review if this is expected and update polkadot/scripts/list-syscalls/*-worker-syscalls as needed.\n";
       fi
   allow_failure: false # this rarely triggers in practice