tests(pd): automated migration testing

Adds intra-CI standalone testing of migration behavior.
This can be run locally on developer workstations,
and also in CI. The job is currently taking >20m, and requires
a lot of disk space, but it's worth it for the assurance.

Building on the smoke-test rewrite to use process-compose,
let's script the migration process, so that we can test
current HEAD of the monorepo against a prior tagged version,
and validate that necessary migrations are in place.

One possible approach is to fetch prebuilt binaries from uploaded
artifacts on Github. That's fine for `pd`, but doesn't work for
running the smoke tests, due to client/server incompatibility.
Therefore we'll clone the entire repo in a git-ignored subdir,
and build the old binaries there. Heavy, but reliable.

Adds a new rust crate, strictly for running the migration-test
suite of integration tests, which is very similar in nature to the
already-existing network-integration tests AKA smoke tests.
Copy/pastes a lot of code from the smokes, which we can always
factor out into reusable utils, but not bothering with that right now.

Refs #4323.

Author: conorsch

.github/workflows/migration.yml

@@ -0,0 +1,45 @@
+---
+# Test suite to verify that migrations apply cleanly.
+# References to the previous stable release's tag, e.g. `v0.76.0`,
+# must be updated manually.
+name: Migration test
+on:
+  # For now, we'll run this on every PR, but it's slow (>20m).
+  pull_request:
+  # Run periodically to check for breakage.
+  schedule:
+    # 16:10 UTC is 9AM PT / 12PM ET.
+    - cron: "10 16 * * *"
+jobs:
+  migration_test:
+    runs-on: buildjet-32vcpu-ubuntu-2204
+    concurrency:
+      group: ${{ github.workflow }}-${{ github.ref }}
+      cancel-in-progress: true
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          lfs: true
+          fetch-tags: true
+          fetch-depth: 0
+
+      - name: Load rust cache
+        uses: astriaorg/[email protected]
+        with:
+          # Cache the git worktree for faster builds
+          workspaces: |
+            . -> ./target
+            ./deployments/worktrees/v0.76.0 -> ./deployments/worktrees/v0.76.0/target
+
+      - name: Install cometbft binary
+        run: ./deployments/scripts/install-cometbft
+
+      - name: Install process-compose
+        run: >-
+          sh -c "$(curl --location https://raw.githubusercontent.com/F1bonacc1/process-compose/main/scripts/get-pc.sh)" --
+          -d -b ~/bin
+
+      - name: Run migration test
+        run: |
+          export PATH="$HOME/bin:$PATH"
+          ./deployments/scripts/migration-test v0.76.0

.gitignore

@@ -52,8 +52,11 @@ deployments/relayer/configs/penumbra-local.json
 
 # Logs, and other files from smoke tests
 deployments/logs/
+deployments/worktrees/
+deployments/bin/
 crates/bin/pcli/proposal.toml
 phase1.bin
+crates/test/migration-test/proposal.toml
 
 # Memory profiler, via bytehound or otherwise
 *.dat

Cargo.lock

@@ -52,7 +52,7 @@ members = [
   "crates/util/tower-trace",
   "crates/view",
   "crates/wallet",
-  "tools/summonerd",
+  "tools/summonerd", "crates/test/migration-test",
 ]
 
 # Config for 'cargo dist'

Cargo.toml

@@ -0,0 +1,35 @@
+[package]
+name = "migration-test"
+description = "Integration test runner for evaluating Penumbra upgrades"
+authors.workspace = true
+edition.workspace = true
+version.workspace = true
+repository.workspace = true
+homepage.workspace = true
+license.workspace = true
+
+# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
+
+[features]
+# By default, don't enable migration-tests: require explicit opt-in via
+# `--features migration-test`.
+default = []
+migration-test = []
+
+[dependencies]
+anyhow = {workspace = true}
+directories = {workspace = true}
+once_cell = {workspace = true}
+penumbra-keys = {workspace = true, default-features = false}
+serde = {workspace = true, features = ["derive"]}
+serde_json = {workspace = true}
+serde_with = {workspace = true, features = ["hex"]}
+toml = {workspace = true, features = ["preserve_order"]}
+tracing = {workspace = true}
+tracing-subscriber = {workspace = true, features = ["env-filter", "ansi"]}
+
+[dev-dependencies]
+assert_cmd = {workspace = true}
+predicates = "2.1"
+regex = {workspace = true}
+tempfile = {workspace = true}

crates/test/migration-test/Cargo.toml

@@ -0,0 +1,3 @@
+fn main() {
+    println!("Hello, world!");
+}