From 71c45a764b1add4b8dd7a4be0aa4632ee5933e1f Mon Sep 17 00:00:00 2001
From: Alex Peters <alpe@users.noreply.github.com>
Date: Thu, 9 Jan 2025 15:47:54 +0100
Subject: [PATCH 1/2] More sims tests
---
scripts/build/simulations.mk | 61 +++----
simapp/v2/sim_runner.go | 294 ++++++++++++++++++-------------
simapp/v2/sim_test.go | 102 ++++++++++-
simsx/environment.go | 3 +
simsx/runner.go | 1 -
simsx/v2/valset_history.go | 17 +-
types/simulation/config.go | 4 +-
x/simulation/client/cli/flags.go | 10 +-
8 files changed, 319 insertions(+), 173 deletions(-)
diff --git a/scripts/build/simulations.mk b/scripts/build/simulations.mk
index 26b4661aa404..6a5a2d6cfe48 100644
--- a/scripts/build/simulations.mk
+++ b/scripts/build/simulations.mk
@@ -2,9 +2,10 @@
#? test-sim-nondeterminism: Run non-determinism test for simapp
test-sim-nondeterminism:
- # @echo "Running non-determinism test..."
- # @cd ${CURRENT_DIR}/simapp && go test -failfast -mod=readonly -timeout=30m -tags='sims' -run TestAppStateDeterminism \
- # -NumBlocks=100 -BlockSize=200 -Period=0
+ @echo "Running non-determinism test..."
+ @cd ${CURRENT_DIR}/simapp/v2 && go test -failfast -mod=readonly -timeout=30m -tags='sims' -run TestAppStateDeterminism \
+ -NumBlocks=100 -BlockSize=200
+
# Requires an exported plugin. See store/streaming/README.md for documentation.
#
@@ -18,51 +19,45 @@ test-sim-nondeterminism:
test-sim-nondeterminism-streaming:
# @echo "Running non-determinism-streaming test..."
# @cd ${CURRENT_DIR}/simapp && go test -failfast -mod=readonly -timeout=30m -tags='sims' -run TestAppStateDeterminism \
- # -NumBlocks=100 -BlockSize=200 -Period=0 -EnableStreaming=true
+ # -NumBlocks=100 -BlockSize=200 -EnableStreaming=true
test-sim-custom-genesis-fast:
# @echo "Running custom genesis simulation..."
# @echo "By default, ${HOME}/.simapp/config/genesis.json will be used."
# @cd ${CURRENT_DIR}/simapp && go test -failfast -mod=readonly -timeout=30m -tags='sims' -run TestFullAppSimulation -Genesis=${HOME}/.simapp/config/genesis.json \
- # -NumBlocks=100 -BlockSize=200 -Seed=99 -Period=5 -SigverifyTx=false
+ # -NumBlocks=100 -BlockSize=200 -Seed=99 -SigverifyTx=false
test-sim-import-export:
- # @echo "Running application import/export simulation. This may take several minutes..."
- # @cd ${CURRENT_DIR}/simapp && go test -failfast -mod=readonly -timeout 20m -tags='sims' -run TestAppImportExport \
- # -NumBlocks=50 -Period=5
+# @echo "Running application import/export simulation. This may take several minutes..."
+# @cd ${CURRENT_DIR}/simapp/v2 && go test -failfast -mod=readonly -timeout 20m -tags='sims' -run TestAppImportExport \
+ # -NumBlocks=50
test-sim-after-import:
- # @echo "Running application simulation-after-import. This may take several minutes..."
- # @cd ${CURRENT_DIR}/simapp && go test -failfast -mod=readonly -timeout 30m -tags='sims' -run TestAppSimulationAfterImport \
- # -NumBlocks=50 -Period=5
+ @echo "Running application simulation-after-import. This may take several minutes..."
+ @cd ${CURRENT_DIR}/simapp/v2 && go test -failfast -mod=readonly -timeout 30m -tags='sims' -run TestAppSimulationAfterImport \
+ -NumBlocks=50
test-sim-custom-genesis-multi-seed:
# @echo "Running multi-seed custom genesis simulation..."
# @echo "By default, ${HOME}/.simapp/config/genesis.json will be used."
- # @cd ${CURRENT_DIR}/simapp && go test -failfast -mod=readonly -timeout 30m -tags='sims' -run TestFullAppSimulation -Genesis=${HOME}/.simapp/config/genesis.json \
- # -NumBlocks=400 -Period=5
+ # @cd ${CURRENT_DIR}/simapp/v2 && go test -failfast -mod=readonly -timeout 30m -tags='sims' -run TestFullAppSimulation -Genesis=${HOME}/.simapp/config/genesis.json \
+ # -NumBlocks=400
test-sim-multi-seed-long:
- # @echo "Running long multi-seed application simulation. This may take awhile!"
- # @cd ${CURRENT_DIR}/simapp && go test -failfast -mod=readonly -timeout=2h -tags='sims' -run TestFullAppSimulation \
- # -NumBlocks=150 -Period=50
-
-test-sim-multi-seed-short: test-v2-sim
- # @echo "Running short multi-seed application simulation. This may take awhile!"
- # @cd ${CURRENT_DIR}/simapp && go test -failfast -mod=readonly -timeout 30m -tags='sims' -run TestFullAppSimulation \
- # -NumBlocks=50 -Period=10 -FauxMerkle=true
+ @echo "Running long multi-seed application simulation. This may take awhile!"
+ @cd ${CURRENT_DIR}/simapp/v2 && go test -failfast -mod=readonly -timeout=2h -tags='sims' -run TestFullAppSimulation \
+ -NumBlocks=150
-.Phony: test-v2-sim
-test-v2-sim:
- @echo "Running short multi-seed application simulation. This may take awhile!"
- @cd ${CURRENT_DIR}/simapp/v2 && go test -failfast -mod=readonly -timeout 30m -tags='sims' -run TestSimsAppV2 \
-# -NumBlocks=50 -Period=10 -FauxMerkle=true
+test-sim-multi-seed-short:
+ @echo "Running short multi-seed application simulation. This may take awhile!"
+ @cd ${CURRENT_DIR}/simapp/v2 && go test -failfast -mod=readonly -timeout 30m -tags='sims' -run TestFullAppSimulation \
+ -NumBlocks=50
test-sim-benchmark-invariants:
# @echo "Running simulation invariant benchmarks..."
# cd ${CURRENT_DIR}/simapp && go test -failfast -mod=readonly -benchmem -bench=BenchmarkInvariants -tags='sims' -run=^$ \
# -Enabled=true -NumBlocks=1000 -BlockSize=200 \
- # -Period=1 -Commit=true -Seed=57 -v -timeout 24h
+ # -Commit=true -Seed=57 -v -timeout 24h
.PHONY: \
test-sim-nondeterminism \
@@ -81,20 +76,20 @@ SIM_COMMIT ?= true
#? test-sim-fuzz: Run fuzz test for simapp
test-sim-fuzz:
- @echo "Running application fuzz for numBlocks=2, blockSize=20. This may take awhile!"
+# @echo "Running application fuzz for numBlocks=2, blockSize=20. This may take awhile!"
#ld flags are a quick fix to make it work on current osx
- @cd ${CURRENT_DIR}/simapp && go test -failfast -mod=readonly -json -tags='sims' -ldflags="-extldflags=-Wl,-ld_classic" -timeout=60m -fuzztime=60m -run=^$$ -fuzz=FuzzFullAppSimulation -GenesisTime=1714720615 -NumBlocks=2 -BlockSize=20
+# @cd ${CURRENT_DIR}/simapp && go test -failfast -mod=readonly -json -tags='sims' -ldflags="-extldflags=-Wl,-ld_classic" -timeout=60m -fuzztime=60m -run=^$$ -fuzz=FuzzFullAppSimulation -GenesisTime=1714720615 -NumBlocks=2 -BlockSize=20
#? test-sim-benchmark: Run benchmark test for simapp
test-sim-benchmark:
- @echo "Running application benchmark for numBlocks=$(SIM_NUM_BLOCKS), blockSize=$(SIM_BLOCK_SIZE). This may take awhile!"
- @cd ${CURRENT_DIR}/simapp && go test -failfast -mod=readonly -tags='sims' -run=^$$ $(.) -bench ^BenchmarkFullAppSimulation$$ \
+# @echo "Running application benchmark for numBlocks=$(SIM_NUM_BLOCKS), blockSize=$(SIM_BLOCK_SIZE). This may take awhile!"
+# @cd ${CURRENT_DIR}/simapp && go test -failfast -mod=readonly -tags='sims' -run=^$$ $(.) -bench ^BenchmarkFullAppSimulation$$ \
-Enabled=true -NumBlocks=$(SIM_NUM_BLOCKS) -BlockSize=$(SIM_BLOCK_SIZE) -Commit=$(SIM_COMMIT) -timeout 24h
test-sim-profile:
- @echo "Running application benchmark for numBlocks=$(SIM_NUM_BLOCKS), blockSize=$(SIM_BLOCK_SIZE). This may take awhile!"
- @cd ${CURRENT_DIR}/simapp && go test -failfast -mod=readonly -tags='sims' -benchmem -run=^$$ $(.) -bench ^BenchmarkFullAppSimulation$$ \
+# @echo "Running application benchmark for numBlocks=$(SIM_NUM_BLOCKS), blockSize=$(SIM_BLOCK_SIZE). This may take awhile!"
+# @cd ${CURRENT_DIR}/simapp && go test -failfast -mod=readonly -tags='sims' -benchmem -run=^$$ $(.) -bench ^BenchmarkFullAppSimulation$$ \
-Enabled=true -NumBlocks=$(SIM_NUM_BLOCKS) -BlockSize=$(SIM_BLOCK_SIZE) -Commit=$(SIM_COMMIT) -timeout 24h -cpuprofile cpu.out -memprofile mem.out
.PHONY: test-sim-profile test-sim-benchmark test-sim-fuzz
diff --git a/simapp/v2/sim_runner.go b/simapp/v2/sim_runner.go
index 3ae56ad77878..3d1bcf21ccab 100644
--- a/simapp/v2/sim_runner.go
+++ b/simapp/v2/sim_runner.go
@@ -27,7 +27,6 @@ import (
"cosmossdk.io/log"
"cosmossdk.io/runtime/v2"
"cosmossdk.io/server/v2/appmanager"
- cometbfttypes "cosmossdk.io/server/v2/cometbft/types"
storev2 "cosmossdk.io/store/v2"
consensustypes "cosmossdk.io/x/consensus/types"
@@ -51,8 +50,8 @@ type (
const SimAppChainID = "simulation-app"
-// this list of seeds was imported from the original simulation runner: https://github.com/cosmos/tools/blob/v1.0.0/cmd/runsim/main.go#L32
-var defaultSeeds = []int64{
+// DefaultSeeds list of seeds was imported from the original simulation runner: https://github.com/cosmos/tools/blob/v1.0.0/cmd/runsim/main.go#L32
+var DefaultSeeds = []int64{
1, 2, 4, 7,
32, 123, 124, 582, 1893, 2989,
3012, 4728, 37827, 981928, 87821, 891823782,
@@ -89,6 +88,14 @@ type (
// SimulationApp abstract blockchain app
SimulationApp[T Tx] interface {
+ appmanager.TransactionFuzzer[T]
+ InitGenesis(
+ ctx context.Context,
+ blockRequest *server.BlockRequest[T],
+ initGenesisJSON []byte,
+ txDecoder transaction.Codec[T],
+ ) (*server.BlockResponse, store.WriterMap, error)
+
GetApp() *runtime.App[T]
TxConfig() client.TxConfig
AppCodec() codec.Codec
@@ -99,6 +106,7 @@ type (
// TestInstance system under test
TestInstance[T Tx] struct {
+ Seed int64
App SimulationApp[T]
TxDecoder transaction.Codec[T]
BankKeeper BankKeeper
@@ -110,18 +118,24 @@ type (
}
AppFactory[T Tx, V SimulationApp[T]] func(config depinject.Config, outputs ...any) (V, error)
+ AppConfigFactory func() depinject.Config
)
// SetupTestInstance initializes and returns the system under test.
-func SetupTestInstance[T Tx, V SimulationApp[T]](t *testing.T, factory AppFactory[T, V], appConfig depinject.Config) TestInstance[T] {
- t.Helper()
+func SetupTestInstance[T Tx, V SimulationApp[T]](
+ tb testing.TB,
+ appFactory AppFactory[T, V],
+ appConfigFactory AppConfigFactory,
+ seed int64,
+) TestInstance[T] {
+ tb.Helper()
vp := viper.New()
vp.Set("store.app-db-backend", "memdb")
- vp.Set("home", t.TempDir())
+ vp.Set("home", tb.TempDir())
depInjCfg := depinject.Configs(
depinject.Supply(log.NewNopLogger(), runtime.GlobalConfig(vp.AllSettings())),
- appConfig,
+ appConfigFactory(),
)
var (
bankKeeper BankKeeper
@@ -134,11 +148,12 @@ func SetupTestInstance[T Tx, V SimulationApp[T]](t *testing.T, factory AppFactor
&bankKeeper,
&stKeeper,
)
- require.NoError(t, err)
+ require.NoError(tb, err)
- xapp, err := factory(depinject.Configs(depinject.Supply(log.NewNopLogger(), runtime.GlobalConfig(vp.AllSettings()))))
- require.NoError(t, err)
+ xapp, err := appFactory(depinject.Configs(depinject.Supply(log.NewNopLogger(), runtime.GlobalConfig(vp.AllSettings()))))
+ require.NoError(tb, err)
return TestInstance[T]{
+ Seed: seed,
App: xapp,
BankKeeper: bankKeeper,
AuthKeeper: authKeeper,
@@ -150,80 +165,128 @@ func SetupTestInstance[T Tx, V SimulationApp[T]](t *testing.T, factory AppFactor
}
}
+// InitializeChain sets up the blockchain with an initial state, validator set, and history using the provided genesis data.
+func (ti TestInstance[T]) InitializeChain(tb testing.TB,
+ ctx context.Context,
+ chainID string,
+ genesisTimestamp time.Time,
+ initialHeight uint64,
+ genesisAppState json.RawMessage,
+) ChainState[T] {
+ tb.Helper()
+ initRsp, stateRoot := doChainInitWithGenesis(
+ tb,
+ ctx,
+ chainID,
+ genesisTimestamp,
+ initialHeight,
+ genesisAppState,
+ ti,
+ )
+ activeValidatorSet := simsxv2.NewValSet().Update(initRsp.ValidatorUpdates)
+ valsetHistory := simsxv2.NewValSetHistory(initialHeight)
+ valsetHistory.Add(genesisTimestamp, activeValidatorSet)
+ return ChainState[T]{
+ ChainID: chainID,
+ BlockTime: genesisTimestamp,
+ BlockHeight: initialHeight,
+ ActiveValidatorSet: activeValidatorSet,
+ ValsetHistory: valsetHistory,
+ AppHash: stateRoot,
+ }
+}
+
// RunWithSeeds runs a series of subtests using the default set of random seeds for deterministic simulation testing.
-func RunWithSeeds[T Tx](
+func RunWithSeeds[T Tx, V SimulationApp[T]](
t *testing.T,
+ appFactory AppFactory[T, V],
+ appConfigFactory AppConfigFactory,
seeds []int64,
- postRunActions ...func(t testing.TB, app TestInstance[T], accs []simtypes.Account),
+ postRunActions ...func(t testing.TB, cs ChainState[T], app TestInstance[T], accs []simtypes.Account),
) {
t.Helper()
cfg := cli.NewConfigFromFlags()
cfg.ChainID = SimAppChainID
- for i := range seeds {
- seed := seeds[i]
+ for _, seed := range seeds {
t.Run(fmt.Sprintf("seed: %d", seed), func(t *testing.T) {
t.Parallel()
- RunWithSeed(t, NewSimApp[T], AppConfig(), cfg, seed, postRunActions...)
+ RunWithSeed(t, appFactory, appConfigFactory, cfg, seed, postRunActions...)
})
}
}
// RunWithSeed initializes and executes a simulation run with the given seed, generating blocks and transactions.
func RunWithSeed[T Tx, V SimulationApp[T]](
- t *testing.T,
+ tb testing.TB,
appFactory AppFactory[T, V],
- appConfig depinject.Config,
+ appConfigFactory AppConfigFactory,
tCfg simtypes.Config,
seed int64,
- postRunActions ...func(t testing.TB, app TestInstance[T], accs []simtypes.Account),
+ postRunActions ...func(t testing.TB, cs ChainState[T], app TestInstance[T], accs []simtypes.Account),
) {
- t.Helper()
- r := rand.New(rand.NewSource(seed))
- testInstance := SetupTestInstance[T, V](t, appFactory, appConfig)
- accounts, genesisAppState, chainID, genesisTimestamp := prepareInitialGenesisState(testInstance.App, r, testInstance.BankKeeper, tCfg, testInstance.ModuleManager)
+ tb.Helper()
+ initialBlockHeight := tCfg.InitialBlockHeight
+ require.NotEmpty(tb, initialBlockHeight, "initial block height must not be 0")
+
+ setupFn := func(ctx context.Context, r *rand.Rand) (TestInstance[T], ChainState[T], []simtypes.Account) {
+ testInstance := SetupTestInstance[T, V](tb, appFactory, appConfigFactory, seed)
+ accounts, genesisAppState, chainID, genesisTimestamp := prepareInitialGenesisState(
+ testInstance.App,
+ r,
+ testInstance.BankKeeper,
+ tCfg,
+ testInstance.ModuleManager,
+ )
+ cs := testInstance.InitializeChain(
+ tb,
+ ctx,
+ chainID,
+ genesisTimestamp,
+ initialBlockHeight,
+ genesisAppState,
+ )
+
+ return testInstance, cs, accounts
+ }
+ RunWithSeedX(tb, tCfg, setupFn, seed, postRunActions...)
+}
- appManager := testInstance.AppManager
- appStore := testInstance.App.Store()
- txConfig := testInstance.App.TxConfig()
+// RunWithSeedX entrypoint for custom chain setups.
+// The function runs the full simulation test circle for the specified seed and setup function, followed by optional post-run actions.
+func RunWithSeedX[T Tx](
+ tb testing.TB,
+ tCfg simtypes.Config,
+ setupChainStateFn func(ctx context.Context, r *rand.Rand) (TestInstance[T], ChainState[T], []simtypes.Account),
+ seed int64,
+ postRunActions ...func(t testing.TB, cs ChainState[T], app TestInstance[T], accs []simtypes.Account),
+) {
+ tb.Helper()
+ r := rand.New(rand.NewSource(seed))
rootCtx, done := context.WithCancel(context.Background())
defer done()
- initRsp, stateRoot := doChainInitWithGenesis(t, rootCtx, chainID, genesisTimestamp, appManager, testInstance.TxDecoder, genesisAppState, appStore)
- activeValidatorSet := simsxv2.NewValSet().Update(initRsp.ValidatorUpdates)
- valsetHistory := simsxv2.NewValSetHistory(1)
- valsetHistory.Add(genesisTimestamp, activeValidatorSet)
+
+ testInstance, chainState, accounts := setupChainStateFn(rootCtx, r)
emptySimParams := make(map[string]json.RawMessage) // todo read sims params from disk as before
modules := testInstance.ModuleManager.Modules()
msgFactoriesFn := prepareSimsMsgFactories(r, modules, simsx.ParamWeightSource(emptySimParams))
- cs := chainState[T]{
- chainID: chainID,
- blockTime: genesisTimestamp,
- activeValidatorSet: activeValidatorSet,
- valsetHistory: valsetHistory,
- stateRoot: stateRoot,
- app: appManager,
- appStore: appStore,
- txConfig: txConfig,
- }
doMainLoop(
- t,
+ tb,
rootCtx,
- cs,
+ testInstance,
+ &chainState,
msgFactoriesFn,
r,
- testInstance.AuthKeeper,
- testInstance.BankKeeper,
+ tCfg,
accounts,
- testInstance.TXBuilder,
- testInstance.StakingKeeper,
)
- require.NoError(t, testInstance.App.Close(), "closing app")
for _, step := range postRunActions {
- step(t, testInstance, accounts)
+ step(tb, chainState, testInstance, accounts)
}
+ require.NoError(tb, testInstance.App.Close(), "closing app")
}
// prepareInitialGenesisState initializes the genesis state for simulation by generating accounts, app state, chain ID, and timestamp.
@@ -257,18 +320,20 @@ func prepareInitialGenesisState[T Tx](
// doChainInitWithGenesis initializes the blockchain state with the provided genesis data and returns the initial block response and state root.
func doChainInitWithGenesis[T Tx](
- t *testing.T,
+ tb testing.TB,
ctx context.Context,
chainID string,
genesisTimestamp time.Time,
- app appmanager.AppManager[T],
- txDecoder transaction.Codec[T],
+ initialHeight uint64,
genesisAppState json.RawMessage,
- appStore cometbfttypes.Store,
+ testInstance TestInstance[T],
) (*server.BlockResponse, store.Hash) {
- t.Helper()
+ tb.Helper()
+ app := testInstance.App
+ txDecoder := testInstance.TxDecoder
+ appStore := testInstance.App.Store()
genesisReq := &server.BlockRequest[T]{
- Height: 0,
+ Height: initialHeight,
Time: genesisTimestamp,
Hash: make([]byte, 32),
ChainId: chainID,
@@ -290,27 +355,25 @@ func doChainInitWithGenesis[T Tx](
}
genesisCtx := context.WithValue(ctx, corecontext.CometParamsInitInfoKey, initialConsensusParams)
initRsp, genesisStateChanges, err := app.InitGenesis(genesisCtx, genesisReq, genesisAppState, txDecoder)
- require.NoError(t, err)
+ require.NoError(tb, err)
- require.NoError(t, appStore.SetInitialVersion(0))
+ require.NoError(tb, appStore.SetInitialVersion(initialHeight-1))
changeSet, err := genesisStateChanges.GetStateChanges()
- require.NoError(t, err)
+ require.NoError(tb, err)
- stateRoot, err := appStore.Commit(&store.Changeset{Changes: changeSet})
- require.NoError(t, err)
+ stateRoot, err := appStore.Commit(&store.Changeset{Changes: changeSet, Version: initialHeight - 1})
+ require.NoError(tb, err)
return initRsp, stateRoot
}
-// chainState represents the state of a blockchain during a simulation run.
-type chainState[T Tx] struct {
- chainID string
- blockTime time.Time
- activeValidatorSet simsxv2.WeightedValidators
- valsetHistory *simsxv2.ValSetHistory
- stateRoot store.Hash
- app appmanager.TransactionFuzzer[T]
- appStore storev2.RootStore
- txConfig client.TxConfig
+// ChainState represents the state of a blockchain during a simulation run.
+type ChainState[T Tx] struct {
+ ChainID string
+ BlockTime time.Time
+ BlockHeight uint64
+ ActiveValidatorSet simsxv2.WeightedValidators
+ ValsetHistory *simsxv2.ValSetHistory
+ AppHash store.Hash
}
// doMainLoop executes the main simulation loop after chain setup with genesis block.
@@ -318,34 +381,23 @@ type chainState[T Tx] struct {
// and executed. Events like validators missing votes or double signing are included in this
// process. The runtime tracks the validator's state and history.
func doMainLoop[T Tx](
- t *testing.T,
+ tb testing.TB,
rootCtx context.Context,
- cs chainState[T],
+ testInstance TestInstance[T],
+ cs *ChainState[T],
nextMsgFactory func() simsx.SimMsgFactoryX,
r *rand.Rand,
- authKeeper AuthKeeper,
- bankKeeper simsx.BalanceSource,
+ tCfg simtypes.Config,
accounts []simtypes.Account,
- txBuilder simsxv2.TXBuilder[T],
- stakingKeeper StakingKeeper,
) {
- t.Helper()
- blockTime := cs.blockTime
- activeValidatorSet := cs.activeValidatorSet
- if len(activeValidatorSet) == 0 {
- t.Fatal("no active validators in chain setup")
+ tb.Helper()
+ if len(cs.ActiveValidatorSet) == 0 {
+ tb.Fatal("no active validators in chain setup")
return
}
- valsetHistory := cs.valsetHistory
- stateRoot := cs.stateRoot
- chainID := cs.chainID
- app := cs.app
- appStore := cs.appStore
-
- const ( // todo: read from CLI instead
- numBlocks = 100 // 500 default
- maxTXPerBlock = 200 // 200 default
- )
+
+ numBlocks := tCfg.NumBlocks
+ maxTXPerBlock := tCfg.BlockSize
var (
txSkippedCounter int
@@ -354,39 +406,39 @@ func doMainLoop[T Tx](
rootReporter := simsx.NewBasicSimulationReporter()
futureOpsReg := simsxv2.NewFutureOpsRegistry()
- for i := 0; i < numBlocks; i++ {
- if len(activeValidatorSet) == 0 {
- t.Skipf("run out of validators in block: %d\n", i+1)
+ for end := cs.BlockHeight + numBlocks; cs.BlockHeight < end; cs.BlockHeight++ {
+ if len(cs.ActiveValidatorSet) == 0 {
+ tb.Skipf("run out of validators in block: %d\n", cs.BlockHeight)
return
}
- blockTime = blockTime.Add(minTimePerBlock)
- blockTime = blockTime.Add(time.Duration(int64(r.Intn(int(timeRangePerBlock/time.Second)))) * time.Second)
- valsetHistory.Add(blockTime, activeValidatorSet)
+ cs.BlockTime = cs.BlockTime.Add(minTimePerBlock).
+ Add(time.Duration(int64(r.Intn(int(timeRangePerBlock/time.Second)))) * time.Second)
+ cs.ValsetHistory.Add(cs.BlockTime, cs.ActiveValidatorSet)
blockReqN := &server.BlockRequest[T]{
- Height: uint64(1 + i),
- Time: blockTime,
- Hash: stateRoot,
- AppHash: stateRoot,
- ChainId: chainID,
+ Height: cs.BlockHeight,
+ Time: cs.BlockTime,
+ Hash: cs.AppHash,
+ AppHash: cs.AppHash,
+ ChainId: cs.ChainID,
}
cometInfo := comet.Info{
ValidatorsHash: nil,
- Evidence: valsetHistory.MissBehaviour(r),
- ProposerAddress: activeValidatorSet[0].Address,
- LastCommit: activeValidatorSet.NewCommitInfo(r),
+ Evidence: cs.ValsetHistory.MissBehaviour(r),
+ ProposerAddress: cs.ActiveValidatorSet[0].Address, // todo: pick random one
+ LastCommit: cs.ActiveValidatorSet.NewCommitInfo(r),
}
- fOps, pos := futureOpsReg.PopScheduledFor(blockTime), 0
- addressCodec := cs.txConfig.SigningContext().AddressCodec()
+ fOps, pos := futureOpsReg.PopScheduledFor(cs.BlockTime), 0
+ addressCodec := testInstance.App.TxConfig().SigningContext().AddressCodec()
simsCtx := context.WithValue(rootCtx, corecontext.CometInfoKey, cometInfo) // required for ContextAwareCometInfoService
resultHandlers := make([]simsx.SimDeliveryResultHandler, 0, maxTXPerBlock)
var txPerBlockCounter int
- blockRsp, updates, err := app.DeliverSims(simsCtx, blockReqN, func(ctx context.Context) iter.Seq[T] {
+ blockRsp, updates, err := testInstance.App.DeliverSims(simsCtx, blockReqN, func(ctx context.Context) iter.Seq[T] {
return func(yield func(T) bool) {
- unbondingTime, err := stakingKeeper.UnbondingTime(ctx)
- require.NoError(t, err)
- valsetHistory.SetMaxHistory(minBlocksInUnbondingPeriod(unbondingTime))
- testData := simsx.NewChainDataSource(ctx, r, authKeeper, bankKeeper, addressCodec, accounts...)
+ unbondingTime, err := testInstance.StakingKeeper.UnbondingTime(ctx)
+ require.NoError(tb, err)
+ cs.ValsetHistory.SetMaxHistory(minBlocksInUnbondingPeriod(unbondingTime))
+ testData := simsx.NewChainDataSource(ctx, r, testInstance.AuthKeeper, testInstance.BankKeeper, addressCodec, accounts...)
for txPerBlockCounter < maxTXPerBlock {
txPerBlockCounter++
@@ -407,36 +459,36 @@ func doMainLoop[T Tx](
signers, msg := mergedMsgFactory.Create()(ctx, testData, reporter)
if reporter.IsSkipped() {
txSkippedCounter++
- require.NoError(t, reporter.Close())
+ require.NoError(tb, reporter.Close())
continue
}
resultHandlers = append(resultHandlers, mergedMsgFactory.DeliveryResultHandler())
reporter.Success(msg)
- require.NoError(t, reporter.Close())
+ require.NoError(tb, reporter.Close())
- tx, err := txBuilder.Build(ctx, authKeeper, signers, msg, r, chainID)
- require.NoError(t, err)
+ tx, err := testInstance.TXBuilder.Build(ctx, testInstance.AuthKeeper, signers, msg, r, cs.ChainID)
+ require.NoError(tb, err)
if !yield(tx) {
return
}
}
}
})
- require.NoError(t, err, "%d, %s", blockReqN.Height, blockReqN.Time)
+ require.NoError(tb, err, "%d, %s", blockReqN.Height, blockReqN.Time)
changeSet, err := updates.GetStateChanges()
- require.NoError(t, err)
- stateRoot, err = appStore.Commit(&store.Changeset{
+ require.NoError(tb, err)
+ cs.AppHash, err = testInstance.App.Store().Commit(&store.Changeset{
Version: blockReqN.Height,
Changes: changeSet,
})
- require.NoError(t, err)
- require.Equal(t, len(resultHandlers), len(blockRsp.TxResults), "txPerBlockCounter: %d, totalSkipped: %d", txPerBlockCounter, txSkippedCounter)
+ require.NoError(tb, err)
+ require.Equal(tb, len(resultHandlers), len(blockRsp.TxResults), "txPerBlockCounter: %d, totalSkipped: %d", txPerBlockCounter, txSkippedCounter)
for i, v := range blockRsp.TxResults {
- require.NoError(t, resultHandlers[i](v.Error))
+ require.NoError(tb, resultHandlers[i](v.Error))
}
txTotalCounter += txPerBlockCounter
- activeValidatorSet = activeValidatorSet.Update(blockRsp.ValidatorUpdates)
+ cs.ActiveValidatorSet = cs.ActiveValidatorSet.Update(blockRsp.ValidatorUpdates)
}
fmt.Println("+++ reporter:\n" + rootReporter.Summary().String())
fmt.Printf("Tx total: %d skipped: %d\n", txTotalCounter, txSkippedCounter)
diff --git a/simapp/v2/sim_test.go b/simapp/v2/sim_test.go
index c92aeaedefc3..0f06a394264a 100644
--- a/simapp/v2/sim_test.go
+++ b/simapp/v2/sim_test.go
@@ -1,7 +1,103 @@
+//go:build sims
+
package simapp
-import "testing"
+import (
+ "bytes"
+ "context"
+ "math/rand"
+ "sync"
+ "testing"
+ "time"
+
+ "github.com/stretchr/testify/require"
+
+ simtypes "github.com/cosmos/cosmos-sdk/types/simulation"
+ v2 "github.com/cosmos/cosmos-sdk/x/genutil/v2"
+ simcli "github.com/cosmos/cosmos-sdk/x/simulation/client/cli"
+)
+
+func init() {
+ simcli.GetSimulatorFlags()
+}
+
+func TestFullAppSimulation(t *testing.T) {
+ RunWithSeeds[Tx](t, NewSimApp[Tx], AppConfig, DefaultSeeds)
+}
+
+func TestAppStateDeterminism(t *testing.T) {
+ var seeds []int64
+ if s := simcli.NewConfigFromFlags().Seed; s != simcli.DefaultSeedValue {
+ // override defaults with user data
+ seeds = []int64{s, s, s} // run same simulation 3 times
+ } else {
+ seeds = []int64{ // some random seeds, tripled to ensure same app-hash on all runs
+ 1, 1, 1,
+ 3, 3, 3,
+ 5, 5, 5,
+ }
+ }
+
+ var mx sync.Mutex
+ appHashResults := make(map[int64][]byte)
+ captureAndCheckHash := func(tb testing.TB, cs ChainState[Tx], ti TestInstance[Tx], _ []simtypes.Account) {
+ tb.Helper()
+ mx.Lock()
+ defer mx.Unlock()
+ otherHashes, ok := appHashResults[ti.Seed]
+ if !ok {
+ appHashResults[ti.Seed] = cs.AppHash
+ return
+ }
+ if !bytes.Equal(otherHashes, cs.AppHash) {
+ tb.Fatalf("non-determinism in seed %d", ti.Seed)
+ }
+ }
+ // run simulations
+ RunWithSeeds(t, NewSimApp[Tx], AppConfig, seeds, captureAndCheckHash)
+}
+
+// ExportableApp defines an interface for exporting application state and validator set.
+type ExportableApp interface {
+ ExportAppStateAndValidators(forZeroHeight bool, jailAllowedAddrs []string) (v2.ExportedApp, error)
+}
+
+// Scenario:
+//
+// Start a fresh node and run n blocks, export state
+// set up a new node instance, Init chain from exported genesis
+// run new instance for n blocks
+func TestAppSimulationAfterImport(t *testing.T) {
+ appFactory := NewSimApp[Tx]
+ cfg := simcli.NewConfigFromFlags()
+ cfg.ChainID = SimAppChainID
+
+ exportAndStartChainFromGenesisPostAction := func(tb testing.TB, cs ChainState[Tx], ti TestInstance[Tx], accs []simtypes.Account) {
+ tb.Helper()
+ tb.Log("exporting genesis...\n")
+ app, ok := ti.App.(ExportableApp)
+ require.True(tb, ok)
+ exported, err := app.ExportAppStateAndValidators(false, []string{})
+ require.NoError(tb, err)
+
+ genesisTimestamp := cs.BlockTime.Add(24 * time.Hour)
+ startHeight := uint64(exported.Height + 1)
+ chainID := SimAppChainID + "_2"
-func TestSimsAppV2(t *testing.T) {
- RunWithSeeds[Tx](t, defaultSeeds)
+ importGenesisChainStateFactory := func(ctx context.Context, r *rand.Rand) (TestInstance[Tx], ChainState[Tx], []simtypes.Account) {
+ testInstance := SetupTestInstance(tb, appFactory, AppConfig, ti.Seed)
+ newCs := testInstance.InitializeChain(
+ tb,
+ ctx,
+ chainID,
+ genesisTimestamp,
+ startHeight,
+ exported.AppState,
+ )
+ return testInstance, newCs, accs
+ }
+ // run sims with new app setup from exported genesis
+ RunWithSeedX[Tx](tb, cfg, importGenesisChainStateFactory, ti.Seed)
+ }
+ RunWithSeeds[Tx, *SimApp[Tx]](t, appFactory, AppConfig, DefaultSeeds, exportAndStartChainFromGenesisPostAction)
}
diff --git a/simsx/environment.go b/simsx/environment.go
index e5302802cce4..b01ad401be72 100644
--- a/simsx/environment.go
+++ b/simsx/environment.go
@@ -263,6 +263,9 @@ func NewChainDataSource(
codec address.Codec,
oldSimAcc ...simtypes.Account,
) *ChainDataSource {
+ if len(oldSimAcc) == 0 {
+ panic("empty accounts")
+ }
acc := make([]SimAccount, len(oldSimAcc))
index := make(map[string]int, len(oldSimAcc))
bank := contextAwareBalanceSource{ctx: ctx, bank: bk}
diff --git a/simsx/runner.go b/simsx/runner.go
index 86992617eac0..548507150eef 100644
--- a/simsx/runner.go
+++ b/simsx/runner.go
@@ -349,7 +349,6 @@ func NewSimulationAppInstance[T SimulationApp](
})
appOptions := make(simtestutil.AppOptionsMap)
appOptions[flags.FlagHome] = workDir
- appOptions[flags.FlagInvCheckPeriod] = cli.FlagPeriodValue
opts := []func(*baseapp.BaseApp){baseapp.SetChainID(tCfg.ChainID)}
if tCfg.FauxMerkle {
opts = append(opts, FauxMerkleModeOpt)
diff --git a/simsx/v2/valset_history.go b/simsx/v2/valset_history.go
index 9c05978ef900..e390f7c83ebd 100644
--- a/simsx/v2/valset_history.go
+++ b/simsx/v2/valset_history.go
@@ -16,15 +16,17 @@ type historicValSet struct {
}
type ValSetHistory struct {
maxElements int
- blockOffset int
+ blockOffset uint64
vals []historicValSet
}
-func NewValSetHistory(maxElements int) *ValSetHistory {
+// NewValSetHistory constructor. The maximum of historic valsets must not exceed the block or time limit for
+// valid evidence.
+func NewValSetHistory(initialHeight uint64) *ValSetHistory {
return &ValSetHistory{
- maxElements: maxElements,
- blockOffset: 1, // start at height 1
- vals: make([]historicValSet, 0, maxElements),
+ maxElements: 1,
+ blockOffset: initialHeight,
+ vals: make([]historicValSet, 0, 1),
}
}
@@ -58,7 +60,7 @@ func (h *ValSetHistory) MissBehaviour(r *rand.Rand) []comet.Evidence {
evidence := comet.Evidence{
Type: comet.DuplicateVote,
Validator: comet.Validator{Address: badVal.Address, Power: badVal.Power},
- Height: int64(h.blockOffset + n),
+ Height: int64(h.blockOffset) + int64(n),
Time: h.vals[n].blockTime,
TotalVotingPower: h.vals[n].vals.TotalPower(),
}
@@ -68,11 +70,12 @@ func (h *ValSetHistory) MissBehaviour(r *rand.Rand) []comet.Evidence {
return []comet.Evidence{evidence}
}
+// SetMaxHistory sets the maximum number of historical validator sets to retain. Reduces retained history if it exceeds the limit.
func (h *ValSetHistory) SetMaxHistory(v int) {
h.maxElements = v
if len(h.vals) > h.maxElements {
diff := len(h.vals) - h.maxElements
h.vals = h.vals[diff:]
- h.blockOffset += diff
+ h.blockOffset += uint64(diff)
}
}
diff --git a/types/simulation/config.go b/types/simulation/config.go
index 6f93684d0822..c5efa35e0d01 100644
--- a/types/simulation/config.go
+++ b/types/simulation/config.go
@@ -13,9 +13,9 @@ type Config struct {
ExportStatsPath string // custom file path to save the exported simulation statistics JSON
Seed int64 // simulation random seed
- InitialBlockHeight int // initial block to start the simulation
+ InitialBlockHeight uint64 // initial block to start the simulation
GenesisTime int64 // genesis time to start the simulation
- NumBlocks int // number of new blocks to simulate from the initial block height
+ NumBlocks uint64 // number of new blocks to simulate from the initial block height
BlockSize int // operations per block
ChainID string // chain-id used on the simulation
diff --git a/x/simulation/client/cli/flags.go b/x/simulation/client/cli/flags.go
index 1dc05fa47f29..4ede979e956c 100644
--- a/x/simulation/client/cli/flags.go
+++ b/x/simulation/client/cli/flags.go
@@ -18,8 +18,8 @@ var (
FlagExportStatePathValue string
FlagExportStatsPathValue string
FlagSeedValue int64
- FlagInitialBlockHeightValue int
- FlagNumBlocksValue int
+ FlagInitialBlockHeightValue uint64
+ FlagNumBlocksValue uint64
FlagBlockSizeValue int
FlagLeanValue bool
FlagCommitValue bool
@@ -27,7 +27,6 @@ var (
FlagEnabledValue bool
FlagVerboseValue bool
- FlagPeriodValue uint
FlagGenesisTimeValue int64
FlagSigverifyTxValue bool
FlagFauxMerkle bool
@@ -42,8 +41,8 @@ func GetSimulatorFlags() {
flag.IntVar(&FlagExportParamsHeightValue, "ExportParamsHeight", 0, "height to which export the randomly generated params")
flag.StringVar(&FlagExportStatePathValue, "ExportStatePath", "", "custom file path to save the exported app state JSON")
flag.Int64Var(&FlagSeedValue, "Seed", DefaultSeedValue, "simulation random seed")
- flag.IntVar(&FlagInitialBlockHeightValue, "InitialBlockHeight", 1, "initial block to start the simulation")
- flag.IntVar(&FlagNumBlocksValue, "NumBlocks", 500, "number of new blocks to simulate from the initial block height")
+ flag.Uint64Var(&FlagInitialBlockHeightValue, "InitialBlockHeight", 1, "initial block to start the simulation")
+ flag.Uint64Var(&FlagNumBlocksValue, "NumBlocks", 500, "number of new blocks to simulate from the initial block height")
flag.IntVar(&FlagBlockSizeValue, "BlockSize", 200, "operations per block")
flag.BoolVar(&FlagLeanValue, "Lean", false, "lean simulation log output")
flag.BoolVar(&FlagCommitValue, "Commit", true, "have the simulation commit")
@@ -52,7 +51,6 @@ func GetSimulatorFlags() {
// simulation flags
flag.BoolVar(&FlagEnabledValue, "Enabled", false, "enable the simulation")
flag.BoolVar(&FlagVerboseValue, "Verbose", false, "verbose log output")
- flag.UintVar(&FlagPeriodValue, "Period", 0, "run slow invariants only once every period assertions")
flag.Int64Var(&FlagGenesisTimeValue, "GenesisTime", time.Now().Unix(), "use current time as genesis UNIX time for default")
flag.BoolVar(&FlagSigverifyTxValue, "SigverifyTx", true, "whether to sigverify check for transaction ")
flag.BoolVar(&FlagFauxMerkle, "FauxMerkle", false, "use faux merkle instead of iavl")
From 4dd80706507cb0e62a3cc89e859bc8f8eeb66d22 Mon Sep 17 00:00:00 2001
From: Alex Peters <alpe@users.noreply.github.com>
Date: Mon, 13 Jan 2025 10:34:55 +0100
Subject: [PATCH 2/2] Review feedback
---
simapp/v2/sim_runner.go | 3 ++-
simapp/v2/sim_test.go | 4 ++--
2 files changed, 4 insertions(+), 3 deletions(-)
diff --git a/simapp/v2/sim_runner.go b/simapp/v2/sim_runner.go
index 3d1bcf21ccab..6610566e0daf 100644
--- a/simapp/v2/sim_runner.go
+++ b/simapp/v2/sim_runner.go
@@ -166,7 +166,8 @@ func SetupTestInstance[T Tx, V SimulationApp[T]](
}
// InitializeChain sets up the blockchain with an initial state, validator set, and history using the provided genesis data.
-func (ti TestInstance[T]) InitializeChain(tb testing.TB,
+func (ti TestInstance[T]) InitializeChain(
+ tb testing.TB,
ctx context.Context,
chainID string,
genesisTimestamp time.Time,
diff --git a/simapp/v2/sim_test.go b/simapp/v2/sim_test.go
index 0f06a394264a..059e8a203f46 100644
--- a/simapp/v2/sim_test.go
+++ b/simapp/v2/sim_test.go
@@ -13,7 +13,7 @@ import (
"github.com/stretchr/testify/require"
simtypes "github.com/cosmos/cosmos-sdk/types/simulation"
- v2 "github.com/cosmos/cosmos-sdk/x/genutil/v2"
+ genutil "github.com/cosmos/cosmos-sdk/x/genutil/v2"
simcli "github.com/cosmos/cosmos-sdk/x/simulation/client/cli"
)
@@ -59,7 +59,7 @@ func TestAppStateDeterminism(t *testing.T) {
// ExportableApp defines an interface for exporting application state and validator set.
type ExportableApp interface {
- ExportAppStateAndValidators(forZeroHeight bool, jailAllowedAddrs []string) (v2.ExportedApp, error)
+ ExportAppStateAndValidators(forZeroHeight bool, jailAllowedAddrs []string) (genutil.ExportedApp, error)
}
// Scenario: