From a727046ce68e171f35ce75cb97be0aced7e4e58e Mon Sep 17 00:00:00 2001 From: Cody Wang Date: Fri, 26 Apr 2024 17:45:58 -0400 Subject: [PATCH 01/38] wip --- op-batcher/batcher/channel_builder.go | 2 +- op-batcher/batcher/channel_builder_test.go | 170 ++++++++++++--------- op-batcher/batcher/channel_config.go | 16 +- op-batcher/batcher/channel_config_test.go | 2 +- op-batcher/batcher/channel_manager_test.go | 10 +- op-batcher/batcher/config.go | 3 + op-batcher/batcher/service.go | 2 +- op-batcher/compressor/config.go | 3 + op-e2e/actions/l2_batcher.go | 2 +- op-e2e/actions/sync_test.go | 2 +- op-node/benchmarks/batchbuilding_test.go | 2 +- op-node/rollup/derive/channel_out_test.go | 4 +- op-node/rollup/derive/span_channel_out.go | 34 ++++- 13 files changed, 159 insertions(+), 93 deletions(-) diff --git a/op-batcher/batcher/channel_builder.go b/op-batcher/batcher/channel_builder.go index e364570d48b0..3fc4c651a5ad 100644 --- a/op-batcher/batcher/channel_builder.go +++ b/op-batcher/batcher/channel_builder.go @@ -86,7 +86,7 @@ func NewChannelBuilder(cfg ChannelConfig, rollupCfg rollup.Config, latestL1Origi } var co derive.ChannelOut if cfg.BatchType == derive.SpanBatchType { - co, err = derive.NewSpanChannelOut(rollupCfg.Genesis.L2Time, rollupCfg.L2ChainID, cfg.CompressorConfig.TargetOutputSize) + co, err = derive.NewSpanChannelOut(rollupCfg.Genesis.L2Time, rollupCfg.L2ChainID, cfg.CompressorConfig.TargetOutputSize, cfg.CompressorConfig.CompressionAlgo, cfg.CompressorConfig.CompressLevel) } else { co, err = derive.NewSingularChannelOut(c) } diff --git a/op-batcher/batcher/channel_builder_test.go b/op-batcher/batcher/channel_builder_test.go index 1d220fe239e5..29c7d21bfd2f 100644 --- a/op-batcher/batcher/channel_builder_test.go +++ b/op-batcher/batcher/channel_builder_test.go @@ -393,7 +393,7 @@ func TestChannelBuilder_OutputFrames(t *testing.T) { channelConfig := defaultTestChannelConfig() channelConfig.MaxFrameSize = derive.FrameV0OverHeadSize + 1 channelConfig.TargetNumFrames = 1000 - channelConfig.InitNoneCompressor() + channelConfig.InitNoneCompressor("brotli", 10) // Construct the channel builder cb, err := NewChannelBuilder(channelConfig, defaultTestRollupConfig, latestL1BlockOrigin) @@ -430,54 +430,72 @@ func TestChannelBuilder_OutputFrames(t *testing.T) { } func TestChannelBuilder_OutputFrames_SpanBatch(t *testing.T) { - channelConfig := defaultTestChannelConfig() - channelConfig.MaxFrameSize = 20 + derive.FrameV0OverHeadSize - channelConfig.TargetNumFrames = 5 - channelConfig.BatchType = derive.SpanBatchType - channelConfig.InitRatioCompressor(1) - - // Construct the channel builder - cb, err := NewChannelBuilder(channelConfig, defaultTestRollupConfig, latestL1BlockOrigin) - require.NoError(t, err) - require.False(t, cb.IsFull()) - require.Equal(t, 0, cb.PendingFrames()) - - // Calling OutputFrames without having called [AddBlock] - // should return no error - require.NoError(t, cb.OutputFrames()) - - // There should be no ready bytes yet - require.Equal(t, 0, cb.co.ReadyBytes()) - - // fill up - for { - err = addMiniBlock(cb) - if err == nil { - require.False(t, cb.IsFull()) - // There should be no ready bytes until the channel is full - require.Equal(t, cb.co.ReadyBytes(), 0) - } else { - require.ErrorIs(t, err, derive.ErrCompressorFull) - break - } + testCases := []struct { + testName string + algo string + compressLevel int + targetNumFrames int + }{ + {testName: "Span Batch output frames with zlib", algo: "zlib", compressLevel: 9, targetNumFrames: 5}, + {testName: "Span Batch output frames with brotli", algo: "brotli", compressLevel: 10, targetNumFrames: 1}, // to fill faster } - require.True(t, cb.IsFull()) - // Check how many ready bytes - require.GreaterOrEqual(t, - cb.co.ReadyBytes()+derive.FrameV0OverHeadSize, - int(channelConfig.MaxFrameSize)) - require.Equal(t, 0, cb.PendingFrames()) - - // We should be able to output the frames - require.NoError(t, cb.OutputFrames()) - - // There should be several frames in the channel builder now - require.Greater(t, cb.PendingFrames(), 1) - for i := 0; i < cb.numFrames-1; i++ { - require.Len(t, cb.frames[i].data, int(channelConfig.MaxFrameSize)) + for _, tc := range testCases { + t.Run(tc.testName, func(t *testing.T) { + channelConfig := defaultTestChannelConfig() + channelConfig.MaxFrameSize = 20 + derive.FrameV0OverHeadSize + channelConfig.TargetNumFrames = tc.targetNumFrames + channelConfig.BatchType = derive.SpanBatchType + channelConfig.InitRatioCompressor(1, tc.algo, tc.compressLevel) + + // Construct the channel builder + cb, err := NewChannelBuilder(channelConfig, defaultTestRollupConfig, latestL1BlockOrigin) + require.NoError(t, err) + require.False(t, cb.IsFull()) + require.Equal(t, 0, cb.PendingFrames()) + + // Calling OutputFrames without having called [AddBlock] + // should return no error + require.NoError(t, cb.OutputFrames()) + + // There should be no ready bytes yet + require.Equal(t, 0, cb.co.ReadyBytes()) + + // fill up + for { + err = addMiniBlock(cb) + if err == nil { + if cb.IsFull() { + // this happens when the data exactly fills the channel + break + } + require.False(t, cb.IsFull()) + // There should be no ready bytes until the channel is full + require.Equal(t, cb.co.ReadyBytes(), 0) + } else { + require.ErrorIs(t, err, derive.ErrCompressorFull) + break + } + } + + require.True(t, cb.IsFull()) + // Check how many ready bytes + require.GreaterOrEqual(t, + cb.co.ReadyBytes()+derive.FrameV0OverHeadSize, + int(channelConfig.MaxFrameSize)) + require.Equal(t, 0, cb.PendingFrames()) + + // We should be able to output the frames + require.NoError(t, cb.OutputFrames()) + + // There should be several frames in the channel builder now + require.Greater(t, cb.PendingFrames(), 1) + for i := 0; i < cb.numFrames-1; i++ { + require.Len(t, cb.frames[i].data, int(channelConfig.MaxFrameSize)) + } + require.LessOrEqual(t, len(cb.frames[len(cb.frames)-1].data), int(channelConfig.MaxFrameSize)) + }) } - require.LessOrEqual(t, len(cb.frames[len(cb.frames)-1].data), int(channelConfig.MaxFrameSize)) } // ChannelBuilder_MaxRLPBytesPerChannel tests the [ChannelBuilder.OutputFrames] @@ -486,7 +504,7 @@ func ChannelBuilder_MaxRLPBytesPerChannel(t *testing.T, batchType uint) { t.Parallel() channelConfig := defaultTestChannelConfig() channelConfig.MaxFrameSize = derive.MaxRLPBytesPerChannel * 2 - channelConfig.InitNoneCompressor() + channelConfig.InitNoneCompressor("brotli", 10) channelConfig.BatchType = batchType // Construct the channel builder @@ -504,7 +522,7 @@ func ChannelBuilder_OutputFramesMaxFrameIndex(t *testing.T, batchType uint) { channelConfig := defaultTestChannelConfig() channelConfig.MaxFrameSize = derive.FrameV0OverHeadSize + 1 channelConfig.TargetNumFrames = math.MaxUint16 + 1 - channelConfig.InitRatioCompressor(.1) + channelConfig.InitRatioCompressor(.1, "brotli", 10) channelConfig.BatchType = batchType rng := rand.New(rand.NewSource(123)) @@ -539,34 +557,50 @@ func ChannelBuilder_OutputFramesMaxFrameIndex(t *testing.T, batchType uint) { // This test fails in multiple places if the subtraction of // [derive.FrameV0OverHeadSize] in [MaxDataSize] is omitted, which has been the // case before it got fixed it #9887. -func TestChannelBuilder_FullShadowCompressor(t *testing.T) { +func TestChannelBuilder_FullShadowCompressor_zlib(t *testing.T) { require := require.New(t) cfg := ChannelConfig{ MaxFrameSize: 752, TargetNumFrames: 1, BatchType: derive.SpanBatchType, } - cfg.InitShadowCompressor() - cb, err := NewChannelBuilder(cfg, defaultTestRollupConfig, latestL1BlockOrigin) - require.NoError(err) + testCases := []struct { + testName string + algo string + compressLevel int + targetNumFrames int + }{ + {testName: "Full shadow compressor with zlib", algo: "zlib", compressLevel: 9}, + {testName: "Full shadow compressor with brotli", algo: "brotli", compressLevel: 10}, + } - rng := rand.New(rand.NewSource(420)) - a := dtest.RandomL2BlockWithChainId(rng, 1, defaultTestRollupConfig.L2ChainID) - _, err = cb.AddBlock(a) - require.NoError(err) - _, err = cb.AddBlock(a) - require.ErrorIs(err, derive.ErrCompressorFull) - // without fix, adding the second block would succeed and then adding a - // third block would fail with full error and the compressor would be full. + for _, tc := range testCases { + t.Run(tc.testName, func(t *testing.T) { + cfg.InitShadowCompressor(tc.algo, tc.compressLevel) - require.NoError(cb.OutputFrames()) + cb, err := NewChannelBuilder(cfg, defaultTestRollupConfig, latestL1BlockOrigin) + require.NoError(err) + + rng := rand.New(rand.NewSource(420)) + a := dtest.RandomL2BlockWithChainId(rng, 1, defaultTestRollupConfig.L2ChainID) + + for { + _, err = cb.AddBlock(a) + if err != nil { + require.ErrorIs(err, derive.ErrCompressorFull) + break + } + } - require.True(cb.HasFrame()) - f := cb.NextFrame() - require.Less(len(f.data), int(cfg.MaxFrameSize)) // would fail without fix, full frame + require.NoError(cb.OutputFrames()) + require.True(cb.HasFrame()) + f := cb.NextFrame() + require.Less(len(f.data), int(cfg.MaxFrameSize)) // would fail without fix, full frame - require.False(cb.HasFrame(), "no leftover frame expected") // would fail without fix + require.False(cb.HasFrame(), "no leftover frame expected") // would fail without fix + }) + } } func ChannelBuilder_AddBlock(t *testing.T, batchType uint) { @@ -577,7 +611,7 @@ func ChannelBuilder_AddBlock(t *testing.T, batchType uint) { channelConfig.MaxFrameSize = 20 + derive.FrameV0OverHeadSize channelConfig.TargetNumFrames = 2 // Configure the Input Threshold params so we observe a full channel - channelConfig.InitRatioCompressor(1) + channelConfig.InitRatioCompressor(1, "brotli", 10) // Construct the channel builder cb, err := NewChannelBuilder(channelConfig, defaultTestRollupConfig, latestL1BlockOrigin) @@ -700,7 +734,7 @@ func ChannelBuilder_PendingFrames_TotalFrames(t *testing.T, batchType uint) { cfg.MaxFrameSize = 1000 cfg.TargetNumFrames = tnf cfg.BatchType = batchType - cfg.InitShadowCompressor() + cfg.InitShadowCompressor("brotli", 10) cb, err := NewChannelBuilder(cfg, defaultTestRollupConfig, latestL1BlockOrigin) require.NoError(err) @@ -782,7 +816,7 @@ func ChannelBuilder_OutputBytes(t *testing.T, batchType uint) { cfg.MaxFrameSize = 1000 cfg.TargetNumFrames = 16 cfg.BatchType = batchType - cfg.InitRatioCompressor(1.0) + cfg.InitRatioCompressor(1.0, "brotli", 10) cb, err := NewChannelBuilder(cfg, defaultTestRollupConfig, latestL1BlockOrigin) require.NoError(err, "NewChannelBuilder") diff --git a/op-batcher/batcher/channel_config.go b/op-batcher/batcher/channel_config.go index c6a0eec4c04c..2ea80d13e84b 100644 --- a/op-batcher/batcher/channel_config.go +++ b/op-batcher/batcher/channel_config.go @@ -53,25 +53,27 @@ type ChannelConfig struct { // value consistent with cc.TargetNumFrames and cc.MaxFrameSize. // comprKind can be the empty string, in which case the default compressor will // be used. -func (cc *ChannelConfig) InitCompressorConfig(approxComprRatio float64, comprKind string) { +func (cc *ChannelConfig) InitCompressorConfig(approxComprRatio float64, comprKind string, compressionAlgo string, compressLevel int) { cc.CompressorConfig = compressor.Config{ // Compressor output size needs to account for frame encoding overhead TargetOutputSize: MaxDataSize(cc.TargetNumFrames, cc.MaxFrameSize), ApproxComprRatio: approxComprRatio, Kind: comprKind, + CompressionAlgo: compressionAlgo, + CompressLevel: compressLevel, } } -func (cc *ChannelConfig) InitRatioCompressor(approxComprRatio float64) { - cc.InitCompressorConfig(approxComprRatio, compressor.RatioKind) +func (cc *ChannelConfig) InitRatioCompressor(approxComprRatio float64, compressionAlgo string, compressLevel int) { + cc.InitCompressorConfig(approxComprRatio, compressor.RatioKind, compressionAlgo, compressLevel) } -func (cc *ChannelConfig) InitShadowCompressor() { - cc.InitCompressorConfig(0, compressor.ShadowKind) +func (cc *ChannelConfig) InitShadowCompressor(compressionAlgo string, compressLevel int) { + cc.InitCompressorConfig(0, compressor.ShadowKind, compressionAlgo, compressLevel) } -func (cc *ChannelConfig) InitNoneCompressor() { - cc.InitCompressorConfig(0, compressor.NoneKind) +func (cc *ChannelConfig) InitNoneCompressor(compressionAlgo string, compressLevel int) { + cc.InitCompressorConfig(0, compressor.NoneKind, compressionAlgo, compressLevel) } func (cc *ChannelConfig) MaxFramesPerTx() int { diff --git a/op-batcher/batcher/channel_config_test.go b/op-batcher/batcher/channel_config_test.go index a89d780ab558..fbb08374fee9 100644 --- a/op-batcher/batcher/channel_config_test.go +++ b/op-batcher/batcher/channel_config_test.go @@ -20,7 +20,7 @@ func defaultTestChannelConfig() ChannelConfig { TargetNumFrames: 1, BatchType: derive.SingularBatchType, } - c.InitRatioCompressor(0.4) + c.InitRatioCompressor(0.4, "brotli", 10) return c } diff --git a/op-batcher/batcher/channel_manager_test.go b/op-batcher/batcher/channel_manager_test.go index e029e69e658c..d1033be1a896 100644 --- a/op-batcher/batcher/channel_manager_test.go +++ b/op-batcher/batcher/channel_manager_test.go @@ -25,7 +25,7 @@ func channelManagerTestConfig(maxFrameSize uint64, batchType uint) ChannelConfig TargetNumFrames: 1, BatchType: batchType, } - cfg.InitRatioCompressor(1) + cfg.InitRatioCompressor(1, "brotli", 10) return cfg } @@ -123,7 +123,7 @@ func ChannelManager_Clear(t *testing.T, batchType uint) { // channels on confirmation. This would result in [TxConfirmed] // clearing confirmed transactions, and resetting the pendingChannels map cfg.ChannelTimeout = 10 - cfg.InitRatioCompressor(1) + cfg.InitRatioCompressor(1, "brotli", 10) m := NewChannelManager(log, metrics.NoopMetrics, cfg, &defaultTestRollupConfig) // Channel Manager state should be empty by default @@ -345,7 +345,7 @@ func TestChannelManager_Close_PartiallyPendingChannel(t *testing.T) { ChannelTimeout: 1000, TargetNumFrames: 100, } - cfg.InitNoneCompressor() + cfg.InitNoneCompressor("brotli", 10) m := NewChannelManager(log, metrics.NoopMetrics, cfg, &defaultTestRollupConfig) m.Clear(eth.BlockID{}) @@ -397,7 +397,7 @@ func ChannelManagerCloseAllTxsFailed(t *testing.T, batchType uint) { log := testlog.Logger(t, log.LevelCrit) cfg := channelManagerTestConfig(100, batchType) cfg.TargetNumFrames = 1000 - cfg.InitNoneCompressor() + cfg.InitNoneCompressor("brotli", 10) m := NewChannelManager(log, metrics.NoopMetrics, cfg, &defaultTestRollupConfig) m.Clear(eth.BlockID{}) @@ -445,7 +445,7 @@ func TestChannelManager_ChannelCreation(t *testing.T) { const maxChannelDuration = 15 cfg := channelManagerTestConfig(1000, derive.SpanBatchType) cfg.MaxChannelDuration = maxChannelDuration - cfg.InitNoneCompressor() + cfg.InitNoneCompressor("brotli", 10) for _, tt := range []struct { name string diff --git a/op-batcher/batcher/config.go b/op-batcher/batcher/config.go index 6f67e3850328..90578611b3a9 100644 --- a/op-batcher/batcher/config.go +++ b/op-batcher/batcher/config.go @@ -67,6 +67,9 @@ type CLIConfig struct { // Type of compressor to use. Must be one of [compressor.KindKeys]. Compressor string + CompressionAlgo string + CompressLevel int + // If Stopped is true, the batcher starts stopped and won't start batching right away. // Batching needs to be started via an admin RPC. Stopped bool diff --git a/op-batcher/batcher/service.go b/op-batcher/batcher/service.go index f9bf8bd8e04f..21e50d8ac717 100644 --- a/op-batcher/batcher/service.go +++ b/op-batcher/batcher/service.go @@ -219,7 +219,7 @@ func (bs *BatcherService) initChannelConfig(cfg *CLIConfig) error { return fmt.Errorf("max frame size %d exceeds plasma max input size %d", cc.MaxFrameSize, plasma.MaxInputSize) } - cc.InitCompressorConfig(cfg.ApproxComprRatio, cfg.Compressor) + cc.InitCompressorConfig(cfg.ApproxComprRatio, cfg.Compressor, cfg.CompressionAlgo, cfg.CompressLevel) if bs.UseBlobs && !bs.RollupConfig.IsEcotone(uint64(time.Now().Unix())) { bs.Log.Error("Cannot use Blob data before Ecotone!") // log only, the batcher may not be actively running. diff --git a/op-batcher/compressor/config.go b/op-batcher/compressor/config.go index 8befc43812ae..09988a26b003 100644 --- a/op-batcher/compressor/config.go +++ b/op-batcher/compressor/config.go @@ -16,6 +16,9 @@ type Config struct { // Kind of compressor to use. Must be one of KindKeys. If unset, NewCompressor // will default to RatioKind. Kind string + + CompressionAlgo string + CompressLevel int } func (c Config) NewCompressor() (derive.Compressor, error) { diff --git a/op-e2e/actions/l2_batcher.go b/op-e2e/actions/l2_batcher.go index 46216d56c6af..0cbf1ab2ba5c 100644 --- a/op-e2e/actions/l2_batcher.go +++ b/op-e2e/actions/l2_batcher.go @@ -200,7 +200,7 @@ func (s *L2Batcher) Buffer(t Testing) error { } else { // use span batch if we're forcing it or if we're at/beyond delta if s.l2BatcherCfg.ForceSubmitSpanBatch || s.rollupCfg.IsDelta(block.Time()) { - ch, err = derive.NewSpanChannelOut(s.rollupCfg.Genesis.L2Time, s.rollupCfg.L2ChainID, target) + ch, err = derive.NewSpanChannelOut(s.rollupCfg.Genesis.L2Time, s.rollupCfg.L2ChainID, target, "brotli", 10) // use singular batches in all other cases } else { ch, err = derive.NewSingularChannelOut(c) diff --git a/op-e2e/actions/sync_test.go b/op-e2e/actions/sync_test.go index 759f51613e0e..9b463b7c2df5 100644 --- a/op-e2e/actions/sync_test.go +++ b/op-e2e/actions/sync_test.go @@ -26,7 +26,7 @@ import ( ) func newSpanChannelOut(t StatefulTesting, e e2eutils.SetupData) derive.ChannelOut { - channelOut, err := derive.NewSpanChannelOut(e.RollupCfg.Genesis.L2Time, e.RollupCfg.L2ChainID, 128_000) + channelOut, err := derive.NewSpanChannelOut(e.RollupCfg.Genesis.L2Time, e.RollupCfg.L2ChainID, 128_000, "brotli", 10) require.NoError(t, err) return channelOut } diff --git a/op-node/benchmarks/batchbuilding_test.go b/op-node/benchmarks/batchbuilding_test.go index 52d6d5721fd8..90a8fae5024a 100644 --- a/op-node/benchmarks/batchbuilding_test.go +++ b/op-node/benchmarks/batchbuilding_test.go @@ -64,7 +64,7 @@ func channelOutByType(batchType uint, compKey string) (derive.ChannelOut, error) return derive.NewSingularChannelOut(compressors[compKey].compressor) } if batchType == derive.SpanBatchType { - return derive.NewSpanChannelOut(0, chainID, compressors[compKey].targetOutput) + return derive.NewSpanChannelOut(0, chainID, compressors[compKey].targetOutput, "brotli", 10) } return nil, fmt.Errorf("unsupported batch type: %d", batchType) } diff --git a/op-node/rollup/derive/channel_out_test.go b/op-node/rollup/derive/channel_out_test.go index 7c4ae92204d0..f560932e4671 100644 --- a/op-node/rollup/derive/channel_out_test.go +++ b/op-node/rollup/derive/channel_out_test.go @@ -52,7 +52,7 @@ var channelTypes = []struct { { Name: "Span", ChannelOut: func(t *testing.T) ChannelOut { - cout, err := NewSpanChannelOut(0, big.NewInt(0), 128_000) + cout, err := NewSpanChannelOut(0, big.NewInt(0), 128_000, "brotli", 10) require.NoError(t, err) return cout }, @@ -225,7 +225,7 @@ func SpanChannelAndBatches(t *testing.T, target uint64, len int) (*SpanChannelOu rng := rand.New(rand.NewSource(0x543331)) chainID := big.NewInt(rng.Int63n(1000)) txCount := 1 - cout, err := NewSpanChannelOut(0, chainID, target) + cout, err := NewSpanChannelOut(0, chainID, target, "brotli", 10) require.NoError(t, err) batches := make([]*SingularBatch, len) // adding the first batch should not cause an error diff --git a/op-node/rollup/derive/span_channel_out.go b/op-node/rollup/derive/span_channel_out.go index e549e862e6bf..7e0bdf6dbde3 100644 --- a/op-node/rollup/derive/span_channel_out.go +++ b/op-node/rollup/derive/span_channel_out.go @@ -8,12 +8,25 @@ import ( "io" "math/big" + "github.com/andybalholm/brotli" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/rlp" "github.com/ethereum-optimism/optimism/op-node/rollup" ) +const ( + ZLIB string = "zlib" + BROTLI string = "brotli" +) + +type CompressorInterace interface { + Write([] byte) (int, error) + Flush() error + Close() error + Reset(io.Writer) +} + type SpanChannelOut struct { id ChannelID // Frame ID of the next frame to emit. Increment after emitting @@ -28,8 +41,8 @@ type SpanChannelOut struct { lastCompressedRLPSize int // compressed contains compressed data for making output frames compressed *bytes.Buffer - // compress is the zlib writer for the channel - compressor *zlib.Writer + // the compressor for the channel + compressor CompressorInterace // target is the target size of the compressed data target uint64 // closed indicates if the channel is closed @@ -49,7 +62,7 @@ func (co *SpanChannelOut) setRandomID() error { return err } -func NewSpanChannelOut(genesisTimestamp uint64, chainID *big.Int, targetOutputSize uint64) (*SpanChannelOut, error) { +func NewSpanChannelOut(genesisTimestamp uint64, chainID *big.Int, targetOutputSize uint64, compressionAlgo string, compressLevel int) (*SpanChannelOut, error) { c := &SpanChannelOut{ id: ChannelID{}, frame: 0, @@ -62,9 +75,20 @@ func NewSpanChannelOut(genesisTimestamp uint64, chainID *big.Int, targetOutputSi if err = c.setRandomID(); err != nil { return nil, err } - if c.compressor, err = zlib.NewWriterLevel(c.compressed, zlib.BestCompression); err != nil { - return nil, err + + if compressionAlgo == ZLIB { + if c.compressor, err = zlib.NewWriterLevel(c.compressed, compressLevel); err != nil { + return nil, err + } + } else if compressionAlgo == BROTLI { + c.compressor = brotli.NewWriterLevel( + c.compressed, + compressLevel, + ) + } else { + return nil, fmt.Errorf("unsupported compression algorithm: %s", compressionAlgo) } + return c, nil } From 9e1673bc9824761861f813c1fe10b4ea968a81f7 Mon Sep 17 00:00:00 2001 From: Cody Wang Date: Tue, 30 Apr 2024 16:04:43 -0400 Subject: [PATCH 02/38] wip --- op-batcher/batcher/channel_builder_test.go | 2 +- op-batcher/batcher/config.go | 11 +++++++ op-batcher/batcher/config_test.go | 2 ++ op-batcher/flags/flags.go | 12 +++++++ op-e2e/sequencer_failover_setup.go | 2 ++ op-e2e/setup.go | 4 +++ op-node/rollup/derive/channel.go | 37 ++++++++++++++++++++-- 7 files changed, 67 insertions(+), 3 deletions(-) diff --git a/op-batcher/batcher/channel_builder_test.go b/op-batcher/batcher/channel_builder_test.go index 29c7d21bfd2f..86da5635931f 100644 --- a/op-batcher/batcher/channel_builder_test.go +++ b/op-batcher/batcher/channel_builder_test.go @@ -557,7 +557,7 @@ func ChannelBuilder_OutputFramesMaxFrameIndex(t *testing.T, batchType uint) { // This test fails in multiple places if the subtraction of // [derive.FrameV0OverHeadSize] in [MaxDataSize] is omitted, which has been the // case before it got fixed it #9887. -func TestChannelBuilder_FullShadowCompressor_zlib(t *testing.T) { +func TestChannelBuilder_FullShadowCompressor(t *testing.T) { require := require.New(t) cfg := ChannelConfig{ MaxFrameSize: 752, diff --git a/op-batcher/batcher/config.go b/op-batcher/batcher/config.go index 90578611b3a9..c5ddd7dcc295 100644 --- a/op-batcher/batcher/config.go +++ b/op-batcher/batcher/config.go @@ -67,7 +67,10 @@ type CLIConfig struct { // Type of compressor to use. Must be one of [compressor.KindKeys]. Compressor string + // Type of compression algorithm to use. Must be one of [zlib, brotli] CompressionAlgo string + + // Levels of compression to use. E.g. 1-11 for brotli, 0-9 for zlib CompressLevel int // If Stopped is true, the batcher starts stopped and won't start batching right away. @@ -127,6 +130,12 @@ func (c *CLIConfig) Check() error { if c.Compressor == compressor.RatioKind && (c.ApproxComprRatio <= 0 || c.ApproxComprRatio > 1) { return fmt.Errorf("invalid ApproxComprRatio %v for ratio compressor", c.ApproxComprRatio) } + if c.CompressionAlgo == "" { + return errors.New("must set compression algo") + } + if (c.CompressionAlgo == "zlib" && (c.CompressLevel < 0 || c.CompressLevel > 9)) || (c.CompressionAlgo == "brotli" && (c.CompressLevel < 1 || c.CompressLevel > 11)) { + return fmt.Errorf("invalid compression level %v for %v", c.CompressLevel, c.CompressionAlgo) + } if c.BatchType > 1 { return fmt.Errorf("unknown batch type: %v", c.BatchType) } @@ -171,6 +180,8 @@ func NewConfig(ctx *cli.Context) *CLIConfig { TargetNumFrames: ctx.Int(flags.TargetNumFramesFlag.Name), ApproxComprRatio: ctx.Float64(flags.ApproxComprRatioFlag.Name), Compressor: ctx.String(flags.CompressorFlag.Name), + CompressionAlgo: ctx.String(flags.CompressionAlgoFlag.Name), + CompressLevel: ctx.Int(flags.CompressLevelFlag.Name), Stopped: ctx.Bool(flags.StoppedFlag.Name), WaitNodeSync: ctx.Bool(flags.WaitNodeSyncFlag.Name), CheckRecentTxsDepth: ctx.Int(flags.CheckRecentTxsDepthFlag.Name), diff --git a/op-batcher/batcher/config_test.go b/op-batcher/batcher/config_test.go index 18a6227e0011..c1f6edc24f14 100644 --- a/op-batcher/batcher/config_test.go +++ b/op-batcher/batcher/config_test.go @@ -36,6 +36,8 @@ func validBatcherConfig() batcher.CLIConfig { PprofConfig: oppprof.DefaultCLIConfig(), // The compressor config is not checked in config.Check() RPC: rpc.DefaultCLIConfig(), + CompressionAlgo: "brotli", + CompressLevel: 9, } } diff --git a/op-batcher/flags/flags.go b/op-batcher/flags/flags.go index 98d2121da0a5..9f5fd36787d2 100644 --- a/op-batcher/flags/flags.go +++ b/op-batcher/flags/flags.go @@ -99,6 +99,18 @@ var ( return nil }, } + CompressionAlgoFlag = &cli.StringFlag{ + Name: "compression-algo", + Usage: "The compression algorithm to use. Valid options: zlib, brotli ", + EnvVars: prefixEnvVars("COMPRESSION_ALGO"), + Value: "zlib", + } + CompressLevelFlag = &cli.IntFlag{ + Name: "compress-level", + Usage: "The compression level to use. E.g. 1-11 for brotli, 0-9 for zlib", + EnvVars: prefixEnvVars("COMPRESS_LEVEL"), + Value: 9, // default to best zlib compression level + } StoppedFlag = &cli.BoolFlag{ Name: "stopped", Usage: "Initialize the batcher in a stopped state. The batcher can be started using the admin_startBatcher RPC", diff --git a/op-e2e/sequencer_failover_setup.go b/op-e2e/sequencer_failover_setup.go index 49a320d4e271..22d110d5e3a3 100644 --- a/op-e2e/sequencer_failover_setup.go +++ b/op-e2e/sequencer_failover_setup.go @@ -291,6 +291,8 @@ func setupBatcher(t *testing.T, sys *System, conductors map[string]*conductor) { BatchType: derive.SpanBatchType, DataAvailabilityType: batcherFlags.CalldataType, ActiveSequencerCheckDuration: 0, + CompressionAlgo: "zlib", + CompressLevel: 9, } batcher, err := bss.BatcherServiceFromCLIConfig(context.Background(), "0.0.1", batcherCLIConfig, sys.Cfg.Loggers["batcher"]) diff --git a/op-e2e/setup.go b/op-e2e/setup.go index 08c2c4805473..8867ecebd300 100644 --- a/op-e2e/setup.go +++ b/op-e2e/setup.go @@ -844,6 +844,10 @@ func (cfg SystemConfig) Start(t *testing.T, _opts ...SystemConfigOption) (*Syste Stopped: sys.Cfg.DisableBatcher, // Batch submitter may be enabled later BatchType: batchType, DataAvailabilityType: sys.Cfg.DataAvailabilityType, + // Add the compression stuff + //CompressionAlgo: "zlib", + CompressionAlgo: "brotli", + CompressLevel: 9, // use brotli 9 for faster e2e tests } // Batch Submitter batcher, err := bss.BatcherServiceFromCLIConfig(context.Background(), "0.0.1", batcherCLIConfig, sys.Cfg.Loggers["batcher"]) diff --git a/op-node/rollup/derive/channel.go b/op-node/rollup/derive/channel.go index 7bd67f184995..ac1de4f7137f 100644 --- a/op-node/rollup/derive/channel.go +++ b/op-node/rollup/derive/channel.go @@ -2,10 +2,15 @@ package derive import ( "bytes" - "compress/zlib" + //"compress/zlib" "fmt" "io" + // "strconv" + // "strings" + + "github.com/andybalholm/brotli" + "github.com/ethereum-optimism/optimism/op-service/eth" "github.com/ethereum/go-ethereum/rlp" ) @@ -147,8 +152,35 @@ func (ch *Channel) Reader() io.Reader { // Warning: the batch reader can read every batch-type. // The caller of the batch-reader should filter the results. func BatchReader(r io.Reader) (func() (*BatchData, error), error) { + // Read the first byte to determine the compression type + // compressionType := make([]byte, 1) + // if _, err := r.Read(compressionType); err != nil { + // return nil, err + // } + + // check if the compression type is 0111 1000 (0x78) + // if no, initialize zlib reader, else initialize brotli reader + reader := func(r io.Reader) (io.Reader, error) { + return brotli.NewReader(r), nil + } + + // fmt.Println("Compression type") + // fmt.Println(strconv.FormatInt(int64(compressionType[0]), 2)) + // fmt.Println(compressionType[0] & 0x0F) + // if compressionType[0] & 0x0F == 8 { + // fmt.Println("Using zlib reader") + // reader = func(r io.Reader) (io.Reader, error) { + // return zlib.NewReader(r) + // } + // } else { + // fmt.Println("Using brotli reader") + // reader = func(r io.Reader) (io.Reader, error) { + // return brotli.NewReader(r), nil + // } + // } + // Setup decompressor stage + RLP reader - zr, err := zlib.NewReader(r) + zr, err := reader(r) if err != nil { return nil, err } @@ -157,6 +189,7 @@ func BatchReader(r io.Reader) (func() (*BatchData, error), error) { return func() (*BatchData, error) { var batchData BatchData if err = rlpReader.Decode(&batchData); err != nil { + fmt.Println("DECODE ERROR") return nil, err } return &batchData, nil From e732b21dff88038179d720a2b6ae08a5ee0d60ca Mon Sep 17 00:00:00 2001 From: Cody Wang Date: Tue, 30 Apr 2024 21:38:26 -0400 Subject: [PATCH 03/38] fix --- op-node/rollup/derive/channel.go | 68 +++++++++++++---------- op-node/rollup/derive/span_channel_out.go | 20 +++++-- 2 files changed, 56 insertions(+), 32 deletions(-) diff --git a/op-node/rollup/derive/channel.go b/op-node/rollup/derive/channel.go index ac1de4f7137f..afe5f771d61b 100644 --- a/op-node/rollup/derive/channel.go +++ b/op-node/rollup/derive/channel.go @@ -2,12 +2,12 @@ package derive import ( "bytes" - //"compress/zlib" + "compress/zlib" "fmt" "io" - // "strconv" - // "strings" + "strconv" + "strings" "github.com/andybalholm/brotli" @@ -152,32 +152,41 @@ func (ch *Channel) Reader() io.Reader { // Warning: the batch reader can read every batch-type. // The caller of the batch-reader should filter the results. func BatchReader(r io.Reader) (func() (*BatchData, error), error) { - // Read the first byte to determine the compression type - // compressionType := make([]byte, 1) - // if _, err := r.Read(compressionType); err != nil { - // return nil, err - // } - - // check if the compression type is 0111 1000 (0x78) - // if no, initialize zlib reader, else initialize brotli reader - reader := func(r io.Reader) (io.Reader, error) { + // Buffer to store the data for reuse + var buffer strings.Builder + if _, err := io.Copy(&buffer, r); err != nil { + return nil, err + } + + tmpReader := strings.NewReader(buffer.String()) + // Read the first byte to be used to determine the compression type + compressionType := make([]byte, 1) + if _, err := tmpReader.Read(compressionType); err != nil { + return nil, err + } + + fmt.Println("Compression type") + fmt.Println(strconv.FormatInt(int64(compressionType[0]), 2)) + fmt.Println(compressionType[0]) + + // Reset the reader with the original data + r = strings.NewReader(buffer.String()) + var reader func(io.Reader) (io.Reader, error) + // If the last 4 bits is 8, then it is a zlib reader + if compressionType[0] & 0x0F == 8 { + fmt.Println("Using zlib reader") + reader = func(r io.Reader) (io.Reader, error) { + return zlib.NewReader(r) + } + // If the bits equal to 2, then it is a brotli reader + } else if compressionType[0] & 0x0F == 2 { + fmt.Println("Using brotli reader") + // remove the first byte by reading it + r.Read(make([]byte, 1)) + reader = func(r io.Reader) (io.Reader, error) { return brotli.NewReader(r), nil } - - // fmt.Println("Compression type") - // fmt.Println(strconv.FormatInt(int64(compressionType[0]), 2)) - // fmt.Println(compressionType[0] & 0x0F) - // if compressionType[0] & 0x0F == 8 { - // fmt.Println("Using zlib reader") - // reader = func(r io.Reader) (io.Reader, error) { - // return zlib.NewReader(r) - // } - // } else { - // fmt.Println("Using brotli reader") - // reader = func(r io.Reader) (io.Reader, error) { - // return brotli.NewReader(r), nil - // } - // } + } // Setup decompressor stage + RLP reader zr, err := reader(r) @@ -189,7 +198,10 @@ func BatchReader(r io.Reader) (func() (*BatchData, error), error) { return func() (*BatchData, error) { var batchData BatchData if err = rlpReader.Decode(&batchData); err != nil { - fmt.Println("DECODE ERROR") + if err != io.EOF { + fmt.Println("DECODE ERROR") + fmt.Println(err) + } return nil, err } return &batchData, nil diff --git a/op-node/rollup/derive/span_channel_out.go b/op-node/rollup/derive/span_channel_out.go index 7e0bdf6dbde3..b8846588a33c 100644 --- a/op-node/rollup/derive/span_channel_out.go +++ b/op-node/rollup/derive/span_channel_out.go @@ -20,7 +20,7 @@ const ( BROTLI string = "brotli" ) -type CompressorInterace interface { +type CompressorInteface interface { Write([] byte) (int, error) Flush() error Close() error @@ -42,7 +42,9 @@ type SpanChannelOut struct { // compressed contains compressed data for making output frames compressed *bytes.Buffer // the compressor for the channel - compressor CompressorInterace + compressor CompressorInteface + // the compression algo used + compressionAlgo string // target is the target size of the compressed data target uint64 // closed indicates if the channel is closed @@ -70,6 +72,7 @@ func NewSpanChannelOut(genesisTimestamp uint64, chainID *big.Int, targetOutputSi rlp: [2]*bytes.Buffer{{}, {}}, compressed: &bytes.Buffer{}, target: targetOutputSize, + compressionAlgo: compressionAlgo, } var err error if err = c.setRandomID(); err != nil { @@ -81,6 +84,7 @@ func NewSpanChannelOut(genesisTimestamp uint64, chainID *big.Int, targetOutputSi return nil, err } } else if compressionAlgo == BROTLI { + c.compressed = bytes.NewBuffer([]byte{0b0010}) c.compressor = brotli.NewWriterLevel( c.compressed, compressLevel, @@ -99,7 +103,11 @@ func (co *SpanChannelOut) Reset() error { co.rlp[0].Reset() co.rlp[1].Reset() co.lastCompressedRLPSize = 0 - co.compressed.Reset() + if co.compressionAlgo == BROTLI { + co.compressed = bytes.NewBuffer([]byte{0b0010}) + } else { + co.compressed.Reset() + } co.compressor.Reset(co.compressed) co.spanBatch = NewSpanBatch(co.spanBatch.GenesisTimestamp, co.spanBatch.ChainID) // setting the new randomID is the only part of the reset that can fail @@ -210,7 +218,11 @@ func (co *SpanChannelOut) AddSingularBatch(batch *SingularBatch, seqNum uint64) // compress compresses the active RLP buffer and checks if the compressed data is over the target size. // it resets all the compression buffers because Span Batches aren't meant to be compressed incrementally. func (co *SpanChannelOut) compress() error { - co.compressed.Reset() + if co.compressionAlgo == BROTLI { + co.compressed = bytes.NewBuffer([]byte{0b0010}) + } else { + co.compressed.Reset() + } co.compressor.Reset(co.compressed) if _, err := co.compressor.Write(co.activeRLP().Bytes()); err != nil { return err From 656628349adadbc7b7db5e7d3f18ef0e8c00e5b7 Mon Sep 17 00:00:00 2001 From: Cody Wang Date: Tue, 30 Apr 2024 21:53:12 -0400 Subject: [PATCH 04/38] fix --- go.mod | 2 +- op-batcher/batcher/config.go | 4 ++-- op-batcher/batcher/config_test.go | 4 ++-- op-e2e/sequencer_failover_setup.go | 4 ++-- op-e2e/setup.go | 2 +- op-node/rollup/derive/channel.go | 25 +++++++---------------- op-node/rollup/derive/span_channel_out.go | 16 +++++++-------- 7 files changed, 23 insertions(+), 34 deletions(-) diff --git a/go.mod b/go.mod index 7be4b2ad787a..5b37b0bba061 100644 --- a/go.mod +++ b/go.mod @@ -4,6 +4,7 @@ go 1.21 require ( github.com/BurntSushi/toml v1.3.2 + github.com/andybalholm/brotli v1.1.0 github.com/btcsuite/btcd v0.24.0 github.com/btcsuite/btcd/chaincfg/chainhash v1.1.0 github.com/cockroachdb/pebble v0.0.0-20231018212520-f6cde3fc2fa4 @@ -64,7 +65,6 @@ require ( github.com/Shopify/goreferrer v0.0.0-20220729165902-8cddb4f5de06 // indirect github.com/VictoriaMetrics/fastcache v1.12.1 // indirect github.com/allegro/bigcache v1.2.1 // indirect - github.com/andybalholm/brotli v1.1.0 // indirect github.com/armon/go-metrics v0.4.1 // indirect github.com/aymerick/douceur v0.2.0 // indirect github.com/benbjohnson/clock v1.3.5 // indirect diff --git a/op-batcher/batcher/config.go b/op-batcher/batcher/config.go index c5ddd7dcc295..7ec8216e3280 100644 --- a/op-batcher/batcher/config.go +++ b/op-batcher/batcher/config.go @@ -71,7 +71,7 @@ type CLIConfig struct { CompressionAlgo string // Levels of compression to use. E.g. 1-11 for brotli, 0-9 for zlib - CompressLevel int + CompressLevel int // If Stopped is true, the batcher starts stopped and won't start batching right away. // Batching needs to be started via an admin RPC. @@ -180,7 +180,7 @@ func NewConfig(ctx *cli.Context) *CLIConfig { TargetNumFrames: ctx.Int(flags.TargetNumFramesFlag.Name), ApproxComprRatio: ctx.Float64(flags.ApproxComprRatioFlag.Name), Compressor: ctx.String(flags.CompressorFlag.Name), - CompressionAlgo: ctx.String(flags.CompressionAlgoFlag.Name), + CompressionAlgo: ctx.String(flags.CompressionAlgoFlag.Name), CompressLevel: ctx.Int(flags.CompressLevelFlag.Name), Stopped: ctx.Bool(flags.StoppedFlag.Name), WaitNodeSync: ctx.Bool(flags.WaitNodeSyncFlag.Name), diff --git a/op-batcher/batcher/config_test.go b/op-batcher/batcher/config_test.go index c1f6edc24f14..10dcdaf21d59 100644 --- a/op-batcher/batcher/config_test.go +++ b/op-batcher/batcher/config_test.go @@ -35,9 +35,9 @@ func validBatcherConfig() batcher.CLIConfig { MetricsConfig: metrics.DefaultCLIConfig(), PprofConfig: oppprof.DefaultCLIConfig(), // The compressor config is not checked in config.Check() - RPC: rpc.DefaultCLIConfig(), + RPC: rpc.DefaultCLIConfig(), CompressionAlgo: "brotli", - CompressLevel: 9, + CompressLevel: 9, } } diff --git a/op-e2e/sequencer_failover_setup.go b/op-e2e/sequencer_failover_setup.go index 22d110d5e3a3..84871caf1e7c 100644 --- a/op-e2e/sequencer_failover_setup.go +++ b/op-e2e/sequencer_failover_setup.go @@ -291,8 +291,8 @@ func setupBatcher(t *testing.T, sys *System, conductors map[string]*conductor) { BatchType: derive.SpanBatchType, DataAvailabilityType: batcherFlags.CalldataType, ActiveSequencerCheckDuration: 0, - CompressionAlgo: "zlib", - CompressLevel: 9, + CompressionAlgo: "zlib", + CompressLevel: 9, } batcher, err := bss.BatcherServiceFromCLIConfig(context.Background(), "0.0.1", batcherCLIConfig, sys.Cfg.Loggers["batcher"]) diff --git a/op-e2e/setup.go b/op-e2e/setup.go index 8867ecebd300..a5a262ce77f3 100644 --- a/op-e2e/setup.go +++ b/op-e2e/setup.go @@ -847,7 +847,7 @@ func (cfg SystemConfig) Start(t *testing.T, _opts ...SystemConfigOption) (*Syste // Add the compression stuff //CompressionAlgo: "zlib", CompressionAlgo: "brotli", - CompressLevel: 9, // use brotli 9 for faster e2e tests + CompressLevel: 9, // use brotli 9 for faster e2e tests } // Batch Submitter batcher, err := bss.BatcherServiceFromCLIConfig(context.Background(), "0.0.1", batcherCLIConfig, sys.Cfg.Loggers["batcher"]) diff --git a/op-node/rollup/derive/channel.go b/op-node/rollup/derive/channel.go index afe5f771d61b..8207f4c60c0a 100644 --- a/op-node/rollup/derive/channel.go +++ b/op-node/rollup/derive/channel.go @@ -6,7 +6,6 @@ import ( "fmt" "io" - "strconv" "strings" "github.com/andybalholm/brotli" @@ -153,10 +152,10 @@ func (ch *Channel) Reader() io.Reader { // The caller of the batch-reader should filter the results. func BatchReader(r io.Reader) (func() (*BatchData, error), error) { // Buffer to store the data for reuse - var buffer strings.Builder - if _, err := io.Copy(&buffer, r); err != nil { - return nil, err - } + var buffer strings.Builder + if _, err := io.Copy(&buffer, r); err != nil { + return nil, err + } tmpReader := strings.NewReader(buffer.String()) // Read the first byte to be used to determine the compression type @@ -165,22 +164,16 @@ func BatchReader(r io.Reader) (func() (*BatchData, error), error) { return nil, err } - fmt.Println("Compression type") - fmt.Println(strconv.FormatInt(int64(compressionType[0]), 2)) - fmt.Println(compressionType[0]) - // Reset the reader with the original data r = strings.NewReader(buffer.String()) var reader func(io.Reader) (io.Reader, error) // If the last 4 bits is 8, then it is a zlib reader - if compressionType[0] & 0x0F == 8 { - fmt.Println("Using zlib reader") + if compressionType[0]&0x0F == 8 { reader = func(r io.Reader) (io.Reader, error) { return zlib.NewReader(r) } - // If the bits equal to 2, then it is a brotli reader - } else if compressionType[0] & 0x0F == 2 { - fmt.Println("Using brotli reader") + // If the bits equal to 2, then it is a brotli reader + } else if compressionType[0]&0x0F == 2 { // remove the first byte by reading it r.Read(make([]byte, 1)) reader = func(r io.Reader) (io.Reader, error) { @@ -198,10 +191,6 @@ func BatchReader(r io.Reader) (func() (*BatchData, error), error) { return func() (*BatchData, error) { var batchData BatchData if err = rlpReader.Decode(&batchData); err != nil { - if err != io.EOF { - fmt.Println("DECODE ERROR") - fmt.Println(err) - } return nil, err } return &batchData, nil diff --git a/op-node/rollup/derive/span_channel_out.go b/op-node/rollup/derive/span_channel_out.go index b8846588a33c..c39199c330a3 100644 --- a/op-node/rollup/derive/span_channel_out.go +++ b/op-node/rollup/derive/span_channel_out.go @@ -16,12 +16,12 @@ import ( ) const ( - ZLIB string = "zlib" + ZLIB string = "zlib" BROTLI string = "brotli" ) type CompressorInteface interface { - Write([] byte) (int, error) + Write([]byte) (int, error) Flush() error Close() error Reset(io.Writer) @@ -66,12 +66,12 @@ func (co *SpanChannelOut) setRandomID() error { func NewSpanChannelOut(genesisTimestamp uint64, chainID *big.Int, targetOutputSize uint64, compressionAlgo string, compressLevel int) (*SpanChannelOut, error) { c := &SpanChannelOut{ - id: ChannelID{}, - frame: 0, - spanBatch: NewSpanBatch(genesisTimestamp, chainID), - rlp: [2]*bytes.Buffer{{}, {}}, - compressed: &bytes.Buffer{}, - target: targetOutputSize, + id: ChannelID{}, + frame: 0, + spanBatch: NewSpanBatch(genesisTimestamp, chainID), + rlp: [2]*bytes.Buffer{{}, {}}, + compressed: &bytes.Buffer{}, + target: targetOutputSize, compressionAlgo: compressionAlgo, } var err error From 4eb83ea2e29759e622c1776ca62ea2313e1ff317 Mon Sep 17 00:00:00 2001 From: Cody Wang Date: Tue, 30 Apr 2024 22:33:47 -0400 Subject: [PATCH 05/38] fix --- op-node/rollup/derive/channel.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/op-node/rollup/derive/channel.go b/op-node/rollup/derive/channel.go index 8207f4c60c0a..e19723dfd5a0 100644 --- a/op-node/rollup/derive/channel.go +++ b/op-node/rollup/derive/channel.go @@ -175,7 +175,10 @@ func BatchReader(r io.Reader) (func() (*BatchData, error), error) { // If the bits equal to 2, then it is a brotli reader } else if compressionType[0]&0x0F == 2 { // remove the first byte by reading it - r.Read(make([]byte, 1)) + _, err := r.Read(make([]byte, 1)) + if err != nil { + return nil ,err + } reader = func(r io.Reader) (io.Reader, error) { return brotli.NewReader(r), nil } From 934ed5e08d1506120e8ea688ea5931b40b72fc1b Mon Sep 17 00:00:00 2001 From: Cody Wang Date: Tue, 30 Apr 2024 23:02:13 -0400 Subject: [PATCH 06/38] fix --- op-node/rollup/derive/channel.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/op-node/rollup/derive/channel.go b/op-node/rollup/derive/channel.go index e19723dfd5a0..8e92e7afb036 100644 --- a/op-node/rollup/derive/channel.go +++ b/op-node/rollup/derive/channel.go @@ -177,7 +177,7 @@ func BatchReader(r io.Reader) (func() (*BatchData, error), error) { // remove the first byte by reading it _, err := r.Read(make([]byte, 1)) if err != nil { - return nil ,err + return nil, err } reader = func(r io.Reader) (io.Reader, error) { return brotli.NewReader(r), nil From ec984a3d7c344853168693091b6ae2c9375ed69f Mon Sep 17 00:00:00 2001 From: Cody Wang Date: Wed, 1 May 2024 17:34:40 -0400 Subject: [PATCH 07/38] address some of the bots comments --- op-batcher/flags/flags.go | 13 +++++++++++++ op-node/rollup/derive/span_channel_out.go | 14 ++++++-------- 2 files changed, 19 insertions(+), 8 deletions(-) diff --git a/op-batcher/flags/flags.go b/op-batcher/flags/flags.go index 9f5fd36787d2..88881fd66c56 100644 --- a/op-batcher/flags/flags.go +++ b/op-batcher/flags/flags.go @@ -104,12 +104,25 @@ var ( Usage: "The compression algorithm to use. Valid options: zlib, brotli ", EnvVars: prefixEnvVars("COMPRESSION_ALGO"), Value: "zlib", + Action: func(c *cli.Context, algo string) error { + if algo != "zlib" && algo != "brotli" { + return fmt.Errorf("unsupported compression algorithm: %s", s) + } + return nil + }, } CompressLevelFlag = &cli.IntFlag{ Name: "compress-level", Usage: "The compression level to use. E.g. 1-11 for brotli, 0-9 for zlib", EnvVars: prefixEnvVars("COMPRESS_LEVEL"), Value: 9, // default to best zlib compression level + Action: func(c *cli.Context, lvl int) error { + algo := c.String("compression-algo") + if (algo == "zlib" && (lvl < 0 || lvl > 9)) || (algo == "brotli" && (lvl < 1 || lvl > 11)) { + return fmt.Errorf("invalid compression level %d for algorithm %s", lvl, algo) + } + return nil + }, } StoppedFlag = &cli.BoolFlag{ Name: "stopped", diff --git a/op-node/rollup/derive/span_channel_out.go b/op-node/rollup/derive/span_channel_out.go index c39199c330a3..cca32057eb70 100644 --- a/op-node/rollup/derive/span_channel_out.go +++ b/op-node/rollup/derive/span_channel_out.go @@ -20,7 +20,7 @@ const ( BROTLI string = "brotli" ) -type CompressorInteface interface { +type CompressorInterface interface { Write([]byte) (int, error) Flush() error Close() error @@ -42,7 +42,7 @@ type SpanChannelOut struct { // compressed contains compressed data for making output frames compressed *bytes.Buffer // the compressor for the channel - compressor CompressorInteface + compressor CompressorInterface // the compression algo used compressionAlgo string // target is the target size of the compressed data @@ -103,10 +103,9 @@ func (co *SpanChannelOut) Reset() error { co.rlp[0].Reset() co.rlp[1].Reset() co.lastCompressedRLPSize = 0 + co.compressed.Reset() if co.compressionAlgo == BROTLI { - co.compressed = bytes.NewBuffer([]byte{0b0010}) - } else { - co.compressed.Reset() + co.compressed.WriteByte(0b0010) } co.compressor.Reset(co.compressed) co.spanBatch = NewSpanBatch(co.spanBatch.GenesisTimestamp, co.spanBatch.ChainID) @@ -218,10 +217,9 @@ func (co *SpanChannelOut) AddSingularBatch(batch *SingularBatch, seqNum uint64) // compress compresses the active RLP buffer and checks if the compressed data is over the target size. // it resets all the compression buffers because Span Batches aren't meant to be compressed incrementally. func (co *SpanChannelOut) compress() error { + co.compressed.Reset() if co.compressionAlgo == BROTLI { - co.compressed = bytes.NewBuffer([]byte{0b0010}) - } else { - co.compressed.Reset() + co.compressed.WriteByte(0b0010) } co.compressor.Reset(co.compressed) if _, err := co.compressor.Write(co.activeRLP().Bytes()); err != nil { From a5ba98e4e7600546739d8d25de3b8e027ed8b43d Mon Sep 17 00:00:00 2001 From: Cody Wang Date: Wed, 1 May 2024 20:12:47 -0400 Subject: [PATCH 08/38] use version bit of 1 --- op-batcher/flags/flags.go | 2 +- op-node/rollup/derive/channel.go | 4 ++-- op-node/rollup/derive/span_channel_out.go | 8 +++++--- 3 files changed, 8 insertions(+), 6 deletions(-) diff --git a/op-batcher/flags/flags.go b/op-batcher/flags/flags.go index 88881fd66c56..f60f8abe4af5 100644 --- a/op-batcher/flags/flags.go +++ b/op-batcher/flags/flags.go @@ -106,7 +106,7 @@ var ( Value: "zlib", Action: func(c *cli.Context, algo string) error { if algo != "zlib" && algo != "brotli" { - return fmt.Errorf("unsupported compression algorithm: %s", s) + return fmt.Errorf("unsupported compression algorithm: %s", algo) } return nil }, diff --git a/op-node/rollup/derive/channel.go b/op-node/rollup/derive/channel.go index 8e92e7afb036..756e99026c09 100644 --- a/op-node/rollup/derive/channel.go +++ b/op-node/rollup/derive/channel.go @@ -172,8 +172,8 @@ func BatchReader(r io.Reader) (func() (*BatchData, error), error) { reader = func(r io.Reader) (io.Reader, error) { return zlib.NewReader(r) } - // If the bits equal to 2, then it is a brotli reader - } else if compressionType[0]&0x0F == 2 { + // If the bits equal to 1, then it is a brotli reader + } else if compressionType[0]&0x0F == 1 { // remove the first byte by reading it _, err := r.Read(make([]byte, 1)) if err != nil { diff --git a/op-node/rollup/derive/span_channel_out.go b/op-node/rollup/derive/span_channel_out.go index cca32057eb70..40baa27c4caf 100644 --- a/op-node/rollup/derive/span_channel_out.go +++ b/op-node/rollup/derive/span_channel_out.go @@ -84,7 +84,8 @@ func NewSpanChannelOut(genesisTimestamp uint64, chainID *big.Int, targetOutputSi return nil, err } } else if compressionAlgo == BROTLI { - c.compressed = bytes.NewBuffer([]byte{0b0010}) + // setting the version bit of 1 + c.compressed = bytes.NewBuffer([]byte{0b0001}) c.compressor = brotli.NewWriterLevel( c.compressed, compressLevel, @@ -105,7 +106,7 @@ func (co *SpanChannelOut) Reset() error { co.lastCompressedRLPSize = 0 co.compressed.Reset() if co.compressionAlgo == BROTLI { - co.compressed.WriteByte(0b0010) + co.compressed.WriteByte(0b0001) } co.compressor.Reset(co.compressed) co.spanBatch = NewSpanBatch(co.spanBatch.GenesisTimestamp, co.spanBatch.ChainID) @@ -219,7 +220,8 @@ func (co *SpanChannelOut) AddSingularBatch(batch *SingularBatch, seqNum uint64) func (co *SpanChannelOut) compress() error { co.compressed.Reset() if co.compressionAlgo == BROTLI { - co.compressed.WriteByte(0b0010) + // reset but still need the version bit of 1 + co.compressed.WriteByte(0b0001) } co.compressor.Reset(co.compressed) if _, err := co.compressor.Write(co.activeRLP().Bytes()); err != nil { From 940f2bc1a7616bb5a5a0bbcd225bbb8bf1fd7447 Mon Sep 17 00:00:00 2001 From: Cody Wang Date: Wed, 1 May 2024 20:31:05 -0400 Subject: [PATCH 09/38] fix lint --- op-batcher/flags/flags.go | 24 ++++++++++++------------ op-node/rollup/derive/channel.go | 2 +- 2 files changed, 13 insertions(+), 13 deletions(-) diff --git a/op-batcher/flags/flags.go b/op-batcher/flags/flags.go index f60f8abe4af5..9da2e6917fcf 100644 --- a/op-batcher/flags/flags.go +++ b/op-batcher/flags/flags.go @@ -105,24 +105,24 @@ var ( EnvVars: prefixEnvVars("COMPRESSION_ALGO"), Value: "zlib", Action: func(c *cli.Context, algo string) error { - if algo != "zlib" && algo != "brotli" { - return fmt.Errorf("unsupported compression algorithm: %s", algo) - } - return nil - }, + if algo != "zlib" && algo != "brotli" { + return fmt.Errorf("unsupported compression algorithm: %s", algo) + } + return nil + }, } CompressLevelFlag = &cli.IntFlag{ Name: "compress-level", Usage: "The compression level to use. E.g. 1-11 for brotli, 0-9 for zlib", EnvVars: prefixEnvVars("COMPRESS_LEVEL"), Value: 9, // default to best zlib compression level - Action: func(c *cli.Context, lvl int) error { - algo := c.String("compression-algo") - if (algo == "zlib" && (lvl < 0 || lvl > 9)) || (algo == "brotli" && (lvl < 1 || lvl > 11)) { - return fmt.Errorf("invalid compression level %d for algorithm %s", lvl, algo) - } - return nil - }, + Action: func(c *cli.Context, lvl int) error { + algo := c.String("compression-algo") + if (algo == "zlib" && (lvl < 0 || lvl > 9)) || (algo == "brotli" && (lvl < 1 || lvl > 11)) { + return fmt.Errorf("invalid compression level %d for algorithm %s", lvl, algo) + } + return nil + }, } StoppedFlag = &cli.BoolFlag{ Name: "stopped", diff --git a/op-node/rollup/derive/channel.go b/op-node/rollup/derive/channel.go index 756e99026c09..039ee2e7a416 100644 --- a/op-node/rollup/derive/channel.go +++ b/op-node/rollup/derive/channel.go @@ -172,7 +172,7 @@ func BatchReader(r io.Reader) (func() (*BatchData, error), error) { reader = func(r io.Reader) (io.Reader, error) { return zlib.NewReader(r) } - // If the bits equal to 1, then it is a brotli reader + // If the bits equal to 1, then it is a brotli reader } else if compressionType[0]&0x0F == 1 { // remove the first byte by reading it _, err := r.Read(make([]byte, 1)) From 642933eb63072b5a0bc5076d7dc396a3c1b096c2 Mon Sep 17 00:00:00 2001 From: Cody Wang Date: Thu, 2 May 2024 17:54:02 -0400 Subject: [PATCH 10/38] adding compression type --- op-batcher/batcher/channel_builder.go | 2 +- op-batcher/batcher/channel_builder_test.go | 33 +++++++------- op-batcher/batcher/channel_config.go | 15 +++---- op-batcher/batcher/channel_config_test.go | 2 +- op-batcher/batcher/channel_manager_test.go | 10 ++--- op-batcher/batcher/config.go | 18 +++----- op-batcher/batcher/config_test.go | 7 ++- op-batcher/batcher/service.go | 2 +- op-batcher/compressor/config.go | 7 ++- op-batcher/flags/flags.go | 31 ++++---------- op-e2e/actions/l2_batcher.go | 2 +- op-e2e/actions/sync_test.go | 2 +- op-e2e/sequencer_failover_setup.go | 3 +- op-e2e/setup.go | 5 +-- op-node/benchmarks/batchbuilding_test.go | 2 +- op-node/rollup/derive/channel_out_test.go | 4 +- op-node/rollup/derive/span_channel_out.go | 25 ++++++----- op-node/rollup/derive/types.go | 50 ++++++++++++++++++++++ 18 files changed, 122 insertions(+), 98 deletions(-) create mode 100644 op-node/rollup/derive/types.go diff --git a/op-batcher/batcher/channel_builder.go b/op-batcher/batcher/channel_builder.go index 3fc4c651a5ad..d63e1d45b5dc 100644 --- a/op-batcher/batcher/channel_builder.go +++ b/op-batcher/batcher/channel_builder.go @@ -86,7 +86,7 @@ func NewChannelBuilder(cfg ChannelConfig, rollupCfg rollup.Config, latestL1Origi } var co derive.ChannelOut if cfg.BatchType == derive.SpanBatchType { - co, err = derive.NewSpanChannelOut(rollupCfg.Genesis.L2Time, rollupCfg.L2ChainID, cfg.CompressorConfig.TargetOutputSize, cfg.CompressorConfig.CompressionAlgo, cfg.CompressorConfig.CompressLevel) + co, err = derive.NewSpanChannelOut(rollupCfg.Genesis.L2Time, rollupCfg.L2ChainID, cfg.CompressorConfig.TargetOutputSize, cfg.CompressorConfig.CompressionAlgo) } else { co, err = derive.NewSingularChannelOut(c) } diff --git a/op-batcher/batcher/channel_builder_test.go b/op-batcher/batcher/channel_builder_test.go index 86da5635931f..90166942f4c3 100644 --- a/op-batcher/batcher/channel_builder_test.go +++ b/op-batcher/batcher/channel_builder_test.go @@ -393,7 +393,7 @@ func TestChannelBuilder_OutputFrames(t *testing.T) { channelConfig := defaultTestChannelConfig() channelConfig.MaxFrameSize = derive.FrameV0OverHeadSize + 1 channelConfig.TargetNumFrames = 1000 - channelConfig.InitNoneCompressor("brotli", 10) + channelConfig.InitNoneCompressor(derive.Brotli10) // Construct the channel builder cb, err := NewChannelBuilder(channelConfig, defaultTestRollupConfig, latestL1BlockOrigin) @@ -432,12 +432,11 @@ func TestChannelBuilder_OutputFrames(t *testing.T) { func TestChannelBuilder_OutputFrames_SpanBatch(t *testing.T) { testCases := []struct { testName string - algo string - compressLevel int + algo derive.CompressionAlgo targetNumFrames int }{ - {testName: "Span Batch output frames with zlib", algo: "zlib", compressLevel: 9, targetNumFrames: 5}, - {testName: "Span Batch output frames with brotli", algo: "brotli", compressLevel: 10, targetNumFrames: 1}, // to fill faster + {testName: "Span Batch output frames with zlib", algo: derive.Zlib, targetNumFrames: 5}, + {testName: "Span Batch output frames with brotli", algo: derive.Brotli10, targetNumFrames: 1}, // to fill faster } for _, tc := range testCases { @@ -446,7 +445,7 @@ func TestChannelBuilder_OutputFrames_SpanBatch(t *testing.T) { channelConfig.MaxFrameSize = 20 + derive.FrameV0OverHeadSize channelConfig.TargetNumFrames = tc.targetNumFrames channelConfig.BatchType = derive.SpanBatchType - channelConfig.InitRatioCompressor(1, tc.algo, tc.compressLevel) + channelConfig.InitRatioCompressor(1, tc.algo) // Construct the channel builder cb, err := NewChannelBuilder(channelConfig, defaultTestRollupConfig, latestL1BlockOrigin) @@ -504,7 +503,7 @@ func ChannelBuilder_MaxRLPBytesPerChannel(t *testing.T, batchType uint) { t.Parallel() channelConfig := defaultTestChannelConfig() channelConfig.MaxFrameSize = derive.MaxRLPBytesPerChannel * 2 - channelConfig.InitNoneCompressor("brotli", 10) + channelConfig.InitNoneCompressor(derive.Brotli10) channelConfig.BatchType = batchType // Construct the channel builder @@ -522,7 +521,7 @@ func ChannelBuilder_OutputFramesMaxFrameIndex(t *testing.T, batchType uint) { channelConfig := defaultTestChannelConfig() channelConfig.MaxFrameSize = derive.FrameV0OverHeadSize + 1 channelConfig.TargetNumFrames = math.MaxUint16 + 1 - channelConfig.InitRatioCompressor(.1, "brotli", 10) + channelConfig.InitRatioCompressor(.1, derive.Brotli10) channelConfig.BatchType = batchType rng := rand.New(rand.NewSource(123)) @@ -566,18 +565,16 @@ func TestChannelBuilder_FullShadowCompressor(t *testing.T) { } testCases := []struct { - testName string - algo string - compressLevel int - targetNumFrames int + testName string + algo derive.CompressionAlgo }{ - {testName: "Full shadow compressor with zlib", algo: "zlib", compressLevel: 9}, - {testName: "Full shadow compressor with brotli", algo: "brotli", compressLevel: 10}, + {testName: "Full shadow compressor with zlib", algo: derive.Zlib}, + {testName: "Full shadow compressor with brotli", algo: derive.Brotli10}, } for _, tc := range testCases { t.Run(tc.testName, func(t *testing.T) { - cfg.InitShadowCompressor(tc.algo, tc.compressLevel) + cfg.InitShadowCompressor(tc.algo) cb, err := NewChannelBuilder(cfg, defaultTestRollupConfig, latestL1BlockOrigin) require.NoError(err) @@ -611,7 +608,7 @@ func ChannelBuilder_AddBlock(t *testing.T, batchType uint) { channelConfig.MaxFrameSize = 20 + derive.FrameV0OverHeadSize channelConfig.TargetNumFrames = 2 // Configure the Input Threshold params so we observe a full channel - channelConfig.InitRatioCompressor(1, "brotli", 10) + channelConfig.InitRatioCompressor(1, derive.Brotli10) // Construct the channel builder cb, err := NewChannelBuilder(channelConfig, defaultTestRollupConfig, latestL1BlockOrigin) @@ -734,7 +731,7 @@ func ChannelBuilder_PendingFrames_TotalFrames(t *testing.T, batchType uint) { cfg.MaxFrameSize = 1000 cfg.TargetNumFrames = tnf cfg.BatchType = batchType - cfg.InitShadowCompressor("brotli", 10) + cfg.InitShadowCompressor(derive.Brotli10) cb, err := NewChannelBuilder(cfg, defaultTestRollupConfig, latestL1BlockOrigin) require.NoError(err) @@ -816,7 +813,7 @@ func ChannelBuilder_OutputBytes(t *testing.T, batchType uint) { cfg.MaxFrameSize = 1000 cfg.TargetNumFrames = 16 cfg.BatchType = batchType - cfg.InitRatioCompressor(1.0, "brotli", 10) + cfg.InitRatioCompressor(1.0, derive.Brotli10) cb, err := NewChannelBuilder(cfg, defaultTestRollupConfig, latestL1BlockOrigin) require.NoError(err, "NewChannelBuilder") diff --git a/op-batcher/batcher/channel_config.go b/op-batcher/batcher/channel_config.go index 2ea80d13e84b..72c5a1afd0c8 100644 --- a/op-batcher/batcher/channel_config.go +++ b/op-batcher/batcher/channel_config.go @@ -53,27 +53,26 @@ type ChannelConfig struct { // value consistent with cc.TargetNumFrames and cc.MaxFrameSize. // comprKind can be the empty string, in which case the default compressor will // be used. -func (cc *ChannelConfig) InitCompressorConfig(approxComprRatio float64, comprKind string, compressionAlgo string, compressLevel int) { +func (cc *ChannelConfig) InitCompressorConfig(approxComprRatio float64, comprKind string, compressionAlgo derive.CompressionAlgo) { cc.CompressorConfig = compressor.Config{ // Compressor output size needs to account for frame encoding overhead TargetOutputSize: MaxDataSize(cc.TargetNumFrames, cc.MaxFrameSize), ApproxComprRatio: approxComprRatio, Kind: comprKind, CompressionAlgo: compressionAlgo, - CompressLevel: compressLevel, } } -func (cc *ChannelConfig) InitRatioCompressor(approxComprRatio float64, compressionAlgo string, compressLevel int) { - cc.InitCompressorConfig(approxComprRatio, compressor.RatioKind, compressionAlgo, compressLevel) +func (cc *ChannelConfig) InitRatioCompressor(approxComprRatio float64, compressionAlgo derive.CompressionAlgo) { + cc.InitCompressorConfig(approxComprRatio, compressor.RatioKind, compressionAlgo) } -func (cc *ChannelConfig) InitShadowCompressor(compressionAlgo string, compressLevel int) { - cc.InitCompressorConfig(0, compressor.ShadowKind, compressionAlgo, compressLevel) +func (cc *ChannelConfig) InitShadowCompressor(compressionAlgo derive.CompressionAlgo) { + cc.InitCompressorConfig(0, compressor.ShadowKind, compressionAlgo) } -func (cc *ChannelConfig) InitNoneCompressor(compressionAlgo string, compressLevel int) { - cc.InitCompressorConfig(0, compressor.NoneKind, compressionAlgo, compressLevel) +func (cc *ChannelConfig) InitNoneCompressor(compressionAlgo derive.CompressionAlgo) { + cc.InitCompressorConfig(0, compressor.NoneKind, compressionAlgo) } func (cc *ChannelConfig) MaxFramesPerTx() int { diff --git a/op-batcher/batcher/channel_config_test.go b/op-batcher/batcher/channel_config_test.go index fbb08374fee9..c4e7594462b2 100644 --- a/op-batcher/batcher/channel_config_test.go +++ b/op-batcher/batcher/channel_config_test.go @@ -20,7 +20,7 @@ func defaultTestChannelConfig() ChannelConfig { TargetNumFrames: 1, BatchType: derive.SingularBatchType, } - c.InitRatioCompressor(0.4, "brotli", 10) + c.InitRatioCompressor(0.4, derive.Brotli10) return c } diff --git a/op-batcher/batcher/channel_manager_test.go b/op-batcher/batcher/channel_manager_test.go index d1033be1a896..f730eb0145d1 100644 --- a/op-batcher/batcher/channel_manager_test.go +++ b/op-batcher/batcher/channel_manager_test.go @@ -25,7 +25,7 @@ func channelManagerTestConfig(maxFrameSize uint64, batchType uint) ChannelConfig TargetNumFrames: 1, BatchType: batchType, } - cfg.InitRatioCompressor(1, "brotli", 10) + cfg.InitRatioCompressor(1, derive.Brotli10) return cfg } @@ -123,7 +123,7 @@ func ChannelManager_Clear(t *testing.T, batchType uint) { // channels on confirmation. This would result in [TxConfirmed] // clearing confirmed transactions, and resetting the pendingChannels map cfg.ChannelTimeout = 10 - cfg.InitRatioCompressor(1, "brotli", 10) + cfg.InitRatioCompressor(1, derive.Brotli10) m := NewChannelManager(log, metrics.NoopMetrics, cfg, &defaultTestRollupConfig) // Channel Manager state should be empty by default @@ -345,7 +345,7 @@ func TestChannelManager_Close_PartiallyPendingChannel(t *testing.T) { ChannelTimeout: 1000, TargetNumFrames: 100, } - cfg.InitNoneCompressor("brotli", 10) + cfg.InitNoneCompressor(derive.Brotli10) m := NewChannelManager(log, metrics.NoopMetrics, cfg, &defaultTestRollupConfig) m.Clear(eth.BlockID{}) @@ -397,7 +397,7 @@ func ChannelManagerCloseAllTxsFailed(t *testing.T, batchType uint) { log := testlog.Logger(t, log.LevelCrit) cfg := channelManagerTestConfig(100, batchType) cfg.TargetNumFrames = 1000 - cfg.InitNoneCompressor("brotli", 10) + cfg.InitNoneCompressor(derive.Brotli10) m := NewChannelManager(log, metrics.NoopMetrics, cfg, &defaultTestRollupConfig) m.Clear(eth.BlockID{}) @@ -445,7 +445,7 @@ func TestChannelManager_ChannelCreation(t *testing.T) { const maxChannelDuration = 15 cfg := channelManagerTestConfig(1000, derive.SpanBatchType) cfg.MaxChannelDuration = maxChannelDuration - cfg.InitNoneCompressor("brotli", 10) + cfg.InitNoneCompressor(derive.Brotli10) for _, tt := range []struct { name string diff --git a/op-batcher/batcher/config.go b/op-batcher/batcher/config.go index 7ec8216e3280..cc37961a056a 100644 --- a/op-batcher/batcher/config.go +++ b/op-batcher/batcher/config.go @@ -10,6 +10,7 @@ import ( "github.com/ethereum-optimism/optimism/op-batcher/compressor" "github.com/ethereum-optimism/optimism/op-batcher/flags" + "github.com/ethereum-optimism/optimism/op-node/rollup/derive" plasma "github.com/ethereum-optimism/optimism/op-plasma" oplog "github.com/ethereum-optimism/optimism/op-service/log" opmetrics "github.com/ethereum-optimism/optimism/op-service/metrics" @@ -67,11 +68,8 @@ type CLIConfig struct { // Type of compressor to use. Must be one of [compressor.KindKeys]. Compressor string - // Type of compression algorithm to use. Must be one of [zlib, brotli] - CompressionAlgo string - - // Levels of compression to use. E.g. 1-11 for brotli, 0-9 for zlib - CompressLevel int + // Type of compression algorithm to use. Must be one of [zlib, brotli, brotli[9-11]] + CompressionAlgo derive.CompressionAlgo // If Stopped is true, the batcher starts stopped and won't start batching right away. // Batching needs to be started via an admin RPC. @@ -130,11 +128,8 @@ func (c *CLIConfig) Check() error { if c.Compressor == compressor.RatioKind && (c.ApproxComprRatio <= 0 || c.ApproxComprRatio > 1) { return fmt.Errorf("invalid ApproxComprRatio %v for ratio compressor", c.ApproxComprRatio) } - if c.CompressionAlgo == "" { - return errors.New("must set compression algo") - } - if (c.CompressionAlgo == "zlib" && (c.CompressLevel < 0 || c.CompressLevel > 9)) || (c.CompressionAlgo == "brotli" && (c.CompressLevel < 1 || c.CompressLevel > 11)) { - return fmt.Errorf("invalid compression level %v for %v", c.CompressLevel, c.CompressionAlgo) + if !derive.ValidCompressionAlgoType(c.CompressionAlgo) { + return fmt.Errorf("invalid compression algo %v", c.CompressionAlgo) } if c.BatchType > 1 { return fmt.Errorf("unknown batch type: %v", c.BatchType) @@ -180,8 +175,7 @@ func NewConfig(ctx *cli.Context) *CLIConfig { TargetNumFrames: ctx.Int(flags.TargetNumFramesFlag.Name), ApproxComprRatio: ctx.Float64(flags.ApproxComprRatioFlag.Name), Compressor: ctx.String(flags.CompressorFlag.Name), - CompressionAlgo: ctx.String(flags.CompressionAlgoFlag.Name), - CompressLevel: ctx.Int(flags.CompressLevelFlag.Name), + CompressionAlgo: derive.CompressionAlgo(ctx.String(flags.CompressionAlgoFlag.Name)), Stopped: ctx.Bool(flags.StoppedFlag.Name), WaitNodeSync: ctx.Bool(flags.WaitNodeSyncFlag.Name), CheckRecentTxsDepth: ctx.Int(flags.CheckRecentTxsDepthFlag.Name), diff --git a/op-batcher/batcher/config_test.go b/op-batcher/batcher/config_test.go index 10dcdaf21d59..ee124644c8fc 100644 --- a/op-batcher/batcher/config_test.go +++ b/op-batcher/batcher/config_test.go @@ -7,6 +7,7 @@ import ( "github.com/ethereum-optimism/optimism/op-batcher/batcher" "github.com/ethereum-optimism/optimism/op-batcher/compressor" "github.com/ethereum-optimism/optimism/op-batcher/flags" + "github.com/ethereum-optimism/optimism/op-node/rollup/derive" "github.com/ethereum-optimism/optimism/op-service/log" "github.com/ethereum-optimism/optimism/op-service/metrics" "github.com/ethereum-optimism/optimism/op-service/oppprof" @@ -34,10 +35,8 @@ func validBatcherConfig() batcher.CLIConfig { LogConfig: log.DefaultCLIConfig(), MetricsConfig: metrics.DefaultCLIConfig(), PprofConfig: oppprof.DefaultCLIConfig(), - // The compressor config is not checked in config.Check() - RPC: rpc.DefaultCLIConfig(), - CompressionAlgo: "brotli", - CompressLevel: 9, + RPC: rpc.DefaultCLIConfig(), + CompressionAlgo: derive.Brotli10, } } diff --git a/op-batcher/batcher/service.go b/op-batcher/batcher/service.go index 21e50d8ac717..6b8f1b2a54b5 100644 --- a/op-batcher/batcher/service.go +++ b/op-batcher/batcher/service.go @@ -219,7 +219,7 @@ func (bs *BatcherService) initChannelConfig(cfg *CLIConfig) error { return fmt.Errorf("max frame size %d exceeds plasma max input size %d", cc.MaxFrameSize, plasma.MaxInputSize) } - cc.InitCompressorConfig(cfg.ApproxComprRatio, cfg.Compressor, cfg.CompressionAlgo, cfg.CompressLevel) + cc.InitCompressorConfig(cfg.ApproxComprRatio, cfg.Compressor, cfg.CompressionAlgo) if bs.UseBlobs && !bs.RollupConfig.IsEcotone(uint64(time.Now().Unix())) { bs.Log.Error("Cannot use Blob data before Ecotone!") // log only, the batcher may not be actively running. diff --git a/op-batcher/compressor/config.go b/op-batcher/compressor/config.go index 09988a26b003..4fca00040792 100644 --- a/op-batcher/compressor/config.go +++ b/op-batcher/compressor/config.go @@ -17,8 +17,11 @@ type Config struct { // will default to RatioKind. Kind string - CompressionAlgo string - CompressLevel int + // Type of compression algorithm to use. Must be one of [zlib, brotli] + CompressionAlgo derive.CompressionAlgo + + // Levels of compression to use. E.g. 1-11 for brotli, 0-9 for zlib + CompressLevel int } func (c Config) NewCompressor() (derive.Compressor, error) { diff --git a/op-batcher/flags/flags.go b/op-batcher/flags/flags.go index 9da2e6917fcf..0dff2297699d 100644 --- a/op-batcher/flags/flags.go +++ b/op-batcher/flags/flags.go @@ -9,6 +9,7 @@ import ( "github.com/urfave/cli/v2" "github.com/ethereum-optimism/optimism/op-batcher/compressor" + "github.com/ethereum-optimism/optimism/op-node/rollup/derive" plasma "github.com/ethereum-optimism/optimism/op-plasma" opservice "github.com/ethereum-optimism/optimism/op-service" openum "github.com/ethereum-optimism/optimism/op-service/enum" @@ -99,30 +100,14 @@ var ( return nil }, } - CompressionAlgoFlag = &cli.StringFlag{ + CompressionAlgoFlag = &cli.GenericFlag{ Name: "compression-algo", - Usage: "The compression algorithm to use. Valid options: zlib, brotli ", - EnvVars: prefixEnvVars("COMPRESSION_ALGO"), - Value: "zlib", - Action: func(c *cli.Context, algo string) error { - if algo != "zlib" && algo != "brotli" { - return fmt.Errorf("unsupported compression algorithm: %s", algo) - } - return nil - }, - } - CompressLevelFlag = &cli.IntFlag{ - Name: "compress-level", - Usage: "The compression level to use. E.g. 1-11 for brotli, 0-9 for zlib", - EnvVars: prefixEnvVars("COMPRESS_LEVEL"), - Value: 9, // default to best zlib compression level - Action: func(c *cli.Context, lvl int) error { - algo := c.String("compression-algo") - if (algo == "zlib" && (lvl < 0 || lvl > 9)) || (algo == "brotli" && (lvl < 1 || lvl > 11)) { - return fmt.Errorf("invalid compression level %d for algorithm %s", lvl, algo) - } - return nil - }, + Usage: "The compression algorithm to use. Valid options: " + openum.EnumString(derive.CompressionAlgoTypes), + EnvVars: prefixEnvVars("COMPRESSION_ALGO_TYPE"), + Value: func() *derive.CompressionAlgo { + out := derive.Zlib + return &out + }(), } StoppedFlag = &cli.BoolFlag{ Name: "stopped", diff --git a/op-e2e/actions/l2_batcher.go b/op-e2e/actions/l2_batcher.go index 0cbf1ab2ba5c..5b03fe963135 100644 --- a/op-e2e/actions/l2_batcher.go +++ b/op-e2e/actions/l2_batcher.go @@ -200,7 +200,7 @@ func (s *L2Batcher) Buffer(t Testing) error { } else { // use span batch if we're forcing it or if we're at/beyond delta if s.l2BatcherCfg.ForceSubmitSpanBatch || s.rollupCfg.IsDelta(block.Time()) { - ch, err = derive.NewSpanChannelOut(s.rollupCfg.Genesis.L2Time, s.rollupCfg.L2ChainID, target, "brotli", 10) + ch, err = derive.NewSpanChannelOut(s.rollupCfg.Genesis.L2Time, s.rollupCfg.L2ChainID, target, derive.Brotli10) // use singular batches in all other cases } else { ch, err = derive.NewSingularChannelOut(c) diff --git a/op-e2e/actions/sync_test.go b/op-e2e/actions/sync_test.go index 9b463b7c2df5..fa86cb5437f8 100644 --- a/op-e2e/actions/sync_test.go +++ b/op-e2e/actions/sync_test.go @@ -26,7 +26,7 @@ import ( ) func newSpanChannelOut(t StatefulTesting, e e2eutils.SetupData) derive.ChannelOut { - channelOut, err := derive.NewSpanChannelOut(e.RollupCfg.Genesis.L2Time, e.RollupCfg.L2ChainID, 128_000, "brotli", 10) + channelOut, err := derive.NewSpanChannelOut(e.RollupCfg.Genesis.L2Time, e.RollupCfg.L2ChainID, 128_000, derive.Brotli10) require.NoError(t, err) return channelOut } diff --git a/op-e2e/sequencer_failover_setup.go b/op-e2e/sequencer_failover_setup.go index 84871caf1e7c..3d8936962067 100644 --- a/op-e2e/sequencer_failover_setup.go +++ b/op-e2e/sequencer_failover_setup.go @@ -291,8 +291,7 @@ func setupBatcher(t *testing.T, sys *System, conductors map[string]*conductor) { BatchType: derive.SpanBatchType, DataAvailabilityType: batcherFlags.CalldataType, ActiveSequencerCheckDuration: 0, - CompressionAlgo: "zlib", - CompressLevel: 9, + CompressionAlgo: derive.Zlib, } batcher, err := bss.BatcherServiceFromCLIConfig(context.Background(), "0.0.1", batcherCLIConfig, sys.Cfg.Loggers["batcher"]) diff --git a/op-e2e/setup.go b/op-e2e/setup.go index a5a262ce77f3..c4053485ea0d 100644 --- a/op-e2e/setup.go +++ b/op-e2e/setup.go @@ -845,9 +845,8 @@ func (cfg SystemConfig) Start(t *testing.T, _opts ...SystemConfigOption) (*Syste BatchType: batchType, DataAvailabilityType: sys.Cfg.DataAvailabilityType, // Add the compression stuff - //CompressionAlgo: "zlib", - CompressionAlgo: "brotli", - CompressLevel: 9, // use brotli 9 for faster e2e tests + //CompressionAlgo: derive.Zlib, + CompressionAlgo: derive.Brotli10, } // Batch Submitter batcher, err := bss.BatcherServiceFromCLIConfig(context.Background(), "0.0.1", batcherCLIConfig, sys.Cfg.Loggers["batcher"]) diff --git a/op-node/benchmarks/batchbuilding_test.go b/op-node/benchmarks/batchbuilding_test.go index 90a8fae5024a..85c80424f085 100644 --- a/op-node/benchmarks/batchbuilding_test.go +++ b/op-node/benchmarks/batchbuilding_test.go @@ -64,7 +64,7 @@ func channelOutByType(batchType uint, compKey string) (derive.ChannelOut, error) return derive.NewSingularChannelOut(compressors[compKey].compressor) } if batchType == derive.SpanBatchType { - return derive.NewSpanChannelOut(0, chainID, compressors[compKey].targetOutput, "brotli", 10) + return derive.NewSpanChannelOut(0, chainID, compressors[compKey].targetOutput, derive.Brotli10) } return nil, fmt.Errorf("unsupported batch type: %d", batchType) } diff --git a/op-node/rollup/derive/channel_out_test.go b/op-node/rollup/derive/channel_out_test.go index f560932e4671..8dba6fbf6af5 100644 --- a/op-node/rollup/derive/channel_out_test.go +++ b/op-node/rollup/derive/channel_out_test.go @@ -52,7 +52,7 @@ var channelTypes = []struct { { Name: "Span", ChannelOut: func(t *testing.T) ChannelOut { - cout, err := NewSpanChannelOut(0, big.NewInt(0), 128_000, "brotli", 10) + cout, err := NewSpanChannelOut(0, big.NewInt(0), 128_000, Brotli10) require.NoError(t, err) return cout }, @@ -225,7 +225,7 @@ func SpanChannelAndBatches(t *testing.T, target uint64, len int) (*SpanChannelOu rng := rand.New(rand.NewSource(0x543331)) chainID := big.NewInt(rng.Int63n(1000)) txCount := 1 - cout, err := NewSpanChannelOut(0, chainID, target, "brotli", 10) + cout, err := NewSpanChannelOut(0, chainID, target, Brotli10) require.NoError(t, err) batches := make([]*SingularBatch, len) // adding the first batch should not cause an error diff --git a/op-node/rollup/derive/span_channel_out.go b/op-node/rollup/derive/span_channel_out.go index 40baa27c4caf..796879a719ff 100644 --- a/op-node/rollup/derive/span_channel_out.go +++ b/op-node/rollup/derive/span_channel_out.go @@ -16,8 +16,7 @@ import ( ) const ( - ZLIB string = "zlib" - BROTLI string = "brotli" + ChannelVersionBrotli byte = 0x01 ) type CompressorInterface interface { @@ -44,7 +43,7 @@ type SpanChannelOut struct { // the compressor for the channel compressor CompressorInterface // the compression algo used - compressionAlgo string + compressionAlgo CompressionAlgo // target is the target size of the compressed data target uint64 // closed indicates if the channel is closed @@ -64,7 +63,7 @@ func (co *SpanChannelOut) setRandomID() error { return err } -func NewSpanChannelOut(genesisTimestamp uint64, chainID *big.Int, targetOutputSize uint64, compressionAlgo string, compressLevel int) (*SpanChannelOut, error) { +func NewSpanChannelOut(genesisTimestamp uint64, chainID *big.Int, targetOutputSize uint64, compressionAlgo CompressionAlgo) (*SpanChannelOut, error) { c := &SpanChannelOut{ id: ChannelID{}, frame: 0, @@ -79,16 +78,16 @@ func NewSpanChannelOut(genesisTimestamp uint64, chainID *big.Int, targetOutputSi return nil, err } - if compressionAlgo == ZLIB { - if c.compressor, err = zlib.NewWriterLevel(c.compressed, compressLevel); err != nil { + if compressionAlgo == Zlib { + if c.compressor, err = zlib.NewWriterLevel(c.compressed, zlib.BestCompression); err != nil { return nil, err } - } else if compressionAlgo == BROTLI { + } else if compressionAlgo == Brotli { // setting the version bit of 1 - c.compressed = bytes.NewBuffer([]byte{0b0001}) + c.compressed = bytes.NewBuffer([]byte{ChannelVersionBrotli}) c.compressor = brotli.NewWriterLevel( c.compressed, - compressLevel, + 10, ) } else { return nil, fmt.Errorf("unsupported compression algorithm: %s", compressionAlgo) @@ -105,8 +104,8 @@ func (co *SpanChannelOut) Reset() error { co.rlp[1].Reset() co.lastCompressedRLPSize = 0 co.compressed.Reset() - if co.compressionAlgo == BROTLI { - co.compressed.WriteByte(0b0001) + if co.compressionAlgo == Brotli { + co.compressed.WriteByte(ChannelVersionBrotli) } co.compressor.Reset(co.compressed) co.spanBatch = NewSpanBatch(co.spanBatch.GenesisTimestamp, co.spanBatch.ChainID) @@ -219,9 +218,9 @@ func (co *SpanChannelOut) AddSingularBatch(batch *SingularBatch, seqNum uint64) // it resets all the compression buffers because Span Batches aren't meant to be compressed incrementally. func (co *SpanChannelOut) compress() error { co.compressed.Reset() - if co.compressionAlgo == BROTLI { + if co.compressionAlgo == Brotli { // reset but still need the version bit of 1 - co.compressed.WriteByte(0b0001) + co.compressed.WriteByte(ChannelVersionBrotli) } co.compressor.Reset(co.compressed) if _, err := co.compressor.Write(co.activeRLP().Bytes()); err != nil { diff --git a/op-node/rollup/derive/types.go b/op-node/rollup/derive/types.go new file mode 100644 index 000000000000..caf82addbc2d --- /dev/null +++ b/op-node/rollup/derive/types.go @@ -0,0 +1,50 @@ +package derive + +import ( + "fmt" +) + +type CompressionAlgo string + +const ( + // compression algo types + Zlib CompressionAlgo = "zlib" + Brotli CompressionAlgo = Brotli10 // default brotli 10 + Brotli9 CompressionAlgo = "brotli-9" + Brotli10 CompressionAlgo = "brotli-10" + Brotli11 CompressionAlgo = "brotli-11" +) + +var CompressionAlgoTypes = []CompressionAlgo{ + Zlib, + Brotli, + Brotli9, + Brotli10, + Brotli11, +} + +func (kind CompressionAlgo) String() string { + return string(kind) +} + +func (kind *CompressionAlgo) Set(value string) error { + if !ValidCompressionAlgoType(CompressionAlgo(value)) { + return fmt.Errorf("unknown compression algo type: %q", value) + } + *kind = CompressionAlgo(value) + return nil +} + +func (kind *CompressionAlgo) Clone() any { + cpy := *kind + return &cpy +} + +func ValidCompressionAlgoType(value CompressionAlgo) bool { + for _, k := range CompressionAlgoTypes { + if k == value { + return true + } + } + return false +} From 8bd2eb6a94e86b9f0ad61582a3ff00862f376ee5 Mon Sep 17 00:00:00 2001 From: Cody Wang Date: Thu, 2 May 2024 18:45:57 -0400 Subject: [PATCH 11/38] update batch reader --- op-node/rollup/derive/channel.go | 39 ++++++++++++++++---------------- 1 file changed, 19 insertions(+), 20 deletions(-) diff --git a/op-node/rollup/derive/channel.go b/op-node/rollup/derive/channel.go index 039ee2e7a416..e3120ac865d9 100644 --- a/op-node/rollup/derive/channel.go +++ b/op-node/rollup/derive/channel.go @@ -1,19 +1,24 @@ package derive import ( + "bufio" "bytes" "compress/zlib" "fmt" "io" - "strings" - "github.com/andybalholm/brotli" "github.com/ethereum-optimism/optimism/op-service/eth" "github.com/ethereum/go-ethereum/rlp" ) +const ( + ZlibCM8 = 8 + ZlibCM15 = 15 + BrotliVersionByte = 0x01 +) + // A Channel is a set of batches that are split into at least one, but possibly multiple frames. // Frames are allowed to be ingested out of order. // Each frame is ingested one by one. Once a frame with `closed` is added to the channel, the @@ -151,41 +156,35 @@ func (ch *Channel) Reader() io.Reader { // Warning: the batch reader can read every batch-type. // The caller of the batch-reader should filter the results. func BatchReader(r io.Reader) (func() (*BatchData, error), error) { - // Buffer to store the data for reuse - var buffer strings.Builder - if _, err := io.Copy(&buffer, r); err != nil { - return nil, err - } - - tmpReader := strings.NewReader(buffer.String()) - // Read the first byte to be used to determine the compression type - compressionType := make([]byte, 1) - if _, err := tmpReader.Read(compressionType); err != nil { + // use buffered reader so can peek the first byte + bufReader := bufio.NewReader(r) + compressionType, err := bufReader.Peek(1) + if err != nil { return nil, err } - // Reset the reader with the original data - r = strings.NewReader(buffer.String()) var reader func(io.Reader) (io.Reader, error) - // If the last 4 bits is 8, then it is a zlib reader - if compressionType[0]&0x0F == 8 { + // For zlib, the last 4 bits must be either 8 or 15 (both are reserved value) + if compressionType[0]&0x0F == ZlibCM8 || compressionType[0]&0x0F == ZlibCM15 { reader = func(r io.Reader) (io.Reader, error) { return zlib.NewReader(r) } // If the bits equal to 1, then it is a brotli reader - } else if compressionType[0]&0x0F == 1 { - // remove the first byte by reading it - _, err := r.Read(make([]byte, 1)) + } else if compressionType[0] == BrotliVersionByte { + // discard the first byte + _, err := bufReader.Discard(1) if err != nil { return nil, err } reader = func(r io.Reader) (io.Reader, error) { return brotli.NewReader(r), nil } + } else { + return nil, fmt.Errorf("cannot distinguish the compression algo used given type byte %v", compressionType[0]) } // Setup decompressor stage + RLP reader - zr, err := reader(r) + zr, err := reader(bufReader) if err != nil { return nil, err } From 314e99da5c29074124e358f962b1eca4f1f1bec3 Mon Sep 17 00:00:00 2001 From: Cody Wang Date: Thu, 2 May 2024 20:59:28 -0400 Subject: [PATCH 12/38] abstract span channel compressor --- op-node/rollup/derive/channel.go | 12 ++- op-node/rollup/derive/channel_out_test.go | 12 +-- .../rollup/derive/span_channel_compressor.go | 81 +++++++++++++++++++ op-node/rollup/derive/span_channel_out.go | 68 ++++------------ op-node/rollup/derive/types.go | 20 +++++ 5 files changed, 130 insertions(+), 63 deletions(-) create mode 100644 op-node/rollup/derive/span_channel_compressor.go diff --git a/op-node/rollup/derive/channel.go b/op-node/rollup/derive/channel.go index e3120ac865d9..11b6eac8fd01 100644 --- a/op-node/rollup/derive/channel.go +++ b/op-node/rollup/derive/channel.go @@ -14,9 +14,8 @@ import ( ) const ( - ZlibCM8 = 8 - ZlibCM15 = 15 - BrotliVersionByte = 0x01 + ZlibCM8 = 8 + ZlibCM15 = 15 ) // A Channel is a set of batches that are split into at least one, but possibly multiple frames. @@ -163,6 +162,8 @@ func BatchReader(r io.Reader) (func() (*BatchData, error), error) { return nil, err } + fmt.Println(compressionType[0]) + var reader func(io.Reader) (io.Reader, error) // For zlib, the last 4 bits must be either 8 or 15 (both are reserved value) if compressionType[0]&0x0F == ZlibCM8 || compressionType[0]&0x0F == ZlibCM15 { @@ -170,10 +171,11 @@ func BatchReader(r io.Reader) (func() (*BatchData, error), error) { return zlib.NewReader(r) } // If the bits equal to 1, then it is a brotli reader - } else if compressionType[0] == BrotliVersionByte { + } else if compressionType[0] == ChannelVersionBrotli { // discard the first byte _, err := bufReader.Discard(1) if err != nil { + fmt.Println("CHECK HERE") return nil, err } reader = func(r io.Reader) (io.Reader, error) { @@ -186,6 +188,7 @@ func BatchReader(r io.Reader) (func() (*BatchData, error), error) { // Setup decompressor stage + RLP reader zr, err := reader(bufReader) if err != nil { + fmt.Println("FAILED READER") return nil, err } rlpReader := rlp.NewStream(zr, MaxRLPBytesPerChannel) @@ -193,6 +196,7 @@ func BatchReader(r io.Reader) (func() (*BatchData, error), error) { return func() (*BatchData, error) { var batchData BatchData if err = rlpReader.Decode(&batchData); err != nil { + fmt.Println("DECODE ERROR") return nil, err } return &batchData, nil diff --git a/op-node/rollup/derive/channel_out_test.go b/op-node/rollup/derive/channel_out_test.go index 8dba6fbf6af5..40346dd1e588 100644 --- a/op-node/rollup/derive/channel_out_test.go +++ b/op-node/rollup/derive/channel_out_test.go @@ -113,7 +113,7 @@ func TestOutputFrameNoEmptyLastFrame(t *testing.T) { // depending on the channel type, determine the size of the written data if span, ok := cout.(*SpanChannelOut); ok { - written = uint64(span.compressed.Len()) + written = uint64(span.compressor.GetCompressedLen()) } else if singular, ok := cout.(*SingularChannelOut); ok { written = uint64(singular.compress.Len()) } @@ -244,7 +244,7 @@ func TestSpanChannelOutCompressionOnlyOneBatch(t *testing.T) { err := cout.AddSingularBatch(singularBatches[0], 0) // confirm compression was not skipped - require.Greater(t, cout.compressed.Len(), 0) + require.Greater(t, cout.compressor.GetCompressedLen(), 0) require.NoError(t, err) // confirm the channel is full @@ -263,14 +263,14 @@ func TestSpanChannelOutCompressionUndo(t *testing.T) { err := cout.AddSingularBatch(singularBatches[0], 0) require.NoError(t, err) // confirm that the first compression was skipped - require.Equal(t, 0, cout.compressed.Len()) + require.Equal(t, 1, cout.compressor.GetCompressedLen()) // 1 because of brotli channel version // record the RLP length to confirm it doesn't change when adding a rejected batch rlp1 := cout.activeRLP().Len() err = cout.AddSingularBatch(singularBatches[1], 0) require.ErrorIs(t, err, ErrCompressorFull) // confirm that the second compression was not skipped - require.Greater(t, cout.compressed.Len(), 0) + require.Greater(t, cout.compressor.GetCompressedLen(), 0) // confirm that the second rlp is tht same size as the first (because the second batch was not added) require.Equal(t, rlp1, cout.activeRLP().Len()) @@ -285,7 +285,7 @@ func TestSpanChannelOutClose(t *testing.T) { err := cout.AddSingularBatch(singularBatches[0], 0) require.NoError(t, err) // confirm no compression has happened yet - require.Equal(t, 0, cout.compressed.Len()) + require.Equal(t, 1, cout.compressor.GetCompressedLen()) // 1 because of brotli channel version // confirm the RLP length is less than the target rlpLen := cout.activeRLP().Len() @@ -295,6 +295,6 @@ func TestSpanChannelOutClose(t *testing.T) { require.NoError(t, cout.Close()) // confirm that the only batch was compressed, and that the RLP did not change - require.Greater(t, cout.compressed.Len(), 0) + require.Greater(t, cout.compressor.GetCompressedLen(), 0) require.Equal(t, rlpLen, cout.activeRLP().Len()) } diff --git a/op-node/rollup/derive/span_channel_compressor.go b/op-node/rollup/derive/span_channel_compressor.go new file mode 100644 index 000000000000..0edf2bc5cb31 --- /dev/null +++ b/op-node/rollup/derive/span_channel_compressor.go @@ -0,0 +1,81 @@ +package derive + +import ( + "bytes" + "compress/zlib" + "fmt" + "io" + + "github.com/andybalholm/brotli" +) + +const ( + ChannelVersionBrotli byte = 0x01 +) + +type CompressorWriter interface { + Write([]byte) (int, error) + Flush() error + Close() error + Reset(io.Writer) +} + +type SpanChannelCompressor struct { + writer CompressorWriter + compressionAlgo CompressionAlgo + compressed *bytes.Buffer +} + +func NewSpanChannelCompressor(algo CompressionAlgo) (*SpanChannelCompressor, error) { + var writer CompressorWriter + var err error + compressed := &bytes.Buffer{} + if algo == Zlib { + writer, err = zlib.NewWriterLevel(compressed, zlib.BestCompression) + } else if algo.IsBrotli() { + compressed.WriteByte(ChannelVersionBrotli) + writer = brotli.NewWriterLevel(compressed, GetBrotliLevel(algo)) + } else { + return nil, fmt.Errorf("unsupported compression algorithm: %s", algo) + } + + if err != nil { + return nil, err + } + + return &SpanChannelCompressor{ + writer: writer, + compressionAlgo: algo, + compressed: compressed, + }, nil + +} + +func (scc *SpanChannelCompressor) Write(data []byte) (int, error) { + return scc.writer.Write(data) +} + +func (scc *SpanChannelCompressor) Flush() error { + return scc.writer.Flush() +} + +func (scc *SpanChannelCompressor) Close() error { + return scc.writer.Close() +} + +func (scc *SpanChannelCompressor) Reset() { + scc.compressed.Reset() + if scc.compressionAlgo.IsBrotli() { + // always add channal version for brotli + scc.compressed.WriteByte(ChannelVersionBrotli) + } + scc.writer.Reset(scc.compressed) +} + +func (scc *SpanChannelCompressor) GetCompressedLen() int { + return scc.compressed.Len() +} + +func (scc *SpanChannelCompressor) GetCompressed() *bytes.Buffer { + return scc.compressed +} diff --git a/op-node/rollup/derive/span_channel_out.go b/op-node/rollup/derive/span_channel_out.go index 796879a719ff..81cfd7b8cea1 100644 --- a/op-node/rollup/derive/span_channel_out.go +++ b/op-node/rollup/derive/span_channel_out.go @@ -2,30 +2,18 @@ package derive import ( "bytes" - "compress/zlib" + "crypto/rand" "fmt" "io" "math/big" - "github.com/andybalholm/brotli" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/rlp" "github.com/ethereum-optimism/optimism/op-node/rollup" ) -const ( - ChannelVersionBrotli byte = 0x01 -) - -type CompressorInterface interface { - Write([]byte) (int, error) - Flush() error - Close() error - Reset(io.Writer) -} - type SpanChannelOut struct { id ChannelID // Frame ID of the next frame to emit. Increment after emitting @@ -38,12 +26,8 @@ type SpanChannelOut struct { // lastCompressedRLPSize tracks the *uncompressed* size of the last RLP buffer that was compressed // it is used to measure the growth of the RLP buffer when adding a new batch to optimize compression lastCompressedRLPSize int - // compressed contains compressed data for making output frames - compressed *bytes.Buffer // the compressor for the channel - compressor CompressorInterface - // the compression algo used - compressionAlgo CompressionAlgo + compressor *SpanChannelCompressor // target is the target size of the compressed data target uint64 // closed indicates if the channel is closed @@ -65,32 +49,19 @@ func (co *SpanChannelOut) setRandomID() error { func NewSpanChannelOut(genesisTimestamp uint64, chainID *big.Int, targetOutputSize uint64, compressionAlgo CompressionAlgo) (*SpanChannelOut, error) { c := &SpanChannelOut{ - id: ChannelID{}, - frame: 0, - spanBatch: NewSpanBatch(genesisTimestamp, chainID), - rlp: [2]*bytes.Buffer{{}, {}}, - compressed: &bytes.Buffer{}, - target: targetOutputSize, - compressionAlgo: compressionAlgo, + id: ChannelID{}, + frame: 0, + spanBatch: NewSpanBatch(genesisTimestamp, chainID), + rlp: [2]*bytes.Buffer{{}, {}}, + target: targetOutputSize, } var err error if err = c.setRandomID(); err != nil { return nil, err } - if compressionAlgo == Zlib { - if c.compressor, err = zlib.NewWriterLevel(c.compressed, zlib.BestCompression); err != nil { - return nil, err - } - } else if compressionAlgo == Brotli { - // setting the version bit of 1 - c.compressed = bytes.NewBuffer([]byte{ChannelVersionBrotli}) - c.compressor = brotli.NewWriterLevel( - c.compressed, - 10, - ) - } else { - return nil, fmt.Errorf("unsupported compression algorithm: %s", compressionAlgo) + if c.compressor, err = NewSpanChannelCompressor(compressionAlgo); err != nil { + return nil, err } return c, nil @@ -103,11 +74,7 @@ func (co *SpanChannelOut) Reset() error { co.rlp[0].Reset() co.rlp[1].Reset() co.lastCompressedRLPSize = 0 - co.compressed.Reset() - if co.compressionAlgo == Brotli { - co.compressed.WriteByte(ChannelVersionBrotli) - } - co.compressor.Reset(co.compressed) + co.compressor.Reset() co.spanBatch = NewSpanBatch(co.spanBatch.GenesisTimestamp, co.spanBatch.ChainID) // setting the new randomID is the only part of the reset that can fail return co.setRandomID() @@ -184,7 +151,7 @@ func (co *SpanChannelOut) AddSingularBatch(batch *SingularBatch, seqNum uint64) // if the compressed data *plus* the new rlp data is under the target size, return early // this optimizes out cases where the compressor will obviously come in under the target size rlpGrowth := co.activeRLP().Len() - co.lastCompressedRLPSize - if uint64(co.compressed.Len()+rlpGrowth) < co.target { + if uint64(co.compressor.GetCompressedLen()+rlpGrowth) < co.target { return nil } @@ -217,12 +184,7 @@ func (co *SpanChannelOut) AddSingularBatch(batch *SingularBatch, seqNum uint64) // compress compresses the active RLP buffer and checks if the compressed data is over the target size. // it resets all the compression buffers because Span Batches aren't meant to be compressed incrementally. func (co *SpanChannelOut) compress() error { - co.compressed.Reset() - if co.compressionAlgo == Brotli { - // reset but still need the version bit of 1 - co.compressed.WriteByte(ChannelVersionBrotli) - } - co.compressor.Reset(co.compressed) + co.compressor.Reset() if _, err := co.compressor.Write(co.activeRLP().Bytes()); err != nil { return err } @@ -242,7 +204,7 @@ func (co *SpanChannelOut) InputBytes() int { // Span Channel Out does not provide early output, so this will always be 0 until the channel is closed or full func (co *SpanChannelOut) ReadyBytes() int { if co.closed || co.FullErr() != nil { - return co.compressed.Len() + return co.compressor.GetCompressedLen() } return 0 } @@ -260,7 +222,7 @@ func (co *SpanChannelOut) checkFull() { if co.full != nil { return } - if uint64(co.compressed.Len()) >= co.target { + if uint64(co.compressor.GetCompressedLen()) >= co.target { co.full = ErrCompressorFull } } @@ -299,7 +261,7 @@ func (co *SpanChannelOut) OutputFrame(w *bytes.Buffer, maxSize uint64) (uint16, f := createEmptyFrame(co.id, co.frame, co.ReadyBytes(), co.closed, maxSize) - if _, err := io.ReadFull(co.compressed, f.Data); err != nil { + if _, err := io.ReadFull(co.compressor.GetCompressed(), f.Data); err != nil { return 0, err } diff --git a/op-node/rollup/derive/types.go b/op-node/rollup/derive/types.go index caf82addbc2d..dbec811b3d38 100644 --- a/op-node/rollup/derive/types.go +++ b/op-node/rollup/derive/types.go @@ -2,6 +2,7 @@ package derive import ( "fmt" + "regexp" ) type CompressionAlgo string @@ -23,6 +24,8 @@ var CompressionAlgoTypes = []CompressionAlgo{ Brotli11, } +var brotliRegexp = regexp.MustCompile(`^brotli-(9|10|11)$`) + func (kind CompressionAlgo) String() string { return string(kind) } @@ -40,6 +43,23 @@ func (kind *CompressionAlgo) Clone() any { return &cpy } +func (kind *CompressionAlgo) IsBrotli() bool { + return brotliRegexp.MatchString(kind.String()) +} + +func GetBrotliLevel(kind CompressionAlgo) int { + switch kind { + case Brotli9: + return 9 + case Brotli10: + return 10 + case Brotli11: + return 11 + default: + panic("Unsupported brotli level") + } +} + func ValidCompressionAlgoType(value CompressionAlgo) bool { for _, k := range CompressionAlgoTypes { if k == value { From fee3e4485db8ed6003bf7d78ea950b21a84392ec Mon Sep 17 00:00:00 2001 From: Cody Wang Date: Sun, 5 May 2024 19:34:55 +0100 Subject: [PATCH 13/38] test and singular batch compressor --- op-batcher/batcher/channel_builder_test.go | 338 ++++++++++-------- op-batcher/batcher/channel_config_test.go | 35 +- op-batcher/batcher/channel_manager_test.go | 87 +++-- op-batcher/batcher/channel_test.go | 56 ++- op-batcher/batcher/config_test.go | 22 +- op-batcher/compressor/config.go | 3 - op-batcher/compressor/ratio_compressor.go | 89 ++++- .../compressor/ratio_compressor_test.go | 28 +- op-batcher/compressor/shadow_compressor.go | 36 +- .../compressor/shadow_compressor_test.go | 2 + op-batcher/flags/flags.go | 2 +- op-node/rollup/derive/channel.go | 12 +- op-node/rollup/derive/channel_test.go | 97 +++++ .../derive/span_channel_compressor_test.go | 60 ++++ op-node/rollup/derive/types.go | 23 +- op-node/rollup/derive/types_test.go | 52 +++ 16 files changed, 665 insertions(+), 277 deletions(-) create mode 100644 op-node/rollup/derive/span_channel_compressor_test.go create mode 100644 op-node/rollup/derive/types_test.go diff --git a/op-batcher/batcher/channel_builder_test.go b/op-batcher/batcher/channel_builder_test.go index 90166942f4c3..1eeb034d23fb 100644 --- a/op-batcher/batcher/channel_builder_test.go +++ b/op-batcher/batcher/channel_builder_test.go @@ -9,6 +9,7 @@ import ( "testing" "time" + "github.com/ethereum-optimism/optimism/op-batcher/compressor" "github.com/ethereum-optimism/optimism/op-node/rollup" "github.com/ethereum-optimism/optimism/op-node/rollup/derive" dtest "github.com/ethereum-optimism/optimism/op-node/rollup/derive/test" @@ -100,10 +101,12 @@ func addTooManyBlocks(cb *ChannelBuilder) error { // is set to 0, the channel builder cannot have a duration timeout. func FuzzDurationTimeoutZeroMaxChannelDuration(f *testing.F) { for i := range [10]int{} { - f.Add(uint64(i)) + for _, algo := range derive.CompressionAlgoTypes { + f.Add(uint64(i), algo.String()) + } } - f.Fuzz(func(t *testing.T, l1BlockNum uint64) { - channelConfig := defaultTestChannelConfig() + f.Fuzz(func(t *testing.T, l1BlockNum uint64, algo string) { + channelConfig := defaultTestChannelConfig(derive.CompressionAlgo(algo)) channelConfig.MaxChannelDuration = 0 cb, err := NewChannelBuilder(channelConfig, defaultTestRollupConfig, latestL1BlockOrigin) require.NoError(t, err) @@ -118,15 +121,17 @@ func FuzzDurationTimeoutZeroMaxChannelDuration(f *testing.F) { // as long as the channel builder's timeout is set to 0. func FuzzChannelBuilder_DurationZero(f *testing.F) { for i := range [10]int{} { - f.Add(uint64(i), uint64(i)) + for _, algo := range derive.CompressionAlgoTypes { + f.Add(uint64(i), uint64(i), algo.String()) + } } - f.Fuzz(func(t *testing.T, l1BlockNum uint64, maxChannelDuration uint64) { + f.Fuzz(func(t *testing.T, l1BlockNum uint64, maxChannelDuration uint64, algo string) { if maxChannelDuration == 0 { t.Skip("Max channel duration cannot be 0") } // Create the channel builder - channelConfig := defaultTestChannelConfig() + channelConfig := defaultTestChannelConfig(derive.CompressionAlgo(algo)) channelConfig.MaxChannelDuration = maxChannelDuration cb, err := NewChannelBuilder(channelConfig, defaultTestRollupConfig, latestL1BlockOrigin) require.NoError(t, err) @@ -145,15 +150,18 @@ func FuzzChannelBuilder_DurationZero(f *testing.F) { func FuzzDurationTimeoutMaxChannelDuration(f *testing.F) { // Set multiple seeds in case fuzzing isn't explicitly used for i := range [10]int{} { - f.Add(uint64(i), uint64(i), uint64(i)) + for _, algo := range derive.CompressionAlgoTypes { + f.Add(uint64(i), uint64(i), uint64(i), algo.String()) + } + } - f.Fuzz(func(t *testing.T, l1BlockNum uint64, maxChannelDuration uint64, timeout uint64) { + f.Fuzz(func(t *testing.T, l1BlockNum uint64, maxChannelDuration uint64, timeout uint64, algo string) { if maxChannelDuration == 0 { t.Skip("Max channel duration cannot be 0") } // Create the channel builder - channelConfig := defaultTestChannelConfig() + channelConfig := defaultTestChannelConfig(derive.CompressionAlgo(algo)) channelConfig.MaxChannelDuration = maxChannelDuration cb, err := NewChannelBuilder(channelConfig, defaultTestRollupConfig, latestL1BlockOrigin) require.NoError(t, err) @@ -182,11 +190,13 @@ func FuzzDurationTimeoutMaxChannelDuration(f *testing.F) { func FuzzChannelCloseTimeout(f *testing.F) { // Set multiple seeds in case fuzzing isn't explicitly used for i := range [10]int{} { - f.Add(uint64(i), uint64(i), uint64(i), uint64(i*5)) + for _, algo := range derive.CompressionAlgoTypes { + f.Add(uint64(i), uint64(i), uint64(i), uint64(i*5), algo.String()) + } } - f.Fuzz(func(t *testing.T, l1BlockNum uint64, channelTimeout uint64, subSafetyMargin uint64, timeout uint64) { + f.Fuzz(func(t *testing.T, l1BlockNum uint64, channelTimeout uint64, subSafetyMargin uint64, timeout uint64, algo string) { // Create the channel builder - channelConfig := defaultTestChannelConfig() + channelConfig := defaultTestChannelConfig(derive.CompressionAlgo(algo)) channelConfig.ChannelTimeout = channelTimeout channelConfig.SubSafetyMargin = subSafetyMargin cb, err := NewChannelBuilder(channelConfig, defaultTestRollupConfig, latestL1BlockOrigin) @@ -210,11 +220,13 @@ func FuzzChannelCloseTimeout(f *testing.F) { func FuzzChannelZeroCloseTimeout(f *testing.F) { // Set multiple seeds in case fuzzing isn't explicitly used for i := range [10]int{} { - f.Add(uint64(i), uint64(i), uint64(i)) + for _, algo := range derive.CompressionAlgoTypes { + f.Add(uint64(i), uint64(i), uint64(i), algo.String()) + } } - f.Fuzz(func(t *testing.T, l1BlockNum uint64, channelTimeout uint64, subSafetyMargin uint64) { + f.Fuzz(func(t *testing.T, l1BlockNum uint64, channelTimeout uint64, subSafetyMargin uint64, algo string) { // Create the channel builder - channelConfig := defaultTestChannelConfig() + channelConfig := defaultTestChannelConfig(derive.CompressionAlgo(algo)) channelConfig.ChannelTimeout = channelTimeout channelConfig.SubSafetyMargin = subSafetyMargin cb, err := NewChannelBuilder(channelConfig, defaultTestRollupConfig, latestL1BlockOrigin) @@ -237,11 +249,13 @@ func FuzzChannelZeroCloseTimeout(f *testing.F) { func FuzzSeqWindowClose(f *testing.F) { // Set multiple seeds in case fuzzing isn't explicitly used for i := range [10]int{} { - f.Add(uint64(i), uint64(i), uint64(i), uint64(i*5)) + for _, algo := range derive.CompressionAlgoTypes { + f.Add(uint64(i), uint64(i), uint64(i), uint64(i*5), algo.String()) + } } - f.Fuzz(func(t *testing.T, epochNum uint64, seqWindowSize uint64, subSafetyMargin uint64, timeout uint64) { + f.Fuzz(func(t *testing.T, epochNum uint64, seqWindowSize uint64, subSafetyMargin uint64, timeout uint64, algo string) { // Create the channel builder - channelConfig := defaultTestChannelConfig() + channelConfig := defaultTestChannelConfig(derive.CompressionAlgo(algo)) channelConfig.SeqWindowSize = seqWindowSize channelConfig.SubSafetyMargin = subSafetyMargin cb, err := NewChannelBuilder(channelConfig, defaultTestRollupConfig, latestL1BlockOrigin) @@ -265,11 +279,13 @@ func FuzzSeqWindowClose(f *testing.F) { func FuzzSeqWindowZeroTimeoutClose(f *testing.F) { // Set multiple seeds in case fuzzing isn't explicitly used for i := range [10]int{} { - f.Add(uint64(i), uint64(i), uint64(i)) + for _, algo := range derive.CompressionAlgoTypes { + f.Add(uint64(i), uint64(i), uint64(i), algo.String()) + } } - f.Fuzz(func(t *testing.T, epochNum uint64, seqWindowSize uint64, subSafetyMargin uint64) { + f.Fuzz(func(t *testing.T, epochNum uint64, seqWindowSize uint64, subSafetyMargin uint64, algo string) { // Create the channel builder - channelConfig := defaultTestChannelConfig() + channelConfig := defaultTestChannelConfig(derive.CompressionAlgo(algo)) channelConfig.SeqWindowSize = seqWindowSize channelConfig.SubSafetyMargin = subSafetyMargin cb, err := NewChannelBuilder(channelConfig, defaultTestRollupConfig, latestL1BlockOrigin) @@ -289,7 +305,7 @@ func FuzzSeqWindowZeroTimeoutClose(f *testing.F) { func TestChannelBuilderBatchType(t *testing.T) { tests := []struct { name string - f func(t *testing.T, batchType uint) + f func(t *testing.T, batchType uint, algo derive.CompressionAlgo) }{ {"ChannelBuilder_MaxRLPBytesPerChannel", ChannelBuilder_MaxRLPBytesPerChannel}, {"ChannelBuilder_OutputFramesMaxFrameIndex", ChannelBuilder_OutputFramesMaxFrameIndex}, @@ -301,22 +317,48 @@ func TestChannelBuilderBatchType(t *testing.T) { } for _, test := range tests { test := test - t.Run(test.name+"_SingularBatch", func(t *testing.T) { - test.f(t, derive.SingularBatchType) - }) + for _, algo := range derive.CompressionAlgoTypes { + t.Run(test.name+"_SingularBatch_"+algo.String(), func(t *testing.T) { + test.f(t, derive.SingularBatchType, algo) + }) + } } for _, test := range tests { test := test - t.Run(test.name+"_SpanBatch", func(t *testing.T) { - test.f(t, derive.SpanBatchType) - }) + for _, algo := range derive.CompressionAlgoTypes { + t.Run(test.name+"_SpanBatch_"+algo.String(), func(t *testing.T) { + test.f(t, derive.SpanBatchType, algo) + }) + } + } +} + +func TestChannelBuilderSingularBatch(t *testing.T) { + tests := []struct { + name string + f func(t *testing.T, algo derive.CompressionAlgo) + }{ + {"ChannelBuilder_NextFrame", ChannelBuilder_NextFrame}, + {"ChannelBuilder_OutputFrames", ChannelBuilder_OutputFrames}, + {"ChannelBuilder_CheckTimeout", ChannelBuilder_CheckTimeout}, + {"ChannelBuilder_CheckTimeoutZeroMaxChannelDuration", ChannelBuilder_CheckTimeoutZeroMaxChannelDuration}, + {"ChannelBuilder_FramePublished", ChannelBuilder_FramePublished}, + {"ChannelBuilder_LatestL1Origin", ChannelBuilder_LatestL1Origin}, + } + for _, test := range tests { + test := test + for _, algo := range derive.CompressionAlgoTypes { + t.Run(test.name+"_"+algo.String(), func(t *testing.T) { + test.f(t, algo) + }) + } } } // TestChannelBuilder_NextFrame tests calling NextFrame on a ChannelBuilder with only one frame -func TestChannelBuilder_NextFrame(t *testing.T) { - channelConfig := defaultTestChannelConfig() +func ChannelBuilder_NextFrame(t *testing.T, algo derive.CompressionAlgo) { + channelConfig := defaultTestChannelConfig(algo) // Create a new channel builder cb, err := NewChannelBuilder(channelConfig, defaultTestRollupConfig, latestL1BlockOrigin) @@ -355,8 +397,8 @@ func TestChannelBuilder_NextFrame(t *testing.T) { } // TestChannelBuilder_OutputWrongFramePanic tests that a panic is thrown when a frame is pushed with an invalid frame id -func ChannelBuilder_OutputWrongFramePanic(t *testing.T, batchType uint) { - channelConfig := defaultTestChannelConfig() +func ChannelBuilder_OutputWrongFramePanic(t *testing.T, batchType uint, algo derive.CompressionAlgo) { + channelConfig := defaultTestChannelConfig(algo) channelConfig.BatchType = batchType // Construct a channel builder @@ -389,11 +431,11 @@ func ChannelBuilder_OutputWrongFramePanic(t *testing.T, batchType uint) { } // TestChannelBuilder_OutputFrames tests [ChannelBuilder.OutputFrames] for singular batches. -func TestChannelBuilder_OutputFrames(t *testing.T) { - channelConfig := defaultTestChannelConfig() +func ChannelBuilder_OutputFrames(t *testing.T, algo derive.CompressionAlgo) { + channelConfig := defaultTestChannelConfig(algo) channelConfig.MaxFrameSize = derive.FrameV0OverHeadSize + 1 channelConfig.TargetNumFrames = 1000 - channelConfig.InitNoneCompressor(derive.Brotli10) + channelConfig.InitNoneCompressor(algo) // Construct the channel builder cb, err := NewChannelBuilder(channelConfig, defaultTestRollupConfig, latestL1BlockOrigin) @@ -413,7 +455,8 @@ func TestChannelBuilder_OutputFrames(t *testing.T) { // Check how many ready bytes require.Greater(t, uint64(cb.co.ReadyBytes()+derive.FrameV0OverHeadSize), channelConfig.MaxFrameSize) - require.Equal(t, 0, cb.PendingFrames()) + + require.Equal(t, 0, cb.PendingFrames()) // always 0 because non compressor // The channel should not be full // but we want to output the frames for testing anyways @@ -430,80 +473,79 @@ func TestChannelBuilder_OutputFrames(t *testing.T) { } func TestChannelBuilder_OutputFrames_SpanBatch(t *testing.T) { - testCases := []struct { - testName string - algo derive.CompressionAlgo - targetNumFrames int - }{ - {testName: "Span Batch output frames with zlib", algo: derive.Zlib, targetNumFrames: 5}, - {testName: "Span Batch output frames with brotli", algo: derive.Brotli10, targetNumFrames: 1}, // to fill faster + for _, algo := range derive.CompressionAlgoTypes { + t.Run("ChannelBuilder_OutputFrames_SpanBatch"+algo.String(), func(t *testing.T) { + if algo.IsBrotli() { + ChannelBuilder_OutputFrames_SpanBatch(t, algo, 1) // to fill faster for brotli + } else { + ChannelBuilder_OutputFrames_SpanBatch(t, algo, 5) + } + }) } +} - for _, tc := range testCases { - t.Run(tc.testName, func(t *testing.T) { - channelConfig := defaultTestChannelConfig() - channelConfig.MaxFrameSize = 20 + derive.FrameV0OverHeadSize - channelConfig.TargetNumFrames = tc.targetNumFrames - channelConfig.BatchType = derive.SpanBatchType - channelConfig.InitRatioCompressor(1, tc.algo) - - // Construct the channel builder - cb, err := NewChannelBuilder(channelConfig, defaultTestRollupConfig, latestL1BlockOrigin) - require.NoError(t, err) - require.False(t, cb.IsFull()) - require.Equal(t, 0, cb.PendingFrames()) - - // Calling OutputFrames without having called [AddBlock] - // should return no error - require.NoError(t, cb.OutputFrames()) - - // There should be no ready bytes yet - require.Equal(t, 0, cb.co.ReadyBytes()) - - // fill up - for { - err = addMiniBlock(cb) - if err == nil { - if cb.IsFull() { - // this happens when the data exactly fills the channel - break - } - require.False(t, cb.IsFull()) - // There should be no ready bytes until the channel is full - require.Equal(t, cb.co.ReadyBytes(), 0) - } else { - require.ErrorIs(t, err, derive.ErrCompressorFull) - break - } - } +func ChannelBuilder_OutputFrames_SpanBatch(t *testing.T, algo derive.CompressionAlgo, targetNumFrames int) { + channelConfig := defaultTestChannelConfig(algo) + channelConfig.MaxFrameSize = 20 + derive.FrameV0OverHeadSize + channelConfig.TargetNumFrames = targetNumFrames + channelConfig.BatchType = derive.SpanBatchType + channelConfig.InitRatioCompressor(1, algo) - require.True(t, cb.IsFull()) - // Check how many ready bytes - require.GreaterOrEqual(t, - cb.co.ReadyBytes()+derive.FrameV0OverHeadSize, - int(channelConfig.MaxFrameSize)) - require.Equal(t, 0, cb.PendingFrames()) + // Construct the channel builder + cb, err := NewChannelBuilder(channelConfig, defaultTestRollupConfig, latestL1BlockOrigin) + require.NoError(t, err) + require.False(t, cb.IsFull()) + require.Equal(t, 0, cb.PendingFrames()) - // We should be able to output the frames - require.NoError(t, cb.OutputFrames()) + // Calling OutputFrames without having called [AddBlock] + // should return no error + require.NoError(t, cb.OutputFrames()) - // There should be several frames in the channel builder now - require.Greater(t, cb.PendingFrames(), 1) - for i := 0; i < cb.numFrames-1; i++ { - require.Len(t, cb.frames[i].data, int(channelConfig.MaxFrameSize)) + // There should be no ready bytes yet + require.Equal(t, 0, cb.co.ReadyBytes()) + + // fill up + for { + err = addMiniBlock(cb) + if err == nil { + if cb.IsFull() { + // this happens when the data exactly fills the channel + break } - require.LessOrEqual(t, len(cb.frames[len(cb.frames)-1].data), int(channelConfig.MaxFrameSize)) - }) + require.False(t, cb.IsFull()) + // There should be no ready bytes until the channel is full + require.Equal(t, cb.co.ReadyBytes(), 0) + } else { + require.ErrorIs(t, err, derive.ErrCompressorFull) + break + } + } + + require.True(t, cb.IsFull()) + // Check how many ready bytes + require.GreaterOrEqual(t, + cb.co.ReadyBytes()+derive.FrameV0OverHeadSize, + int(channelConfig.MaxFrameSize)) + require.Equal(t, 0, cb.PendingFrames()) + + // We should be able to output the frames + require.NoError(t, cb.OutputFrames()) + + // There should be several frames in the channel builder now + require.Greater(t, cb.PendingFrames(), 1) + for i := 0; i < cb.numFrames-1; i++ { + require.Len(t, cb.frames[i].data, int(channelConfig.MaxFrameSize)) } + require.LessOrEqual(t, len(cb.frames[len(cb.frames)-1].data), int(channelConfig.MaxFrameSize)) } // ChannelBuilder_MaxRLPBytesPerChannel tests the [ChannelBuilder.OutputFrames] // function errors when the max RLP bytes per channel is reached. -func ChannelBuilder_MaxRLPBytesPerChannel(t *testing.T, batchType uint) { +func ChannelBuilder_MaxRLPBytesPerChannel(t *testing.T, batchType uint, algo derive.CompressionAlgo) { t.Parallel() - channelConfig := defaultTestChannelConfig() + channelConfig := defaultTestChannelConfig(algo) channelConfig.MaxFrameSize = derive.MaxRLPBytesPerChannel * 2 - channelConfig.InitNoneCompressor(derive.Brotli10) + channelConfig.InitNoneCompressor(algo) channelConfig.BatchType = batchType // Construct the channel builder @@ -517,11 +559,11 @@ func ChannelBuilder_MaxRLPBytesPerChannel(t *testing.T, batchType uint) { // ChannelBuilder_OutputFramesMaxFrameIndex tests the [ChannelBuilder.OutputFrames] // function errors when the max frame index is reached. -func ChannelBuilder_OutputFramesMaxFrameIndex(t *testing.T, batchType uint) { - channelConfig := defaultTestChannelConfig() +func ChannelBuilder_OutputFramesMaxFrameIndex(t *testing.T, batchType uint, algo derive.CompressionAlgo) { + channelConfig := defaultTestChannelConfig(algo) channelConfig.MaxFrameSize = derive.FrameV0OverHeadSize + 1 channelConfig.TargetNumFrames = math.MaxUint16 + 1 - channelConfig.InitRatioCompressor(.1, derive.Brotli10) + channelConfig.InitRatioCompressor(.1, algo) channelConfig.BatchType = batchType rng := rand.New(rand.NewSource(123)) @@ -557,58 +599,56 @@ func ChannelBuilder_OutputFramesMaxFrameIndex(t *testing.T, batchType uint) { // [derive.FrameV0OverHeadSize] in [MaxDataSize] is omitted, which has been the // case before it got fixed it #9887. func TestChannelBuilder_FullShadowCompressor(t *testing.T) { + for _, algo := range derive.CompressionAlgoTypes { + t.Run("ChannelBuilder_FullShadowCompressor"+algo.String(), func(t *testing.T) { + ChannelBuilder_FullShadowCompressor(t, algo) + }) + } +} + +func ChannelBuilder_FullShadowCompressor(t *testing.T, algo derive.CompressionAlgo) { require := require.New(t) cfg := ChannelConfig{ MaxFrameSize: 752, TargetNumFrames: 1, BatchType: derive.SpanBatchType, + CompressorConfig: compressor.Config{ + CompressionAlgo: algo, + }, } - testCases := []struct { - testName string - algo derive.CompressionAlgo - }{ - {testName: "Full shadow compressor with zlib", algo: derive.Zlib}, - {testName: "Full shadow compressor with brotli", algo: derive.Brotli10}, - } - - for _, tc := range testCases { - t.Run(tc.testName, func(t *testing.T) { - cfg.InitShadowCompressor(tc.algo) - - cb, err := NewChannelBuilder(cfg, defaultTestRollupConfig, latestL1BlockOrigin) - require.NoError(err) + cfg.InitShadowCompressor(algo) + cb, err := NewChannelBuilder(cfg, defaultTestRollupConfig, latestL1BlockOrigin) + require.NoError(err) - rng := rand.New(rand.NewSource(420)) - a := dtest.RandomL2BlockWithChainId(rng, 1, defaultTestRollupConfig.L2ChainID) + rng := rand.New(rand.NewSource(420)) + a := dtest.RandomL2BlockWithChainId(rng, 1, defaultTestRollupConfig.L2ChainID) - for { - _, err = cb.AddBlock(a) - if err != nil { - require.ErrorIs(err, derive.ErrCompressorFull) - break - } - } + for { + _, err = cb.AddBlock(a) + if err != nil { + require.ErrorIs(err, derive.ErrCompressorFull) + break + } + } - require.NoError(cb.OutputFrames()) - require.True(cb.HasFrame()) - f := cb.NextFrame() - require.Less(len(f.data), int(cfg.MaxFrameSize)) // would fail without fix, full frame + require.NoError(cb.OutputFrames()) + require.True(cb.HasFrame()) + f := cb.NextFrame() + require.Less(len(f.data), int(cfg.MaxFrameSize)) // would fail without fix, full frame - require.False(cb.HasFrame(), "no leftover frame expected") // would fail without fix - }) - } + require.False(cb.HasFrame(), "no leftover frame expected") // would fail without fix } -func ChannelBuilder_AddBlock(t *testing.T, batchType uint) { - channelConfig := defaultTestChannelConfig() +func ChannelBuilder_AddBlock(t *testing.T, batchType uint, algo derive.CompressionAlgo) { + channelConfig := defaultTestChannelConfig(algo) channelConfig.BatchType = batchType // Lower the max frame size so that we can batch channelConfig.MaxFrameSize = 20 + derive.FrameV0OverHeadSize channelConfig.TargetNumFrames = 2 // Configure the Input Threshold params so we observe a full channel - channelConfig.InitRatioCompressor(1, derive.Brotli10) + channelConfig.InitRatioCompressor(1, algo) // Construct the channel builder cb, err := NewChannelBuilder(channelConfig, defaultTestRollupConfig, latestL1BlockOrigin) @@ -633,8 +673,8 @@ func ChannelBuilder_AddBlock(t *testing.T, batchType uint) { require.ErrorIs(t, addMiniBlock(cb), derive.ErrCompressorFull) } -func TestChannelBuilder_CheckTimeout(t *testing.T) { - channelConfig := defaultTestChannelConfig() +func ChannelBuilder_CheckTimeout(t *testing.T, algo derive.CompressionAlgo) { + channelConfig := defaultTestChannelConfig(algo) // Construct the channel builder cb, err := NewChannelBuilder(channelConfig, defaultTestRollupConfig, latestL1BlockOrigin) @@ -655,8 +695,8 @@ func TestChannelBuilder_CheckTimeout(t *testing.T) { require.ErrorIs(t, cb.FullErr(), ErrMaxDurationReached) } -func TestChannelBuilder_CheckTimeoutZeroMaxChannelDuration(t *testing.T) { - channelConfig := defaultTestChannelConfig() +func ChannelBuilder_CheckTimeoutZeroMaxChannelDuration(t *testing.T, algo derive.CompressionAlgo) { + channelConfig := defaultTestChannelConfig(algo) // Set the max channel duration to 0 channelConfig.MaxChannelDuration = 0 @@ -678,8 +718,8 @@ func TestChannelBuilder_CheckTimeoutZeroMaxChannelDuration(t *testing.T) { require.Equal(t, uint64(0), cb.timeout) } -func TestChannelBuilder_FramePublished(t *testing.T) { - cfg := defaultTestChannelConfig() +func ChannelBuilder_FramePublished(t *testing.T, algo derive.CompressionAlgo) { + cfg := defaultTestChannelConfig(algo) cfg.MaxChannelDuration = 10_000 cfg.ChannelTimeout = 1000 cfg.SubSafetyMargin = 100 @@ -701,8 +741,8 @@ func TestChannelBuilder_FramePublished(t *testing.T) { require.Less(t, cb.timeout, priorTimeout) } -func TestChannelBuilder_LatestL1Origin(t *testing.T) { - cb, err := NewChannelBuilder(defaultTestChannelConfig(), defaultTestRollupConfig, latestL1BlockOrigin) +func ChannelBuilder_LatestL1Origin(t *testing.T, algo derive.CompressionAlgo) { + cb, err := NewChannelBuilder(defaultTestChannelConfig(algo), defaultTestRollupConfig, latestL1BlockOrigin) require.NoError(t, err) require.Equal(t, eth.BlockID{}, cb.LatestL1Origin()) @@ -723,15 +763,15 @@ func TestChannelBuilder_LatestL1Origin(t *testing.T) { require.Equal(t, uint64(2), cb.LatestL1Origin().Number) } -func ChannelBuilder_PendingFrames_TotalFrames(t *testing.T, batchType uint) { +func ChannelBuilder_PendingFrames_TotalFrames(t *testing.T, batchType uint, algo derive.CompressionAlgo) { const tnf = 9 rng := rand.New(rand.NewSource(94572314)) require := require.New(t) - cfg := defaultTestChannelConfig() + cfg := defaultTestChannelConfig(algo) cfg.MaxFrameSize = 1000 cfg.TargetNumFrames = tnf cfg.BatchType = batchType - cfg.InitShadowCompressor(derive.Brotli10) + cfg.InitShadowCompressor(algo) cb, err := NewChannelBuilder(cfg, defaultTestRollupConfig, latestL1BlockOrigin) require.NoError(err) @@ -767,10 +807,10 @@ func ChannelBuilder_PendingFrames_TotalFrames(t *testing.T, batchType uint) { } } -func ChannelBuilder_InputBytes(t *testing.T, batchType uint) { +func ChannelBuilder_InputBytes(t *testing.T, batchType uint, algo derive.CompressionAlgo) { require := require.New(t) rng := rand.New(rand.NewSource(4982432)) - cfg := defaultTestChannelConfig() + cfg := defaultTestChannelConfig(algo) cfg.BatchType = batchType var spanBatch *derive.SpanBatch if batchType == derive.SpanBatchType { @@ -806,14 +846,14 @@ func ChannelBuilder_InputBytes(t *testing.T, batchType uint) { } } -func ChannelBuilder_OutputBytes(t *testing.T, batchType uint) { +func ChannelBuilder_OutputBytes(t *testing.T, batchType uint, algo derive.CompressionAlgo) { require := require.New(t) rng := rand.New(rand.NewSource(9860372)) - cfg := defaultTestChannelConfig() + cfg := defaultTestChannelConfig(algo) cfg.MaxFrameSize = 1000 cfg.TargetNumFrames = 16 cfg.BatchType = batchType - cfg.InitRatioCompressor(1.0, derive.Brotli10) + cfg.InitRatioCompressor(1.0, algo) cb, err := NewChannelBuilder(cfg, defaultTestRollupConfig, latestL1BlockOrigin) require.NoError(err, "NewChannelBuilder") diff --git a/op-batcher/batcher/channel_config_test.go b/op-batcher/batcher/channel_config_test.go index c4e7594462b2..d8419740d504 100644 --- a/op-batcher/batcher/channel_config_test.go +++ b/op-batcher/batcher/channel_config_test.go @@ -5,12 +5,13 @@ import ( "math" "testing" + "github.com/ethereum-optimism/optimism/op-batcher/compressor" "github.com/ethereum-optimism/optimism/op-node/rollup/derive" "github.com/stretchr/testify/require" ) -func defaultTestChannelConfig() ChannelConfig { +func defaultTestChannelConfig(algo derive.CompressionAlgo) ChannelConfig { c := ChannelConfig{ SeqWindowSize: 15, ChannelTimeout: 40, @@ -19,27 +20,30 @@ func defaultTestChannelConfig() ChannelConfig { MaxFrameSize: 120_000, TargetNumFrames: 1, BatchType: derive.SingularBatchType, + CompressorConfig: compressor.Config{ + CompressionAlgo: algo, + }, } - c.InitRatioCompressor(0.4, derive.Brotli10) + c.InitRatioCompressor(0.4, algo) return c } func TestChannelConfig_Check(t *testing.T) { type test struct { - input func() ChannelConfig + input func(algo derive.CompressionAlgo) ChannelConfig assertion func(error) } tests := []test{ { - input: defaultTestChannelConfig, + input: func(algo derive.CompressionAlgo) ChannelConfig { return defaultTestChannelConfig(algo) }, assertion: func(output error) { require.NoError(t, output) }, }, { - input: func() ChannelConfig { - cfg := defaultTestChannelConfig() + input: func(algo derive.CompressionAlgo) ChannelConfig { + cfg := defaultTestChannelConfig(algo) cfg.ChannelTimeout = 0 cfg.SubSafetyMargin = 1 return cfg @@ -53,8 +57,8 @@ func TestChannelConfig_Check(t *testing.T) { expectedErr := fmt.Sprintf("max frame size %d is less than the minimum 23", i) i := i // need to udpate Go version... tests = append(tests, test{ - input: func() ChannelConfig { - cfg := defaultTestChannelConfig() + input: func(algo derive.CompressionAlgo) ChannelConfig { + cfg := defaultTestChannelConfig(algo) cfg.MaxFrameSize = uint64(i) return cfg }, @@ -66,8 +70,10 @@ func TestChannelConfig_Check(t *testing.T) { // Run the table tests for _, test := range tests { - cfg := test.input() - test.assertion(cfg.Check()) + for _, algo := range derive.CompressionAlgoTypes { + cfg := test.input(algo) + test.assertion(cfg.Check()) + } } } @@ -76,9 +82,12 @@ func TestChannelConfig_Check(t *testing.T) { // the ChannelTimeout is less than the SubSafetyMargin. func FuzzChannelConfig_CheckTimeout(f *testing.F) { for i := range [10]int{} { - f.Add(uint64(i+1), uint64(i)) + for _, algo := range derive.CompressionAlgoTypes { + f.Add(uint64(i+1), uint64(i), algo.String()) + } + } - f.Fuzz(func(t *testing.T, channelTimeout uint64, subSafetyMargin uint64) { + f.Fuzz(func(t *testing.T, channelTimeout uint64, subSafetyMargin uint64, algo string) { // We only test where [ChannelTimeout] is less than the [SubSafetyMargin] // So we cannot have [ChannelTimeout] be [math.MaxUint64] if channelTimeout == math.MaxUint64 { @@ -88,7 +97,7 @@ func FuzzChannelConfig_CheckTimeout(f *testing.F) { subSafetyMargin = channelTimeout + 1 } - channelConfig := defaultTestChannelConfig() + channelConfig := defaultTestChannelConfig(derive.CompressionAlgo(algo)) channelConfig.ChannelTimeout = channelTimeout channelConfig.SubSafetyMargin = subSafetyMargin require.ErrorIs(t, channelConfig.Check(), ErrInvalidChannelTimeout) diff --git a/op-batcher/batcher/channel_manager_test.go b/op-batcher/batcher/channel_manager_test.go index f730eb0145d1..f42e62f709c4 100644 --- a/op-batcher/batcher/channel_manager_test.go +++ b/op-batcher/batcher/channel_manager_test.go @@ -7,6 +7,7 @@ import ( "testing" "time" + "github.com/ethereum-optimism/optimism/op-batcher/compressor" "github.com/ethereum-optimism/optimism/op-batcher/metrics" "github.com/ethereum-optimism/optimism/op-node/rollup" "github.com/ethereum-optimism/optimism/op-node/rollup/derive" @@ -19,20 +20,23 @@ import ( "github.com/stretchr/testify/require" ) -func channelManagerTestConfig(maxFrameSize uint64, batchType uint) ChannelConfig { +func channelManagerTestConfig(maxFrameSize uint64, batchType uint, algo derive.CompressionAlgo) ChannelConfig { cfg := ChannelConfig{ MaxFrameSize: maxFrameSize, TargetNumFrames: 1, BatchType: batchType, + CompressorConfig: compressor.Config{ + CompressionAlgo: algo, + }, } - cfg.InitRatioCompressor(1, derive.Brotli10) + cfg.InitRatioCompressor(1, algo) return cfg } func TestChannelManagerBatchType(t *testing.T) { tests := []struct { name string - f func(t *testing.T, batchType uint) + f func(t *testing.T, batchType uint, algo derive.CompressionAlgo) }{ {"ChannelManagerReturnsErrReorg", ChannelManagerReturnsErrReorg}, {"ChannelManagerReturnsErrReorgWhenDrained", ChannelManagerReturnsErrReorgWhenDrained}, @@ -45,24 +49,28 @@ func TestChannelManagerBatchType(t *testing.T) { } for _, test := range tests { test := test - t.Run(test.name+"_SingularBatch", func(t *testing.T) { - test.f(t, derive.SingularBatchType) - }) + for _, algo := range derive.CompressionAlgoTypes { + t.Run(test.name+"_SingularBatch_"+algo.String(), func(t *testing.T) { + test.f(t, derive.SingularBatchType, algo) + }) + } } for _, test := range tests { test := test - t.Run(test.name+"_SpanBatch", func(t *testing.T) { - test.f(t, derive.SpanBatchType) - }) + for _, algo := range derive.CompressionAlgoTypes { + t.Run(test.name+"_SpanBatch"+algo.String(), func(t *testing.T) { + test.f(t, derive.SpanBatchType, algo) + }) + } } } // ChannelManagerReturnsErrReorg ensures that the channel manager // detects a reorg when it has cached L1 blocks. -func ChannelManagerReturnsErrReorg(t *testing.T, batchType uint) { +func ChannelManagerReturnsErrReorg(t *testing.T, batchType uint, algo derive.CompressionAlgo) { log := testlog.Logger(t, log.LevelCrit) - m := NewChannelManager(log, metrics.NoopMetrics, ChannelConfig{BatchType: batchType}, &rollup.Config{}) + m := NewChannelManager(log, metrics.NoopMetrics, ChannelConfig{BatchType: batchType, CompressorConfig: compressor.Config{CompressionAlgo: algo}}, &rollup.Config{}) m.Clear(eth.BlockID{}) a := types.NewBlock(&types.Header{ @@ -91,9 +99,9 @@ func ChannelManagerReturnsErrReorg(t *testing.T, batchType uint) { // ChannelManagerReturnsErrReorgWhenDrained ensures that the channel manager // detects a reorg even if it does not have any blocks inside it. -func ChannelManagerReturnsErrReorgWhenDrained(t *testing.T, batchType uint) { +func ChannelManagerReturnsErrReorgWhenDrained(t *testing.T, batchType uint, algo derive.CompressionAlgo) { log := testlog.Logger(t, log.LevelCrit) - cfg := channelManagerTestConfig(120_000, batchType) + cfg := channelManagerTestConfig(120_000, batchType, algo) cfg.CompressorConfig.TargetOutputSize = 1 // full on first block m := NewChannelManager(log, metrics.NoopMetrics, cfg, &rollup.Config{}) m.Clear(eth.BlockID{}) @@ -112,18 +120,18 @@ func ChannelManagerReturnsErrReorgWhenDrained(t *testing.T, batchType uint) { } // ChannelManager_Clear tests clearing the channel manager. -func ChannelManager_Clear(t *testing.T, batchType uint) { +func ChannelManager_Clear(t *testing.T, batchType uint, algo derive.CompressionAlgo) { require := require.New(t) // Create a channel manager log := testlog.Logger(t, log.LevelCrit) rng := rand.New(rand.NewSource(time.Now().UnixNano())) - cfg := channelManagerTestConfig(derive.FrameV0OverHeadSize+1, batchType) + cfg := channelManagerTestConfig(derive.FrameV0OverHeadSize+1, batchType, algo) // Need to set the channel timeout here so we don't clear pending // channels on confirmation. This would result in [TxConfirmed] // clearing confirmed transactions, and resetting the pendingChannels map cfg.ChannelTimeout = 10 - cfg.InitRatioCompressor(1, derive.Brotli10) + cfg.InitRatioCompressor(1, algo) m := NewChannelManager(log, metrics.NoopMetrics, cfg, &defaultTestRollupConfig) // Channel Manager state should be empty by default @@ -189,11 +197,11 @@ func ChannelManager_Clear(t *testing.T, batchType uint) { require.Empty(m.txChannels) } -func ChannelManager_TxResend(t *testing.T, batchType uint) { +func ChannelManager_TxResend(t *testing.T, batchType uint, algo derive.CompressionAlgo) { require := require.New(t) rng := rand.New(rand.NewSource(time.Now().UnixNano())) log := testlog.Logger(t, log.LevelError) - cfg := channelManagerTestConfig(120_000, batchType) + cfg := channelManagerTestConfig(120_000, batchType, algo) cfg.CompressorConfig.TargetOutputSize = 1 // full on first block m := NewChannelManager(log, metrics.NoopMetrics, cfg, &defaultTestRollupConfig) m.Clear(eth.BlockID{}) @@ -228,12 +236,12 @@ func ChannelManager_TxResend(t *testing.T, batchType uint) { // ChannelManagerCloseBeforeFirstUse ensures that the channel manager // will not produce any frames if closed immediately. -func ChannelManagerCloseBeforeFirstUse(t *testing.T, batchType uint) { +func ChannelManagerCloseBeforeFirstUse(t *testing.T, batchType uint, algo derive.CompressionAlgo) { require := require.New(t) rng := rand.New(rand.NewSource(time.Now().UnixNano())) log := testlog.Logger(t, log.LevelCrit) m := NewChannelManager(log, metrics.NoopMetrics, - channelManagerTestConfig(10000, batchType), + channelManagerTestConfig(10000, batchType, algo), &defaultTestRollupConfig, ) m.Clear(eth.BlockID{}) @@ -252,10 +260,10 @@ func ChannelManagerCloseBeforeFirstUse(t *testing.T, batchType uint) { // ChannelManagerCloseNoPendingChannel ensures that the channel manager // can gracefully close with no pending channels, and will not emit any new // channel frames. -func ChannelManagerCloseNoPendingChannel(t *testing.T, batchType uint) { +func ChannelManagerCloseNoPendingChannel(t *testing.T, batchType uint, algo derive.CompressionAlgo) { require := require.New(t) log := testlog.Logger(t, log.LevelCrit) - cfg := channelManagerTestConfig(10000, batchType) + cfg := channelManagerTestConfig(10000, batchType, algo) cfg.CompressorConfig.TargetOutputSize = 1 // full on first block cfg.ChannelTimeout = 1000 m := NewChannelManager(log, metrics.NoopMetrics, cfg, &defaultTestRollupConfig) @@ -286,13 +294,13 @@ func ChannelManagerCloseNoPendingChannel(t *testing.T, batchType uint) { // ChannelManagerCloseNoPendingChannel ensures that the channel manager // can gracefully close with a pending channel, and will not produce any // new channel frames after this point. -func ChannelManagerClosePendingChannel(t *testing.T, batchType uint) { +func ChannelManagerClosePendingChannel(t *testing.T, batchType uint, algo derive.CompressionAlgo) { require := require.New(t) // The number of batch txs depends on compression of the random data, hence the static test RNG seed. // Example of different RNG seed that creates less than 2 frames: 1698700588902821588 rng := rand.New(rand.NewSource(123)) log := testlog.Logger(t, log.LevelError) - cfg := channelManagerTestConfig(10_000, batchType) + cfg := channelManagerTestConfig(10_000, batchType, algo) cfg.ChannelTimeout = 1000 m := NewChannelManager(log, metrics.NoopMetrics, cfg, &defaultTestRollupConfig) m.Clear(eth.BlockID{}) @@ -335,6 +343,14 @@ func ChannelManagerClosePendingChannel(t *testing.T, batchType uint) { // Couldn't get the test to work even with modifying NonCompressor // to flush half-way through writing to the compressor... func TestChannelManager_Close_PartiallyPendingChannel(t *testing.T) { + for _, algo := range derive.CompressionAlgoTypes { + t.Run("ChannelManager_Close_PartiallyPendingChannel"+algo.String(), func(t *testing.T) { + ChannelManager_Close_PartiallyPendingChannel(t, algo) + }) + } +} + +func ChannelManager_Close_PartiallyPendingChannel(t *testing.T, algo derive.CompressionAlgo) { require := require.New(t) // The number of batch txs depends on compression of the random data, hence the static test RNG seed. // Example of different RNG seed that creates less than 2 frames: 1698700588902821588 @@ -344,8 +360,11 @@ func TestChannelManager_Close_PartiallyPendingChannel(t *testing.T) { MaxFrameSize: 2200, ChannelTimeout: 1000, TargetNumFrames: 100, + CompressorConfig: compressor.Config{ + CompressionAlgo: algo, + }, } - cfg.InitNoneCompressor(derive.Brotli10) + cfg.InitNoneCompressor(algo) m := NewChannelManager(log, metrics.NoopMetrics, cfg, &defaultTestRollupConfig) m.Clear(eth.BlockID{}) @@ -391,13 +410,13 @@ func TestChannelManager_Close_PartiallyPendingChannel(t *testing.T) { // ChannelManagerCloseAllTxsFailed ensures that the channel manager // can gracefully close after producing transaction frames if none of these // have successfully landed on chain. -func ChannelManagerCloseAllTxsFailed(t *testing.T, batchType uint) { +func ChannelManagerCloseAllTxsFailed(t *testing.T, batchType uint, algo derive.CompressionAlgo) { require := require.New(t) rng := rand.New(rand.NewSource(1357)) log := testlog.Logger(t, log.LevelCrit) - cfg := channelManagerTestConfig(100, batchType) + cfg := channelManagerTestConfig(100, batchType, algo) cfg.TargetNumFrames = 1000 - cfg.InitNoneCompressor(derive.Brotli10) + cfg.InitNoneCompressor(algo) m := NewChannelManager(log, metrics.NoopMetrics, cfg, &defaultTestRollupConfig) m.Clear(eth.BlockID{}) @@ -441,11 +460,19 @@ func ChannelManagerCloseAllTxsFailed(t *testing.T, batchType uint) { } func TestChannelManager_ChannelCreation(t *testing.T) { + for _, algo := range derive.CompressionAlgoTypes { + t.Run("ChannelManager_ChannelCreation"+algo.String(), func(t *testing.T) { + ChannelManager_ChannelCreation(t, algo) + }) + } +} + +func ChannelManager_ChannelCreation(t *testing.T, algo derive.CompressionAlgo) { l := testlog.Logger(t, log.LevelCrit) const maxChannelDuration = 15 - cfg := channelManagerTestConfig(1000, derive.SpanBatchType) + cfg := channelManagerTestConfig(1000, derive.SpanBatchType, algo) cfg.MaxChannelDuration = maxChannelDuration - cfg.InitNoneCompressor(derive.Brotli10) + cfg.InitNoneCompressor(algo) for _, tt := range []struct { name string diff --git a/op-batcher/batcher/channel_test.go b/op-batcher/batcher/channel_test.go index 71903eb370ca..0c366226a119 100644 --- a/op-batcher/batcher/channel_test.go +++ b/op-batcher/batcher/channel_test.go @@ -4,6 +4,7 @@ import ( "io" "testing" + "github.com/ethereum-optimism/optimism/op-batcher/compressor" "github.com/ethereum-optimism/optimism/op-batcher/metrics" "github.com/ethereum-optimism/optimism/op-node/rollup" "github.com/ethereum-optimism/optimism/op-node/rollup/derive" @@ -22,13 +23,39 @@ func zeroFrameTxID(fn uint16) txID { return txID{frameID{frameNumber: fn}} } +func TestChannel(t *testing.T) { + tests := []struct { + name string + f func(t *testing.T, algo derive.CompressionAlgo) + }{ + {"ChannelTimeout", ChannelTimeout}, + {"ChannelManager_NextTxData", ChannelManager_NextTxData}, + {"Channel_NextTxData_singleFrameTx", Channel_NextTxData_singleFrameTx}, + {"Channel_NextTxData_multiFrameTx", Channel_NextTxData_multiFrameTx}, + {"ChannelTxConfirmed", ChannelTxConfirmed}, + {"ChannelTxFailed", ChannelTxFailed}, + } + + for _, test := range tests { + test := test + for _, algo := range derive.CompressionAlgoTypes { + t.Run(test.name+"_"+algo.String(), func(t *testing.T) { + test.f(t, algo) + }) + } + } +} + // TestChannelTimeout tests that the channel manager // correctly identifies when a pending channel is timed out. -func TestChannelTimeout(t *testing.T) { +func ChannelTimeout(t *testing.T, algo derive.CompressionAlgo) { // Create a new channel manager with a ChannelTimeout log := testlog.Logger(t, log.LevelCrit) m := NewChannelManager(log, metrics.NoopMetrics, ChannelConfig{ ChannelTimeout: 100, + CompressorConfig: compressor.Config{ + CompressionAlgo: algo, + }, }, &rollup.Config{}) m.Clear(eth.BlockID{}) @@ -69,9 +96,11 @@ func TestChannelTimeout(t *testing.T) { } // TestChannelManager_NextTxData tests the nextTxData function. -func TestChannelManager_NextTxData(t *testing.T) { +func ChannelManager_NextTxData(t *testing.T, algo derive.CompressionAlgo) { log := testlog.Logger(t, log.LevelCrit) - m := NewChannelManager(log, metrics.NoopMetrics, ChannelConfig{}, &rollup.Config{}) + m := NewChannelManager(log, metrics.NoopMetrics, ChannelConfig{CompressorConfig: compressor.Config{ + CompressionAlgo: algo, + }}, &rollup.Config{}) m.Clear(eth.BlockID{}) // Nil pending channel should return EOF @@ -111,13 +140,16 @@ func TestChannelManager_NextTxData(t *testing.T) { require.Equal(t, expectedTxData, channel.pendingTransactions[expectedChannelID]) } -func TestChannel_NextTxData_singleFrameTx(t *testing.T) { +func Channel_NextTxData_singleFrameTx(t *testing.T, algo derive.CompressionAlgo) { require := require.New(t) const n = 6 lgr := testlog.Logger(t, log.LevelWarn) ch, err := newChannel(lgr, metrics.NoopMetrics, ChannelConfig{ MultiFrameTxs: false, TargetNumFrames: n, + CompressorConfig: compressor.Config{ + CompressionAlgo: algo, + }, }, &rollup.Config{}, latestL1BlockOrigin) require.NoError(err) chID := ch.ID() @@ -149,13 +181,16 @@ func TestChannel_NextTxData_singleFrameTx(t *testing.T) { require.False(ch.HasTxData()) } -func TestChannel_NextTxData_multiFrameTx(t *testing.T) { +func Channel_NextTxData_multiFrameTx(t *testing.T, algo derive.CompressionAlgo) { require := require.New(t) const n = 6 lgr := testlog.Logger(t, log.LevelWarn) ch, err := newChannel(lgr, metrics.NoopMetrics, ChannelConfig{ MultiFrameTxs: true, TargetNumFrames: n, + CompressorConfig: compressor.Config{ + CompressionAlgo: algo, + }, }, &rollup.Config{}, latestL1BlockOrigin) require.NoError(err) chID := ch.ID() @@ -194,7 +229,7 @@ func makeMockFrameDatas(id derive.ChannelID, n int) []frameData { } // TestChannelTxConfirmed checks the [ChannelManager.TxConfirmed] function. -func TestChannelTxConfirmed(t *testing.T) { +func ChannelTxConfirmed(t *testing.T, algo derive.CompressionAlgo) { // Create a channel manager log := testlog.Logger(t, log.LevelCrit) m := NewChannelManager(log, metrics.NoopMetrics, ChannelConfig{ @@ -202,6 +237,9 @@ func TestChannelTxConfirmed(t *testing.T) { // channels on confirmation. This would result in [TxConfirmed] // clearing confirmed transactions, and resetting the pendingChannels map ChannelTimeout: 10, + CompressorConfig: compressor.Config{ + CompressionAlgo: algo, + }, }, &rollup.Config{}) m.Clear(eth.BlockID{}) @@ -248,10 +286,12 @@ func TestChannelTxConfirmed(t *testing.T) { } // TestChannelTxFailed checks the [ChannelManager.TxFailed] function. -func TestChannelTxFailed(t *testing.T) { +func ChannelTxFailed(t *testing.T, algo derive.CompressionAlgo) { // Create a channel manager log := testlog.Logger(t, log.LevelCrit) - m := NewChannelManager(log, metrics.NoopMetrics, ChannelConfig{}, &rollup.Config{}) + m := NewChannelManager(log, metrics.NoopMetrics, ChannelConfig{CompressorConfig: compressor.Config{ + CompressionAlgo: algo, + }}, &rollup.Config{}) m.Clear(eth.BlockID{}) // Let's add a valid pending transaction to the channel diff --git a/op-batcher/batcher/config_test.go b/op-batcher/batcher/config_test.go index ee124644c8fc..bb454158e1fb 100644 --- a/op-batcher/batcher/config_test.go +++ b/op-batcher/batcher/config_test.go @@ -16,7 +16,7 @@ import ( "github.com/stretchr/testify/require" ) -func validBatcherConfig() batcher.CLIConfig { +func validBatcherConfig(algo derive.CompressionAlgo) batcher.CLIConfig { return batcher.CLIConfig{ L1EthRpc: "fake", L2EthRpc: "fake", @@ -36,13 +36,15 @@ func validBatcherConfig() batcher.CLIConfig { MetricsConfig: metrics.DefaultCLIConfig(), PprofConfig: oppprof.DefaultCLIConfig(), RPC: rpc.DefaultCLIConfig(), - CompressionAlgo: derive.Brotli10, + CompressionAlgo: algo, } } func TestValidBatcherConfig(t *testing.T) { - cfg := validBatcherConfig() - require.NoError(t, cfg.Check(), "valid config should pass the check function") + for _, algo := range derive.CompressionAlgoTypes { + cfg := validBatcherConfig(algo) + require.NoError(t, cfg.Check(), "valid config should pass the check function") + } } func TestBatcherConfig(t *testing.T) { @@ -116,10 +118,12 @@ func TestBatcherConfig(t *testing.T) { for _, test := range tests { tc := test - t.Run(tc.name, func(t *testing.T) { - cfg := validBatcherConfig() - tc.override(&cfg) - require.ErrorContains(t, cfg.Check(), tc.errString) - }) + for _, algo := range derive.CompressionAlgoTypes { + t.Run(tc.name+"_"+algo.String(), func(t *testing.T) { + cfg := validBatcherConfig(algo) + tc.override(&cfg) + require.ErrorContains(t, cfg.Check(), tc.errString) + }) + } } } diff --git a/op-batcher/compressor/config.go b/op-batcher/compressor/config.go index 4fca00040792..3bf9534c91b4 100644 --- a/op-batcher/compressor/config.go +++ b/op-batcher/compressor/config.go @@ -19,9 +19,6 @@ type Config struct { // Type of compression algorithm to use. Must be one of [zlib, brotli] CompressionAlgo derive.CompressionAlgo - - // Levels of compression to use. E.g. 1-11 for brotli, 0-9 for zlib - CompressLevel int } func (c Config) NewCompressor() (derive.Compressor, error) { diff --git a/op-batcher/compressor/ratio_compressor.go b/op-batcher/compressor/ratio_compressor.go index 6844062e44e8..c4440acbeeb9 100644 --- a/op-batcher/compressor/ratio_compressor.go +++ b/op-batcher/compressor/ratio_compressor.go @@ -3,16 +3,84 @@ package compressor import ( "bytes" "compress/zlib" + "fmt" + "io" + "github.com/andybalholm/brotli" "github.com/ethereum-optimism/optimism/op-node/rollup/derive" ) +type CompressorWriter interface { + Write([]byte) (int, error) + Flush() error + Close() error + Reset(io.Writer) +} + +type AlgoCompressor struct { + compressed *bytes.Buffer + writer CompressorWriter + algo derive.CompressionAlgo +} + +func NewAlgoCompressor(algo derive.CompressionAlgo) (*AlgoCompressor, error) { + var writer CompressorWriter + var err error + compressed := &bytes.Buffer{} + if algo == derive.Zlib { + writer, err = zlib.NewWriterLevel(compressed, zlib.BestCompression) + } else if algo.IsBrotli() { + compressed.WriteByte(derive.ChannelVersionBrotli) + writer = brotli.NewWriterLevel(compressed, derive.GetBrotliLevel(algo)) + } else { + return nil, fmt.Errorf("unsupported compression algorithm: %s", algo) + } + + if err != nil { + return nil, err + } + + return &AlgoCompressor{ + writer: writer, + compressed: compressed, + algo: algo, + }, nil +} + +func (ac *AlgoCompressor) Write(data []byte) (int, error) { + return ac.writer.Write(data) +} + +func (ac *AlgoCompressor) Flush() error { + return ac.writer.Flush() +} + +func (ac *AlgoCompressor) Close() error { + return ac.writer.Close() +} + +func (ac *AlgoCompressor) Reset() { + ac.compressed.Reset() + if ac.algo.IsBrotli() { + // always add channal version for brotli + ac.compressed.WriteByte(derive.ChannelVersionBrotli) + } + ac.writer.Reset(ac.compressed) +} + +func (ac *AlgoCompressor) Len() int { + return ac.compressed.Len() +} + +func (ac *AlgoCompressor) Read(p []byte) (int, error) { + return ac.compressed.Read(p) +} + type RatioCompressor struct { config Config inputBytes int - buf bytes.Buffer - compress *zlib.Writer + compressor *AlgoCompressor } // NewRatioCompressor creates a new derive.Compressor implementation that uses the target @@ -25,11 +93,11 @@ func NewRatioCompressor(config Config) (derive.Compressor, error) { config: config, } - compress, err := zlib.NewWriterLevel(&c.buf, zlib.BestCompression) + compressor, err := NewAlgoCompressor(config.CompressionAlgo) if err != nil { return nil, err } - c.compress = compress + c.compressor = compressor return c, nil } @@ -39,29 +107,28 @@ func (t *RatioCompressor) Write(p []byte) (int, error) { return 0, err } t.inputBytes += len(p) - return t.compress.Write(p) + return t.compressor.Write(p) } func (t *RatioCompressor) Close() error { - return t.compress.Close() + return t.compressor.Close() } func (t *RatioCompressor) Read(p []byte) (int, error) { - return t.buf.Read(p) + return t.compressor.Read(p) } func (t *RatioCompressor) Reset() { - t.buf.Reset() - t.compress.Reset(&t.buf) + t.compressor.Reset() t.inputBytes = 0 } func (t *RatioCompressor) Len() int { - return t.buf.Len() + return t.compressor.Len() } func (t *RatioCompressor) Flush() error { - return t.compress.Flush() + return t.compressor.Flush() } func (t *RatioCompressor) FullErr() error { diff --git a/op-batcher/compressor/ratio_compressor_test.go b/op-batcher/compressor/ratio_compressor_test.go index 27e377a234d6..7b827a000668 100644 --- a/op-batcher/compressor/ratio_compressor_test.go +++ b/op-batcher/compressor/ratio_compressor_test.go @@ -6,6 +6,7 @@ import ( "testing" "github.com/ethereum-optimism/optimism/op-batcher/compressor" + "github.com/ethereum-optimism/optimism/op-node/rollup/derive" "github.com/stretchr/testify/require" ) @@ -58,18 +59,21 @@ func TestChannelConfig_InputThreshold(t *testing.T) { // Validate each test case for i, tt := range tests { - t.Run(fmt.Sprintf("test-%d", i), func(t *testing.T) { - comp, err := compressor.NewRatioCompressor(compressor.Config{ - TargetOutputSize: tt.targetOutputSize, - ApproxComprRatio: tt.approxComprRatio, + for _, algo := range derive.CompressionAlgoTypes { + t.Run(fmt.Sprintf("test-%d", i), func(t *testing.T) { + comp, err := compressor.NewRatioCompressor(compressor.Config{ + TargetOutputSize: tt.targetOutputSize, + ApproxComprRatio: tt.approxComprRatio, + CompressionAlgo: algo, + }) + require.NoError(t, err) + got := comp.(*compressor.RatioCompressor).InputThreshold() + if tt.assertion != nil { + tt.assertion(got) + } else { + require.Equal(t, tt.expInputThreshold, got) + } }) - require.NoError(t, err) - got := comp.(*compressor.RatioCompressor).InputThreshold() - if tt.assertion != nil { - tt.assertion(got) - } else { - require.Equal(t, tt.expInputThreshold, got) - } - }) + } } } diff --git a/op-batcher/compressor/shadow_compressor.go b/op-batcher/compressor/shadow_compressor.go index 7e6172460392..f79f8f621056 100644 --- a/op-batcher/compressor/shadow_compressor.go +++ b/op-batcher/compressor/shadow_compressor.go @@ -1,9 +1,6 @@ package compressor import ( - "bytes" - "compress/zlib" - "github.com/ethereum-optimism/optimism/op-node/rollup/derive" ) @@ -21,11 +18,8 @@ const ( type ShadowCompressor struct { config Config - buf bytes.Buffer - compress *zlib.Writer - - shadowBuf bytes.Buffer - shadowCompress *zlib.Writer + compressor *AlgoCompressor + shadowCompressor *AlgoCompressor fullErr error @@ -45,11 +39,11 @@ func NewShadowCompressor(config Config) (derive.Compressor, error) { } var err error - c.compress, err = zlib.NewWriterLevel(&c.buf, zlib.BestCompression) + c.compressor, err = NewAlgoCompressor(config.CompressionAlgo) if err != nil { return nil, err } - c.shadowCompress, err = zlib.NewWriterLevel(&c.shadowBuf, zlib.BestCompression) + c.shadowCompressor, err = NewAlgoCompressor(config.CompressionAlgo) if err != nil { return nil, err } @@ -62,7 +56,7 @@ func (t *ShadowCompressor) Write(p []byte) (int, error) { if t.fullErr != nil { return 0, t.fullErr } - _, err := t.shadowCompress.Write(p) + _, err := t.shadowCompressor.Write(p) if err != nil { return 0, err } @@ -71,10 +65,10 @@ func (t *ShadowCompressor) Write(p []byte) (int, error) { // Do not flush the buffer unless there's some chance we will be over the size limit. // This reduces CPU but more importantly it makes the shadow compression ratio more // closely reflect the ultimate compression ratio. - if err = t.shadowCompress.Flush(); err != nil { + if err = t.shadowCompressor.Flush(); err != nil { return 0, err } - newBound = uint64(t.shadowBuf.Len()) + CloseOverheadZlib + newBound = uint64(t.shadowCompressor.Len()) + CloseOverheadZlib if newBound > t.config.TargetOutputSize { t.fullErr = derive.ErrCompressorFull if t.Len() > 0 { @@ -85,32 +79,30 @@ func (t *ShadowCompressor) Write(p []byte) (int, error) { } } t.bound = newBound - return t.compress.Write(p) + return t.compressor.Write(p) } func (t *ShadowCompressor) Close() error { - return t.compress.Close() + return t.compressor.Close() } func (t *ShadowCompressor) Read(p []byte) (int, error) { - return t.buf.Read(p) + return t.compressor.Read(p) } func (t *ShadowCompressor) Reset() { - t.buf.Reset() - t.compress.Reset(&t.buf) - t.shadowBuf.Reset() - t.shadowCompress.Reset(&t.shadowBuf) + t.compressor.Reset() + t.shadowCompressor.Reset() t.fullErr = nil t.bound = safeCompressionOverhead } func (t *ShadowCompressor) Len() int { - return t.buf.Len() + return t.compressor.Len() } func (t *ShadowCompressor) Flush() error { - return t.compress.Flush() + return t.compressor.Flush() } func (t *ShadowCompressor) FullErr() error { diff --git a/op-batcher/compressor/shadow_compressor_test.go b/op-batcher/compressor/shadow_compressor_test.go index 1b300bcc3321..c29daeaad322 100644 --- a/op-batcher/compressor/shadow_compressor_test.go +++ b/op-batcher/compressor/shadow_compressor_test.go @@ -63,6 +63,7 @@ func TestShadowCompressor(t *testing.T) { sc, err := NewShadowCompressor(Config{ TargetOutputSize: test.targetOutputSize, + CompressionAlgo: derive.Zlib, }) require.NoError(t, err) @@ -115,6 +116,7 @@ func TestBoundInaccurateForLargeRandomData(t *testing.T) { sc, err := NewShadowCompressor(Config{ TargetOutputSize: sizeLimit + 100, + CompressionAlgo: derive.Zlib, }) require.NoError(t, err) diff --git a/op-batcher/flags/flags.go b/op-batcher/flags/flags.go index 0dff2297699d..da7dc1e9ed96 100644 --- a/op-batcher/flags/flags.go +++ b/op-batcher/flags/flags.go @@ -103,7 +103,7 @@ var ( CompressionAlgoFlag = &cli.GenericFlag{ Name: "compression-algo", Usage: "The compression algorithm to use. Valid options: " + openum.EnumString(derive.CompressionAlgoTypes), - EnvVars: prefixEnvVars("COMPRESSION_ALGO_TYPE"), + EnvVars: prefixEnvVars("COMPRESSION_ALGO"), Value: func() *derive.CompressionAlgo { out := derive.Zlib return &out diff --git a/op-node/rollup/derive/channel.go b/op-node/rollup/derive/channel.go index 11b6eac8fd01..8490817407c9 100644 --- a/op-node/rollup/derive/channel.go +++ b/op-node/rollup/derive/channel.go @@ -8,7 +8,6 @@ import ( "io" "github.com/andybalholm/brotli" - "github.com/ethereum-optimism/optimism/op-service/eth" "github.com/ethereum/go-ethereum/rlp" ) @@ -162,6 +161,9 @@ func BatchReader(r io.Reader) (func() (*BatchData, error), error) { return nil, err } + fmt.Println("buf reader size start") + fmt.Println(bufReader.Size()) + fmt.Println(compressionType[0]) var reader func(io.Reader) (io.Reader, error) @@ -175,7 +177,6 @@ func BatchReader(r io.Reader) (func() (*BatchData, error), error) { // discard the first byte _, err := bufReader.Discard(1) if err != nil { - fmt.Println("CHECK HERE") return nil, err } reader = func(r io.Reader) (io.Reader, error) { @@ -185,10 +186,12 @@ func BatchReader(r io.Reader) (func() (*BatchData, error), error) { return nil, fmt.Errorf("cannot distinguish the compression algo used given type byte %v", compressionType[0]) } + fmt.Println("buf reader size") + fmt.Println(bufReader.Size()) // Setup decompressor stage + RLP reader zr, err := reader(bufReader) + // zr, err := zlib.NewReader(r) if err != nil { - fmt.Println("FAILED READER") return nil, err } rlpReader := rlp.NewStream(zr, MaxRLPBytesPerChannel) @@ -196,7 +199,8 @@ func BatchReader(r io.Reader) (func() (*BatchData, error), error) { return func() (*BatchData, error) { var batchData BatchData if err = rlpReader.Decode(&batchData); err != nil { - fmt.Println("DECODE ERROR") + fmt.Println(batchData) + fmt.Println("error decoding batch data") return nil, err } return &batchData, nil diff --git a/op-node/rollup/derive/channel_test.go b/op-node/rollup/derive/channel_test.go index fdd6e4065e25..4a4cfed28e2a 100644 --- a/op-node/rollup/derive/channel_test.go +++ b/op-node/rollup/derive/channel_test.go @@ -4,6 +4,8 @@ import ( "testing" "github.com/ethereum-optimism/optimism/op-service/eth" + + //"github.com/ethereum/go-ethereum/rlp" "github.com/stretchr/testify/require" ) @@ -99,3 +101,98 @@ func TestFrameValidity(t *testing.T) { t.Run(tc.name, tc.Run) } } + +// func TestBatchReader(t *testing.T) { +// // Get batch data +// rng := rand.New(rand.NewSource(0x543331)) +// singularBatch := RandomSingularBatch(rng, 5000, big.NewInt(333)) +// batchDataInput := NewBatchData(singularBatch) + +// encodedBatch := &bytes.Buffer{} +// // Get the encoded data of the batch data +// batchDataInput.encodeTyped(encodedBatch) + +// fmt.Println(encodedBatch.Bytes()[:10]) + +// var testCases = []struct { +// name string +// algo func(buf *bytes.Buffer) () +// }{{ +// name: "zlib", +// algo: func(buf *bytes.Buffer) { +// writer := zlib.NewWriter(buf) +// writer.Write(encodedBatch.Bytes()) +// }, +// }, +// // }, { +// // name: "brotli10", +// // algo: func(buf *bytes.Buffer) { +// // buf.WriteByte(ChannelVersionBrotli) +// // writer := brotli.NewWriterLevel(buf, 10) +// // writer.Write(encodedBatch.Bytes()) +// // }, +// // }, { +// // name: "zstd", +// // algo: func(buf *bytes.Buffer) { +// // writer := zstd.NewWriter(buf) +// // writer.Write(encodedBatch.Bytes()) +// // }, +// //}} +// } + +// for _, tc := range testCases { +// compressed := bytes.NewBuffer([]byte{}) +// tc := tc +// t.Run(tc.name, func(t *testing.T) { +// tc.algo(compressed) +// fmt.Println("compressed len") +// fmt.Println(compressed.Len()) +// fmt.Println(compressed.Bytes()[:10]) + +// //r, err := zlib.NewReader(bytes.NewReader(compressed.Bytes())) + +// enc, err := batchDataInput.MarshalBinary() +// buf := &bytes.Buffer{} +// writer := zlib.NewWriter(buf) +// writer.Write(enc) +// require.NoError(t, err) + +// reader, err := zlib.NewReader(buf) +// require.NoError(t, err) + +// rlpReader := rlp.NewStream(reader, MaxRLPBytesPerChannel) + +// // require.NoError(t, err) +// var dec BatchData +// // err = dec.UnmarshalBinary(encoded) +// err = rlpReader.Decode(&dec) +// // require.NoError(t, err) +// //require.Equal(t, batchDataInput, &dec) + +// // read from r the compressed data then rlp decode this +// // uncompressed, err := io.ReadAll(r) +// // require.NoError(t, err) +// // fmt.Println("uncompressed len") +// // fmt.Println(len(uncompressed)) + +// // var batchData BatchData +// // rlp.Decode(bytes.NewReader(uncompressed), &batchData) + +// // require.Equal(t, batchDataInput, batchData) + +// // reader, err := BatchReader(bytes.NewReader(compressed.Bytes())) +// // if tc.name == "zstd" { +// // require.NotNil(t, err) +// // return +// // } +// // require.Nil(t, err) + +// // // read the batch data +// // batchData, err := reader() +// // require.Nil(t, err) +// // require.NotNil(t, batchData) +// // require.Equal(t, batchDataInput, batchData) +// }) +// } + +// } diff --git a/op-node/rollup/derive/span_channel_compressor_test.go b/op-node/rollup/derive/span_channel_compressor_test.go new file mode 100644 index 000000000000..36d8fe8ac0fb --- /dev/null +++ b/op-node/rollup/derive/span_channel_compressor_test.go @@ -0,0 +1,60 @@ +package derive + +import ( + "math/rand" + "testing" + + "github.com/stretchr/testify/require" +) + +var r = rand.New(rand.NewSource(99)) + +func randomBytes(length int) []byte { + b := make([]byte, length) + _, err := r.Read(b) + // Rand.Read always returns nil error + if err != nil { + panic(err) + } + return b +} + +func TestSpanChannelCompressor(t *testing.T) { + testCases := []struct { + name string + algo CompressionAlgo + expectedCompressedSize int + }{{ + name: "zlib", + algo: Zlib, + expectedCompressedSize: 0, + }, + { + name: "brotli10", + algo: Brotli10, + expectedCompressedSize: 1, + }, + { + name: "zstd", + algo: CompressionAlgo("zstd"), + expectedCompressedSize: 0, + }} + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + scc, err := NewSpanChannelCompressor(tc.algo) + if tc.name == "zstd" { + require.Error(t, err) + return + } + require.NoError(t, err) + require.Equal(t, tc.expectedCompressedSize, scc.GetCompressedLen()) + + scc.Write(randomBytes(10000000)) + require.Greater(t, scc.GetCompressedLen(), tc.expectedCompressedSize) + + scc.Reset() + require.Equal(t, tc.expectedCompressedSize, scc.GetCompressedLen()) + }) + } +} diff --git a/op-node/rollup/derive/types.go b/op-node/rollup/derive/types.go index dbec811b3d38..97abe5ed965d 100644 --- a/op-node/rollup/derive/types.go +++ b/op-node/rollup/derive/types.go @@ -10,7 +10,6 @@ type CompressionAlgo string const ( // compression algo types Zlib CompressionAlgo = "zlib" - Brotli CompressionAlgo = Brotli10 // default brotli 10 Brotli9 CompressionAlgo = "brotli-9" Brotli10 CompressionAlgo = "brotli-10" Brotli11 CompressionAlgo = "brotli-11" @@ -18,7 +17,6 @@ const ( var CompressionAlgoTypes = []CompressionAlgo{ Zlib, - Brotli, Brotli9, Brotli10, Brotli11, @@ -26,29 +24,24 @@ var CompressionAlgoTypes = []CompressionAlgo{ var brotliRegexp = regexp.MustCompile(`^brotli-(9|10|11)$`) -func (kind CompressionAlgo) String() string { - return string(kind) +func (algo CompressionAlgo) String() string { + return string(algo) } -func (kind *CompressionAlgo) Set(value string) error { +func (algo *CompressionAlgo) Set(value string) error { if !ValidCompressionAlgoType(CompressionAlgo(value)) { return fmt.Errorf("unknown compression algo type: %q", value) } - *kind = CompressionAlgo(value) + *algo = CompressionAlgo(value) return nil } -func (kind *CompressionAlgo) Clone() any { - cpy := *kind - return &cpy +func (algo *CompressionAlgo) IsBrotli() bool { + return brotliRegexp.MatchString(algo.String()) } -func (kind *CompressionAlgo) IsBrotli() bool { - return brotliRegexp.MatchString(kind.String()) -} - -func GetBrotliLevel(kind CompressionAlgo) int { - switch kind { +func GetBrotliLevel(algo CompressionAlgo) int { + switch algo { case Brotli9: return 9 case Brotli10: diff --git a/op-node/rollup/derive/types_test.go b/op-node/rollup/derive/types_test.go new file mode 100644 index 000000000000..7ab49d7ad204 --- /dev/null +++ b/op-node/rollup/derive/types_test.go @@ -0,0 +1,52 @@ +package derive + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +func TestIsBrotli(t *testing.T) { + testCases := []struct { + name string + algo CompressionAlgo + expectedResult bool + isValidCompressionAlgoType bool + }{{ + name: "zlib", + algo: Zlib, + expectedResult: false, + isValidCompressionAlgoType: true, + }, + { + name: "brotli-9", + algo: Brotli9, + expectedResult: true, + isValidCompressionAlgoType: true, + }, + { + name: "brotli-10", + algo: Brotli10, + expectedResult: true, + isValidCompressionAlgoType: true, + }, + { + name: "brotli-11", + algo: Brotli11, + expectedResult: true, + isValidCompressionAlgoType: true, + }, + { + name: "invalid", + algo: CompressionAlgo("invalid"), + expectedResult: false, + isValidCompressionAlgoType: false, + }} + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + require.Equal(t, tc.expectedResult, tc.algo.IsBrotli()) + require.Equal(t, tc.isValidCompressionAlgoType, ValidCompressionAlgoType(tc.algo)) + }) + } +} From 633aae24709628179e1dcd5071084dcdba5b495f Mon Sep 17 00:00:00 2001 From: Cody Wang Date: Sun, 5 May 2024 20:12:08 +0100 Subject: [PATCH 14/38] fix --- op-batcher/compressor/ratio_compressor.go | 76 +-------- op-batcher/compressor/shadow_compressor.go | 8 +- op-node/rollup/derive/channel.go | 10 -- ...el_compressor.go => channel_compressor.go} | 40 ++--- ...sor_test.go => channel_compressor_test.go} | 8 +- op-node/rollup/derive/channel_out_test.go | 12 +- op-node/rollup/derive/channel_test.go | 151 +++++++----------- op-node/rollup/derive/span_channel_out.go | 10 +- 8 files changed, 102 insertions(+), 213 deletions(-) rename op-node/rollup/derive/{span_channel_compressor.go => channel_compressor.go} (54%) rename op-node/rollup/derive/{span_channel_compressor_test.go => channel_compressor_test.go} (79%) diff --git a/op-batcher/compressor/ratio_compressor.go b/op-batcher/compressor/ratio_compressor.go index c4440acbeeb9..bd174d61eca7 100644 --- a/op-batcher/compressor/ratio_compressor.go +++ b/op-batcher/compressor/ratio_compressor.go @@ -1,86 +1,14 @@ package compressor import ( - "bytes" - "compress/zlib" - "fmt" - "io" - - "github.com/andybalholm/brotli" "github.com/ethereum-optimism/optimism/op-node/rollup/derive" ) -type CompressorWriter interface { - Write([]byte) (int, error) - Flush() error - Close() error - Reset(io.Writer) -} - -type AlgoCompressor struct { - compressed *bytes.Buffer - writer CompressorWriter - algo derive.CompressionAlgo -} - -func NewAlgoCompressor(algo derive.CompressionAlgo) (*AlgoCompressor, error) { - var writer CompressorWriter - var err error - compressed := &bytes.Buffer{} - if algo == derive.Zlib { - writer, err = zlib.NewWriterLevel(compressed, zlib.BestCompression) - } else if algo.IsBrotli() { - compressed.WriteByte(derive.ChannelVersionBrotli) - writer = brotli.NewWriterLevel(compressed, derive.GetBrotliLevel(algo)) - } else { - return nil, fmt.Errorf("unsupported compression algorithm: %s", algo) - } - - if err != nil { - return nil, err - } - - return &AlgoCompressor{ - writer: writer, - compressed: compressed, - algo: algo, - }, nil -} - -func (ac *AlgoCompressor) Write(data []byte) (int, error) { - return ac.writer.Write(data) -} - -func (ac *AlgoCompressor) Flush() error { - return ac.writer.Flush() -} - -func (ac *AlgoCompressor) Close() error { - return ac.writer.Close() -} - -func (ac *AlgoCompressor) Reset() { - ac.compressed.Reset() - if ac.algo.IsBrotli() { - // always add channal version for brotli - ac.compressed.WriteByte(derive.ChannelVersionBrotli) - } - ac.writer.Reset(ac.compressed) -} - -func (ac *AlgoCompressor) Len() int { - return ac.compressed.Len() -} - -func (ac *AlgoCompressor) Read(p []byte) (int, error) { - return ac.compressed.Read(p) -} - type RatioCompressor struct { config Config inputBytes int - compressor *AlgoCompressor + compressor *derive.ChannelCompressor } // NewRatioCompressor creates a new derive.Compressor implementation that uses the target @@ -93,7 +21,7 @@ func NewRatioCompressor(config Config) (derive.Compressor, error) { config: config, } - compressor, err := NewAlgoCompressor(config.CompressionAlgo) + compressor, err := derive.NewChannelCompressor(config.CompressionAlgo) if err != nil { return nil, err } diff --git a/op-batcher/compressor/shadow_compressor.go b/op-batcher/compressor/shadow_compressor.go index f79f8f621056..940a782a5ca6 100644 --- a/op-batcher/compressor/shadow_compressor.go +++ b/op-batcher/compressor/shadow_compressor.go @@ -18,8 +18,8 @@ const ( type ShadowCompressor struct { config Config - compressor *AlgoCompressor - shadowCompressor *AlgoCompressor + compressor *derive.ChannelCompressor + shadowCompressor *derive.ChannelCompressor fullErr error @@ -39,11 +39,11 @@ func NewShadowCompressor(config Config) (derive.Compressor, error) { } var err error - c.compressor, err = NewAlgoCompressor(config.CompressionAlgo) + c.compressor, err = derive.NewChannelCompressor(config.CompressionAlgo) if err != nil { return nil, err } - c.shadowCompressor, err = NewAlgoCompressor(config.CompressionAlgo) + c.shadowCompressor, err = derive.NewChannelCompressor(config.CompressionAlgo) if err != nil { return nil, err } diff --git a/op-node/rollup/derive/channel.go b/op-node/rollup/derive/channel.go index 8490817407c9..c4272c3623b0 100644 --- a/op-node/rollup/derive/channel.go +++ b/op-node/rollup/derive/channel.go @@ -161,11 +161,6 @@ func BatchReader(r io.Reader) (func() (*BatchData, error), error) { return nil, err } - fmt.Println("buf reader size start") - fmt.Println(bufReader.Size()) - - fmt.Println(compressionType[0]) - var reader func(io.Reader) (io.Reader, error) // For zlib, the last 4 bits must be either 8 or 15 (both are reserved value) if compressionType[0]&0x0F == ZlibCM8 || compressionType[0]&0x0F == ZlibCM15 { @@ -186,11 +181,8 @@ func BatchReader(r io.Reader) (func() (*BatchData, error), error) { return nil, fmt.Errorf("cannot distinguish the compression algo used given type byte %v", compressionType[0]) } - fmt.Println("buf reader size") - fmt.Println(bufReader.Size()) // Setup decompressor stage + RLP reader zr, err := reader(bufReader) - // zr, err := zlib.NewReader(r) if err != nil { return nil, err } @@ -199,8 +191,6 @@ func BatchReader(r io.Reader) (func() (*BatchData, error), error) { return func() (*BatchData, error) { var batchData BatchData if err = rlpReader.Decode(&batchData); err != nil { - fmt.Println(batchData) - fmt.Println("error decoding batch data") return nil, err } return &batchData, nil diff --git a/op-node/rollup/derive/span_channel_compressor.go b/op-node/rollup/derive/channel_compressor.go similarity index 54% rename from op-node/rollup/derive/span_channel_compressor.go rename to op-node/rollup/derive/channel_compressor.go index 0edf2bc5cb31..e0f701b95c55 100644 --- a/op-node/rollup/derive/span_channel_compressor.go +++ b/op-node/rollup/derive/channel_compressor.go @@ -20,13 +20,13 @@ type CompressorWriter interface { Reset(io.Writer) } -type SpanChannelCompressor struct { +type ChannelCompressor struct { writer CompressorWriter compressionAlgo CompressionAlgo compressed *bytes.Buffer } -func NewSpanChannelCompressor(algo CompressionAlgo) (*SpanChannelCompressor, error) { +func NewChannelCompressor(algo CompressionAlgo) (*ChannelCompressor, error) { var writer CompressorWriter var err error compressed := &bytes.Buffer{} @@ -43,7 +43,7 @@ func NewSpanChannelCompressor(algo CompressionAlgo) (*SpanChannelCompressor, err return nil, err } - return &SpanChannelCompressor{ + return &ChannelCompressor{ writer: writer, compressionAlgo: algo, compressed: compressed, @@ -51,31 +51,35 @@ func NewSpanChannelCompressor(algo CompressionAlgo) (*SpanChannelCompressor, err } -func (scc *SpanChannelCompressor) Write(data []byte) (int, error) { - return scc.writer.Write(data) +func (cc *ChannelCompressor) Write(data []byte) (int, error) { + return cc.writer.Write(data) } -func (scc *SpanChannelCompressor) Flush() error { - return scc.writer.Flush() +func (cc *ChannelCompressor) Flush() error { + return cc.writer.Flush() } -func (scc *SpanChannelCompressor) Close() error { - return scc.writer.Close() +func (cc *ChannelCompressor) Close() error { + return cc.writer.Close() } -func (scc *SpanChannelCompressor) Reset() { - scc.compressed.Reset() - if scc.compressionAlgo.IsBrotli() { +func (cc *ChannelCompressor) Reset() { + cc.compressed.Reset() + if cc.compressionAlgo.IsBrotli() { // always add channal version for brotli - scc.compressed.WriteByte(ChannelVersionBrotli) + cc.compressed.WriteByte(ChannelVersionBrotli) } - scc.writer.Reset(scc.compressed) + cc.writer.Reset(cc.compressed) } -func (scc *SpanChannelCompressor) GetCompressedLen() int { - return scc.compressed.Len() +func (cc *ChannelCompressor) Len() int { + return cc.compressed.Len() } -func (scc *SpanChannelCompressor) GetCompressed() *bytes.Buffer { - return scc.compressed +func (cc *ChannelCompressor) GetCompressed() *bytes.Buffer { + return cc.compressed +} + +func (cc *ChannelCompressor) Read(p []byte) (int, error) { + return cc.compressed.Read(p) } diff --git a/op-node/rollup/derive/span_channel_compressor_test.go b/op-node/rollup/derive/channel_compressor_test.go similarity index 79% rename from op-node/rollup/derive/span_channel_compressor_test.go rename to op-node/rollup/derive/channel_compressor_test.go index 36d8fe8ac0fb..dd1f351760dc 100644 --- a/op-node/rollup/derive/span_channel_compressor_test.go +++ b/op-node/rollup/derive/channel_compressor_test.go @@ -42,19 +42,19 @@ func TestSpanChannelCompressor(t *testing.T) { for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { - scc, err := NewSpanChannelCompressor(tc.algo) + scc, err := NewChannelCompressor(tc.algo) if tc.name == "zstd" { require.Error(t, err) return } require.NoError(t, err) - require.Equal(t, tc.expectedCompressedSize, scc.GetCompressedLen()) + require.Equal(t, tc.expectedCompressedSize, scc.Len()) scc.Write(randomBytes(10000000)) - require.Greater(t, scc.GetCompressedLen(), tc.expectedCompressedSize) + require.Greater(t, scc.Len(), tc.expectedCompressedSize) scc.Reset() - require.Equal(t, tc.expectedCompressedSize, scc.GetCompressedLen()) + require.Equal(t, tc.expectedCompressedSize, scc.Len()) }) } } diff --git a/op-node/rollup/derive/channel_out_test.go b/op-node/rollup/derive/channel_out_test.go index 40346dd1e588..9c1cd6070383 100644 --- a/op-node/rollup/derive/channel_out_test.go +++ b/op-node/rollup/derive/channel_out_test.go @@ -113,7 +113,7 @@ func TestOutputFrameNoEmptyLastFrame(t *testing.T) { // depending on the channel type, determine the size of the written data if span, ok := cout.(*SpanChannelOut); ok { - written = uint64(span.compressor.GetCompressedLen()) + written = uint64(span.compressor.Len()) } else if singular, ok := cout.(*SingularChannelOut); ok { written = uint64(singular.compress.Len()) } @@ -244,7 +244,7 @@ func TestSpanChannelOutCompressionOnlyOneBatch(t *testing.T) { err := cout.AddSingularBatch(singularBatches[0], 0) // confirm compression was not skipped - require.Greater(t, cout.compressor.GetCompressedLen(), 0) + require.Greater(t, cout.compressor.Len(), 0) require.NoError(t, err) // confirm the channel is full @@ -263,14 +263,14 @@ func TestSpanChannelOutCompressionUndo(t *testing.T) { err := cout.AddSingularBatch(singularBatches[0], 0) require.NoError(t, err) // confirm that the first compression was skipped - require.Equal(t, 1, cout.compressor.GetCompressedLen()) // 1 because of brotli channel version + require.Equal(t, 1, cout.compressor.Len()) // 1 because of brotli channel version // record the RLP length to confirm it doesn't change when adding a rejected batch rlp1 := cout.activeRLP().Len() err = cout.AddSingularBatch(singularBatches[1], 0) require.ErrorIs(t, err, ErrCompressorFull) // confirm that the second compression was not skipped - require.Greater(t, cout.compressor.GetCompressedLen(), 0) + require.Greater(t, cout.compressor.Len(), 0) // confirm that the second rlp is tht same size as the first (because the second batch was not added) require.Equal(t, rlp1, cout.activeRLP().Len()) @@ -285,7 +285,7 @@ func TestSpanChannelOutClose(t *testing.T) { err := cout.AddSingularBatch(singularBatches[0], 0) require.NoError(t, err) // confirm no compression has happened yet - require.Equal(t, 1, cout.compressor.GetCompressedLen()) // 1 because of brotli channel version + require.Equal(t, 1, cout.compressor.Len()) // 1 because of brotli channel version // confirm the RLP length is less than the target rlpLen := cout.activeRLP().Len() @@ -295,6 +295,6 @@ func TestSpanChannelOutClose(t *testing.T) { require.NoError(t, cout.Close()) // confirm that the only batch was compressed, and that the RLP did not change - require.Greater(t, cout.compressor.GetCompressedLen(), 0) + require.Greater(t, cout.compressor.Len(), 0) require.Equal(t, rlpLen, cout.activeRLP().Len()) } diff --git a/op-node/rollup/derive/channel_test.go b/op-node/rollup/derive/channel_test.go index 4a4cfed28e2a..1e8426154199 100644 --- a/op-node/rollup/derive/channel_test.go +++ b/op-node/rollup/derive/channel_test.go @@ -1,11 +1,16 @@ package derive import ( + "bytes" + "compress/zlib" + "math/big" + "math/rand" "testing" + "github.com/DataDog/zstd" + "github.com/andybalholm/brotli" "github.com/ethereum-optimism/optimism/op-service/eth" - //"github.com/ethereum/go-ethereum/rlp" "github.com/stretchr/testify/require" ) @@ -102,97 +107,59 @@ func TestFrameValidity(t *testing.T) { } } -// func TestBatchReader(t *testing.T) { -// // Get batch data -// rng := rand.New(rand.NewSource(0x543331)) -// singularBatch := RandomSingularBatch(rng, 5000, big.NewInt(333)) -// batchDataInput := NewBatchData(singularBatch) - -// encodedBatch := &bytes.Buffer{} -// // Get the encoded data of the batch data -// batchDataInput.encodeTyped(encodedBatch) - -// fmt.Println(encodedBatch.Bytes()[:10]) - -// var testCases = []struct { -// name string -// algo func(buf *bytes.Buffer) () -// }{{ -// name: "zlib", -// algo: func(buf *bytes.Buffer) { -// writer := zlib.NewWriter(buf) -// writer.Write(encodedBatch.Bytes()) -// }, -// }, -// // }, { -// // name: "brotli10", -// // algo: func(buf *bytes.Buffer) { -// // buf.WriteByte(ChannelVersionBrotli) -// // writer := brotli.NewWriterLevel(buf, 10) -// // writer.Write(encodedBatch.Bytes()) -// // }, -// // }, { -// // name: "zstd", -// // algo: func(buf *bytes.Buffer) { -// // writer := zstd.NewWriter(buf) -// // writer.Write(encodedBatch.Bytes()) -// // }, -// //}} -// } - -// for _, tc := range testCases { -// compressed := bytes.NewBuffer([]byte{}) -// tc := tc -// t.Run(tc.name, func(t *testing.T) { -// tc.algo(compressed) -// fmt.Println("compressed len") -// fmt.Println(compressed.Len()) -// fmt.Println(compressed.Bytes()[:10]) - -// //r, err := zlib.NewReader(bytes.NewReader(compressed.Bytes())) - -// enc, err := batchDataInput.MarshalBinary() -// buf := &bytes.Buffer{} -// writer := zlib.NewWriter(buf) -// writer.Write(enc) -// require.NoError(t, err) - -// reader, err := zlib.NewReader(buf) -// require.NoError(t, err) - -// rlpReader := rlp.NewStream(reader, MaxRLPBytesPerChannel) - -// // require.NoError(t, err) -// var dec BatchData -// // err = dec.UnmarshalBinary(encoded) -// err = rlpReader.Decode(&dec) -// // require.NoError(t, err) -// //require.Equal(t, batchDataInput, &dec) - -// // read from r the compressed data then rlp decode this -// // uncompressed, err := io.ReadAll(r) -// // require.NoError(t, err) -// // fmt.Println("uncompressed len") -// // fmt.Println(len(uncompressed)) - -// // var batchData BatchData -// // rlp.Decode(bytes.NewReader(uncompressed), &batchData) - -// // require.Equal(t, batchDataInput, batchData) +func TestBatchReader(t *testing.T) { + // Get batch data + rng := rand.New(rand.NewSource(0x543331)) + singularBatch := RandomSingularBatch(rng, 5000, big.NewInt(333)) + batchDataInput := NewBatchData(singularBatch) + + encodedBatch := &bytes.Buffer{} + // Get the encoded data of the batch data + batchDataInput.encodeTyped(encodedBatch) + + var testCases = []struct { + name string + algo func(buf *bytes.Buffer) () + }{{ + name: "zlib", + algo: func(buf *bytes.Buffer) { + writer := zlib.NewWriter(buf) + writer.Write(encodedBatch.Bytes()) + }, + }, + { + name: "brotli10", + algo: func(buf *bytes.Buffer) { + buf.WriteByte(ChannelVersionBrotli) + writer := brotli.NewWriterLevel(buf, 10) + writer.Write(encodedBatch.Bytes()) + }, + }, { + name: "zstd", + algo: func(buf *bytes.Buffer) { + writer := zstd.NewWriter(buf) + writer.Write(encodedBatch.Bytes()) + }, + }} -// // reader, err := BatchReader(bytes.NewReader(compressed.Bytes())) -// // if tc.name == "zstd" { -// // require.NotNil(t, err) -// // return -// // } -// // require.Nil(t, err) + for _, tc := range testCases { + compressed := bytes.NewBuffer([]byte{}) + tc := tc + t.Run(tc.name, func(t *testing.T) { + tc.algo(compressed) + reader, err := BatchReader(bytes.NewReader(compressed.Bytes())) + if tc.name == "zstd" { + require.NotNil(t, err) + return + } + require.Nil(t, err) -// // // read the batch data -// // batchData, err := reader() -// // require.Nil(t, err) -// // require.NotNil(t, batchData) -// // require.Equal(t, batchDataInput, batchData) -// }) -// } + // read the batch data + batchData, err := reader() + require.Nil(t, err) + require.NotNil(t, batchData) + require.Equal(t, batchDataInput, batchData) + }) + } -// } +} diff --git a/op-node/rollup/derive/span_channel_out.go b/op-node/rollup/derive/span_channel_out.go index 81cfd7b8cea1..98303839fb73 100644 --- a/op-node/rollup/derive/span_channel_out.go +++ b/op-node/rollup/derive/span_channel_out.go @@ -27,7 +27,7 @@ type SpanChannelOut struct { // it is used to measure the growth of the RLP buffer when adding a new batch to optimize compression lastCompressedRLPSize int // the compressor for the channel - compressor *SpanChannelCompressor + compressor *ChannelCompressor // target is the target size of the compressed data target uint64 // closed indicates if the channel is closed @@ -60,7 +60,7 @@ func NewSpanChannelOut(genesisTimestamp uint64, chainID *big.Int, targetOutputSi return nil, err } - if c.compressor, err = NewSpanChannelCompressor(compressionAlgo); err != nil { + if c.compressor, err = NewChannelCompressor(compressionAlgo); err != nil { return nil, err } @@ -151,7 +151,7 @@ func (co *SpanChannelOut) AddSingularBatch(batch *SingularBatch, seqNum uint64) // if the compressed data *plus* the new rlp data is under the target size, return early // this optimizes out cases where the compressor will obviously come in under the target size rlpGrowth := co.activeRLP().Len() - co.lastCompressedRLPSize - if uint64(co.compressor.GetCompressedLen()+rlpGrowth) < co.target { + if uint64(co.compressor.Len()+rlpGrowth) < co.target { return nil } @@ -204,7 +204,7 @@ func (co *SpanChannelOut) InputBytes() int { // Span Channel Out does not provide early output, so this will always be 0 until the channel is closed or full func (co *SpanChannelOut) ReadyBytes() int { if co.closed || co.FullErr() != nil { - return co.compressor.GetCompressedLen() + return co.compressor.Len() } return 0 } @@ -222,7 +222,7 @@ func (co *SpanChannelOut) checkFull() { if co.full != nil { return } - if uint64(co.compressor.GetCompressedLen()) >= co.target { + if uint64(co.compressor.Len()) >= co.target { co.full = ErrCompressorFull } } From bf3675a72dd7f9664d24b535c03083659c256b68 Mon Sep 17 00:00:00 2001 From: Cody Wang Date: Sun, 5 May 2024 20:12:41 +0100 Subject: [PATCH 15/38] lint --- op-node/rollup/derive/channel_test.go | 30 +++++++++++++-------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/op-node/rollup/derive/channel_test.go b/op-node/rollup/derive/channel_test.go index 1e8426154199..ec75ebf52525 100644 --- a/op-node/rollup/derive/channel_test.go +++ b/op-node/rollup/derive/channel_test.go @@ -119,7 +119,7 @@ func TestBatchReader(t *testing.T) { var testCases = []struct { name string - algo func(buf *bytes.Buffer) () + algo func(buf *bytes.Buffer) }{{ name: "zlib", algo: func(buf *bytes.Buffer) { @@ -127,20 +127,20 @@ func TestBatchReader(t *testing.T) { writer.Write(encodedBatch.Bytes()) }, }, - { - name: "brotli10", - algo: func(buf *bytes.Buffer) { - buf.WriteByte(ChannelVersionBrotli) - writer := brotli.NewWriterLevel(buf, 10) - writer.Write(encodedBatch.Bytes()) - }, - }, { - name: "zstd", - algo: func(buf *bytes.Buffer) { - writer := zstd.NewWriter(buf) - writer.Write(encodedBatch.Bytes()) - }, - }} + { + name: "brotli10", + algo: func(buf *bytes.Buffer) { + buf.WriteByte(ChannelVersionBrotli) + writer := brotli.NewWriterLevel(buf, 10) + writer.Write(encodedBatch.Bytes()) + }, + }, { + name: "zstd", + algo: func(buf *bytes.Buffer) { + writer := zstd.NewWriter(buf) + writer.Write(encodedBatch.Bytes()) + }, + }} for _, tc := range testCases { compressed := bytes.NewBuffer([]byte{}) From 1efd338129e57aa495074281b2c43f03bcb6402c Mon Sep 17 00:00:00 2001 From: Cody Wang Date: Tue, 7 May 2024 17:35:25 -0400 Subject: [PATCH 16/38] move channel compressor as interface --- op-batcher/compressor/ratio_compressor.go | 2 +- .../compressor/ratio_compressor_test.go | 2 +- op-batcher/compressor/shadow_compressor.go | 4 +- op-node/benchmarks/batchbuilding_test.go | 153 +++++++++--------- op-node/rollup/derive/channel_compressor.go | 120 ++++++++------ op-node/rollup/derive/span_channel_out.go | 2 +- 6 files changed, 160 insertions(+), 123 deletions(-) diff --git a/op-batcher/compressor/ratio_compressor.go b/op-batcher/compressor/ratio_compressor.go index bd174d61eca7..516cd05f6a2c 100644 --- a/op-batcher/compressor/ratio_compressor.go +++ b/op-batcher/compressor/ratio_compressor.go @@ -8,7 +8,7 @@ type RatioCompressor struct { config Config inputBytes int - compressor *derive.ChannelCompressor + compressor derive.ChannelCompressor } // NewRatioCompressor creates a new derive.Compressor implementation that uses the target diff --git a/op-batcher/compressor/ratio_compressor_test.go b/op-batcher/compressor/ratio_compressor_test.go index 7b827a000668..089e8856f5d9 100644 --- a/op-batcher/compressor/ratio_compressor_test.go +++ b/op-batcher/compressor/ratio_compressor_test.go @@ -60,7 +60,7 @@ func TestChannelConfig_InputThreshold(t *testing.T) { // Validate each test case for i, tt := range tests { for _, algo := range derive.CompressionAlgoTypes { - t.Run(fmt.Sprintf("test-%d", i), func(t *testing.T) { + t.Run(fmt.Sprintf("test-%d-%s", i, algo.String()), func(t *testing.T) { comp, err := compressor.NewRatioCompressor(compressor.Config{ TargetOutputSize: tt.targetOutputSize, ApproxComprRatio: tt.approxComprRatio, diff --git a/op-batcher/compressor/shadow_compressor.go b/op-batcher/compressor/shadow_compressor.go index 940a782a5ca6..b5cd9fe6f199 100644 --- a/op-batcher/compressor/shadow_compressor.go +++ b/op-batcher/compressor/shadow_compressor.go @@ -18,8 +18,8 @@ const ( type ShadowCompressor struct { config Config - compressor *derive.ChannelCompressor - shadowCompressor *derive.ChannelCompressor + compressor derive.ChannelCompressor + shadowCompressor derive.ChannelCompressor fullErr error diff --git a/op-node/benchmarks/batchbuilding_test.go b/op-node/benchmarks/batchbuilding_test.go index 85c80424f085..75f96a349504 100644 --- a/op-node/benchmarks/batchbuilding_test.go +++ b/op-node/benchmarks/batchbuilding_test.go @@ -58,13 +58,13 @@ type compressorAndTarget struct { } // channelOutByType returns a channel out of the given type as a helper for the benchmarks -func channelOutByType(batchType uint, compKey string) (derive.ChannelOut, error) { +func channelOutByType(batchType uint, compKey string, algo derive.CompressionAlgo) (derive.ChannelOut, error) { chainID := big.NewInt(333) if batchType == derive.SingularBatchType { return derive.NewSingularChannelOut(compressors[compKey].compressor) } if batchType == derive.SpanBatchType { - return derive.NewSpanChannelOut(0, chainID, compressors[compKey].targetOutput, derive.Brotli10) + return derive.NewSpanChannelOut(0, chainID, compressors[compKey].targetOutput, algo) } return nil, fmt.Errorf("unsupported batch type: %d", batchType) } @@ -129,25 +129,28 @@ func BenchmarkFinalBatchChannelOut(b *testing.B) { // to leverage optimizations in the Batch Linked List batches[i].Timestamp = uint64(t.Add(time.Duration(i) * time.Second).Unix()) } - b.Run(tc.String(), func(b *testing.B) { - // reset the compressor used in the test case - for bn := 0; bn < b.N; bn++ { - // don't measure the setup time - b.StopTimer() - compressors[tc.compKey].compressor.Reset() - cout, _ := channelOutByType(tc.BatchType, tc.compKey) - // add all but the final batch to the channel out - for i := 0; i < tc.BatchCount-1; i++ { - err := cout.AddSingularBatch(batches[i], 0) + for _, algo := range derive.CompressionAlgoTypes { + b.Run(tc.String()+"_"+algo.String(), func(b *testing.B) { + // reset the compressor used in the test case + for bn := 0; bn < b.N; bn++ { + // don't measure the setup time + b.StopTimer() + compressors[tc.compKey].compressor.Reset() + cout, _ := channelOutByType(tc.BatchType, tc.compKey, algo) + // add all but the final batch to the channel out + for i := 0; i < tc.BatchCount-1; i++ { + err := cout.AddSingularBatch(batches[i], 0) + require.NoError(b, err) + } + // measure the time to add the final batch + b.StartTimer() + // add the final batch to the channel out + err := cout.AddSingularBatch(batches[tc.BatchCount-1], 0) require.NoError(b, err) } - // measure the time to add the final batch - b.StartTimer() - // add the final batch to the channel out - err := cout.AddSingularBatch(batches[tc.BatchCount-1], 0) - require.NoError(b, err) - } - }) + }) + } + } } @@ -165,35 +168,37 @@ func BenchmarkIncremental(b *testing.B) { {derive.SpanBatchType, 5, 1, "RealBlindCompressor"}, //{derive.SingularBatchType, 100, 1, "RealShadowCompressor"}, } - for _, tc := range tcs { - cout, err := channelOutByType(tc.BatchType, tc.compKey) - if err != nil { - b.Fatal(err) - } - done := false - for base := 0; !done; base += tc.BatchCount { - rangeName := fmt.Sprintf("Incremental %s: %d-%d", tc.String(), base, base+tc.BatchCount) - b.Run(rangeName, func(b *testing.B) { - b.StopTimer() - // prepare the batches - t := time.Now() - batches := make([]*derive.SingularBatch, tc.BatchCount) - for i := 0; i < tc.BatchCount; i++ { - t := t.Add(time.Second) - batches[i] = derive.RandomSingularBatch(rng, tc.txPerBatch, chainID) - // set the timestamp to increase with each batch - // to leverage optimizations in the Batch Linked List - batches[i].Timestamp = uint64(t.Unix()) - } - b.StartTimer() - for i := 0; i < tc.BatchCount; i++ { - err := cout.AddSingularBatch(batches[i], 0) - if err != nil { - done = true - return + for _, algo := range derive.CompressionAlgoTypes { + for _, tc := range tcs { + cout, err := channelOutByType(tc.BatchType, tc.compKey, algo) + if err != nil { + b.Fatal(err) + } + done := false + for base := 0; !done; base += tc.BatchCount { + rangeName := fmt.Sprintf("Incremental %s: %d-%d", tc.String(), base, base+tc.BatchCount) + b.Run(rangeName+"_"+algo.String(), func(b *testing.B) { + b.StopTimer() + // prepare the batches + t := time.Now() + batches := make([]*derive.SingularBatch, tc.BatchCount) + for i := 0; i < tc.BatchCount; i++ { + t := t.Add(time.Second) + batches[i] = derive.RandomSingularBatch(rng, tc.txPerBatch, chainID) + // set the timestamp to increase with each batch + // to leverage optimizations in the Batch Linked List + batches[i].Timestamp = uint64(t.Unix()) } - } - }) + b.StartTimer() + for i := 0; i < tc.BatchCount; i++ { + err := cout.AddSingularBatch(batches[i], 0) + if err != nil { + done = true + return + } + } + }) + } } } } @@ -226,33 +231,35 @@ func BenchmarkAllBatchesChannelOut(b *testing.B) { } } - for _, tc := range tests { - chainID := big.NewInt(333) - rng := rand.New(rand.NewSource(0x543331)) - // pre-generate batches to keep the benchmark from including the random generation - batches := make([]*derive.SingularBatch, tc.BatchCount) - t := time.Now() - for i := 0; i < tc.BatchCount; i++ { - batches[i] = derive.RandomSingularBatch(rng, tc.txPerBatch, chainID) - // set the timestamp to increase with each batch - // to leverage optimizations in the Batch Linked List - batches[i].Timestamp = uint64(t.Add(time.Duration(i) * time.Second).Unix()) - } - b.Run(tc.String(), func(b *testing.B) { - // reset the compressor used in the test case - for bn := 0; bn < b.N; bn++ { - // don't measure the setup time - b.StopTimer() - compressors[tc.compKey].compressor.Reset() - cout, _ := channelOutByType(tc.BatchType, tc.compKey) - b.StartTimer() - // add all batches to the channel out - for i := 0; i < tc.BatchCount; i++ { - err := cout.AddSingularBatch(batches[i], 0) - require.NoError(b, err) - } + for _, algo := range derive.CompressionAlgoTypes { + for _, tc := range tests { + chainID := big.NewInt(333) + rng := rand.New(rand.NewSource(0x543331)) + // pre-generate batches to keep the benchmark from including the random generation + batches := make([]*derive.SingularBatch, tc.BatchCount) + t := time.Now() + for i := 0; i < tc.BatchCount; i++ { + batches[i] = derive.RandomSingularBatch(rng, tc.txPerBatch, chainID) + // set the timestamp to increase with each batch + // to leverage optimizations in the Batch Linked List + batches[i].Timestamp = uint64(t.Add(time.Duration(i) * time.Second).Unix()) } - }) + b.Run(tc.String()+"_"+algo.String(), func(b *testing.B) { + // reset the compressor used in the test case + for bn := 0; bn < b.N; bn++ { + // don't measure the setup time + b.StopTimer() + compressors[tc.compKey].compressor.Reset() + cout, _ := channelOutByType(tc.BatchType, tc.compKey, algo) + b.StartTimer() + // add all batches to the channel out + for i := 0; i < tc.BatchCount; i++ { + err := cout.AddSingularBatch(batches[i], 0) + require.NoError(b, err) + } + } + }) + } } } diff --git a/op-node/rollup/derive/channel_compressor.go b/op-node/rollup/derive/channel_compressor.go index e0f701b95c55..11cb8991581c 100644 --- a/op-node/rollup/derive/channel_compressor.go +++ b/op-node/rollup/derive/channel_compressor.go @@ -4,7 +4,6 @@ import ( "bytes" "compress/zlib" "fmt" - "io" "github.com/andybalholm/brotli" ) @@ -13,73 +12,104 @@ const ( ChannelVersionBrotli byte = 0x01 ) -type CompressorWriter interface { +type ChannelCompressor interface { Write([]byte) (int, error) Flush() error Close() error - Reset(io.Writer) + Reset() + Len() int + Read([]byte) (int, error) + GetCompressed() *bytes.Buffer } -type ChannelCompressor struct { - writer CompressorWriter - compressionAlgo CompressionAlgo +type ZlibCompressor struct { + writer *zlib.Writer compressed *bytes.Buffer } -func NewChannelCompressor(algo CompressionAlgo) (*ChannelCompressor, error) { - var writer CompressorWriter - var err error - compressed := &bytes.Buffer{} - if algo == Zlib { - writer, err = zlib.NewWriterLevel(compressed, zlib.BestCompression) - } else if algo.IsBrotli() { - compressed.WriteByte(ChannelVersionBrotli) - writer = brotli.NewWriterLevel(compressed, GetBrotliLevel(algo)) - } else { - return nil, fmt.Errorf("unsupported compression algorithm: %s", algo) - } +func (zc *ZlibCompressor) Write(data []byte) (int, error) { + return zc.writer.Write(data) +} - if err != nil { - return nil, err - } +func (zc *ZlibCompressor) Flush() error { + return zc.writer.Flush() +} - return &ChannelCompressor{ - writer: writer, - compressionAlgo: algo, - compressed: compressed, - }, nil +func (zc *ZlibCompressor) Close() error { + return zc.writer.Close() +} +func (zc *ZlibCompressor) Reset() { + zc.compressed.Reset() + zc.writer.Reset(zc.compressed) } -func (cc *ChannelCompressor) Write(data []byte) (int, error) { - return cc.writer.Write(data) +func (zc *ZlibCompressor) Len() int { + return zc.compressed.Len() } -func (cc *ChannelCompressor) Flush() error { - return cc.writer.Flush() +func (zc *ZlibCompressor) Read(p []byte) (int, error) { + return zc.compressed.Read(p) } -func (cc *ChannelCompressor) Close() error { - return cc.writer.Close() +func (zc *ZlibCompressor) GetCompressed() *bytes.Buffer { + return zc.compressed } -func (cc *ChannelCompressor) Reset() { - cc.compressed.Reset() - if cc.compressionAlgo.IsBrotli() { - // always add channal version for brotli - cc.compressed.WriteByte(ChannelVersionBrotli) - } - cc.writer.Reset(cc.compressed) +type BrotliCompressor struct { + writer *brotli.Writer + compressed *bytes.Buffer +} + +func (bc *BrotliCompressor) Write(data []byte) (int, error) { + return bc.writer.Write(data) } -func (cc *ChannelCompressor) Len() int { - return cc.compressed.Len() +func (bc *BrotliCompressor) Flush() error { + return bc.writer.Flush() } -func (cc *ChannelCompressor) GetCompressed() *bytes.Buffer { - return cc.compressed +func (bc *BrotliCompressor) Close() error { + return bc.writer.Close() } -func (cc *ChannelCompressor) Read(p []byte) (int, error) { - return cc.compressed.Read(p) +func (bc *BrotliCompressor) Len() int { + return bc.compressed.Len() +} + +func (bc *BrotliCompressor) Read(p []byte) (int, error) { + return bc.compressed.Read(p) +} + +func (bc *BrotliCompressor) Reset() { + bc.compressed.Reset() + bc.compressed.WriteByte(ChannelVersionBrotli) + bc.writer.Reset(bc.compressed) +} + +func (bc *BrotliCompressor) GetCompressed() *bytes.Buffer { + return bc.compressed +} + +func NewChannelCompressor(algo CompressionAlgo) (ChannelCompressor, error) { + compressed := &bytes.Buffer{} + if algo == Zlib { + writer, err := zlib.NewWriterLevel(compressed, zlib.BestCompression) + if err != nil { + return nil, err + } + return &ZlibCompressor{ + writer: writer, + compressed: compressed, + }, nil + } else if algo.IsBrotli() { + compressed.WriteByte(ChannelVersionBrotli) + writer := brotli.NewWriterLevel(compressed, GetBrotliLevel(algo)) + return &BrotliCompressor{ + writer: writer, + compressed: compressed, + }, nil + } else { + return nil, fmt.Errorf("unsupported compression algorithm: %s", algo) + } } diff --git a/op-node/rollup/derive/span_channel_out.go b/op-node/rollup/derive/span_channel_out.go index 98303839fb73..d252c6fb3cf1 100644 --- a/op-node/rollup/derive/span_channel_out.go +++ b/op-node/rollup/derive/span_channel_out.go @@ -27,7 +27,7 @@ type SpanChannelOut struct { // it is used to measure the growth of the RLP buffer when adding a new batch to optimize compression lastCompressedRLPSize int // the compressor for the channel - compressor *ChannelCompressor + compressor ChannelCompressor // target is the target size of the compressed data target uint64 // closed indicates if the channel is closed From 42ade28d65c6876711925fc32a238de066e60a74 Mon Sep 17 00:00:00 2001 From: Cody Wang Date: Tue, 7 May 2024 17:45:09 -0400 Subject: [PATCH 17/38] add base class --- op-node/rollup/derive/channel_compressor.go | 81 +++++++++------------ 1 file changed, 36 insertions(+), 45 deletions(-) diff --git a/op-node/rollup/derive/channel_compressor.go b/op-node/rollup/derive/channel_compressor.go index 11cb8991581c..e541617ea00f 100644 --- a/op-node/rollup/derive/channel_compressor.go +++ b/op-node/rollup/derive/channel_compressor.go @@ -4,6 +4,7 @@ import ( "bytes" "compress/zlib" "fmt" + "io" "github.com/andybalholm/brotli" ) @@ -22,63 +23,53 @@ type ChannelCompressor interface { GetCompressed() *bytes.Buffer } -type ZlibCompressor struct { - writer *zlib.Writer - compressed *bytes.Buffer +type CompressorWriter interface { + Write([]byte) (int, error) + Flush() error + Close() error + Reset(io.Writer) } -func (zc *ZlibCompressor) Write(data []byte) (int, error) { - return zc.writer.Write(data) +type BaseChannelCompressor struct { + compressed *bytes.Buffer + writer CompressorWriter } -func (zc *ZlibCompressor) Flush() error { - return zc.writer.Flush() +func (bcc *BaseChannelCompressor) Write(data []byte) (int, error) { + return bcc.writer.Write(data) } -func (zc *ZlibCompressor) Close() error { - return zc.writer.Close() +func (bcc *BaseChannelCompressor) Flush() error { + return bcc.writer.Flush() } -func (zc *ZlibCompressor) Reset() { - zc.compressed.Reset() - zc.writer.Reset(zc.compressed) +func (bcc *BaseChannelCompressor) Close() error { + return bcc.writer.Close() } -func (zc *ZlibCompressor) Len() int { - return zc.compressed.Len() +func (bcc *BaseChannelCompressor) Len() int { + return bcc.compressed.Len() } -func (zc *ZlibCompressor) Read(p []byte) (int, error) { - return zc.compressed.Read(p) +func (bcc *BaseChannelCompressor) Read(p []byte) (int, error) { + return bcc.compressed.Read(p) } -func (zc *ZlibCompressor) GetCompressed() *bytes.Buffer { - return zc.compressed +func (bcc *BaseChannelCompressor) GetCompressed() *bytes.Buffer { + return bcc.compressed } -type BrotliCompressor struct { - writer *brotli.Writer - compressed *bytes.Buffer -} - -func (bc *BrotliCompressor) Write(data []byte) (int, error) { - return bc.writer.Write(data) -} - -func (bc *BrotliCompressor) Flush() error { - return bc.writer.Flush() -} - -func (bc *BrotliCompressor) Close() error { - return bc.writer.Close() +type ZlibCompressor struct { + BaseChannelCompressor } -func (bc *BrotliCompressor) Len() int { - return bc.compressed.Len() +func (zc *ZlibCompressor) Reset() { + zc.compressed.Reset() + zc.writer.Reset(zc.compressed) } -func (bc *BrotliCompressor) Read(p []byte) (int, error) { - return bc.compressed.Read(p) +type BrotliCompressor struct { + BaseChannelCompressor } func (bc *BrotliCompressor) Reset() { @@ -87,10 +78,6 @@ func (bc *BrotliCompressor) Reset() { bc.writer.Reset(bc.compressed) } -func (bc *BrotliCompressor) GetCompressed() *bytes.Buffer { - return bc.compressed -} - func NewChannelCompressor(algo CompressionAlgo) (ChannelCompressor, error) { compressed := &bytes.Buffer{} if algo == Zlib { @@ -99,15 +86,19 @@ func NewChannelCompressor(algo CompressionAlgo) (ChannelCompressor, error) { return nil, err } return &ZlibCompressor{ - writer: writer, - compressed: compressed, + BaseChannelCompressor{ + writer: writer, + compressed: compressed, + }, }, nil } else if algo.IsBrotli() { compressed.WriteByte(ChannelVersionBrotli) writer := brotli.NewWriterLevel(compressed, GetBrotliLevel(algo)) return &BrotliCompressor{ - writer: writer, - compressed: compressed, + BaseChannelCompressor{ + writer: writer, + compressed: compressed, + }, }, nil } else { return nil, fmt.Errorf("unsupported compression algorithm: %s", algo) From 6e09caf8f1b93a602e5cec51a3b410e962203c0a Mon Sep 17 00:00:00 2001 From: Cody Wang Date: Wed, 8 May 2024 14:04:28 -0400 Subject: [PATCH 18/38] fix go mod --- go.mod | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go.mod b/go.mod index 5b37b0bba061..0057af12f3ab 100644 --- a/go.mod +++ b/go.mod @@ -4,6 +4,7 @@ go 1.21 require ( github.com/BurntSushi/toml v1.3.2 + github.com/DataDog/zstd v1.5.2 github.com/andybalholm/brotli v1.1.0 github.com/btcsuite/btcd v0.24.0 github.com/btcsuite/btcd/chaincfg/chainhash v1.1.0 @@ -59,7 +60,6 @@ require ( require ( github.com/CloudyKit/fastprinter v0.0.0-20200109182630-33d98a066a53 // indirect github.com/CloudyKit/jet/v6 v6.2.0 // indirect - github.com/DataDog/zstd v1.5.2 // indirect github.com/Joker/jade v1.1.3 // indirect github.com/Microsoft/go-winio v0.6.1 // indirect github.com/Shopify/goreferrer v0.0.0-20220729165902-8cddb4f5de06 // indirect From e5f7e6c7f3641f5342ec1f0d0b9452c3d0f60833 Mon Sep 17 00:00:00 2001 From: Roberto Bayardo Date: Wed, 8 May 2024 12:22:45 -0700 Subject: [PATCH 19/38] test fixes --- op-node/rollup/derive/channel.go | 18 +++++++----------- op-node/rollup/derive/channel_test.go | 10 ++++++++-- 2 files changed, 15 insertions(+), 13 deletions(-) diff --git a/op-node/rollup/derive/channel.go b/op-node/rollup/derive/channel.go index c4272c3623b0..92fb331ed11a 100644 --- a/op-node/rollup/derive/channel.go +++ b/op-node/rollup/derive/channel.go @@ -161,11 +161,13 @@ func BatchReader(r io.Reader) (func() (*BatchData, error), error) { return nil, err } - var reader func(io.Reader) (io.Reader, error) + var zr io.Reader // For zlib, the last 4 bits must be either 8 or 15 (both are reserved value) if compressionType[0]&0x0F == ZlibCM8 || compressionType[0]&0x0F == ZlibCM15 { - reader = func(r io.Reader) (io.Reader, error) { - return zlib.NewReader(r) + var err error + zr, err = zlib.NewReader(bufReader) + if err != nil { + return nil, err } // If the bits equal to 1, then it is a brotli reader } else if compressionType[0] == ChannelVersionBrotli { @@ -174,23 +176,17 @@ func BatchReader(r io.Reader) (func() (*BatchData, error), error) { if err != nil { return nil, err } - reader = func(r io.Reader) (io.Reader, error) { - return brotli.NewReader(r), nil - } + zr = brotli.NewReader(bufReader) } else { return nil, fmt.Errorf("cannot distinguish the compression algo used given type byte %v", compressionType[0]) } // Setup decompressor stage + RLP reader - zr, err := reader(bufReader) - if err != nil { - return nil, err - } rlpReader := rlp.NewStream(zr, MaxRLPBytesPerChannel) // Read each batch iteratively return func() (*BatchData, error) { var batchData BatchData - if err = rlpReader.Decode(&batchData); err != nil { + if err := rlpReader.Decode(&batchData); err != nil { return nil, err } return &batchData, nil diff --git a/op-node/rollup/derive/channel_test.go b/op-node/rollup/derive/channel_test.go index ec75ebf52525..58284aa1af97 100644 --- a/op-node/rollup/derive/channel_test.go +++ b/op-node/rollup/derive/channel_test.go @@ -10,7 +10,7 @@ import ( "github.com/DataDog/zstd" "github.com/andybalholm/brotli" "github.com/ethereum-optimism/optimism/op-service/eth" - + "github.com/ethereum/go-ethereum/rlp" "github.com/stretchr/testify/require" ) @@ -114,8 +114,11 @@ func TestBatchReader(t *testing.T) { batchDataInput := NewBatchData(singularBatch) encodedBatch := &bytes.Buffer{} + buf := &bytes.Buffer{} // Get the encoded data of the batch data - batchDataInput.encodeTyped(encodedBatch) + batchDataInput.encodeTyped(buf) + err := rlp.Encode(encodedBatch, buf.Bytes()) + require.NoError(t, err) var testCases = []struct { name string @@ -125,6 +128,7 @@ func TestBatchReader(t *testing.T) { algo: func(buf *bytes.Buffer) { writer := zlib.NewWriter(buf) writer.Write(encodedBatch.Bytes()) + writer.Close() }, }, { @@ -133,12 +137,14 @@ func TestBatchReader(t *testing.T) { buf.WriteByte(ChannelVersionBrotli) writer := brotli.NewWriterLevel(buf, 10) writer.Write(encodedBatch.Bytes()) + writer.Close() }, }, { name: "zstd", algo: func(buf *bytes.Buffer) { writer := zstd.NewWriter(buf) writer.Write(encodedBatch.Bytes()) + writer.Close() }, }} From c5f28f31b79df71f61ba2e8b38f26364172108b3 Mon Sep 17 00:00:00 2001 From: Cody Wang Date: Thu, 9 May 2024 14:59:49 -0700 Subject: [PATCH 20/38] address comments --- op-batcher/batcher/channel_builder_test.go | 138 +++++++----------- op-batcher/batcher/channel_config.go | 4 +- op-batcher/batcher/channel_config_test.go | 35 ++--- op-batcher/batcher/channel_manager_test.go | 60 ++++---- op-batcher/batcher/channel_test.go | 47 ++---- op-batcher/batcher/config_test.go | 25 ++-- op-batcher/compressor/config.go | 2 +- op-e2e/actions/l2_batcher.go | 3 +- op-e2e/actions/sync_test.go | 2 +- op-e2e/setup.go | 4 +- op-node/benchmarks/batchbuilding_test.go | 2 +- op-node/rollup/derive/channel_compressor.go | 20 +-- .../rollup/derive/channel_compressor_test.go | 41 +++--- op-node/rollup/derive/channel_out_test.go | 50 +++++-- op-node/rollup/derive/channel_test.go | 75 +++++++--- op-node/rollup/derive/types_test.go | 32 ++-- 16 files changed, 268 insertions(+), 272 deletions(-) diff --git a/op-batcher/batcher/channel_builder_test.go b/op-batcher/batcher/channel_builder_test.go index 1eeb034d23fb..9853bbe4c80f 100644 --- a/op-batcher/batcher/channel_builder_test.go +++ b/op-batcher/batcher/channel_builder_test.go @@ -9,7 +9,6 @@ import ( "testing" "time" - "github.com/ethereum-optimism/optimism/op-batcher/compressor" "github.com/ethereum-optimism/optimism/op-node/rollup" "github.com/ethereum-optimism/optimism/op-node/rollup/derive" dtest "github.com/ethereum-optimism/optimism/op-node/rollup/derive/test" @@ -101,12 +100,10 @@ func addTooManyBlocks(cb *ChannelBuilder) error { // is set to 0, the channel builder cannot have a duration timeout. func FuzzDurationTimeoutZeroMaxChannelDuration(f *testing.F) { for i := range [10]int{} { - for _, algo := range derive.CompressionAlgoTypes { - f.Add(uint64(i), algo.String()) - } + f.Add(uint64(i)) } - f.Fuzz(func(t *testing.T, l1BlockNum uint64, algo string) { - channelConfig := defaultTestChannelConfig(derive.CompressionAlgo(algo)) + f.Fuzz(func(t *testing.T, l1BlockNum uint64) { + channelConfig := defaultTestChannelConfig() channelConfig.MaxChannelDuration = 0 cb, err := NewChannelBuilder(channelConfig, defaultTestRollupConfig, latestL1BlockOrigin) require.NoError(t, err) @@ -121,17 +118,15 @@ func FuzzDurationTimeoutZeroMaxChannelDuration(f *testing.F) { // as long as the channel builder's timeout is set to 0. func FuzzChannelBuilder_DurationZero(f *testing.F) { for i := range [10]int{} { - for _, algo := range derive.CompressionAlgoTypes { - f.Add(uint64(i), uint64(i), algo.String()) - } + f.Add(uint64(i), uint64(i)) } - f.Fuzz(func(t *testing.T, l1BlockNum uint64, maxChannelDuration uint64, algo string) { + f.Fuzz(func(t *testing.T, l1BlockNum uint64, maxChannelDuration uint64) { if maxChannelDuration == 0 { t.Skip("Max channel duration cannot be 0") } // Create the channel builder - channelConfig := defaultTestChannelConfig(derive.CompressionAlgo(algo)) + channelConfig := defaultTestChannelConfig() channelConfig.MaxChannelDuration = maxChannelDuration cb, err := NewChannelBuilder(channelConfig, defaultTestRollupConfig, latestL1BlockOrigin) require.NoError(t, err) @@ -150,18 +145,15 @@ func FuzzChannelBuilder_DurationZero(f *testing.F) { func FuzzDurationTimeoutMaxChannelDuration(f *testing.F) { // Set multiple seeds in case fuzzing isn't explicitly used for i := range [10]int{} { - for _, algo := range derive.CompressionAlgoTypes { - f.Add(uint64(i), uint64(i), uint64(i), algo.String()) - } - + f.Add(uint64(i), uint64(i), uint64(i)) } - f.Fuzz(func(t *testing.T, l1BlockNum uint64, maxChannelDuration uint64, timeout uint64, algo string) { + f.Fuzz(func(t *testing.T, l1BlockNum uint64, maxChannelDuration uint64, timeout uint64) { if maxChannelDuration == 0 { t.Skip("Max channel duration cannot be 0") } // Create the channel builder - channelConfig := defaultTestChannelConfig(derive.CompressionAlgo(algo)) + channelConfig := defaultTestChannelConfig() channelConfig.MaxChannelDuration = maxChannelDuration cb, err := NewChannelBuilder(channelConfig, defaultTestRollupConfig, latestL1BlockOrigin) require.NoError(t, err) @@ -190,13 +182,11 @@ func FuzzDurationTimeoutMaxChannelDuration(f *testing.F) { func FuzzChannelCloseTimeout(f *testing.F) { // Set multiple seeds in case fuzzing isn't explicitly used for i := range [10]int{} { - for _, algo := range derive.CompressionAlgoTypes { - f.Add(uint64(i), uint64(i), uint64(i), uint64(i*5), algo.String()) - } + f.Add(uint64(i), uint64(i), uint64(i), uint64(i*5)) } - f.Fuzz(func(t *testing.T, l1BlockNum uint64, channelTimeout uint64, subSafetyMargin uint64, timeout uint64, algo string) { + f.Fuzz(func(t *testing.T, l1BlockNum uint64, channelTimeout uint64, subSafetyMargin uint64, timeout uint64) { // Create the channel builder - channelConfig := defaultTestChannelConfig(derive.CompressionAlgo(algo)) + channelConfig := defaultTestChannelConfig() channelConfig.ChannelTimeout = channelTimeout channelConfig.SubSafetyMargin = subSafetyMargin cb, err := NewChannelBuilder(channelConfig, defaultTestRollupConfig, latestL1BlockOrigin) @@ -220,13 +210,11 @@ func FuzzChannelCloseTimeout(f *testing.F) { func FuzzChannelZeroCloseTimeout(f *testing.F) { // Set multiple seeds in case fuzzing isn't explicitly used for i := range [10]int{} { - for _, algo := range derive.CompressionAlgoTypes { - f.Add(uint64(i), uint64(i), uint64(i), algo.String()) - } + f.Add(uint64(i), uint64(i), uint64(i)) } - f.Fuzz(func(t *testing.T, l1BlockNum uint64, channelTimeout uint64, subSafetyMargin uint64, algo string) { + f.Fuzz(func(t *testing.T, l1BlockNum uint64, channelTimeout uint64, subSafetyMargin uint64) { // Create the channel builder - channelConfig := defaultTestChannelConfig(derive.CompressionAlgo(algo)) + channelConfig := defaultTestChannelConfig() channelConfig.ChannelTimeout = channelTimeout channelConfig.SubSafetyMargin = subSafetyMargin cb, err := NewChannelBuilder(channelConfig, defaultTestRollupConfig, latestL1BlockOrigin) @@ -249,13 +237,11 @@ func FuzzChannelZeroCloseTimeout(f *testing.F) { func FuzzSeqWindowClose(f *testing.F) { // Set multiple seeds in case fuzzing isn't explicitly used for i := range [10]int{} { - for _, algo := range derive.CompressionAlgoTypes { - f.Add(uint64(i), uint64(i), uint64(i), uint64(i*5), algo.String()) - } + f.Add(uint64(i), uint64(i), uint64(i), uint64(i*5)) } - f.Fuzz(func(t *testing.T, epochNum uint64, seqWindowSize uint64, subSafetyMargin uint64, timeout uint64, algo string) { + f.Fuzz(func(t *testing.T, epochNum uint64, seqWindowSize uint64, subSafetyMargin uint64, timeout uint64) { // Create the channel builder - channelConfig := defaultTestChannelConfig(derive.CompressionAlgo(algo)) + channelConfig := defaultTestChannelConfig() channelConfig.SeqWindowSize = seqWindowSize channelConfig.SubSafetyMargin = subSafetyMargin cb, err := NewChannelBuilder(channelConfig, defaultTestRollupConfig, latestL1BlockOrigin) @@ -279,13 +265,11 @@ func FuzzSeqWindowClose(f *testing.F) { func FuzzSeqWindowZeroTimeoutClose(f *testing.F) { // Set multiple seeds in case fuzzing isn't explicitly used for i := range [10]int{} { - for _, algo := range derive.CompressionAlgoTypes { - f.Add(uint64(i), uint64(i), uint64(i), algo.String()) - } + f.Add(uint64(i), uint64(i), uint64(i)) } - f.Fuzz(func(t *testing.T, epochNum uint64, seqWindowSize uint64, subSafetyMargin uint64, algo string) { + f.Fuzz(func(t *testing.T, epochNum uint64, seqWindowSize uint64, subSafetyMargin uint64) { // Create the channel builder - channelConfig := defaultTestChannelConfig(derive.CompressionAlgo(algo)) + channelConfig := defaultTestChannelConfig() channelConfig.SeqWindowSize = seqWindowSize channelConfig.SubSafetyMargin = subSafetyMargin cb, err := NewChannelBuilder(channelConfig, defaultTestRollupConfig, latestL1BlockOrigin) @@ -341,10 +325,6 @@ func TestChannelBuilderSingularBatch(t *testing.T) { }{ {"ChannelBuilder_NextFrame", ChannelBuilder_NextFrame}, {"ChannelBuilder_OutputFrames", ChannelBuilder_OutputFrames}, - {"ChannelBuilder_CheckTimeout", ChannelBuilder_CheckTimeout}, - {"ChannelBuilder_CheckTimeoutZeroMaxChannelDuration", ChannelBuilder_CheckTimeoutZeroMaxChannelDuration}, - {"ChannelBuilder_FramePublished", ChannelBuilder_FramePublished}, - {"ChannelBuilder_LatestL1Origin", ChannelBuilder_LatestL1Origin}, } for _, test := range tests { test := test @@ -358,7 +338,7 @@ func TestChannelBuilderSingularBatch(t *testing.T) { // TestChannelBuilder_NextFrame tests calling NextFrame on a ChannelBuilder with only one frame func ChannelBuilder_NextFrame(t *testing.T, algo derive.CompressionAlgo) { - channelConfig := defaultTestChannelConfig(algo) + channelConfig := defaultTestChannelConfig() // Create a new channel builder cb, err := NewChannelBuilder(channelConfig, defaultTestRollupConfig, latestL1BlockOrigin) @@ -398,7 +378,7 @@ func ChannelBuilder_NextFrame(t *testing.T, algo derive.CompressionAlgo) { // TestChannelBuilder_OutputWrongFramePanic tests that a panic is thrown when a frame is pushed with an invalid frame id func ChannelBuilder_OutputWrongFramePanic(t *testing.T, batchType uint, algo derive.CompressionAlgo) { - channelConfig := defaultTestChannelConfig(algo) + channelConfig := defaultTestChannelConfig() channelConfig.BatchType = batchType // Construct a channel builder @@ -432,10 +412,10 @@ func ChannelBuilder_OutputWrongFramePanic(t *testing.T, batchType uint, algo der // TestChannelBuilder_OutputFrames tests [ChannelBuilder.OutputFrames] for singular batches. func ChannelBuilder_OutputFrames(t *testing.T, algo derive.CompressionAlgo) { - channelConfig := defaultTestChannelConfig(algo) + channelConfig := defaultTestChannelConfig() channelConfig.MaxFrameSize = derive.FrameV0OverHeadSize + 1 channelConfig.TargetNumFrames = 1000 - channelConfig.InitNoneCompressor(algo) + channelConfig.InitNoneCompressor() // Construct the channel builder cb, err := NewChannelBuilder(channelConfig, defaultTestRollupConfig, latestL1BlockOrigin) @@ -476,18 +456,18 @@ func TestChannelBuilder_OutputFrames_SpanBatch(t *testing.T) { for _, algo := range derive.CompressionAlgoTypes { t.Run("ChannelBuilder_OutputFrames_SpanBatch"+algo.String(), func(t *testing.T) { if algo.IsBrotli() { - ChannelBuilder_OutputFrames_SpanBatch(t, algo, 1) // to fill faster for brotli + ChannelBuilder_OutputFrames_SpanBatch(t, algo) // to fill faster for brotli } else { - ChannelBuilder_OutputFrames_SpanBatch(t, algo, 5) + ChannelBuilder_OutputFrames_SpanBatch(t, algo) } }) } } -func ChannelBuilder_OutputFrames_SpanBatch(t *testing.T, algo derive.CompressionAlgo, targetNumFrames int) { - channelConfig := defaultTestChannelConfig(algo) +func ChannelBuilder_OutputFrames_SpanBatch(t *testing.T, algo derive.CompressionAlgo) { + channelConfig := defaultTestChannelConfig() channelConfig.MaxFrameSize = 20 + derive.FrameV0OverHeadSize - channelConfig.TargetNumFrames = targetNumFrames + channelConfig.TargetNumFrames = 5 channelConfig.BatchType = derive.SpanBatchType channelConfig.InitRatioCompressor(1, algo) @@ -543,9 +523,9 @@ func ChannelBuilder_OutputFrames_SpanBatch(t *testing.T, algo derive.Compression // function errors when the max RLP bytes per channel is reached. func ChannelBuilder_MaxRLPBytesPerChannel(t *testing.T, batchType uint, algo derive.CompressionAlgo) { t.Parallel() - channelConfig := defaultTestChannelConfig(algo) + channelConfig := defaultTestChannelConfig() channelConfig.MaxFrameSize = derive.MaxRLPBytesPerChannel * 2 - channelConfig.InitNoneCompressor(algo) + channelConfig.InitNoneCompressor() channelConfig.BatchType = batchType // Construct the channel builder @@ -560,7 +540,7 @@ func ChannelBuilder_MaxRLPBytesPerChannel(t *testing.T, batchType uint, algo der // ChannelBuilder_OutputFramesMaxFrameIndex tests the [ChannelBuilder.OutputFrames] // function errors when the max frame index is reached. func ChannelBuilder_OutputFramesMaxFrameIndex(t *testing.T, batchType uint, algo derive.CompressionAlgo) { - channelConfig := defaultTestChannelConfig(algo) + channelConfig := defaultTestChannelConfig() channelConfig.MaxFrameSize = derive.FrameV0OverHeadSize + 1 channelConfig.TargetNumFrames = math.MaxUint16 + 1 channelConfig.InitRatioCompressor(.1, algo) @@ -599,39 +579,25 @@ func ChannelBuilder_OutputFramesMaxFrameIndex(t *testing.T, batchType uint, algo // [derive.FrameV0OverHeadSize] in [MaxDataSize] is omitted, which has been the // case before it got fixed it #9887. func TestChannelBuilder_FullShadowCompressor(t *testing.T) { - for _, algo := range derive.CompressionAlgoTypes { - t.Run("ChannelBuilder_FullShadowCompressor"+algo.String(), func(t *testing.T) { - ChannelBuilder_FullShadowCompressor(t, algo) - }) - } -} - -func ChannelBuilder_FullShadowCompressor(t *testing.T, algo derive.CompressionAlgo) { require := require.New(t) cfg := ChannelConfig{ MaxFrameSize: 752, TargetNumFrames: 1, BatchType: derive.SpanBatchType, - CompressorConfig: compressor.Config{ - CompressionAlgo: algo, - }, } - cfg.InitShadowCompressor(algo) + cfg.InitShadowCompressor(derive.Zlib) cb, err := NewChannelBuilder(cfg, defaultTestRollupConfig, latestL1BlockOrigin) require.NoError(err) rng := rand.New(rand.NewSource(420)) a := dtest.RandomL2BlockWithChainId(rng, 1, defaultTestRollupConfig.L2ChainID) - - for { - _, err = cb.AddBlock(a) - if err != nil { - require.ErrorIs(err, derive.ErrCompressorFull) - break - } - } - + _, err = cb.AddBlock(a) + require.NoError(err) + _, err = cb.AddBlock(a) + require.ErrorIs(err, derive.ErrCompressorFull) + // without fix, adding the second block would succeed and then adding a + // third block would fail with full error and the compressor would be full. require.NoError(cb.OutputFrames()) require.True(cb.HasFrame()) f := cb.NextFrame() @@ -641,7 +607,7 @@ func ChannelBuilder_FullShadowCompressor(t *testing.T, algo derive.CompressionAl } func ChannelBuilder_AddBlock(t *testing.T, batchType uint, algo derive.CompressionAlgo) { - channelConfig := defaultTestChannelConfig(algo) + channelConfig := defaultTestChannelConfig() channelConfig.BatchType = batchType // Lower the max frame size so that we can batch @@ -673,8 +639,8 @@ func ChannelBuilder_AddBlock(t *testing.T, batchType uint, algo derive.Compressi require.ErrorIs(t, addMiniBlock(cb), derive.ErrCompressorFull) } -func ChannelBuilder_CheckTimeout(t *testing.T, algo derive.CompressionAlgo) { - channelConfig := defaultTestChannelConfig(algo) +func TestChannelBuilder_CheckTimeout(t *testing.T) { + channelConfig := defaultTestChannelConfig() // Construct the channel builder cb, err := NewChannelBuilder(channelConfig, defaultTestRollupConfig, latestL1BlockOrigin) @@ -695,8 +661,8 @@ func ChannelBuilder_CheckTimeout(t *testing.T, algo derive.CompressionAlgo) { require.ErrorIs(t, cb.FullErr(), ErrMaxDurationReached) } -func ChannelBuilder_CheckTimeoutZeroMaxChannelDuration(t *testing.T, algo derive.CompressionAlgo) { - channelConfig := defaultTestChannelConfig(algo) +func TestChannelBuilder_CheckTimeoutZeroMaxChannelDuration(t *testing.T) { + channelConfig := defaultTestChannelConfig() // Set the max channel duration to 0 channelConfig.MaxChannelDuration = 0 @@ -718,8 +684,8 @@ func ChannelBuilder_CheckTimeoutZeroMaxChannelDuration(t *testing.T, algo derive require.Equal(t, uint64(0), cb.timeout) } -func ChannelBuilder_FramePublished(t *testing.T, algo derive.CompressionAlgo) { - cfg := defaultTestChannelConfig(algo) +func TestChannelBuilder_FramePublished(t *testing.T) { + cfg := defaultTestChannelConfig() cfg.MaxChannelDuration = 10_000 cfg.ChannelTimeout = 1000 cfg.SubSafetyMargin = 100 @@ -741,8 +707,8 @@ func ChannelBuilder_FramePublished(t *testing.T, algo derive.CompressionAlgo) { require.Less(t, cb.timeout, priorTimeout) } -func ChannelBuilder_LatestL1Origin(t *testing.T, algo derive.CompressionAlgo) { - cb, err := NewChannelBuilder(defaultTestChannelConfig(algo), defaultTestRollupConfig, latestL1BlockOrigin) +func TestChannelBuilder_LatestL1Origin(t *testing.T) { + cb, err := NewChannelBuilder(defaultTestChannelConfig(), defaultTestRollupConfig, latestL1BlockOrigin) require.NoError(t, err) require.Equal(t, eth.BlockID{}, cb.LatestL1Origin()) @@ -767,7 +733,7 @@ func ChannelBuilder_PendingFrames_TotalFrames(t *testing.T, batchType uint, algo const tnf = 9 rng := rand.New(rand.NewSource(94572314)) require := require.New(t) - cfg := defaultTestChannelConfig(algo) + cfg := defaultTestChannelConfig() cfg.MaxFrameSize = 1000 cfg.TargetNumFrames = tnf cfg.BatchType = batchType @@ -810,7 +776,7 @@ func ChannelBuilder_PendingFrames_TotalFrames(t *testing.T, batchType uint, algo func ChannelBuilder_InputBytes(t *testing.T, batchType uint, algo derive.CompressionAlgo) { require := require.New(t) rng := rand.New(rand.NewSource(4982432)) - cfg := defaultTestChannelConfig(algo) + cfg := defaultTestChannelConfig() cfg.BatchType = batchType var spanBatch *derive.SpanBatch if batchType == derive.SpanBatchType { @@ -849,7 +815,7 @@ func ChannelBuilder_InputBytes(t *testing.T, batchType uint, algo derive.Compres func ChannelBuilder_OutputBytes(t *testing.T, batchType uint, algo derive.CompressionAlgo) { require := require.New(t) rng := rand.New(rand.NewSource(9860372)) - cfg := defaultTestChannelConfig(algo) + cfg := defaultTestChannelConfig() cfg.MaxFrameSize = 1000 cfg.TargetNumFrames = 16 cfg.BatchType = batchType diff --git a/op-batcher/batcher/channel_config.go b/op-batcher/batcher/channel_config.go index 72c5a1afd0c8..b504d2480f3d 100644 --- a/op-batcher/batcher/channel_config.go +++ b/op-batcher/batcher/channel_config.go @@ -71,8 +71,8 @@ func (cc *ChannelConfig) InitShadowCompressor(compressionAlgo derive.Compression cc.InitCompressorConfig(0, compressor.ShadowKind, compressionAlgo) } -func (cc *ChannelConfig) InitNoneCompressor(compressionAlgo derive.CompressionAlgo) { - cc.InitCompressorConfig(0, compressor.NoneKind, compressionAlgo) +func (cc *ChannelConfig) InitNoneCompressor() { + cc.InitCompressorConfig(0, compressor.NoneKind, derive.Zlib) } func (cc *ChannelConfig) MaxFramesPerTx() int { diff --git a/op-batcher/batcher/channel_config_test.go b/op-batcher/batcher/channel_config_test.go index d8419740d504..d7f3c2cc5ea5 100644 --- a/op-batcher/batcher/channel_config_test.go +++ b/op-batcher/batcher/channel_config_test.go @@ -5,13 +5,12 @@ import ( "math" "testing" - "github.com/ethereum-optimism/optimism/op-batcher/compressor" "github.com/ethereum-optimism/optimism/op-node/rollup/derive" "github.com/stretchr/testify/require" ) -func defaultTestChannelConfig(algo derive.CompressionAlgo) ChannelConfig { +func defaultTestChannelConfig() ChannelConfig { c := ChannelConfig{ SeqWindowSize: 15, ChannelTimeout: 40, @@ -20,30 +19,27 @@ func defaultTestChannelConfig(algo derive.CompressionAlgo) ChannelConfig { MaxFrameSize: 120_000, TargetNumFrames: 1, BatchType: derive.SingularBatchType, - CompressorConfig: compressor.Config{ - CompressionAlgo: algo, - }, } - c.InitRatioCompressor(0.4, algo) + c.InitRatioCompressor(0.4, derive.Zlib) return c } func TestChannelConfig_Check(t *testing.T) { type test struct { - input func(algo derive.CompressionAlgo) ChannelConfig + input func() ChannelConfig assertion func(error) } tests := []test{ { - input: func(algo derive.CompressionAlgo) ChannelConfig { return defaultTestChannelConfig(algo) }, + input: defaultTestChannelConfig, assertion: func(output error) { require.NoError(t, output) }, }, { - input: func(algo derive.CompressionAlgo) ChannelConfig { - cfg := defaultTestChannelConfig(algo) + input: func() ChannelConfig { + cfg := defaultTestChannelConfig() cfg.ChannelTimeout = 0 cfg.SubSafetyMargin = 1 return cfg @@ -57,8 +53,8 @@ func TestChannelConfig_Check(t *testing.T) { expectedErr := fmt.Sprintf("max frame size %d is less than the minimum 23", i) i := i // need to udpate Go version... tests = append(tests, test{ - input: func(algo derive.CompressionAlgo) ChannelConfig { - cfg := defaultTestChannelConfig(algo) + input: func() ChannelConfig { + cfg := defaultTestChannelConfig() cfg.MaxFrameSize = uint64(i) return cfg }, @@ -70,10 +66,8 @@ func TestChannelConfig_Check(t *testing.T) { // Run the table tests for _, test := range tests { - for _, algo := range derive.CompressionAlgoTypes { - cfg := test.input(algo) - test.assertion(cfg.Check()) - } + cfg := test.input() + test.assertion(cfg.Check()) } } @@ -82,12 +76,9 @@ func TestChannelConfig_Check(t *testing.T) { // the ChannelTimeout is less than the SubSafetyMargin. func FuzzChannelConfig_CheckTimeout(f *testing.F) { for i := range [10]int{} { - for _, algo := range derive.CompressionAlgoTypes { - f.Add(uint64(i+1), uint64(i), algo.String()) - } - + f.Add(uint64(i+1), uint64(i)) } - f.Fuzz(func(t *testing.T, channelTimeout uint64, subSafetyMargin uint64, algo string) { + f.Fuzz(func(t *testing.T, channelTimeout uint64, subSafetyMargin uint64) { // We only test where [ChannelTimeout] is less than the [SubSafetyMargin] // So we cannot have [ChannelTimeout] be [math.MaxUint64] if channelTimeout == math.MaxUint64 { @@ -97,7 +88,7 @@ func FuzzChannelConfig_CheckTimeout(f *testing.F) { subSafetyMargin = channelTimeout + 1 } - channelConfig := defaultTestChannelConfig(derive.CompressionAlgo(algo)) + channelConfig := defaultTestChannelConfig() channelConfig.ChannelTimeout = channelTimeout channelConfig.SubSafetyMargin = subSafetyMargin require.ErrorIs(t, channelConfig.Check(), ErrInvalidChannelTimeout) diff --git a/op-batcher/batcher/channel_manager_test.go b/op-batcher/batcher/channel_manager_test.go index f42e62f709c4..ff13bc2eefde 100644 --- a/op-batcher/batcher/channel_manager_test.go +++ b/op-batcher/batcher/channel_manager_test.go @@ -25,15 +25,12 @@ func channelManagerTestConfig(maxFrameSize uint64, batchType uint, algo derive.C MaxFrameSize: maxFrameSize, TargetNumFrames: 1, BatchType: batchType, - CompressorConfig: compressor.Config{ - CompressionAlgo: algo, - }, } cfg.InitRatioCompressor(1, algo) return cfg } -func TestChannelManagerBatchType(t *testing.T) { +func TestChannelManagerBatchTypeAndAlgo(t *testing.T) { tests := []struct { name string f func(t *testing.T, batchType uint, algo derive.CompressionAlgo) @@ -45,7 +42,6 @@ func TestChannelManagerBatchType(t *testing.T) { {"ChannelManagerCloseBeforeFirstUse", ChannelManagerCloseBeforeFirstUse}, {"ChannelManagerCloseNoPendingChannel", ChannelManagerCloseNoPendingChannel}, {"ChannelManagerClosePendingChannel", ChannelManagerClosePendingChannel}, - {"ChannelManagerCloseAllTxsFailed", ChannelManagerCloseAllTxsFailed}, } for _, test := range tests { test := test @@ -66,6 +62,29 @@ func TestChannelManagerBatchType(t *testing.T) { } } +func TestChannelManagerBatchType(t *testing.T) { + tests := []struct { + name string + f func(t *testing.T, batchType uint) + }{ + {"ChannelManagerCloseAllTxsFailed", ChannelManagerCloseAllTxsFailed}, + } + for _, test := range tests { + test := test + t.Run(test.name+"_SingularBatch_",func(t *testing.T) { + test.f(t, derive.SingularBatchType) + }) + } + + for _, test := range tests { + test := test + t.Run(test.name+"_SpanBatch", func(t *testing.T) { + test.f(t, derive.SpanBatchType) + }) + } +} + + // ChannelManagerReturnsErrReorg ensures that the channel manager // detects a reorg when it has cached L1 blocks. func ChannelManagerReturnsErrReorg(t *testing.T, batchType uint, algo derive.CompressionAlgo) { @@ -343,14 +362,6 @@ func ChannelManagerClosePendingChannel(t *testing.T, batchType uint, algo derive // Couldn't get the test to work even with modifying NonCompressor // to flush half-way through writing to the compressor... func TestChannelManager_Close_PartiallyPendingChannel(t *testing.T) { - for _, algo := range derive.CompressionAlgoTypes { - t.Run("ChannelManager_Close_PartiallyPendingChannel"+algo.String(), func(t *testing.T) { - ChannelManager_Close_PartiallyPendingChannel(t, algo) - }) - } -} - -func ChannelManager_Close_PartiallyPendingChannel(t *testing.T, algo derive.CompressionAlgo) { require := require.New(t) // The number of batch txs depends on compression of the random data, hence the static test RNG seed. // Example of different RNG seed that creates less than 2 frames: 1698700588902821588 @@ -360,11 +371,8 @@ func ChannelManager_Close_PartiallyPendingChannel(t *testing.T, algo derive.Comp MaxFrameSize: 2200, ChannelTimeout: 1000, TargetNumFrames: 100, - CompressorConfig: compressor.Config{ - CompressionAlgo: algo, - }, } - cfg.InitNoneCompressor(algo) + cfg.InitNoneCompressor() m := NewChannelManager(log, metrics.NoopMetrics, cfg, &defaultTestRollupConfig) m.Clear(eth.BlockID{}) @@ -410,13 +418,13 @@ func ChannelManager_Close_PartiallyPendingChannel(t *testing.T, algo derive.Comp // ChannelManagerCloseAllTxsFailed ensures that the channel manager // can gracefully close after producing transaction frames if none of these // have successfully landed on chain. -func ChannelManagerCloseAllTxsFailed(t *testing.T, batchType uint, algo derive.CompressionAlgo) { +func ChannelManagerCloseAllTxsFailed(t *testing.T, batchType uint) { require := require.New(t) rng := rand.New(rand.NewSource(1357)) log := testlog.Logger(t, log.LevelCrit) - cfg := channelManagerTestConfig(100, batchType, algo) + cfg := channelManagerTestConfig(100, batchType, derive.Zlib) cfg.TargetNumFrames = 1000 - cfg.InitNoneCompressor(algo) + cfg.InitNoneCompressor() m := NewChannelManager(log, metrics.NoopMetrics, cfg, &defaultTestRollupConfig) m.Clear(eth.BlockID{}) @@ -460,19 +468,11 @@ func ChannelManagerCloseAllTxsFailed(t *testing.T, batchType uint, algo derive.C } func TestChannelManager_ChannelCreation(t *testing.T) { - for _, algo := range derive.CompressionAlgoTypes { - t.Run("ChannelManager_ChannelCreation"+algo.String(), func(t *testing.T) { - ChannelManager_ChannelCreation(t, algo) - }) - } -} - -func ChannelManager_ChannelCreation(t *testing.T, algo derive.CompressionAlgo) { l := testlog.Logger(t, log.LevelCrit) const maxChannelDuration = 15 - cfg := channelManagerTestConfig(1000, derive.SpanBatchType, algo) + cfg := channelManagerTestConfig(1000, derive.SpanBatchType, derive.Zlib) cfg.MaxChannelDuration = maxChannelDuration - cfg.InitNoneCompressor(algo) + cfg.InitNoneCompressor() for _, tt := range []struct { name string diff --git a/op-batcher/batcher/channel_test.go b/op-batcher/batcher/channel_test.go index 0c366226a119..3d3e813d0ae8 100644 --- a/op-batcher/batcher/channel_test.go +++ b/op-batcher/batcher/channel_test.go @@ -23,38 +23,15 @@ func zeroFrameTxID(fn uint16) txID { return txID{frameID{frameNumber: fn}} } -func TestChannel(t *testing.T) { - tests := []struct { - name string - f func(t *testing.T, algo derive.CompressionAlgo) - }{ - {"ChannelTimeout", ChannelTimeout}, - {"ChannelManager_NextTxData", ChannelManager_NextTxData}, - {"Channel_NextTxData_singleFrameTx", Channel_NextTxData_singleFrameTx}, - {"Channel_NextTxData_multiFrameTx", Channel_NextTxData_multiFrameTx}, - {"ChannelTxConfirmed", ChannelTxConfirmed}, - {"ChannelTxFailed", ChannelTxFailed}, - } - - for _, test := range tests { - test := test - for _, algo := range derive.CompressionAlgoTypes { - t.Run(test.name+"_"+algo.String(), func(t *testing.T) { - test.f(t, algo) - }) - } - } -} - // TestChannelTimeout tests that the channel manager // correctly identifies when a pending channel is timed out. -func ChannelTimeout(t *testing.T, algo derive.CompressionAlgo) { +func TestChannelTimeout(t *testing.T) { // Create a new channel manager with a ChannelTimeout log := testlog.Logger(t, log.LevelCrit) m := NewChannelManager(log, metrics.NoopMetrics, ChannelConfig{ ChannelTimeout: 100, CompressorConfig: compressor.Config{ - CompressionAlgo: algo, + CompressionAlgo: derive.Zlib, }, }, &rollup.Config{}) m.Clear(eth.BlockID{}) @@ -96,10 +73,10 @@ func ChannelTimeout(t *testing.T, algo derive.CompressionAlgo) { } // TestChannelManager_NextTxData tests the nextTxData function. -func ChannelManager_NextTxData(t *testing.T, algo derive.CompressionAlgo) { +func TestChannelManager_NextTxData(t *testing.T) { log := testlog.Logger(t, log.LevelCrit) m := NewChannelManager(log, metrics.NoopMetrics, ChannelConfig{CompressorConfig: compressor.Config{ - CompressionAlgo: algo, + CompressionAlgo: derive.Zlib, }}, &rollup.Config{}) m.Clear(eth.BlockID{}) @@ -140,7 +117,7 @@ func ChannelManager_NextTxData(t *testing.T, algo derive.CompressionAlgo) { require.Equal(t, expectedTxData, channel.pendingTransactions[expectedChannelID]) } -func Channel_NextTxData_singleFrameTx(t *testing.T, algo derive.CompressionAlgo) { +func TestChannel_NextTxData_singleFrameTx(t *testing.T) { require := require.New(t) const n = 6 lgr := testlog.Logger(t, log.LevelWarn) @@ -148,7 +125,7 @@ func Channel_NextTxData_singleFrameTx(t *testing.T, algo derive.CompressionAlgo) MultiFrameTxs: false, TargetNumFrames: n, CompressorConfig: compressor.Config{ - CompressionAlgo: algo, + CompressionAlgo: derive.Zlib, }, }, &rollup.Config{}, latestL1BlockOrigin) require.NoError(err) @@ -181,7 +158,7 @@ func Channel_NextTxData_singleFrameTx(t *testing.T, algo derive.CompressionAlgo) require.False(ch.HasTxData()) } -func Channel_NextTxData_multiFrameTx(t *testing.T, algo derive.CompressionAlgo) { +func TestChannel_NextTxData_multiFrameTx(t *testing.T) { require := require.New(t) const n = 6 lgr := testlog.Logger(t, log.LevelWarn) @@ -189,7 +166,7 @@ func Channel_NextTxData_multiFrameTx(t *testing.T, algo derive.CompressionAlgo) MultiFrameTxs: true, TargetNumFrames: n, CompressorConfig: compressor.Config{ - CompressionAlgo: algo, + CompressionAlgo: derive.Zlib, }, }, &rollup.Config{}, latestL1BlockOrigin) require.NoError(err) @@ -229,7 +206,7 @@ func makeMockFrameDatas(id derive.ChannelID, n int) []frameData { } // TestChannelTxConfirmed checks the [ChannelManager.TxConfirmed] function. -func ChannelTxConfirmed(t *testing.T, algo derive.CompressionAlgo) { +func TestChannelTxConfirmed(t *testing.T) { // Create a channel manager log := testlog.Logger(t, log.LevelCrit) m := NewChannelManager(log, metrics.NoopMetrics, ChannelConfig{ @@ -238,7 +215,7 @@ func ChannelTxConfirmed(t *testing.T, algo derive.CompressionAlgo) { // clearing confirmed transactions, and resetting the pendingChannels map ChannelTimeout: 10, CompressorConfig: compressor.Config{ - CompressionAlgo: algo, + CompressionAlgo: derive.Zlib, }, }, &rollup.Config{}) m.Clear(eth.BlockID{}) @@ -286,11 +263,11 @@ func ChannelTxConfirmed(t *testing.T, algo derive.CompressionAlgo) { } // TestChannelTxFailed checks the [ChannelManager.TxFailed] function. -func ChannelTxFailed(t *testing.T, algo derive.CompressionAlgo) { +func TestChannelTxFailed(t *testing.T) { // Create a channel manager log := testlog.Logger(t, log.LevelCrit) m := NewChannelManager(log, metrics.NoopMetrics, ChannelConfig{CompressorConfig: compressor.Config{ - CompressionAlgo: algo, + CompressionAlgo: derive.Zlib, }}, &rollup.Config{}) m.Clear(eth.BlockID{}) diff --git a/op-batcher/batcher/config_test.go b/op-batcher/batcher/config_test.go index bb454158e1fb..f8fb08a703da 100644 --- a/op-batcher/batcher/config_test.go +++ b/op-batcher/batcher/config_test.go @@ -16,7 +16,7 @@ import ( "github.com/stretchr/testify/require" ) -func validBatcherConfig(algo derive.CompressionAlgo) batcher.CLIConfig { +func validBatcherConfig() batcher.CLIConfig { return batcher.CLIConfig{ L1EthRpc: "fake", L2EthRpc: "fake", @@ -35,16 +35,15 @@ func validBatcherConfig(algo derive.CompressionAlgo) batcher.CLIConfig { LogConfig: log.DefaultCLIConfig(), MetricsConfig: metrics.DefaultCLIConfig(), PprofConfig: oppprof.DefaultCLIConfig(), - RPC: rpc.DefaultCLIConfig(), - CompressionAlgo: algo, + // The compressor config is not checked in config.Check() + RPC: rpc.DefaultCLIConfig(), + CompressionAlgo: derive.Zlib, } } func TestValidBatcherConfig(t *testing.T) { - for _, algo := range derive.CompressionAlgoTypes { - cfg := validBatcherConfig(algo) - require.NoError(t, cfg.Check(), "valid config should pass the check function") - } + cfg := validBatcherConfig() + require.NoError(t, cfg.Check(), "valid config should pass the check function") } func TestBatcherConfig(t *testing.T) { @@ -118,12 +117,10 @@ func TestBatcherConfig(t *testing.T) { for _, test := range tests { tc := test - for _, algo := range derive.CompressionAlgoTypes { - t.Run(tc.name+"_"+algo.String(), func(t *testing.T) { - cfg := validBatcherConfig(algo) - tc.override(&cfg) - require.ErrorContains(t, cfg.Check(), tc.errString) - }) - } + t.Run(tc.name, func(t *testing.T) { + cfg := validBatcherConfig() + tc.override(&cfg) + require.ErrorContains(t, cfg.Check(), tc.errString) + }) } } diff --git a/op-batcher/compressor/config.go b/op-batcher/compressor/config.go index 3bf9534c91b4..e4078c8373a6 100644 --- a/op-batcher/compressor/config.go +++ b/op-batcher/compressor/config.go @@ -17,7 +17,7 @@ type Config struct { // will default to RatioKind. Kind string - // Type of compression algorithm to use. Must be one of [zlib, brotli] + // Type of compression algorithm to use. Must be one of [zlib, brotli-(9|10|11)] CompressionAlgo derive.CompressionAlgo } diff --git a/op-e2e/actions/l2_batcher.go b/op-e2e/actions/l2_batcher.go index 5b03fe963135..a4970b95f02d 100644 --- a/op-e2e/actions/l2_batcher.go +++ b/op-e2e/actions/l2_batcher.go @@ -192,6 +192,7 @@ func (s *L2Batcher) Buffer(t Testing) error { target := batcher.MaxDataSize(1, s.l2BatcherCfg.MaxL1TxSize) c, e := compressor.NewShadowCompressor(compressor.Config{ TargetOutputSize: target, + CompressionAlgo: derive.Zlib, }) require.NoError(t, e, "failed to create compressor") @@ -200,7 +201,7 @@ func (s *L2Batcher) Buffer(t Testing) error { } else { // use span batch if we're forcing it or if we're at/beyond delta if s.l2BatcherCfg.ForceSubmitSpanBatch || s.rollupCfg.IsDelta(block.Time()) { - ch, err = derive.NewSpanChannelOut(s.rollupCfg.Genesis.L2Time, s.rollupCfg.L2ChainID, target, derive.Brotli10) + ch, err = derive.NewSpanChannelOut(s.rollupCfg.Genesis.L2Time, s.rollupCfg.L2ChainID, target, derive.Zlib) // use singular batches in all other cases } else { ch, err = derive.NewSingularChannelOut(c) diff --git a/op-e2e/actions/sync_test.go b/op-e2e/actions/sync_test.go index fa86cb5437f8..e7521bdd8c94 100644 --- a/op-e2e/actions/sync_test.go +++ b/op-e2e/actions/sync_test.go @@ -26,7 +26,7 @@ import ( ) func newSpanChannelOut(t StatefulTesting, e e2eutils.SetupData) derive.ChannelOut { - channelOut, err := derive.NewSpanChannelOut(e.RollupCfg.Genesis.L2Time, e.RollupCfg.L2ChainID, 128_000, derive.Brotli10) + channelOut, err := derive.NewSpanChannelOut(e.RollupCfg.Genesis.L2Time, e.RollupCfg.L2ChainID, 128_000, derive.Zlib) require.NoError(t, err) return channelOut } diff --git a/op-e2e/setup.go b/op-e2e/setup.go index c4053485ea0d..8ee03c3ac712 100644 --- a/op-e2e/setup.go +++ b/op-e2e/setup.go @@ -844,9 +844,7 @@ func (cfg SystemConfig) Start(t *testing.T, _opts ...SystemConfigOption) (*Syste Stopped: sys.Cfg.DisableBatcher, // Batch submitter may be enabled later BatchType: batchType, DataAvailabilityType: sys.Cfg.DataAvailabilityType, - // Add the compression stuff - //CompressionAlgo: derive.Zlib, - CompressionAlgo: derive.Brotli10, + CompressionAlgo: derive.Zlib, } // Batch Submitter batcher, err := bss.BatcherServiceFromCLIConfig(context.Background(), "0.0.1", batcherCLIConfig, sys.Cfg.Loggers["batcher"]) diff --git a/op-node/benchmarks/batchbuilding_test.go b/op-node/benchmarks/batchbuilding_test.go index 75f96a349504..c8a47d549498 100644 --- a/op-node/benchmarks/batchbuilding_test.go +++ b/op-node/benchmarks/batchbuilding_test.go @@ -176,7 +176,7 @@ func BenchmarkIncremental(b *testing.B) { } done := false for base := 0; !done; base += tc.BatchCount { - rangeName := fmt.Sprintf("Incremental %s: %d-%d", tc.String(), base, base+tc.BatchCount) + rangeName := fmt.Sprintf("Incremental %s-%s: %d-%d", algo, tc.String(), base, base+tc.BatchCount) b.Run(rangeName+"_"+algo.String(), func(b *testing.B) { b.StopTimer() // prepare the batches diff --git a/op-node/rollup/derive/channel_compressor.go b/op-node/rollup/derive/channel_compressor.go index e541617ea00f..eab350922f41 100644 --- a/op-node/rollup/derive/channel_compressor.go +++ b/op-node/rollup/derive/channel_compressor.go @@ -32,19 +32,19 @@ type CompressorWriter interface { type BaseChannelCompressor struct { compressed *bytes.Buffer - writer CompressorWriter + CompressorWriter } func (bcc *BaseChannelCompressor) Write(data []byte) (int, error) { - return bcc.writer.Write(data) + return bcc.CompressorWriter.Write(data) } func (bcc *BaseChannelCompressor) Flush() error { - return bcc.writer.Flush() + return bcc.CompressorWriter.Flush() } func (bcc *BaseChannelCompressor) Close() error { - return bcc.writer.Close() + return bcc.CompressorWriter.Close() } func (bcc *BaseChannelCompressor) Len() int { @@ -65,7 +65,7 @@ type ZlibCompressor struct { func (zc *ZlibCompressor) Reset() { zc.compressed.Reset() - zc.writer.Reset(zc.compressed) + zc.CompressorWriter.Reset(zc.compressed) } type BrotliCompressor struct { @@ -75,7 +75,7 @@ type BrotliCompressor struct { func (bc *BrotliCompressor) Reset() { bc.compressed.Reset() bc.compressed.WriteByte(ChannelVersionBrotli) - bc.writer.Reset(bc.compressed) + bc.CompressorWriter.Reset(bc.compressed) } func NewChannelCompressor(algo CompressionAlgo) (ChannelCompressor, error) { @@ -87,8 +87,8 @@ func NewChannelCompressor(algo CompressionAlgo) (ChannelCompressor, error) { } return &ZlibCompressor{ BaseChannelCompressor{ - writer: writer, - compressed: compressed, + CompressorWriter: writer, + compressed: compressed, }, }, nil } else if algo.IsBrotli() { @@ -96,8 +96,8 @@ func NewChannelCompressor(algo CompressionAlgo) (ChannelCompressor, error) { writer := brotli.NewWriterLevel(compressed, GetBrotliLevel(algo)) return &BrotliCompressor{ BaseChannelCompressor{ - writer: writer, - compressed: compressed, + CompressorWriter: writer, + compressed: compressed, }, }, nil } else { diff --git a/op-node/rollup/derive/channel_compressor_test.go b/op-node/rollup/derive/channel_compressor_test.go index dd1f351760dc..a85198d81fad 100644 --- a/op-node/rollup/derive/channel_compressor_test.go +++ b/op-node/rollup/derive/channel_compressor_test.go @@ -19,42 +19,47 @@ func randomBytes(length int) []byte { return b } -func TestSpanChannelCompressor(t *testing.T) { +func TestChannelCompressor_NewReset(t *testing.T) { testCases := []struct { - name string - algo CompressionAlgo - expectedCompressedSize int + name string + algo CompressionAlgo + expectedResetSize int + expectErr bool }{{ - name: "zlib", - algo: Zlib, - expectedCompressedSize: 0, + name: "zlib", + algo: Zlib, + expectedResetSize: 0, }, { - name: "brotli10", - algo: Brotli10, - expectedCompressedSize: 1, + name: "brotli10", + algo: Brotli10, + expectedResetSize: 1, }, { - name: "zstd", - algo: CompressionAlgo("zstd"), - expectedCompressedSize: 0, + name: "zstd", + algo: CompressionAlgo("zstd"), + expectedResetSize: 0, + expectErr: true, }} for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { scc, err := NewChannelCompressor(tc.algo) - if tc.name == "zstd" { + if tc.expectErr { require.Error(t, err) return } require.NoError(t, err) - require.Equal(t, tc.expectedCompressedSize, scc.Len()) + require.Equal(t, tc.expectedResetSize, scc.Len()) - scc.Write(randomBytes(10000000)) - require.Greater(t, scc.Len(), tc.expectedCompressedSize) + _, err = scc.Write(randomBytes(10)) + require.NoError(t, err) + err = scc.Flush() + require.NoError(t, err) + require.Greater(t, scc.Len(), tc.expectedResetSize) scc.Reset() - require.Equal(t, tc.expectedCompressedSize, scc.Len()) + require.Equal(t, tc.expectedResetSize, scc.Len()) }) } } diff --git a/op-node/rollup/derive/channel_out_test.go b/op-node/rollup/derive/channel_out_test.go index 9c1cd6070383..00e0a8b14487 100644 --- a/op-node/rollup/derive/channel_out_test.go +++ b/op-node/rollup/derive/channel_out_test.go @@ -52,7 +52,7 @@ var channelTypes = []struct { { Name: "Span", ChannelOut: func(t *testing.T) ChannelOut { - cout, err := NewSpanChannelOut(0, big.NewInt(0), 128_000, Brotli10) + cout, err := NewSpanChannelOut(0, big.NewInt(0), 128_000, Zlib) require.NoError(t, err) return cout }, @@ -220,12 +220,12 @@ func TestBlockToBatchValidity(t *testing.T) { require.ErrorContains(t, err, "has no transactions") } -func SpanChannelAndBatches(t *testing.T, target uint64, len int) (*SpanChannelOut, []*SingularBatch) { +func SpanChannelAndBatches(t *testing.T, target uint64, len int, algo CompressionAlgo) (*SpanChannelOut, []*SingularBatch) { // target is larger than one batch, but smaller than two batches rng := rand.New(rand.NewSource(0x543331)) chainID := big.NewInt(rng.Int63n(1000)) txCount := 1 - cout, err := NewSpanChannelOut(0, chainID, target, Brotli10) + cout, err := NewSpanChannelOut(0, chainID, target, algo) require.NoError(t, err) batches := make([]*SingularBatch, len) // adding the first batch should not cause an error @@ -237,10 +237,29 @@ func SpanChannelAndBatches(t *testing.T, target uint64, len int) (*SpanChannelOu return cout, batches } +func TestSpanChannelOut(t *testing.T) { + tests := []struct { + name string + f func(t *testing.T, algo CompressionAlgo) + }{ + {"SpanChannelOutCompressionOnlyOneBatch", SpanChannelOutCompressionOnlyOneBatch}, + {"SpanChannelOutCompressionUndo", SpanChannelOutCompressionUndo}, + {"SpanChannelOutClose", SpanChannelOutClose}, + } + for _, test := range tests { + test := test + for _, algo := range CompressionAlgoTypes { + t.Run(test.name+"_"+algo.String(), func(t *testing.T) { + test.f(t, algo) + }) + } + } +} + // TestSpanChannelOutCompressionOnlyOneBatch tests that the SpanChannelOut compression works as expected when there is only one batch // and it is larger than the target size. The single batch should be compressed, and the channel should now be full -func TestSpanChannelOutCompressionOnlyOneBatch(t *testing.T) { - cout, singularBatches := SpanChannelAndBatches(t, 300, 2) +func SpanChannelOutCompressionOnlyOneBatch(t *testing.T, algo CompressionAlgo) { + cout, singularBatches := SpanChannelAndBatches(t, 300, 2, algo) err := cout.AddSingularBatch(singularBatches[0], 0) // confirm compression was not skipped @@ -256,14 +275,18 @@ func TestSpanChannelOutCompressionOnlyOneBatch(t *testing.T) { } // TestSpanChannelOutCompressionUndo tests that the SpanChannelOut compression rejects a batch that would cause the channel to be overfull -func TestSpanChannelOutCompressionUndo(t *testing.T) { +func SpanChannelOutCompressionUndo(t *testing.T, algo CompressionAlgo) { // target is larger than one batch, but smaller than two batches - cout, singularBatches := SpanChannelAndBatches(t, 750, 2) + cout, singularBatches := SpanChannelAndBatches(t, 750, 2, algo) err := cout.AddSingularBatch(singularBatches[0], 0) require.NoError(t, err) // confirm that the first compression was skipped - require.Equal(t, 1, cout.compressor.Len()) // 1 because of brotli channel version + if algo == Zlib { + require.Equal(t, 0, cout.compressor.Len()) + } else { + require.Equal(t, 1, cout.compressor.Len()) // 1 because of brotli channel version + } // record the RLP length to confirm it doesn't change when adding a rejected batch rlp1 := cout.activeRLP().Len() @@ -278,14 +301,19 @@ func TestSpanChannelOutCompressionUndo(t *testing.T) { // TestSpanChannelOutClose tests that the SpanChannelOut compression works as expected when the channel is closed. // it should compress the batch even if it is smaller than the target size because the channel is closing -func TestSpanChannelOutClose(t *testing.T) { +func SpanChannelOutClose(t *testing.T, algo CompressionAlgo) { target := uint64(600) - cout, singularBatches := SpanChannelAndBatches(t, target, 1) + cout, singularBatches := SpanChannelAndBatches(t, target, 1, algo) err := cout.AddSingularBatch(singularBatches[0], 0) require.NoError(t, err) // confirm no compression has happened yet - require.Equal(t, 1, cout.compressor.Len()) // 1 because of brotli channel version + + if algo == Zlib { + require.Equal(t, 0, cout.compressor.Len()) + } else { + require.Equal(t, 1, cout.compressor.Len()) // 1 because of brotli channel version + } // confirm the RLP length is less than the target rlpLen := cout.activeRLP().Len() diff --git a/op-node/rollup/derive/channel_test.go b/op-node/rollup/derive/channel_test.go index 58284aa1af97..ddd9877a5dfa 100644 --- a/op-node/rollup/derive/channel_test.go +++ b/op-node/rollup/derive/channel_test.go @@ -110,62 +110,89 @@ func TestFrameValidity(t *testing.T) { func TestBatchReader(t *testing.T) { // Get batch data rng := rand.New(rand.NewSource(0x543331)) - singularBatch := RandomSingularBatch(rng, 5000, big.NewInt(333)) + singularBatch := RandomSingularBatch(rng, 20, big.NewInt(333)) batchDataInput := NewBatchData(singularBatch) encodedBatch := &bytes.Buffer{} buf := &bytes.Buffer{} // Get the encoded data of the batch data - batchDataInput.encodeTyped(buf) - err := rlp.Encode(encodedBatch, buf.Bytes()) + err := batchDataInput.encodeTyped(buf) + require.NoError(t, err) + err = rlp.Encode(encodedBatch, buf.Bytes()) require.NoError(t, err) var testCases = []struct { - name string - algo func(buf *bytes.Buffer) - }{{ - name: "zlib", - algo: func(buf *bytes.Buffer) { - writer := zlib.NewWriter(buf) - writer.Write(encodedBatch.Bytes()) - writer.Close() + name string + algo func(buf *bytes.Buffer, t *testing.T) + expectErr bool + }{ + { + name: "zlib", + algo: func(buf *bytes.Buffer, t *testing.T) { + writer := zlib.NewWriter(buf) + _, err := writer.Write(encodedBatch.Bytes()) + require.NoError(t, err) + writer.Close() + }, + }, + { + name: "brotli9", + algo: func(buf *bytes.Buffer, t *testing.T) { + buf.WriteByte(ChannelVersionBrotli) + writer := brotli.NewWriterLevel(buf, 9) + _, err := writer.Write(encodedBatch.Bytes()) + require.NoError(t, err) + writer.Close() + }, }, - }, { name: "brotli10", - algo: func(buf *bytes.Buffer) { + algo: func(buf *bytes.Buffer, t *testing.T) { buf.WriteByte(ChannelVersionBrotli) writer := brotli.NewWriterLevel(buf, 10) - writer.Write(encodedBatch.Bytes()) + _, err := writer.Write(encodedBatch.Bytes()) + require.NoError(t, err) writer.Close() }, - }, { + }, + { + name: "brotli11", + algo: func(buf *bytes.Buffer, t *testing.T) { + buf.WriteByte(ChannelVersionBrotli) + writer := brotli.NewWriterLevel(buf, 11) + _, err := writer.Write(encodedBatch.Bytes()) + require.NoError(t, err) + writer.Close() + }, + }, + { name: "zstd", - algo: func(buf *bytes.Buffer) { + algo: func(buf *bytes.Buffer, t *testing.T) { writer := zstd.NewWriter(buf) - writer.Write(encodedBatch.Bytes()) + _, err := writer.Write(encodedBatch.Bytes()) + require.NoError(t, err) writer.Close() }, + expectErr: true, }} for _, tc := range testCases { - compressed := bytes.NewBuffer([]byte{}) + compressed := new(bytes.Buffer) tc := tc t.Run(tc.name, func(t *testing.T) { - tc.algo(compressed) + tc.algo(compressed, t) reader, err := BatchReader(bytes.NewReader(compressed.Bytes())) - if tc.name == "zstd" { - require.NotNil(t, err) + if tc.expectErr { + require.Error(t, err) return } - require.Nil(t, err) + require.NoError(t, err) // read the batch data batchData, err := reader() - require.Nil(t, err) + require.NoError(t, err) require.NotNil(t, batchData) require.Equal(t, batchDataInput, batchData) }) } - } diff --git a/op-node/rollup/derive/types_test.go b/op-node/rollup/derive/types_test.go index 7ab49d7ad204..5b9c1d94ed50 100644 --- a/op-node/rollup/derive/types_test.go +++ b/op-node/rollup/derive/types_test.go @@ -6,46 +6,52 @@ import ( "github.com/stretchr/testify/require" ) -func TestIsBrotli(t *testing.T) { +func TestCompressionAlgo(t *testing.T) { testCases := []struct { name string algo CompressionAlgo - expectedResult bool + isBrotli bool isValidCompressionAlgoType bool - }{{ - name: "zlib", - algo: Zlib, - expectedResult: false, - isValidCompressionAlgoType: true, - }, + }{ + { + name: "zlib", + algo: Zlib, + isBrotli: false, + isValidCompressionAlgoType: true, + }, { name: "brotli-9", algo: Brotli9, - expectedResult: true, + isBrotli: true, isValidCompressionAlgoType: true, }, { name: "brotli-10", algo: Brotli10, - expectedResult: true, + isBrotli: true, isValidCompressionAlgoType: true, }, { name: "brotli-11", algo: Brotli11, - expectedResult: true, + isBrotli: true, isValidCompressionAlgoType: true, }, { name: "invalid", algo: CompressionAlgo("invalid"), - expectedResult: false, + isBrotli: false, isValidCompressionAlgoType: false, }} for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { - require.Equal(t, tc.expectedResult, tc.algo.IsBrotli()) + require.Equal(t, tc.isBrotli, tc.algo.IsBrotli()) + if tc.isBrotli { + require.NotPanics(t, func() { GetBrotliLevel((tc.algo)) }) + } else { + require.Panics(t, func() { GetBrotliLevel(tc.algo) }) + } require.Equal(t, tc.isValidCompressionAlgoType, ValidCompressionAlgoType(tc.algo)) }) } From aade67d54e0c2fd786e7bd6fcaaa681fc91902fb Mon Sep 17 00:00:00 2001 From: Cody Wang Date: Thu, 9 May 2024 16:02:59 -0700 Subject: [PATCH 21/38] fix --- op-batcher/batcher/channel_builder_test.go | 49 +++++++++++-------- op-batcher/batcher/channel_manager_test.go | 13 +++-- op-e2e/actions/l2_batcher.go | 2 +- .../rollup/derive/channel_compressor_test.go | 14 +++--- 4 files changed, 44 insertions(+), 34 deletions(-) diff --git a/op-batcher/batcher/channel_builder_test.go b/op-batcher/batcher/channel_builder_test.go index 9853bbe4c80f..3e34ce071adf 100644 --- a/op-batcher/batcher/channel_builder_test.go +++ b/op-batcher/batcher/channel_builder_test.go @@ -286,18 +286,15 @@ func FuzzSeqWindowZeroTimeoutClose(f *testing.F) { }) } -func TestChannelBuilderBatchType(t *testing.T) { +func TestChannelBuilderBatchTypeAndAlgo(t *testing.T) { tests := []struct { name string f func(t *testing.T, batchType uint, algo derive.CompressionAlgo) }{ - {"ChannelBuilder_MaxRLPBytesPerChannel", ChannelBuilder_MaxRLPBytesPerChannel}, {"ChannelBuilder_OutputFramesMaxFrameIndex", ChannelBuilder_OutputFramesMaxFrameIndex}, {"ChannelBuilder_AddBlock", ChannelBuilder_AddBlock}, {"ChannelBuilder_PendingFrames_TotalFrames", ChannelBuilder_PendingFrames_TotalFrames}, - {"ChannelBuilder_InputBytes", ChannelBuilder_InputBytes}, {"ChannelBuilder_OutputBytes", ChannelBuilder_OutputBytes}, - {"ChannelBuilder_OutputWrongFramePanic", ChannelBuilder_OutputWrongFramePanic}, } for _, test := range tests { test := test @@ -318,26 +315,32 @@ func TestChannelBuilderBatchType(t *testing.T) { } } -func TestChannelBuilderSingularBatch(t *testing.T) { +func TestChannelBuilderBatchType(t *testing.T) { tests := []struct { name string - f func(t *testing.T, algo derive.CompressionAlgo) + f func(t *testing.T, batchType uint) }{ - {"ChannelBuilder_NextFrame", ChannelBuilder_NextFrame}, - {"ChannelBuilder_OutputFrames", ChannelBuilder_OutputFrames}, + {"ChannelBuilder_MaxRLPBytesPerChannel", ChannelBuilder_MaxRLPBytesPerChannel}, + {"ChannelBuilder_InputBytes", ChannelBuilder_InputBytes}, + {"ChannelBuilder_OutputWrongFramePanic", ChannelBuilder_OutputWrongFramePanic}, } for _, test := range tests { test := test - for _, algo := range derive.CompressionAlgoTypes { - t.Run(test.name+"_"+algo.String(), func(t *testing.T) { - test.f(t, algo) - }) - } + t.Run(test.name+"_SingularBatch", func(t *testing.T) { + test.f(t, derive.SingularBatchType) + }) + } + + for _, test := range tests { + test := test + t.Run(test.name+"_SpanBatch_", func(t *testing.T) { + test.f(t, derive.SpanBatchType) + }) } } // TestChannelBuilder_NextFrame tests calling NextFrame on a ChannelBuilder with only one frame -func ChannelBuilder_NextFrame(t *testing.T, algo derive.CompressionAlgo) { +func TestChannelBuilder_NextFrame(t *testing.T) { channelConfig := defaultTestChannelConfig() // Create a new channel builder @@ -377,7 +380,7 @@ func ChannelBuilder_NextFrame(t *testing.T, algo derive.CompressionAlgo) { } // TestChannelBuilder_OutputWrongFramePanic tests that a panic is thrown when a frame is pushed with an invalid frame id -func ChannelBuilder_OutputWrongFramePanic(t *testing.T, batchType uint, algo derive.CompressionAlgo) { +func ChannelBuilder_OutputWrongFramePanic(t *testing.T, batchType uint) { channelConfig := defaultTestChannelConfig() channelConfig.BatchType = batchType @@ -411,7 +414,7 @@ func ChannelBuilder_OutputWrongFramePanic(t *testing.T, batchType uint, algo der } // TestChannelBuilder_OutputFrames tests [ChannelBuilder.OutputFrames] for singular batches. -func ChannelBuilder_OutputFrames(t *testing.T, algo derive.CompressionAlgo) { +func TestChannelBuilder_OutputFrames(t *testing.T) { channelConfig := defaultTestChannelConfig() channelConfig.MaxFrameSize = derive.FrameV0OverHeadSize + 1 channelConfig.TargetNumFrames = 1000 @@ -454,7 +457,7 @@ func ChannelBuilder_OutputFrames(t *testing.T, algo derive.CompressionAlgo) { func TestChannelBuilder_OutputFrames_SpanBatch(t *testing.T) { for _, algo := range derive.CompressionAlgoTypes { - t.Run("ChannelBuilder_OutputFrames_SpanBatch"+algo.String(), func(t *testing.T) { + t.Run("ChannelBuilder_OutputFrames_SpanBatch_"+algo.String(), func(t *testing.T) { if algo.IsBrotli() { ChannelBuilder_OutputFrames_SpanBatch(t, algo) // to fill faster for brotli } else { @@ -467,7 +470,11 @@ func TestChannelBuilder_OutputFrames_SpanBatch(t *testing.T) { func ChannelBuilder_OutputFrames_SpanBatch(t *testing.T, algo derive.CompressionAlgo) { channelConfig := defaultTestChannelConfig() channelConfig.MaxFrameSize = 20 + derive.FrameV0OverHeadSize - channelConfig.TargetNumFrames = 5 + if algo.IsBrotli() { + channelConfig.TargetNumFrames = 3 + } else { + channelConfig.TargetNumFrames = 5 + } channelConfig.BatchType = derive.SpanBatchType channelConfig.InitRatioCompressor(1, algo) @@ -521,7 +528,7 @@ func ChannelBuilder_OutputFrames_SpanBatch(t *testing.T, algo derive.Compression // ChannelBuilder_MaxRLPBytesPerChannel tests the [ChannelBuilder.OutputFrames] // function errors when the max RLP bytes per channel is reached. -func ChannelBuilder_MaxRLPBytesPerChannel(t *testing.T, batchType uint, algo derive.CompressionAlgo) { +func ChannelBuilder_MaxRLPBytesPerChannel(t *testing.T, batchType uint) { t.Parallel() channelConfig := defaultTestChannelConfig() channelConfig.MaxFrameSize = derive.MaxRLPBytesPerChannel * 2 @@ -598,7 +605,9 @@ func TestChannelBuilder_FullShadowCompressor(t *testing.T) { require.ErrorIs(err, derive.ErrCompressorFull) // without fix, adding the second block would succeed and then adding a // third block would fail with full error and the compressor would be full. + require.NoError(cb.OutputFrames()) + require.True(cb.HasFrame()) f := cb.NextFrame() require.Less(len(f.data), int(cfg.MaxFrameSize)) // would fail without fix, full frame @@ -773,7 +782,7 @@ func ChannelBuilder_PendingFrames_TotalFrames(t *testing.T, batchType uint, algo } } -func ChannelBuilder_InputBytes(t *testing.T, batchType uint, algo derive.CompressionAlgo) { +func ChannelBuilder_InputBytes(t *testing.T, batchType uint) { require := require.New(t) rng := rand.New(rand.NewSource(4982432)) cfg := defaultTestChannelConfig() diff --git a/op-batcher/batcher/channel_manager_test.go b/op-batcher/batcher/channel_manager_test.go index ff13bc2eefde..aadd45e9fbab 100644 --- a/op-batcher/batcher/channel_manager_test.go +++ b/op-batcher/batcher/channel_manager_test.go @@ -71,20 +71,19 @@ func TestChannelManagerBatchType(t *testing.T) { } for _, test := range tests { test := test - t.Run(test.name+"_SingularBatch_",func(t *testing.T) { - test.f(t, derive.SingularBatchType) - }) + t.Run(test.name+"_SingularBatch_", func(t *testing.T) { + test.f(t, derive.SingularBatchType) + }) } for _, test := range tests { test := test - t.Run(test.name+"_SpanBatch", func(t *testing.T) { - test.f(t, derive.SpanBatchType) - }) + t.Run(test.name+"_SpanBatch", func(t *testing.T) { + test.f(t, derive.SpanBatchType) + }) } } - // ChannelManagerReturnsErrReorg ensures that the channel manager // detects a reorg when it has cached L1 blocks. func ChannelManagerReturnsErrReorg(t *testing.T, batchType uint, algo derive.CompressionAlgo) { diff --git a/op-e2e/actions/l2_batcher.go b/op-e2e/actions/l2_batcher.go index a4970b95f02d..16cab1a1d0d4 100644 --- a/op-e2e/actions/l2_batcher.go +++ b/op-e2e/actions/l2_batcher.go @@ -192,7 +192,7 @@ func (s *L2Batcher) Buffer(t Testing) error { target := batcher.MaxDataSize(1, s.l2BatcherCfg.MaxL1TxSize) c, e := compressor.NewShadowCompressor(compressor.Config{ TargetOutputSize: target, - CompressionAlgo: derive.Zlib, + CompressionAlgo: derive.Zlib, }) require.NoError(t, e, "failed to create compressor") diff --git a/op-node/rollup/derive/channel_compressor_test.go b/op-node/rollup/derive/channel_compressor_test.go index a85198d81fad..3224b89d9808 100644 --- a/op-node/rollup/derive/channel_compressor_test.go +++ b/op-node/rollup/derive/channel_compressor_test.go @@ -25,11 +25,12 @@ func TestChannelCompressor_NewReset(t *testing.T) { algo CompressionAlgo expectedResetSize int expectErr bool - }{{ - name: "zlib", - algo: Zlib, - expectedResetSize: 0, - }, + }{ + { + name: "zlib", + algo: Zlib, + expectedResetSize: 0, + }, { name: "brotli10", algo: Brotli10, @@ -40,7 +41,8 @@ func TestChannelCompressor_NewReset(t *testing.T) { algo: CompressionAlgo("zstd"), expectedResetSize: 0, expectErr: true, - }} + }, + } for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { From ca47d0fbe07cd1198943ca84ce950387417093f7 Mon Sep 17 00:00:00 2001 From: Cody Wang Date: Thu, 9 May 2024 17:05:36 -0700 Subject: [PATCH 22/38] fix --- op-batcher/batcher/channel_builder_test.go | 2 +- op-batcher/batcher/channel_manager_test.go | 69 +++++++--------------- op-node/rollup/derive/channel_test.go | 2 +- 3 files changed, 24 insertions(+), 49 deletions(-) diff --git a/op-batcher/batcher/channel_builder_test.go b/op-batcher/batcher/channel_builder_test.go index af24b5712a2b..20423c871710 100644 --- a/op-batcher/batcher/channel_builder_test.go +++ b/op-batcher/batcher/channel_builder_test.go @@ -333,7 +333,7 @@ func TestChannelBuilderBatchType(t *testing.T) { for _, test := range tests { test := test - t.Run(test.name+"_SpanBatch_", func(t *testing.T) { + t.Run(test.name+"_SpanBatch", func(t *testing.T) { test.f(t, derive.SpanBatchType) }) } diff --git a/op-batcher/batcher/channel_manager_test.go b/op-batcher/batcher/channel_manager_test.go index aadd45e9fbab..24dcc4b27a48 100644 --- a/op-batcher/batcher/channel_manager_test.go +++ b/op-batcher/batcher/channel_manager_test.go @@ -20,20 +20,20 @@ import ( "github.com/stretchr/testify/require" ) -func channelManagerTestConfig(maxFrameSize uint64, batchType uint, algo derive.CompressionAlgo) ChannelConfig { +func channelManagerTestConfig(maxFrameSize uint64, batchType uint) ChannelConfig { cfg := ChannelConfig{ MaxFrameSize: maxFrameSize, TargetNumFrames: 1, BatchType: batchType, } - cfg.InitRatioCompressor(1, algo) + cfg.InitRatioCompressor(1, derive.Zlib) return cfg } -func TestChannelManagerBatchTypeAndAlgo(t *testing.T) { +func TestChannelManagerBatchType(t *testing.T) { tests := []struct { name string - f func(t *testing.T, batchType uint, algo derive.CompressionAlgo) + f func(t *testing.T, batchType uint) }{ {"ChannelManagerReturnsErrReorg", ChannelManagerReturnsErrReorg}, {"ChannelManagerReturnsErrReorgWhenDrained", ChannelManagerReturnsErrReorgWhenDrained}, @@ -42,36 +42,11 @@ func TestChannelManagerBatchTypeAndAlgo(t *testing.T) { {"ChannelManagerCloseBeforeFirstUse", ChannelManagerCloseBeforeFirstUse}, {"ChannelManagerCloseNoPendingChannel", ChannelManagerCloseNoPendingChannel}, {"ChannelManagerClosePendingChannel", ChannelManagerClosePendingChannel}, - } - for _, test := range tests { - test := test - for _, algo := range derive.CompressionAlgoTypes { - t.Run(test.name+"_SingularBatch_"+algo.String(), func(t *testing.T) { - test.f(t, derive.SingularBatchType, algo) - }) - } - } - - for _, test := range tests { - test := test - for _, algo := range derive.CompressionAlgoTypes { - t.Run(test.name+"_SpanBatch"+algo.String(), func(t *testing.T) { - test.f(t, derive.SpanBatchType, algo) - }) - } - } -} - -func TestChannelManagerBatchType(t *testing.T) { - tests := []struct { - name string - f func(t *testing.T, batchType uint) - }{ {"ChannelManagerCloseAllTxsFailed", ChannelManagerCloseAllTxsFailed}, } for _, test := range tests { test := test - t.Run(test.name+"_SingularBatch_", func(t *testing.T) { + t.Run(test.name+"_SingularBatch", func(t *testing.T) { test.f(t, derive.SingularBatchType) }) } @@ -86,9 +61,9 @@ func TestChannelManagerBatchType(t *testing.T) { // ChannelManagerReturnsErrReorg ensures that the channel manager // detects a reorg when it has cached L1 blocks. -func ChannelManagerReturnsErrReorg(t *testing.T, batchType uint, algo derive.CompressionAlgo) { +func ChannelManagerReturnsErrReorg(t *testing.T, batchType uint) { log := testlog.Logger(t, log.LevelCrit) - m := NewChannelManager(log, metrics.NoopMetrics, ChannelConfig{BatchType: batchType, CompressorConfig: compressor.Config{CompressionAlgo: algo}}, &rollup.Config{}) + m := NewChannelManager(log, metrics.NoopMetrics, ChannelConfig{BatchType: batchType, CompressorConfig: compressor.Config{CompressionAlgo: derive.Zlib}}, &rollup.Config{}) m.Clear(eth.BlockID{}) a := types.NewBlock(&types.Header{ @@ -117,9 +92,9 @@ func ChannelManagerReturnsErrReorg(t *testing.T, batchType uint, algo derive.Com // ChannelManagerReturnsErrReorgWhenDrained ensures that the channel manager // detects a reorg even if it does not have any blocks inside it. -func ChannelManagerReturnsErrReorgWhenDrained(t *testing.T, batchType uint, algo derive.CompressionAlgo) { +func ChannelManagerReturnsErrReorgWhenDrained(t *testing.T, batchType uint) { log := testlog.Logger(t, log.LevelCrit) - cfg := channelManagerTestConfig(120_000, batchType, algo) + cfg := channelManagerTestConfig(120_000, batchType) cfg.CompressorConfig.TargetOutputSize = 1 // full on first block m := NewChannelManager(log, metrics.NoopMetrics, cfg, &rollup.Config{}) m.Clear(eth.BlockID{}) @@ -138,18 +113,18 @@ func ChannelManagerReturnsErrReorgWhenDrained(t *testing.T, batchType uint, algo } // ChannelManager_Clear tests clearing the channel manager. -func ChannelManager_Clear(t *testing.T, batchType uint, algo derive.CompressionAlgo) { +func ChannelManager_Clear(t *testing.T, batchType uint) { require := require.New(t) // Create a channel manager log := testlog.Logger(t, log.LevelCrit) rng := rand.New(rand.NewSource(time.Now().UnixNano())) - cfg := channelManagerTestConfig(derive.FrameV0OverHeadSize+1, batchType, algo) + cfg := channelManagerTestConfig(derive.FrameV0OverHeadSize+1, batchType) // Need to set the channel timeout here so we don't clear pending // channels on confirmation. This would result in [TxConfirmed] // clearing confirmed transactions, and resetting the pendingChannels map cfg.ChannelTimeout = 10 - cfg.InitRatioCompressor(1, algo) + cfg.InitRatioCompressor(1, derive.Zlib) m := NewChannelManager(log, metrics.NoopMetrics, cfg, &defaultTestRollupConfig) // Channel Manager state should be empty by default @@ -215,11 +190,11 @@ func ChannelManager_Clear(t *testing.T, batchType uint, algo derive.CompressionA require.Empty(m.txChannels) } -func ChannelManager_TxResend(t *testing.T, batchType uint, algo derive.CompressionAlgo) { +func ChannelManager_TxResend(t *testing.T, batchType uint) { require := require.New(t) rng := rand.New(rand.NewSource(time.Now().UnixNano())) log := testlog.Logger(t, log.LevelError) - cfg := channelManagerTestConfig(120_000, batchType, algo) + cfg := channelManagerTestConfig(120_000, batchType) cfg.CompressorConfig.TargetOutputSize = 1 // full on first block m := NewChannelManager(log, metrics.NoopMetrics, cfg, &defaultTestRollupConfig) m.Clear(eth.BlockID{}) @@ -254,12 +229,12 @@ func ChannelManager_TxResend(t *testing.T, batchType uint, algo derive.Compressi // ChannelManagerCloseBeforeFirstUse ensures that the channel manager // will not produce any frames if closed immediately. -func ChannelManagerCloseBeforeFirstUse(t *testing.T, batchType uint, algo derive.CompressionAlgo) { +func ChannelManagerCloseBeforeFirstUse(t *testing.T, batchType uint) { require := require.New(t) rng := rand.New(rand.NewSource(time.Now().UnixNano())) log := testlog.Logger(t, log.LevelCrit) m := NewChannelManager(log, metrics.NoopMetrics, - channelManagerTestConfig(10000, batchType, algo), + channelManagerTestConfig(10000, batchType), &defaultTestRollupConfig, ) m.Clear(eth.BlockID{}) @@ -278,10 +253,10 @@ func ChannelManagerCloseBeforeFirstUse(t *testing.T, batchType uint, algo derive // ChannelManagerCloseNoPendingChannel ensures that the channel manager // can gracefully close with no pending channels, and will not emit any new // channel frames. -func ChannelManagerCloseNoPendingChannel(t *testing.T, batchType uint, algo derive.CompressionAlgo) { +func ChannelManagerCloseNoPendingChannel(t *testing.T, batchType uint) { require := require.New(t) log := testlog.Logger(t, log.LevelCrit) - cfg := channelManagerTestConfig(10000, batchType, algo) + cfg := channelManagerTestConfig(10000, batchType) cfg.CompressorConfig.TargetOutputSize = 1 // full on first block cfg.ChannelTimeout = 1000 m := NewChannelManager(log, metrics.NoopMetrics, cfg, &defaultTestRollupConfig) @@ -312,13 +287,13 @@ func ChannelManagerCloseNoPendingChannel(t *testing.T, batchType uint, algo deri // ChannelManagerCloseNoPendingChannel ensures that the channel manager // can gracefully close with a pending channel, and will not produce any // new channel frames after this point. -func ChannelManagerClosePendingChannel(t *testing.T, batchType uint, algo derive.CompressionAlgo) { +func ChannelManagerClosePendingChannel(t *testing.T, batchType uint) { require := require.New(t) // The number of batch txs depends on compression of the random data, hence the static test RNG seed. // Example of different RNG seed that creates less than 2 frames: 1698700588902821588 rng := rand.New(rand.NewSource(123)) log := testlog.Logger(t, log.LevelError) - cfg := channelManagerTestConfig(10_000, batchType, algo) + cfg := channelManagerTestConfig(10_000, batchType) cfg.ChannelTimeout = 1000 m := NewChannelManager(log, metrics.NoopMetrics, cfg, &defaultTestRollupConfig) m.Clear(eth.BlockID{}) @@ -421,7 +396,7 @@ func ChannelManagerCloseAllTxsFailed(t *testing.T, batchType uint) { require := require.New(t) rng := rand.New(rand.NewSource(1357)) log := testlog.Logger(t, log.LevelCrit) - cfg := channelManagerTestConfig(100, batchType, derive.Zlib) + cfg := channelManagerTestConfig(100, batchType) cfg.TargetNumFrames = 1000 cfg.InitNoneCompressor() m := NewChannelManager(log, metrics.NoopMetrics, cfg, &defaultTestRollupConfig) @@ -469,7 +444,7 @@ func ChannelManagerCloseAllTxsFailed(t *testing.T, batchType uint) { func TestChannelManager_ChannelCreation(t *testing.T) { l := testlog.Logger(t, log.LevelCrit) const maxChannelDuration = 15 - cfg := channelManagerTestConfig(1000, derive.SpanBatchType, derive.Zlib) + cfg := channelManagerTestConfig(1000, derive.SpanBatchType) cfg.MaxChannelDuration = maxChannelDuration cfg.InitNoneCompressor() diff --git a/op-node/rollup/derive/channel_test.go b/op-node/rollup/derive/channel_test.go index ddd9877a5dfa..69b3feb29fe0 100644 --- a/op-node/rollup/derive/channel_test.go +++ b/op-node/rollup/derive/channel_test.go @@ -181,7 +181,7 @@ func TestBatchReader(t *testing.T) { tc := tc t.Run(tc.name, func(t *testing.T) { tc.algo(compressed, t) - reader, err := BatchReader(bytes.NewReader(compressed.Bytes())) + reader, err := BatchReader(bytes.NewReader(compressed.Bytes()), 120000) if tc.expectErr { require.Error(t, err) return From 729616fdcb10b580d6b241087db791903fbd1bec Mon Sep 17 00:00:00 2001 From: Cody Wang Date: Thu, 9 May 2024 19:31:23 -0700 Subject: [PATCH 23/38] revert channel builder test --- op-batcher/batcher/channel_builder_test.go | 50 ++++++---------------- 1 file changed, 12 insertions(+), 38 deletions(-) diff --git a/op-batcher/batcher/channel_builder_test.go b/op-batcher/batcher/channel_builder_test.go index 20423c871710..cb4c3ee9b7bd 100644 --- a/op-batcher/batcher/channel_builder_test.go +++ b/op-batcher/batcher/channel_builder_test.go @@ -286,43 +286,17 @@ func FuzzSeqWindowZeroTimeoutClose(f *testing.F) { }) } -func TestChannelBuilderBatchTypeAndAlgo(t *testing.T) { - tests := []struct { - name string - f func(t *testing.T, batchType uint, algo derive.CompressionAlgo) - }{ - {"ChannelBuilder_OutputFramesMaxFrameIndex", ChannelBuilder_OutputFramesMaxFrameIndex}, - {"ChannelBuilder_AddBlock", ChannelBuilder_AddBlock}, - {"ChannelBuilder_PendingFrames_TotalFrames", ChannelBuilder_PendingFrames_TotalFrames}, - {"ChannelBuilder_OutputBytes", ChannelBuilder_OutputBytes}, - } - for _, test := range tests { - test := test - for _, algo := range derive.CompressionAlgoTypes { - t.Run(test.name+"_SingularBatch_"+algo.String(), func(t *testing.T) { - test.f(t, derive.SingularBatchType, algo) - }) - } - } - - for _, test := range tests { - test := test - for _, algo := range derive.CompressionAlgoTypes { - t.Run(test.name+"_SpanBatch_"+algo.String(), func(t *testing.T) { - test.f(t, derive.SpanBatchType, algo) - }) - } - } -} - func TestChannelBuilderBatchType(t *testing.T) { tests := []struct { name string f func(t *testing.T, batchType uint) }{ {"ChannelBuilder_MaxRLPBytesPerChannel", ChannelBuilder_MaxRLPBytesPerChannel}, + {"ChannelBuilder_OutputFramesMaxFrameIndex", ChannelBuilder_OutputFramesMaxFrameIndex}, + {"ChannelBuilder_AddBlock", ChannelBuilder_AddBlock}, + {"ChannelBuilder_PendingFrames_TotalFrames", ChannelBuilder_PendingFrames_TotalFrames}, {"ChannelBuilder_InputBytes", ChannelBuilder_InputBytes}, - {"ChannelBuilder_OutputWrongFramePanic", ChannelBuilder_OutputWrongFramePanic}, + {"ChannelBuilder_OutputBytes", ChannelBuilder_OutputBytes}, } for _, test := range tests { test := test @@ -546,11 +520,11 @@ func ChannelBuilder_MaxRLPBytesPerChannel(t *testing.T, batchType uint) { // ChannelBuilder_OutputFramesMaxFrameIndex tests the [ChannelBuilder.OutputFrames] // function errors when the max frame index is reached. -func ChannelBuilder_OutputFramesMaxFrameIndex(t *testing.T, batchType uint, algo derive.CompressionAlgo) { +func ChannelBuilder_OutputFramesMaxFrameIndex(t *testing.T, batchType uint) { channelConfig := defaultTestChannelConfig() channelConfig.MaxFrameSize = derive.FrameV0OverHeadSize + 1 channelConfig.TargetNumFrames = math.MaxUint16 + 1 - channelConfig.InitRatioCompressor(.1, algo) + channelConfig.InitRatioCompressor(.1, derive.Zlib) channelConfig.BatchType = batchType rng := rand.New(rand.NewSource(123)) @@ -615,7 +589,7 @@ func TestChannelBuilder_FullShadowCompressor(t *testing.T) { require.False(cb.HasFrame(), "no leftover frame expected") // would fail without fix } -func ChannelBuilder_AddBlock(t *testing.T, batchType uint, algo derive.CompressionAlgo) { +func ChannelBuilder_AddBlock(t *testing.T, batchType uint) { channelConfig := defaultTestChannelConfig() channelConfig.BatchType = batchType @@ -623,7 +597,7 @@ func ChannelBuilder_AddBlock(t *testing.T, batchType uint, algo derive.Compressi channelConfig.MaxFrameSize = 20 + derive.FrameV0OverHeadSize channelConfig.TargetNumFrames = 2 // Configure the Input Threshold params so we observe a full channel - channelConfig.InitRatioCompressor(1, algo) + channelConfig.InitRatioCompressor(1, derive.Zlib) // Construct the channel builder cb, err := NewChannelBuilder(channelConfig, defaultTestRollupConfig, latestL1BlockOrigin) @@ -738,7 +712,7 @@ func TestChannelBuilder_LatestL1Origin(t *testing.T) { require.Equal(t, uint64(2), cb.LatestL1Origin().Number) } -func ChannelBuilder_PendingFrames_TotalFrames(t *testing.T, batchType uint, algo derive.CompressionAlgo) { +func ChannelBuilder_PendingFrames_TotalFrames(t *testing.T, batchType uint) { const tnf = 9 rng := rand.New(rand.NewSource(94572314)) require := require.New(t) @@ -746,7 +720,7 @@ func ChannelBuilder_PendingFrames_TotalFrames(t *testing.T, batchType uint, algo cfg.MaxFrameSize = 1000 cfg.TargetNumFrames = tnf cfg.BatchType = batchType - cfg.InitShadowCompressor(algo) + cfg.InitShadowCompressor(derive.Zlib) cb, err := NewChannelBuilder(cfg, defaultTestRollupConfig, latestL1BlockOrigin) require.NoError(err) @@ -821,14 +795,14 @@ func ChannelBuilder_InputBytes(t *testing.T, batchType uint) { } } -func ChannelBuilder_OutputBytes(t *testing.T, batchType uint, algo derive.CompressionAlgo) { +func ChannelBuilder_OutputBytes(t *testing.T, batchType uint) { require := require.New(t) rng := rand.New(rand.NewSource(9860372)) cfg := defaultTestChannelConfig() cfg.MaxFrameSize = 1000 cfg.TargetNumFrames = 16 cfg.BatchType = batchType - cfg.InitRatioCompressor(1.0, algo) + cfg.InitRatioCompressor(1.0, derive.Zlib) cb, err := NewChannelBuilder(cfg, defaultTestRollupConfig, latestL1BlockOrigin) require.NoError(err, "NewChannelBuilder") From 7aaa4ed0c2a526ab1001b183565d4745df498f20 Mon Sep 17 00:00:00 2001 From: Cody Wang Date: Thu, 9 May 2024 19:43:06 -0700 Subject: [PATCH 24/38] revert ratio compressor test --- .../compressor/ratio_compressor_test.go | 28 +++++++++---------- 1 file changed, 13 insertions(+), 15 deletions(-) diff --git a/op-batcher/compressor/ratio_compressor_test.go b/op-batcher/compressor/ratio_compressor_test.go index 089e8856f5d9..d1b0d4b936c7 100644 --- a/op-batcher/compressor/ratio_compressor_test.go +++ b/op-batcher/compressor/ratio_compressor_test.go @@ -59,21 +59,19 @@ func TestChannelConfig_InputThreshold(t *testing.T) { // Validate each test case for i, tt := range tests { - for _, algo := range derive.CompressionAlgoTypes { - t.Run(fmt.Sprintf("test-%d-%s", i, algo.String()), func(t *testing.T) { - comp, err := compressor.NewRatioCompressor(compressor.Config{ - TargetOutputSize: tt.targetOutputSize, - ApproxComprRatio: tt.approxComprRatio, - CompressionAlgo: algo, - }) - require.NoError(t, err) - got := comp.(*compressor.RatioCompressor).InputThreshold() - if tt.assertion != nil { - tt.assertion(got) - } else { - require.Equal(t, tt.expInputThreshold, got) - } + t.Run(fmt.Sprintf("test-%d", i), func(t *testing.T) { + comp, err := compressor.NewRatioCompressor(compressor.Config{ + TargetOutputSize: tt.targetOutputSize, + ApproxComprRatio: tt.approxComprRatio, + CompressionAlgo: derive.Zlib, }) - } + require.NoError(t, err) + got := comp.(*compressor.RatioCompressor).InputThreshold() + if tt.assertion != nil { + tt.assertion(got) + } else { + require.Equal(t, tt.expInputThreshold, got) + } + }) } } From da699a967c15483145456ca362cdb6eb23210aa1 Mon Sep 17 00:00:00 2001 From: Cody Wang Date: Fri, 10 May 2024 11:10:15 -0700 Subject: [PATCH 25/38] add checks to accept brotli only post fjord --- op-batcher/batcher/service.go | 5 +++ .../batch_decoder/reassemble/reassemble.go | 2 +- op-node/rollup/derive/channel.go | 8 +++- op-node/rollup/derive/channel_in_reader.go | 2 +- op-node/rollup/derive/channel_test.go | 40 ++++++++++++++++--- 5 files changed, 47 insertions(+), 10 deletions(-) diff --git a/op-batcher/batcher/service.go b/op-batcher/batcher/service.go index 6b8f1b2a54b5..4203f442c78e 100644 --- a/op-batcher/batcher/service.go +++ b/op-batcher/batcher/service.go @@ -228,6 +228,11 @@ func (bs *BatcherService) initChannelConfig(cfg *CLIConfig) error { bs.Log.Warn("Ecotone upgrade is active, but batcher is not configured to use Blobs!") } + // Checking for brotli compression only post Fjord + if bs.ChannelConfig.CompressorConfig.CompressionAlgo.IsBrotli() && !bs.RollupConfig.IsFjord(uint64(time.Now().Unix())) { + return fmt.Errorf("cannot use brotli compression before Fjord") + } + if err := cc.Check(); err != nil { return fmt.Errorf("invalid channel configuration: %w", err) } diff --git a/op-node/cmd/batch_decoder/reassemble/reassemble.go b/op-node/cmd/batch_decoder/reassemble/reassemble.go index c4a76493944c..23b4323b3e0d 100644 --- a/op-node/cmd/batch_decoder/reassemble/reassemble.go +++ b/op-node/cmd/batch_decoder/reassemble/reassemble.go @@ -111,7 +111,7 @@ func processFrames(cfg Config, rollupCfg *rollup.Config, id derive.ChannelID, fr var batchTypes []int invalidBatches := false if ch.IsReady() { - br, err := derive.BatchReader(ch.Reader(), spec.MaxRLPBytesPerChannel(ch.HighestBlock().Time)) + br, err := derive.BatchReader(ch.Reader(), spec.MaxRLPBytesPerChannel(ch.HighestBlock().Time), rollupCfg.IsFjord(ch.HighestBlock().Time)) if err == nil { for batchData, err := br(); err != io.EOF; batchData, err = br() { if err != nil { diff --git a/op-node/rollup/derive/channel.go b/op-node/rollup/derive/channel.go index 86fe39e90827..b41623fbe4c3 100644 --- a/op-node/rollup/derive/channel.go +++ b/op-node/rollup/derive/channel.go @@ -158,7 +158,7 @@ func (ch *Channel) Reader() io.Reader { // The L1Inclusion block is also provided at creation time. // Warning: the batch reader can read every batch-type. // The caller of the batch-reader should filter the results. -func BatchReader(r io.Reader, maxRLPBytesPerChannel uint64) (func() (*BatchData, error), error) { +func BatchReader(r io.Reader, maxRLPBytesPerChannel uint64, isFjord bool) (func() (*BatchData, error), error) { // use buffered reader so can peek the first byte bufReader := bufio.NewReader(r) compressionType, err := bufReader.Peek(1) @@ -174,13 +174,17 @@ func BatchReader(r io.Reader, maxRLPBytesPerChannel uint64) (func() (*BatchData, if err != nil { return nil, err } - // If the bits equal to 1, then it is a brotli reader + // If the bits equal to 1, then it is a brotli reader } else if compressionType[0] == ChannelVersionBrotli { // discard the first byte _, err := bufReader.Discard(1) if err != nil { return nil, err } + // If before Fjord, we cannot accept brotli compressed batch + if !isFjord { + return nil, fmt.Errorf("cannot accept zlib compressed batch after Fjord") + } zr = brotli.NewReader(bufReader) } else { return nil, fmt.Errorf("cannot distinguish the compression algo used given type byte %v", compressionType[0]) diff --git a/op-node/rollup/derive/channel_in_reader.go b/op-node/rollup/derive/channel_in_reader.go index 31a5746bb737..cd739e95fb34 100644 --- a/op-node/rollup/derive/channel_in_reader.go +++ b/op-node/rollup/derive/channel_in_reader.go @@ -44,7 +44,7 @@ func (cr *ChannelInReader) Origin() eth.L1BlockRef { // TODO: Take full channel for better logging func (cr *ChannelInReader) WriteChannel(data []byte) error { - if f, err := BatchReader(bytes.NewBuffer(data), cr.spec.MaxRLPBytesPerChannel(cr.prev.Origin().Time)); err == nil { + if f, err := BatchReader(bytes.NewBuffer(data), cr.spec.MaxRLPBytesPerChannel(cr.prev.Origin().Time), cr.cfg.IsFjord(cr.prev.Origin().Time)); err == nil { cr.nextBatchFn = f cr.metrics.RecordChannelInputBytes(len(data)) return nil diff --git a/op-node/rollup/derive/channel_test.go b/op-node/rollup/derive/channel_test.go index 69b3feb29fe0..41547cb41372 100644 --- a/op-node/rollup/derive/channel_test.go +++ b/op-node/rollup/derive/channel_test.go @@ -124,19 +124,42 @@ func TestBatchReader(t *testing.T) { var testCases = []struct { name string algo func(buf *bytes.Buffer, t *testing.T) + isFjord bool expectErr bool }{ { - name: "zlib", + name: "zlib-post-fjord", algo: func(buf *bytes.Buffer, t *testing.T) { writer := zlib.NewWriter(buf) _, err := writer.Write(encodedBatch.Bytes()) require.NoError(t, err) writer.Close() }, + isFjord: true, }, { - name: "brotli9", + name: "zlib-pre-fjord", + algo: func(buf *bytes.Buffer, t *testing.T) { + writer := zlib.NewWriter(buf) + _, err := writer.Write(encodedBatch.Bytes()) + require.NoError(t, err) + writer.Close() + }, + isFjord: false, + }, + { + name: "brotli9-post-fjord", + algo: func(buf *bytes.Buffer, t *testing.T) { + buf.WriteByte(ChannelVersionBrotli) + writer := brotli.NewWriterLevel(buf, 9) + _, err := writer.Write(encodedBatch.Bytes()) + require.NoError(t, err) + writer.Close() + }, + isFjord: true, + }, + { + name: "brotli9-pre-fjord", algo: func(buf *bytes.Buffer, t *testing.T) { buf.WriteByte(ChannelVersionBrotli) writer := brotli.NewWriterLevel(buf, 9) @@ -144,9 +167,11 @@ func TestBatchReader(t *testing.T) { require.NoError(t, err) writer.Close() }, + isFjord: false, + expectErr: true, // expect an error because brotli is not supported before Fjord }, { - name: "brotli10", + name: "brotli10-post-fjord", algo: func(buf *bytes.Buffer, t *testing.T) { buf.WriteByte(ChannelVersionBrotli) writer := brotli.NewWriterLevel(buf, 10) @@ -154,9 +179,10 @@ func TestBatchReader(t *testing.T) { require.NoError(t, err) writer.Close() }, + isFjord: true, }, { - name: "brotli11", + name: "brotli11-post-fjord", algo: func(buf *bytes.Buffer, t *testing.T) { buf.WriteByte(ChannelVersionBrotli) writer := brotli.NewWriterLevel(buf, 11) @@ -164,9 +190,10 @@ func TestBatchReader(t *testing.T) { require.NoError(t, err) writer.Close() }, + isFjord: true, }, { - name: "zstd", + name: "zstd-post-fjord", algo: func(buf *bytes.Buffer, t *testing.T) { writer := zstd.NewWriter(buf) _, err := writer.Write(encodedBatch.Bytes()) @@ -174,6 +201,7 @@ func TestBatchReader(t *testing.T) { writer.Close() }, expectErr: true, + isFjord: true, }} for _, tc := range testCases { @@ -181,7 +209,7 @@ func TestBatchReader(t *testing.T) { tc := tc t.Run(tc.name, func(t *testing.T) { tc.algo(compressed, t) - reader, err := BatchReader(bytes.NewReader(compressed.Bytes()), 120000) + reader, err := BatchReader(bytes.NewReader(compressed.Bytes()), 120000, tc.isFjord) if tc.expectErr { require.Error(t, err) return From abdea7d8c420229235111ccc89790742ee6b89d4 Mon Sep 17 00:00:00 2001 From: Cody Wang Date: Fri, 10 May 2024 11:12:10 -0700 Subject: [PATCH 26/38] revemo unnecessary in test --- op-batcher/batcher/channel_manager_test.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/op-batcher/batcher/channel_manager_test.go b/op-batcher/batcher/channel_manager_test.go index 24dcc4b27a48..eafcc49d1143 100644 --- a/op-batcher/batcher/channel_manager_test.go +++ b/op-batcher/batcher/channel_manager_test.go @@ -7,7 +7,6 @@ import ( "testing" "time" - "github.com/ethereum-optimism/optimism/op-batcher/compressor" "github.com/ethereum-optimism/optimism/op-batcher/metrics" "github.com/ethereum-optimism/optimism/op-node/rollup" "github.com/ethereum-optimism/optimism/op-node/rollup/derive" @@ -63,7 +62,7 @@ func TestChannelManagerBatchType(t *testing.T) { // detects a reorg when it has cached L1 blocks. func ChannelManagerReturnsErrReorg(t *testing.T, batchType uint) { log := testlog.Logger(t, log.LevelCrit) - m := NewChannelManager(log, metrics.NoopMetrics, ChannelConfig{BatchType: batchType, CompressorConfig: compressor.Config{CompressionAlgo: derive.Zlib}}, &rollup.Config{}) + m := NewChannelManager(log, metrics.NoopMetrics, ChannelConfig{BatchType: batchType}, &rollup.Config{}) m.Clear(eth.BlockID{}) a := types.NewBlock(&types.Header{ From 5f1b1981ad0bfccea6671212e4a0a620282455ae Mon Sep 17 00:00:00 2001 From: Cody Wang Date: Fri, 10 May 2024 11:32:42 -0700 Subject: [PATCH 27/38] fix forge-std --- packages/contracts-bedrock/lib/forge-std | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/contracts-bedrock/lib/forge-std b/packages/contracts-bedrock/lib/forge-std index bb4ceea94d6f..2d8b7b876a5b 160000 --- a/packages/contracts-bedrock/lib/forge-std +++ b/packages/contracts-bedrock/lib/forge-std @@ -1 +1 @@ -Subproject commit bb4ceea94d6f10eeb5b41dc2391c6c8bf8e734ef +Subproject commit 2d8b7b876a5b328d6a73e13c4740ed7a0d72d5f4 From 7298161f134367cdd16f3e720295397931aa11de Mon Sep 17 00:00:00 2001 From: Cody Wang Date: Fri, 10 May 2024 11:37:04 -0700 Subject: [PATCH 28/38] gofmt --- op-node/rollup/derive/channel.go | 2 +- op-node/rollup/derive/channel_test.go | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/op-node/rollup/derive/channel.go b/op-node/rollup/derive/channel.go index b41623fbe4c3..51e58684b3bd 100644 --- a/op-node/rollup/derive/channel.go +++ b/op-node/rollup/derive/channel.go @@ -174,7 +174,7 @@ func BatchReader(r io.Reader, maxRLPBytesPerChannel uint64, isFjord bool) (func( if err != nil { return nil, err } - // If the bits equal to 1, then it is a brotli reader + // If the bits equal to 1, then it is a brotli reader } else if compressionType[0] == ChannelVersionBrotli { // discard the first byte _, err := bufReader.Discard(1) diff --git a/op-node/rollup/derive/channel_test.go b/op-node/rollup/derive/channel_test.go index 41547cb41372..7a9dbb3b5d58 100644 --- a/op-node/rollup/derive/channel_test.go +++ b/op-node/rollup/derive/channel_test.go @@ -124,7 +124,7 @@ func TestBatchReader(t *testing.T) { var testCases = []struct { name string algo func(buf *bytes.Buffer, t *testing.T) - isFjord bool + isFjord bool expectErr bool }{ { @@ -167,7 +167,7 @@ func TestBatchReader(t *testing.T) { require.NoError(t, err) writer.Close() }, - isFjord: false, + isFjord: false, expectErr: true, // expect an error because brotli is not supported before Fjord }, { @@ -201,7 +201,7 @@ func TestBatchReader(t *testing.T) { writer.Close() }, expectErr: true, - isFjord: true, + isFjord: true, }} for _, tc := range testCases { From aa0823bc1b57a6c6757aaadac0178dfb7af60ce9 Mon Sep 17 00:00:00 2001 From: Cody Wang Date: Sat, 11 May 2024 16:33:27 -0700 Subject: [PATCH 29/38] address comments --- op-node/rollup/derive/channel.go | 8 ++++---- op-node/rollup/derive/channel_test.go | 7 +------ 2 files changed, 5 insertions(+), 10 deletions(-) diff --git a/op-node/rollup/derive/channel.go b/op-node/rollup/derive/channel.go index 51e58684b3bd..b3c2f2c0067c 100644 --- a/op-node/rollup/derive/channel.go +++ b/op-node/rollup/derive/channel.go @@ -176,15 +176,15 @@ func BatchReader(r io.Reader, maxRLPBytesPerChannel uint64, isFjord bool) (func( } // If the bits equal to 1, then it is a brotli reader } else if compressionType[0] == ChannelVersionBrotli { + // If before Fjord, we cannot accept brotli compressed batch + if !isFjord { + return nil, fmt.Errorf("cannot accept brotli compressed batch after Fjord") + } // discard the first byte _, err := bufReader.Discard(1) if err != nil { return nil, err } - // If before Fjord, we cannot accept brotli compressed batch - if !isFjord { - return nil, fmt.Errorf("cannot accept zlib compressed batch after Fjord") - } zr = brotli.NewReader(bufReader) } else { return nil, fmt.Errorf("cannot distinguish the compression algo used given type byte %v", compressionType[0]) diff --git a/op-node/rollup/derive/channel_test.go b/op-node/rollup/derive/channel_test.go index 7a9dbb3b5d58..d52fa84e6ca2 100644 --- a/op-node/rollup/derive/channel_test.go +++ b/op-node/rollup/derive/channel_test.go @@ -10,7 +10,6 @@ import ( "github.com/DataDog/zstd" "github.com/andybalholm/brotli" "github.com/ethereum-optimism/optimism/op-service/eth" - "github.com/ethereum/go-ethereum/rlp" "github.com/stretchr/testify/require" ) @@ -114,11 +113,7 @@ func TestBatchReader(t *testing.T) { batchDataInput := NewBatchData(singularBatch) encodedBatch := &bytes.Buffer{} - buf := &bytes.Buffer{} - // Get the encoded data of the batch data - err := batchDataInput.encodeTyped(buf) - require.NoError(t, err) - err = rlp.Encode(encodedBatch, buf.Bytes()) + err := batchDataInput.EncodeRLP(encodedBatch) require.NoError(t, err) var testCases = []struct { From 010ed14b3f15940301e02732d01609f4bdf62053 Mon Sep 17 00:00:00 2001 From: Cody Wang Date: Sat, 11 May 2024 16:40:50 -0700 Subject: [PATCH 30/38] remove methods in compressor --- op-node/rollup/derive/channel_compressor.go | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/op-node/rollup/derive/channel_compressor.go b/op-node/rollup/derive/channel_compressor.go index eab350922f41..341dd13d825e 100644 --- a/op-node/rollup/derive/channel_compressor.go +++ b/op-node/rollup/derive/channel_compressor.go @@ -35,18 +35,6 @@ type BaseChannelCompressor struct { CompressorWriter } -func (bcc *BaseChannelCompressor) Write(data []byte) (int, error) { - return bcc.CompressorWriter.Write(data) -} - -func (bcc *BaseChannelCompressor) Flush() error { - return bcc.CompressorWriter.Flush() -} - -func (bcc *BaseChannelCompressor) Close() error { - return bcc.CompressorWriter.Close() -} - func (bcc *BaseChannelCompressor) Len() int { return bcc.compressed.Len() } From d50841257e382375e4d907beee7b021c0d05c77f Mon Sep 17 00:00:00 2001 From: Cody Wang Date: Sat, 11 May 2024 16:41:27 -0700 Subject: [PATCH 31/38] fix error msg --- op-node/rollup/derive/channel.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/op-node/rollup/derive/channel.go b/op-node/rollup/derive/channel.go index b3c2f2c0067c..58238133081e 100644 --- a/op-node/rollup/derive/channel.go +++ b/op-node/rollup/derive/channel.go @@ -178,7 +178,7 @@ func BatchReader(r io.Reader, maxRLPBytesPerChannel uint64, isFjord bool) (func( } else if compressionType[0] == ChannelVersionBrotli { // If before Fjord, we cannot accept brotli compressed batch if !isFjord { - return nil, fmt.Errorf("cannot accept brotli compressed batch after Fjord") + return nil, fmt.Errorf("cannot accept brotli compressed batch before Fjord") } // discard the first byte _, err := bufReader.Discard(1) From be6629e9b7341911abf116a6887f75d6d761580d Mon Sep 17 00:00:00 2001 From: Cody Wang Date: Mon, 13 May 2024 13:54:09 -0700 Subject: [PATCH 32/38] add compression algo flag to optional flags --- op-batcher/flags/flags.go | 1 + 1 file changed, 1 insertion(+) diff --git a/op-batcher/flags/flags.go b/op-batcher/flags/flags.go index da7dc1e9ed96..c91234b96a29 100644 --- a/op-batcher/flags/flags.go +++ b/op-batcher/flags/flags.go @@ -177,6 +177,7 @@ var optionalFlags = []cli.Flag{ BatchTypeFlag, DataAvailabilityTypeFlag, ActiveSequencerCheckDurationFlag, + CompressionAlgoFlag, } func init() { From eadc24ef27602e7ad87a38397de904fc56e300d2 Mon Sep 17 00:00:00 2001 From: Cody Wang Date: Mon, 13 May 2024 14:56:43 -0700 Subject: [PATCH 33/38] add Clone() function --- op-node/rollup/derive/types.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/op-node/rollup/derive/types.go b/op-node/rollup/derive/types.go index 97abe5ed965d..a17c1c9a9a6a 100644 --- a/op-node/rollup/derive/types.go +++ b/op-node/rollup/derive/types.go @@ -36,6 +36,11 @@ func (algo *CompressionAlgo) Set(value string) error { return nil } +func (algo *CompressionAlgo) Clone() any { + cpy := *algo + return &cpy +} + func (algo *CompressionAlgo) IsBrotli() bool { return brotliRegexp.MatchString(algo.String()) } From ade9cc1860f0df76188367389ec80a9717e9286c Mon Sep 17 00:00:00 2001 From: Cody Wang Date: Mon, 13 May 2024 20:37:55 -0700 Subject: [PATCH 34/38] brotli e2e test --- op-e2e/brotli_batcher_test.go | 159 ++++++++++++++++++++++++++++++++++ op-e2e/setup.go | 10 ++- 2 files changed, 168 insertions(+), 1 deletion(-) create mode 100644 op-e2e/brotli_batcher_test.go diff --git a/op-e2e/brotli_batcher_test.go b/op-e2e/brotli_batcher_test.go new file mode 100644 index 000000000000..7576dec360c2 --- /dev/null +++ b/op-e2e/brotli_batcher_test.go @@ -0,0 +1,159 @@ +package op_e2e + +import ( + "context" + "crypto/ecdsa" + "fmt" + "math/big" + "testing" + "time" + + "github.com/stretchr/testify/require" + + "github.com/ethereum/go-ethereum/accounts/abi/bind" + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/common/hexutil" + "github.com/ethereum/go-ethereum/core" + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/log" + "github.com/ethereum/go-ethereum/rpc" + + batcherFlags "github.com/ethereum-optimism/optimism/op-batcher/flags" + gethutils "github.com/ethereum-optimism/optimism/op-e2e/e2eutils/geth" + "github.com/ethereum-optimism/optimism/op-e2e/e2eutils/wait" + "github.com/ethereum-optimism/optimism/op-service/client" + "github.com/ethereum-optimism/optimism/op-service/sources" + "github.com/ethereum-optimism/optimism/op-service/testlog" +) + +// TestSystem4844E2E runs the SystemE2E test with 4844 enabled on L1, +// and active on the rollup in the op-batcher and verifier. +// func TestSystem4844E2E(t *testing.T) { +// t.Run("single-blob", func(t *testing.T) { testSystem4844E2E(t, false) }) +// t.Run("multi-blob", func(t *testing.T) { testSystem4844E2E(t, true) }) +// } + +func setupAliceAccount(t *testing.T, cfg SystemConfig, sys *System, ethPrivKey *ecdsa.PrivateKey) { + l1Client := sys.Clients["l1"] + l2Verif := sys.Clients["verifier"] + + // Send Transaction & wait for success + fromAddr := cfg.Secrets.Addresses().Alice + log.Info("alice", "addr", fromAddr) + + ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second) + defer cancel() + startBalance, err := l2Verif.BalanceAt(ctx, fromAddr, nil) + require.NoError(t, err) + + // Send deposit transaction + opts, err := bind.NewKeyedTransactorWithChainID(ethPrivKey, cfg.L1ChainIDBig()) + require.NoError(t, err) + mintAmount := big.NewInt(1_000_000_000_000) + opts.Value = mintAmount + SendDepositTx(t, cfg, l1Client, l2Verif, opts, func(l2Opts *DepositTxOpts) {}) + + // Confirm balance + ctx, cancel = context.WithTimeout(context.Background(), 15*time.Second) + defer cancel() + endBalance, err := wait.ForBalanceChange(ctx, l2Verif, fromAddr, startBalance) + require.NoError(t, err) + + diff := new(big.Int).Sub(endBalance, startBalance) + require.Equal(t, mintAmount, diff, "Did not get expected balance change") +} + +func TestBrotliBatcherFjord(t *testing.T,) { + InitParallel(t) + + cfg := DefaultSystemConfig(t) + cfg.DataAvailabilityType = batcherFlags.BlobsType + + genesisActivation := hexutil.Uint64(0) + cfg.DeployConfig.L1CancunTimeOffset = &genesisActivation + cfg.DeployConfig.L2GenesisDeltaTimeOffset = &genesisActivation + cfg.DeployConfig.L2GenesisEcotoneTimeOffset = &genesisActivation + cfg.DeployConfig.L2GenesisFjordTimeOffset = &genesisActivation + + // set up batcher to use brotli + sys, err := cfg.Start(t, SystemConfigOption{"compressionAlgo", "brotli", nil} ) + require.Nil(t, err, "Error starting up system") + defer sys.Close() + + log := testlog.Logger(t, log.LevelInfo) + log.Info("genesis", "l2", sys.RollupConfig.Genesis.L2, "l1", sys.RollupConfig.Genesis.L1, "l2_time", sys.RollupConfig.Genesis.L2Time) + + l1Client := sys.Clients["l1"] + l2Seq := sys.Clients["sequencer"] + l2Verif := sys.Clients["verifier"] + + // Transactor Account and set up the account + ethPrivKey := cfg.Secrets.Alice + setupAliceAccount(t, cfg, sys, ethPrivKey) + + // Submit TX to L2 sequencer node + receipt := SendL2Tx(t, cfg, l2Seq, ethPrivKey, func(opts *TxOpts) { + opts.Value = big.NewInt(1_000_000_000) + opts.Nonce = 1 // Already have deposit + opts.ToAddr = &common.Address{0xff, 0xff} + opts.Gas, err = core.IntrinsicGas(opts.Data, nil, false, true, true, false) + require.NoError(t, err) + opts.VerifyOnClients(l2Verif) + }) + + // Verify blocks match after batch submission on verifiers and sequencers + verifBlock, err := l2Verif.BlockByNumber(context.Background(), receipt.BlockNumber) + require.NoError(t, err) + require.Equal(t, verifBlock.Hash(), receipt.BlockHash, "must be same block") + seqBlock, err := l2Seq.BlockByNumber(context.Background(), receipt.BlockNumber) + require.NoError(t, err) + require.Equal(t, seqBlock.Hash(), receipt.BlockHash, "must be same block") + require.Equal(t, verifBlock.NumberU64(), seqBlock.NumberU64(), "Verifier and sequencer blocks not the same after including a batch tx") + require.Equal(t, verifBlock.ParentHash(), seqBlock.ParentHash(), "Verifier and sequencer blocks parent hashes not the same after including a batch tx") + require.Equal(t, verifBlock.Hash(), seqBlock.Hash(), "Verifier and sequencer blocks not the same after including a batch tx") + + rollupRPCClient, err := rpc.DialContext(context.Background(), sys.RollupNodes["sequencer"].HTTPEndpoint()) + require.NoError(t, err) + rollupClient := sources.NewRollupClient(client.NewBaseRPCClient(rollupRPCClient)) + // basic check that sync status works + seqStatus, err := rollupClient.SyncStatus(context.Background()) + require.NoError(t, err) + require.LessOrEqual(t, seqBlock.NumberU64(), seqStatus.UnsafeL2.Number) + // basic check that version endpoint works + seqVersion, err := rollupClient.Version(context.Background()) + require.NoError(t, err) + require.NotEqual(t, "", seqVersion) + + // quick check that the batch submitter works + require.Eventually(t, func() bool { + // wait for chain to be marked as "safe" (i.e. confirm batch-submission works) + stat, err := rollupClient.SyncStatus(context.Background()) + require.NoError(t, err) + return stat.SafeL2.Number >= receipt.BlockNumber.Uint64() + }, time.Second*20, time.Second, "expected L2 to be batch-submitted and labeled as safe") + + // check that the L2 tx is still canonical + seqBlock, err = l2Seq.BlockByNumber(context.Background(), receipt.BlockNumber) + require.NoError(t, err) + require.Equal(t, seqBlock.Hash(), receipt.BlockHash, "receipt block must match canonical block at tx inclusion height") + + // find L1 block that contained the blob(s) batch tx + tip, err := l1Client.HeaderByNumber(context.Background(), nil) + require.NoError(t, err) + var blobTx *types.Transaction + _, err = gethutils.FindBlock(l1Client, int(tip.Number.Int64()), 0, 5*time.Second, + func(b *types.Block) (bool, error) { + for _, tx := range b.Transactions() { + if tx.Type() != types.BlobTxType { + continue + } + blobTx = tx + return true, nil + } + return false, nil + }) + require.NoError(t, err) + + // check blob tx is the same as receipt + require.Equal(t, blobTx.BlobHashes()[0], receipt.TxHash) +} diff --git a/op-e2e/setup.go b/op-e2e/setup.go index 3aa833f58c46..976cef9c18d8 100644 --- a/op-e2e/setup.go +++ b/op-e2e/setup.go @@ -825,6 +825,14 @@ func (cfg SystemConfig) Start(t *testing.T, _opts ...SystemConfigOption) (*Syste if batcherTargetNumFrames == 0 { batcherTargetNumFrames = 1 } + + var compressionAlgo derive.CompressionAlgo = derive.Zlib + // If setting has brotli key, set the compression algo as brotli + if _, ok := opts.Get("compressionAlgo", "brotli"); ok { + fmt.Println("setting as brotli") + compressionAlgo = derive.Brotli10 + } + batcherCLIConfig := &bss.CLIConfig{ L1EthRpc: sys.EthInstances["l1"].WSEndpoint(), L2EthRpc: sys.EthInstances["sequencer"].WSEndpoint(), @@ -845,7 +853,7 @@ func (cfg SystemConfig) Start(t *testing.T, _opts ...SystemConfigOption) (*Syste Stopped: sys.Cfg.DisableBatcher, // Batch submitter may be enabled later BatchType: batchType, DataAvailabilityType: sys.Cfg.DataAvailabilityType, - CompressionAlgo: derive.Zlib, + CompressionAlgo: compressionAlgo, } // Batch Submitter batcher, err := bss.BatcherServiceFromCLIConfig(context.Background(), "0.0.1", batcherCLIConfig, sys.Cfg.Loggers["batcher"]) From 760b8c90ffd19128818b566bc4781841922488b4 Mon Sep 17 00:00:00 2001 From: Cody Wang Date: Tue, 14 May 2024 01:19:06 -0700 Subject: [PATCH 35/38] fix fmt --- op-e2e/brotli_batcher_test.go | 19 +++++-------------- 1 file changed, 5 insertions(+), 14 deletions(-) diff --git a/op-e2e/brotli_batcher_test.go b/op-e2e/brotli_batcher_test.go index 7576dec360c2..f6ecf2bd907e 100644 --- a/op-e2e/brotli_batcher_test.go +++ b/op-e2e/brotli_batcher_test.go @@ -3,7 +3,6 @@ package op_e2e import ( "context" "crypto/ecdsa" - "fmt" "math/big" "testing" "time" @@ -63,7 +62,7 @@ func setupAliceAccount(t *testing.T, cfg SystemConfig, sys *System, ethPrivKey * require.Equal(t, mintAmount, diff, "Did not get expected balance change") } -func TestBrotliBatcherFjord(t *testing.T,) { +func TestBrotliBatcherFjord(t *testing.T) { InitParallel(t) cfg := DefaultSystemConfig(t) @@ -76,7 +75,7 @@ func TestBrotliBatcherFjord(t *testing.T,) { cfg.DeployConfig.L2GenesisFjordTimeOffset = &genesisActivation // set up batcher to use brotli - sys, err := cfg.Start(t, SystemConfigOption{"compressionAlgo", "brotli", nil} ) + sys, err := cfg.Start(t, SystemConfigOption{"compressionAlgo", "brotli", nil}) require.Nil(t, err, "Error starting up system") defer sys.Close() @@ -140,20 +139,12 @@ func TestBrotliBatcherFjord(t *testing.T,) { // find L1 block that contained the blob(s) batch tx tip, err := l1Client.HeaderByNumber(context.Background(), nil) require.NoError(t, err) - var blobTx *types.Transaction _, err = gethutils.FindBlock(l1Client, int(tip.Number.Int64()), 0, 5*time.Second, func(b *types.Block) (bool, error) { - for _, tx := range b.Transactions() { - if tx.Type() != types.BlobTxType { - continue - } - blobTx = tx - return true, nil - } - return false, nil + // Check that the transaction exists in the L1 block + require.Equal(t, b.Transactions().Len(), 1) + return true, nil }) require.NoError(t, err) - // check blob tx is the same as receipt - require.Equal(t, blobTx.BlobHashes()[0], receipt.TxHash) } From 4926e1c533ef14439e9f01c5d8fe806514649bcc Mon Sep 17 00:00:00 2001 From: Cody Wang Date: Tue, 14 May 2024 02:20:55 -0700 Subject: [PATCH 36/38] fix comments --- op-e2e/brotli_batcher_test.go | 10 +--------- op-e2e/setup.go | 3 +-- 2 files changed, 2 insertions(+), 11 deletions(-) diff --git a/op-e2e/brotli_batcher_test.go b/op-e2e/brotli_batcher_test.go index f6ecf2bd907e..a7d64f3b6dfd 100644 --- a/op-e2e/brotli_batcher_test.go +++ b/op-e2e/brotli_batcher_test.go @@ -25,13 +25,6 @@ import ( "github.com/ethereum-optimism/optimism/op-service/testlog" ) -// TestSystem4844E2E runs the SystemE2E test with 4844 enabled on L1, -// and active on the rollup in the op-batcher and verifier. -// func TestSystem4844E2E(t *testing.T) { -// t.Run("single-blob", func(t *testing.T) { testSystem4844E2E(t, false) }) -// t.Run("multi-blob", func(t *testing.T) { testSystem4844E2E(t, true) }) -// } - func setupAliceAccount(t *testing.T, cfg SystemConfig, sys *System, ethPrivKey *ecdsa.PrivateKey) { l1Client := sys.Clients["l1"] l2Verif := sys.Clients["verifier"] @@ -141,10 +134,9 @@ func TestBrotliBatcherFjord(t *testing.T) { require.NoError(t, err) _, err = gethutils.FindBlock(l1Client, int(tip.Number.Int64()), 0, 5*time.Second, func(b *types.Block) (bool, error) { - // Check that the transaction exists in the L1 block + // check that the transaction exists in the L1 block require.Equal(t, b.Transactions().Len(), 1) return true, nil }) require.NoError(t, err) - } diff --git a/op-e2e/setup.go b/op-e2e/setup.go index 976cef9c18d8..4e4692babe40 100644 --- a/op-e2e/setup.go +++ b/op-e2e/setup.go @@ -827,9 +827,8 @@ func (cfg SystemConfig) Start(t *testing.T, _opts ...SystemConfigOption) (*Syste } var compressionAlgo derive.CompressionAlgo = derive.Zlib - // If setting has brotli key, set the compression algo as brotli + // if opt has brotli key, set the compression algo as brotli if _, ok := opts.Get("compressionAlgo", "brotli"); ok { - fmt.Println("setting as brotli") compressionAlgo = derive.Brotli10 } From c687a5f332adf2802679c0eb0e072802f46f9db4 Mon Sep 17 00:00:00 2001 From: Cody Wang Date: Tue, 14 May 2024 02:42:56 -0700 Subject: [PATCH 37/38] fix --- op-e2e/brotli_batcher_test.go | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/op-e2e/brotli_batcher_test.go b/op-e2e/brotli_batcher_test.go index a7d64f3b6dfd..a74239d8b3e9 100644 --- a/op-e2e/brotli_batcher_test.go +++ b/op-e2e/brotli_batcher_test.go @@ -111,10 +111,6 @@ func TestBrotliBatcherFjord(t *testing.T) { seqStatus, err := rollupClient.SyncStatus(context.Background()) require.NoError(t, err) require.LessOrEqual(t, seqBlock.NumberU64(), seqStatus.UnsafeL2.Number) - // basic check that version endpoint works - seqVersion, err := rollupClient.Version(context.Background()) - require.NoError(t, err) - require.NotEqual(t, "", seqVersion) // quick check that the batch submitter works require.Eventually(t, func() bool { @@ -129,13 +125,14 @@ func TestBrotliBatcherFjord(t *testing.T) { require.NoError(t, err) require.Equal(t, seqBlock.Hash(), receipt.BlockHash, "receipt block must match canonical block at tx inclusion height") - // find L1 block that contained the blob(s) batch tx + // find L1 block that contained the blob batch tx tip, err := l1Client.HeaderByNumber(context.Background(), nil) require.NoError(t, err) _, err = gethutils.FindBlock(l1Client, int(tip.Number.Int64()), 0, 5*time.Second, func(b *types.Block) (bool, error) { - // check that the transaction exists in the L1 block + // check that the blob transaction exists in the L1 block require.Equal(t, b.Transactions().Len(), 1) + require.Equal(t, int(b.Transactions()[0].Type()), types.BlobTxType) return true, nil }) require.NoError(t, err) From dd5754be77aaf8f4be66e207663b63c1b1d7d96b Mon Sep 17 00:00:00 2001 From: Cody Wang Date: Tue, 14 May 2024 02:53:03 -0700 Subject: [PATCH 38/38] update --- op-e2e/brotli_batcher_test.go | 16 +--------------- 1 file changed, 1 insertion(+), 15 deletions(-) diff --git a/op-e2e/brotli_batcher_test.go b/op-e2e/brotli_batcher_test.go index a74239d8b3e9..97211c471ba0 100644 --- a/op-e2e/brotli_batcher_test.go +++ b/op-e2e/brotli_batcher_test.go @@ -13,12 +13,10 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common/hexutil" "github.com/ethereum/go-ethereum/core" - "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/rpc" batcherFlags "github.com/ethereum-optimism/optimism/op-batcher/flags" - gethutils "github.com/ethereum-optimism/optimism/op-e2e/e2eutils/geth" "github.com/ethereum-optimism/optimism/op-e2e/e2eutils/wait" "github.com/ethereum-optimism/optimism/op-service/client" "github.com/ethereum-optimism/optimism/op-service/sources" @@ -75,7 +73,6 @@ func TestBrotliBatcherFjord(t *testing.T) { log := testlog.Logger(t, log.LevelInfo) log.Info("genesis", "l2", sys.RollupConfig.Genesis.L2, "l1", sys.RollupConfig.Genesis.L1, "l2_time", sys.RollupConfig.Genesis.L2Time) - l1Client := sys.Clients["l1"] l2Seq := sys.Clients["sequencer"] l2Verif := sys.Clients["verifier"] @@ -121,19 +118,8 @@ func TestBrotliBatcherFjord(t *testing.T) { }, time.Second*20, time.Second, "expected L2 to be batch-submitted and labeled as safe") // check that the L2 tx is still canonical + // safe and canonical => the block was batched successfully with brotli seqBlock, err = l2Seq.BlockByNumber(context.Background(), receipt.BlockNumber) require.NoError(t, err) require.Equal(t, seqBlock.Hash(), receipt.BlockHash, "receipt block must match canonical block at tx inclusion height") - - // find L1 block that contained the blob batch tx - tip, err := l1Client.HeaderByNumber(context.Background(), nil) - require.NoError(t, err) - _, err = gethutils.FindBlock(l1Client, int(tip.Number.Int64()), 0, 5*time.Second, - func(b *types.Block) (bool, error) { - // check that the blob transaction exists in the L1 block - require.Equal(t, b.Transactions().Len(), 1) - require.Equal(t, int(b.Transactions()[0].Type()), types.BlobTxType) - return true, nil - }) - require.NoError(t, err) }