diff --git a/cmd/utils/flags.go b/cmd/utils/flags.go index 7371a3770074..a5a06faa9d37 100644 --- a/cmd/utils/flags.go +++ b/cmd/utils/flags.go @@ -898,6 +898,12 @@ var ( Name: "da.sync", Usage: "Enable node syncing from DA", } + DAMissingHeaderFieldsBaseURLFlag = cli.StringFlag{ + Name: "da.missingheaderfields.baseurl", + Usage: "Base URL for fetching missing header fields for pre-EuclidV2 blocks", + Value: "https://scroll-block-missing-metadata.s3.us-west-2.amazonaws.com/", + } + DABlobScanAPIEndpointFlag = cli.StringFlag{ Name: "da.blob.blobscan", Usage: "BlobScan blob API endpoint", @@ -1382,6 +1388,11 @@ func SetNodeConfig(ctx *cli.Context, cfg *node.Config) { cfg.DaSyncingEnabled = ctx.Bool(DASyncEnabledFlag.Name) } + cfg.DAMissingHeaderFieldsBaseURL = DAMissingHeaderFieldsBaseURLFlag.Value + if ctx.GlobalIsSet(DAMissingHeaderFieldsBaseURLFlag.Name) { + cfg.DAMissingHeaderFieldsBaseURL = ctx.GlobalString(DAMissingHeaderFieldsBaseURLFlag.Name) + } + if ctx.GlobalIsSet(ExternalSignerFlag.Name) { cfg.ExternalSigner = ctx.GlobalString(ExternalSignerFlag.Name) } diff --git a/core/blockchain.go b/core/blockchain.go index 57a803198fa1..2da580a26372 100644 --- a/core/blockchain.go +++ b/core/blockchain.go @@ -1880,15 +1880,17 @@ func (bc *BlockChain) BuildAndWriteBlock(parentBlock *types.Block, header *types header.ParentHash = parentBlock.Hash() + // sanitize base fee + if header.BaseFee != nil && header.BaseFee.Cmp(common.Big0) == 0 { + header.BaseFee = nil + } + tempBlock := types.NewBlockWithHeader(header).WithBody(txs, nil) receipts, logs, gasUsed, err := bc.processor.Process(tempBlock, statedb, bc.vmConfig) if err != nil { return nil, NonStatTy, fmt.Errorf("error processing block: %w", err) } - // TODO: once we have the extra and difficulty we need to verify the signature of the block with Clique - // This should be done with https://github.com/scroll-tech/go-ethereum/pull/913. - if sign { // Prevent Engine from overriding timestamp. originalTime := header.Time @@ -1901,7 +1903,11 @@ func (bc *BlockChain) BuildAndWriteBlock(parentBlock *types.Block, header *types // finalize and assemble block as fullBlock: replicates consensus.FinalizeAndAssemble() header.GasUsed = gasUsed - header.Root = statedb.IntermediateRoot(bc.chainConfig.IsEIP158(header.Number)) + + // state root might be set from partial header. If it is not set, we calculate it. + if header.Root == (common.Hash{}) { + header.Root = statedb.IntermediateRoot(bc.chainConfig.IsEIP158(header.Number)) + } fullBlock := types.NewBlock(header, txs, nil, receipts, trie.NewStackTrie(nil)) diff --git a/eth/backend.go b/eth/backend.go index ad765b835b0a..3fbefcfe63d2 100644 --- a/eth/backend.go +++ b/eth/backend.go @@ -22,6 +22,9 @@ import ( "errors" "fmt" "math/big" + "net/url" + "path" + "path/filepath" "runtime" "sync" "sync/atomic" @@ -61,6 +64,7 @@ import ( "github.com/scroll-tech/go-ethereum/rollup/ccc" "github.com/scroll-tech/go-ethereum/rollup/da_syncer" "github.com/scroll-tech/go-ethereum/rollup/l1" + "github.com/scroll-tech/go-ethereum/rollup/missing_header_fields" "github.com/scroll-tech/go-ethereum/rollup/rollup_sync_service" "github.com/scroll-tech/go-ethereum/rollup/sync_service" "github.com/scroll-tech/go-ethereum/rpc" @@ -241,7 +245,12 @@ func New(stack *node.Node, config *ethconfig.Config, l1Client l1.Client) (*Ether if config.EnableDASyncing { // Do not start syncing pipeline if we are producing blocks for permissionless batches. if !config.DA.ProduceBlocks { - eth.syncingPipeline, err = da_syncer.NewSyncingPipeline(context.Background(), eth.blockchain, chainConfig, eth.chainDb, l1Client, stack.Config().L1DeploymentBlock, config.DA) + missingHeaderFieldsManager, err := createMissingHeaderFieldsManager(stack, chainConfig) + if err != nil { + return nil, fmt.Errorf("cannot create missing header fields manager: %w", err) + } + + eth.syncingPipeline, err = da_syncer.NewSyncingPipeline(context.Background(), eth.blockchain, chainConfig, eth.chainDb, l1Client, stack.Config().L1DeploymentBlock, config.DA, missingHeaderFieldsManager) if err != nil { return nil, fmt.Errorf("cannot initialize da syncer: %w", err) } @@ -337,6 +346,22 @@ func New(stack *node.Node, config *ethconfig.Config, l1Client l1.Client) (*Ether return eth, nil } +func createMissingHeaderFieldsManager(stack *node.Node, chainConfig *params.ChainConfig) (*missing_header_fields.Manager, error) { + downloadURL, err := url.Parse(stack.Config().DAMissingHeaderFieldsBaseURL) + if err != nil { + return nil, fmt.Errorf("invalid DAMissingHeaderFieldsBaseURL: %w", err) + } + downloadURL.Path = path.Join(downloadURL.Path, chainConfig.ChainID.String()+".bin") + + expectedSHA256Checksum := chainConfig.Scroll.MissingHeaderFieldsSHA256 + if expectedSHA256Checksum == nil { + return nil, fmt.Errorf("missing expected SHA256 checksum for missing header fields file in chain config") + } + + filePath := filepath.Join(stack.Config().DataDir, fmt.Sprintf("missing-header-fields-%s-%s", chainConfig.ChainID, expectedSHA256Checksum.Hex())) + return missing_header_fields.NewManager(context.Background(), filePath, downloadURL.String(), *expectedSHA256Checksum), nil +} + func makeExtraData(extra []byte) []byte { if len(extra) == 0 { // create default extradata diff --git a/node/config.go b/node/config.go index 6e7133a95129..d7212389684e 100644 --- a/node/config.go +++ b/node/config.go @@ -201,6 +201,8 @@ type Config struct { L1DisableMessageQueueV2 bool `toml:",omitempty"` // Is daSyncingEnabled DaSyncingEnabled bool `toml:",omitempty"` + // Base URL for missing header fields file + DAMissingHeaderFieldsBaseURL string `toml:",omitempty"` } // IPCEndpoint resolves an IPC endpoint based on a configured value, taking into diff --git a/params/config.go b/params/config.go index 96b77e04a7f4..d85198b9e4aa 100644 --- a/params/config.go +++ b/params/config.go @@ -30,16 +30,18 @@ import ( // Genesis hashes to enforce below configs on. var ( - MainnetGenesisHash = common.HexToHash("0xd4e56740f876aef8c010b86a40d5f56745a118d0906a34e69aec8c0db1cb8fa3") - RopstenGenesisHash = common.HexToHash("0x41941023680923e0fe4d74a34bdac8141f2540e3ae90623718e47d66d1ca4a2d") - SepoliaGenesisHash = common.HexToHash("0x25a5cc106eea7138acab33231d7160d69cb777ee0c2c553fcddf5138993e6dd9") - RinkebyGenesisHash = common.HexToHash("0x6341fd3daf94b748c72ced5a5b26028f2474f5f00d824504e4fa37a75767e177") - GoerliGenesisHash = common.HexToHash("0xbf7e331f7f7c1dd2e05159666b3bf8bc7a8a3a9eb1d518969eab529dd9b88c1a") - ScrollAlphaGenesisHash = common.HexToHash("0xa4fc62b9b0643e345bdcebe457b3ae898bef59c7203c3db269200055e037afda") - ScrollSepoliaGenesisHash = common.HexToHash("0xaa62d1a8b2bffa9e5d2368b63aae0d98d54928bd713125e3fd9e5c896c68592c") - ScrollMainnetGenesisHash = common.HexToHash("0xbbc05efd412b7cd47a2ed0e5ddfcf87af251e414ea4c801d78b6784513180a80") - ScrollSepoliaGenesisState = common.HexToHash("0x20695989e9038823e35f0e88fbc44659ffdbfa1fe89fbeb2689b43f15fa64cb5") - ScrollMainnetGenesisState = common.HexToHash("0x08d535cc60f40af5dd3b31e0998d7567c2d568b224bed2ba26070aeb078d1339") + MainnetGenesisHash = common.HexToHash("0xd4e56740f876aef8c010b86a40d5f56745a118d0906a34e69aec8c0db1cb8fa3") + RopstenGenesisHash = common.HexToHash("0x41941023680923e0fe4d74a34bdac8141f2540e3ae90623718e47d66d1ca4a2d") + SepoliaGenesisHash = common.HexToHash("0x25a5cc106eea7138acab33231d7160d69cb777ee0c2c553fcddf5138993e6dd9") + RinkebyGenesisHash = common.HexToHash("0x6341fd3daf94b748c72ced5a5b26028f2474f5f00d824504e4fa37a75767e177") + GoerliGenesisHash = common.HexToHash("0xbf7e331f7f7c1dd2e05159666b3bf8bc7a8a3a9eb1d518969eab529dd9b88c1a") + ScrollAlphaGenesisHash = common.HexToHash("0xa4fc62b9b0643e345bdcebe457b3ae898bef59c7203c3db269200055e037afda") + ScrollSepoliaGenesisHash = common.HexToHash("0xaa62d1a8b2bffa9e5d2368b63aae0d98d54928bd713125e3fd9e5c896c68592c") + ScrollMainnetGenesisHash = common.HexToHash("0xbbc05efd412b7cd47a2ed0e5ddfcf87af251e414ea4c801d78b6784513180a80") + ScrollSepoliaGenesisState = common.HexToHash("0x20695989e9038823e35f0e88fbc44659ffdbfa1fe89fbeb2689b43f15fa64cb5") + ScrollMainnetGenesisState = common.HexToHash("0x08d535cc60f40af5dd3b31e0998d7567c2d568b224bed2ba26070aeb078d1339") + ScrollMainnetMissingHeaderFieldsSHA256 = common.HexToHash("0x9062e2fa1200dca63bee1d18d429572f134f5f0c98cb4852f62fc394e33cf6e6") + ScrollSepoliaMissingHeaderFieldsSHA256 = common.HexToHash("0x3629f5e53250a526ffc46806c4d74b9c52c9209a6d45ecdfebdef5d596bb3f40") ) func newUint64(val uint64) *uint64 { return &val } @@ -353,7 +355,8 @@ var ( ScrollChainAddress: common.HexToAddress("0x2D567EcE699Eabe5afCd141eDB7A4f2D0D6ce8a0"), L2SystemConfigAddress: common.HexToAddress("0xF444cF06A3E3724e20B35c2989d3942ea8b59124"), }, - GenesisStateRoot: &ScrollSepoliaGenesisState, + GenesisStateRoot: &ScrollSepoliaGenesisState, + MissingHeaderFieldsSHA256: &ScrollSepoliaMissingHeaderFieldsSHA256, }, } @@ -404,7 +407,8 @@ var ( ScrollChainAddress: common.HexToAddress("0xa13BAF47339d63B743e7Da8741db5456DAc1E556"), L2SystemConfigAddress: common.HexToAddress("0x331A873a2a85219863d80d248F9e2978fE88D0Ea"), }, - GenesisStateRoot: &ScrollMainnetGenesisState, + GenesisStateRoot: &ScrollMainnetGenesisState, + MissingHeaderFieldsSHA256: &ScrollMainnetMissingHeaderFieldsSHA256, }, } @@ -706,6 +710,9 @@ type ScrollConfig struct { // Genesis State Root for MPT clients GenesisStateRoot *common.Hash `json:"genesisStateRoot,omitempty"` + + // MissingHeaderFieldsSHA256 is the SHA256 hash of the missing header fields file. + MissingHeaderFieldsSHA256 *common.Hash `json:"missingHeaderFieldsSHA256,omitempty"` } // L1Config contains the l1 parameters needed to sync l1 contract events (e.g., l1 messages, commit/revert/finalize batches) in the sequencer @@ -756,8 +763,13 @@ func (s ScrollConfig) String() string { genesisStateRoot = fmt.Sprintf("%v", *s.GenesisStateRoot) } - return fmt.Sprintf("{useZktrie: %v, maxTxPerBlock: %v, MaxTxPayloadBytesPerBlock: %v, feeVaultAddress: %v, l1Config: %v, genesisStateRoot: %v}", - s.UseZktrie, maxTxPerBlock, maxTxPayloadBytesPerBlock, s.FeeVaultAddress, s.L1Config.String(), genesisStateRoot) + missingHeaderFieldsSHA256 := "" + if s.MissingHeaderFieldsSHA256 != nil { + missingHeaderFieldsSHA256 = fmt.Sprintf("%v", *s.MissingHeaderFieldsSHA256) + } + + return fmt.Sprintf("{useZktrie: %v, maxTxPerBlock: %v, MaxTxPayloadBytesPerBlock: %v, feeVaultAddress: %v, l1Config: %v, genesisStateRoot: %v, missingHeaderFieldsSHA256: %v}", + s.UseZktrie, maxTxPerBlock, maxTxPayloadBytesPerBlock, s.FeeVaultAddress, s.L1Config.String(), genesisStateRoot, missingHeaderFieldsSHA256) } // IsValidTxCount returns whether the given block's transaction count is below the limit. diff --git a/rollup/da_syncer/block_queue.go b/rollup/da_syncer/block_queue.go index 630382f001c0..30d53bfcb6a5 100644 --- a/rollup/da_syncer/block_queue.go +++ b/rollup/da_syncer/block_queue.go @@ -6,19 +6,22 @@ import ( "github.com/scroll-tech/go-ethereum/core/rawdb" "github.com/scroll-tech/go-ethereum/rollup/da_syncer/da" + "github.com/scroll-tech/go-ethereum/rollup/missing_header_fields" ) // BlockQueue is a pipeline stage that reads batches from BatchQueue, extracts all da.PartialBlock from it and // provides them to the next stage one-by-one. type BlockQueue struct { - batchQueue *BatchQueue - blocks []*da.PartialBlock + batchQueue *BatchQueue + blocks []*da.PartialBlock + missingHeaderFieldsManager *missing_header_fields.Manager } -func NewBlockQueue(batchQueue *BatchQueue) *BlockQueue { +func NewBlockQueue(batchQueue *BatchQueue, missingHeaderFieldsManager *missing_header_fields.Manager) *BlockQueue { return &BlockQueue{ - batchQueue: batchQueue, - blocks: make([]*da.PartialBlock, 0), + batchQueue: batchQueue, + blocks: make([]*da.PartialBlock, 0), + missingHeaderFieldsManager: missingHeaderFieldsManager, } } @@ -40,7 +43,7 @@ func (bq *BlockQueue) getBlocksFromBatch(ctx context.Context) error { return err } - bq.blocks, err = entryWithBlocks.Blocks() + bq.blocks, err = entryWithBlocks.Blocks(bq.missingHeaderFieldsManager) if err != nil { return fmt.Errorf("failed to get blocks from entry: %w", err) } diff --git a/rollup/da_syncer/da/commitV0.go b/rollup/da_syncer/da/commitV0.go index c99a474b256b..15f96ed4ed02 100644 --- a/rollup/da_syncer/da/commitV0.go +++ b/rollup/da_syncer/da/commitV0.go @@ -13,6 +13,7 @@ import ( "github.com/scroll-tech/go-ethereum/log" "github.com/scroll-tech/go-ethereum/rollup/da_syncer/serrors" "github.com/scroll-tech/go-ethereum/rollup/l1" + "github.com/scroll-tech/go-ethereum/rollup/missing_header_fields" ) type CommitBatchDAV0 struct { @@ -109,7 +110,7 @@ func (c *CommitBatchDAV0) CompareTo(other Entry) int { return 0 } -func (c *CommitBatchDAV0) Blocks() ([]*PartialBlock, error) { +func (c *CommitBatchDAV0) Blocks(manager *missing_header_fields.Manager) ([]*PartialBlock, error) { l1Txs, err := getL1Messages(c.db, c.parentTotalL1MessagePopped, c.skippedL1MessageBitmap, c.l1MessagesPopped) if err != nil { return nil, fmt.Errorf("failed to get L1 messages for v0 batch %d: %w", c.batchIndex, err) @@ -120,7 +121,7 @@ func (c *CommitBatchDAV0) Blocks() ([]*PartialBlock, error) { curL1TxIndex := c.parentTotalL1MessagePopped for _, chunk := range c.chunks { - for blockId, daBlock := range chunk.Blocks { + for blockIndex, daBlock := range chunk.Blocks { // create txs txs := make(types.Transactions, 0, daBlock.NumTransactions()) // insert l1 msgs @@ -132,7 +133,12 @@ func (c *CommitBatchDAV0) Blocks() ([]*PartialBlock, error) { curL1TxIndex += uint64(daBlock.NumL1Messages()) // insert l2 txs - txs = append(txs, chunk.Transactions[blockId]...) + txs = append(txs, chunk.Transactions[blockIndex]...) + + difficulty, stateRoot, extraData, err := manager.GetMissingHeaderFields(daBlock.Number()) + if err != nil { + return nil, fmt.Errorf("failed to get missing header fields for block %d: %w", daBlock.Number(), err) + } block := NewPartialBlock( &PartialHeader{ @@ -140,8 +146,9 @@ func (c *CommitBatchDAV0) Blocks() ([]*PartialBlock, error) { Time: daBlock.Timestamp(), BaseFee: daBlock.BaseFee(), GasLimit: daBlock.GasLimit(), - Difficulty: 10, // TODO: replace with real difficulty - ExtraData: []byte{1, 2, 3, 4, 5, 6, 7, 8}, // TODO: replace with real extra data + Difficulty: difficulty, + ExtraData: extraData, + StateRoot: stateRoot, }, txs) blocks = append(blocks, block) diff --git a/rollup/da_syncer/da/commitV7.go b/rollup/da_syncer/da/commitV7.go index 6048b853aee7..37a4b948df6c 100644 --- a/rollup/da_syncer/da/commitV7.go +++ b/rollup/da_syncer/da/commitV7.go @@ -13,6 +13,7 @@ import ( "github.com/scroll-tech/go-ethereum/rollup/da_syncer/blob_client" "github.com/scroll-tech/go-ethereum/rollup/da_syncer/serrors" "github.com/scroll-tech/go-ethereum/rollup/l1" + "github.com/scroll-tech/go-ethereum/rollup/missing_header_fields" "github.com/scroll-tech/go-ethereum/common" "github.com/scroll-tech/go-ethereum/crypto/kzg4844" @@ -113,7 +114,7 @@ func (c *CommitBatchDAV7) Event() l1.RollupEvent { return c.event } -func (c *CommitBatchDAV7) Blocks() ([]*PartialBlock, error) { +func (c *CommitBatchDAV7) Blocks(_ *missing_header_fields.Manager) ([]*PartialBlock, error) { initialL1MessageIndex := c.parentTotalL1MessagePopped l1Txs, err := getL1MessagesV7(c.db, c.blobPayload.Blocks(), initialL1MessageIndex) diff --git a/rollup/da_syncer/da/da.go b/rollup/da_syncer/da/da.go index fe72473451b6..3a56d326d931 100644 --- a/rollup/da_syncer/da/da.go +++ b/rollup/da_syncer/da/da.go @@ -8,6 +8,7 @@ import ( "github.com/scroll-tech/go-ethereum/common" "github.com/scroll-tech/go-ethereum/core/types" "github.com/scroll-tech/go-ethereum/rollup/l1" + "github.com/scroll-tech/go-ethereum/rollup/missing_header_fields" ) type Type int @@ -34,7 +35,7 @@ type Entry interface { type EntryWithBlocks interface { Entry - Blocks() ([]*PartialBlock, error) + Blocks(manager *missing_header_fields.Manager) ([]*PartialBlock, error) Version() encoding.CodecVersion Chunks() []*encoding.DAChunkRawTx BlobVersionedHashes() []common.Hash @@ -53,6 +54,7 @@ type PartialHeader struct { GasLimit uint64 Difficulty uint64 ExtraData []byte + StateRoot common.Hash } func (h *PartialHeader) ToHeader() *types.Header { @@ -63,6 +65,7 @@ func (h *PartialHeader) ToHeader() *types.Header { GasLimit: h.GasLimit, Difficulty: new(big.Int).SetUint64(h.Difficulty), Extra: h.ExtraData, + Root: h.StateRoot, } } diff --git a/rollup/da_syncer/syncing_pipeline.go b/rollup/da_syncer/syncing_pipeline.go index 080179107f8c..19b7bcbc07de 100644 --- a/rollup/da_syncer/syncing_pipeline.go +++ b/rollup/da_syncer/syncing_pipeline.go @@ -16,6 +16,7 @@ import ( "github.com/scroll-tech/go-ethereum/rollup/da_syncer/blob_client" "github.com/scroll-tech/go-ethereum/rollup/da_syncer/serrors" "github.com/scroll-tech/go-ethereum/rollup/l1" + "github.com/scroll-tech/go-ethereum/rollup/missing_header_fields" ) // Config is the configuration parameters of data availability syncing. @@ -50,7 +51,7 @@ type SyncingPipeline struct { daQueue *DAQueue } -func NewSyncingPipeline(ctx context.Context, blockchain *core.BlockChain, genesisConfig *params.ChainConfig, db ethdb.Database, ethClient l1.Client, l1DeploymentBlock uint64, config Config) (*SyncingPipeline, error) { +func NewSyncingPipeline(ctx context.Context, blockchain *core.BlockChain, genesisConfig *params.ChainConfig, db ethdb.Database, ethClient l1.Client, l1DeploymentBlock uint64, config Config, missingHeaderFieldsManager *missing_header_fields.Manager) (*SyncingPipeline, error) { l1Reader, err := l1.NewReader(ctx, l1.Config{ ScrollChainAddress: genesisConfig.Scroll.L1Config.ScrollChainAddress, L1MessageQueueAddress: genesisConfig.Scroll.L1Config.L1MessageQueueAddress, @@ -124,7 +125,7 @@ func NewSyncingPipeline(ctx context.Context, blockchain *core.BlockChain, genesi daQueue := NewDAQueue(lastProcessedBatchMeta.L1BlockNumber, dataSourceFactory) batchQueue := NewBatchQueue(daQueue, db, lastProcessedBatchMeta) - blockQueue := NewBlockQueue(batchQueue) + blockQueue := NewBlockQueue(batchQueue, missingHeaderFieldsManager) daSyncer := NewDASyncer(blockchain, config.L2EndBlock) ctx, cancel := context.WithCancel(ctx) diff --git a/rollup/missing_header_fields/manager.go b/rollup/missing_header_fields/manager.go new file mode 100644 index 000000000000..17763c653b8d --- /dev/null +++ b/rollup/missing_header_fields/manager.go @@ -0,0 +1,186 @@ +package missing_header_fields + +import ( + "bytes" + "context" + "crypto/sha256" + "errors" + "fmt" + "io" + "net/http" + "os" + "time" + + "github.com/scroll-tech/go-ethereum/common" + "github.com/scroll-tech/go-ethereum/log" +) + +const timeoutDownload = 30 * time.Minute + +// Manager is responsible for managing the missing header fields file. +// It lazily downloads the file if it doesn't exist, verifies its expectedChecksum and provides the missing header fields. +type Manager struct { + ctx context.Context + filePath string + downloadURL string + expectedChecksum common.Hash + + reader *Reader +} + +func NewManager(ctx context.Context, filePath string, downloadURL string, expectedChecksum common.Hash) *Manager { + return &Manager{ + ctx: ctx, + filePath: filePath, + downloadURL: downloadURL, + expectedChecksum: expectedChecksum, + } +} + +func (m *Manager) GetMissingHeaderFields(headerNum uint64) (difficulty uint64, stateRoot common.Hash, extraData []byte, err error) { + // lazy initialization: if the reader is not initialized this is the first time we read from the file + if m.reader == nil { + if err = m.initialize(); err != nil { + return 0, common.Hash{}, nil, fmt.Errorf("failed to initialize missing header reader: %v", err) + } + } + + return m.reader.Read(headerNum) +} + +func (m *Manager) initialize() error { + // if the file doesn't exist, download it + if _, err := os.Stat(m.filePath); errors.Is(err, os.ErrNotExist) { + if err = m.downloadFile(); err != nil { + return fmt.Errorf("failed to download file: %v", err) + } + } + + // verify the expectedChecksum + f, err := os.Open(m.filePath) + if err != nil { + return fmt.Errorf("failed to open file: %v", err) + } + + h := sha256.New() + if _, err = io.Copy(h, f); err != nil { + return fmt.Errorf("failed to copy file: %v", err) + } + if err = f.Close(); err != nil { + return fmt.Errorf("failed to close file: %v", err) + } + computedChecksum := h.Sum(nil) + if !bytes.Equal(computedChecksum, m.expectedChecksum[:]) { + return fmt.Errorf("expectedChecksum mismatch, expected %x, got %x", m.expectedChecksum, computedChecksum) + } + + // finally initialize the reader + reader, err := NewReader(m.filePath) + if err != nil { + return fmt.Errorf("failed to create reader: %v", err) + } + + m.reader = reader + return nil +} +func (m *Manager) Close() error { + if m.reader != nil { + return m.reader.Close() + } + return nil +} + +func (m *Manager) downloadFile() error { + log.Info("Downloading missing header fields. This might take a while...", "url", m.downloadURL) + + downloadCtx, downloadCtxCancel := context.WithTimeout(m.ctx, timeoutDownload) + defer downloadCtxCancel() + + req, err := http.NewRequestWithContext(downloadCtx, http.MethodGet, m.downloadURL, nil) + if err != nil { + return fmt.Errorf("failed to create download request: %v", err) + } + + resp, err := http.DefaultClient.Do(req) + if err != nil { + return fmt.Errorf("failed to download file: %v", err) + } + + if resp.StatusCode != http.StatusOK { + return fmt.Errorf("server returned status code %d", resp.StatusCode) + } + + // create a temporary file + tmpFilePath := m.filePath + ".tmp" // append .tmp to the file path + tmpFile, err := os.Create(tmpFilePath) + if err != nil { + return fmt.Errorf("failed to create temporary file: %v", err) + } + var ok bool + defer func() { + if !ok { + _ = os.Remove(tmpFilePath) + } + }() + + // copy the response body to the temporary file and print progress + writeCounter := NewWriteCounter(m.ctx, uint64(resp.ContentLength)) + if _, err = io.Copy(tmpFile, io.TeeReader(resp.Body, writeCounter)); err != nil { + return fmt.Errorf("failed to copy response body: %v", err) + } + + if err = tmpFile.Close(); err != nil { + return fmt.Errorf("failed to close temporary file: %v", err) + } + + // rename the temporary file to the final file path + if err = os.Rename(tmpFilePath, m.filePath); err != nil { + return fmt.Errorf("failed to rename temporary file: %v", err) + } + + ok = true + return nil +} + +type WriteCounter struct { + ctx context.Context + total uint64 + written uint64 + lastProgressPrinted time.Time +} + +func NewWriteCounter(ctx context.Context, total uint64) *WriteCounter { + return &WriteCounter{ + ctx: ctx, + total: total, + } +} + +func (wc *WriteCounter) Write(p []byte) (int, error) { + n := len(p) + wc.written += uint64(n) + + // check if the context is done and return early + select { + case <-wc.ctx.Done(): + return n, wc.ctx.Err() + default: + } + + wc.printProgress() + + return n, nil +} + +func (wc *WriteCounter) printProgress() { + if time.Since(wc.lastProgressPrinted) < 5*time.Second { + return + } + wc.lastProgressPrinted = time.Now() + + log.Info(fmt.Sprintf("Downloading missing header fields... %d MB / %d MB", toMB(wc.written), toMB(wc.total))) +} + +func toMB(bytes uint64) uint64 { + return bytes / 1024 / 1024 +} diff --git a/rollup/missing_header_fields/manager_test.go b/rollup/missing_header_fields/manager_test.go new file mode 100644 index 000000000000..f675c3077d38 --- /dev/null +++ b/rollup/missing_header_fields/manager_test.go @@ -0,0 +1,60 @@ +package missing_header_fields + +import ( + "context" + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/require" + + "github.com/scroll-tech/go-ethereum/common" + "github.com/scroll-tech/go-ethereum/log" + "github.com/scroll-tech/go-ethereum/params" +) + +func TestManagerDownload(t *testing.T) { + t.Skip("skipping test due to long runtime/downloading file") + log.Root().SetHandler(log.StdoutHandler) + + sha256 := *params.ScrollSepoliaChainConfig.Scroll.MissingHeaderFieldsSHA256 + downloadURL := "https://scroll-block-missing-metadata.s3.us-west-2.amazonaws.com/" + params.ScrollSepoliaChainConfig.ChainID.String() + ".bin" + filePath := filepath.Join(t.TempDir(), "test_file_path") + manager := NewManager(context.Background(), filePath, downloadURL, sha256) + + _, _, _, err := manager.GetMissingHeaderFields(0) + require.NoError(t, err) + + // Check if the file was downloaded and tmp file was removed + _, err = os.Stat(filePath) + require.NoError(t, err) + _, err = os.Stat(filePath + ".tmp") + require.Error(t, err) +} + +func TestManagerChecksum(t *testing.T) { + downloadURL := "" // since the file exists we don't need to download it + filePath := filepath.Join("testdata", "missing-headers.bin") + + // Checksum doesn't match + { + sha256 := [32]byte(common.FromHex("0xaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa")) + + manager := NewManager(context.Background(), filePath, downloadURL, sha256) + + _, _, _, err := manager.GetMissingHeaderFields(0) + require.ErrorContains(t, err, "expectedChecksum mismatch") + } + + // Checksum matches + { + sha256 := [32]byte(common.FromHex("e5a1e71338cd899e46ff28a9ae81b8f2579e429e18cec463104fb246a6e23502")) + manager := NewManager(context.Background(), filePath, downloadURL, sha256) + + difficulty, stateRoot, extra, err := manager.GetMissingHeaderFields(0) + require.NoError(t, err) + require.Equal(t, expectedMissingHeaders[0].difficulty, difficulty) + require.Equal(t, expectedMissingHeaders[0].stateRoot, stateRoot) + require.Equal(t, expectedMissingHeaders[0].extra, extra) + } +} diff --git a/rollup/missing_header_fields/reader.go b/rollup/missing_header_fields/reader.go new file mode 100644 index 000000000000..d078ecfdcb40 --- /dev/null +++ b/rollup/missing_header_fields/reader.go @@ -0,0 +1,186 @@ +package missing_header_fields + +import ( + "bufio" + "bytes" + "fmt" + "io" + "os" + + "github.com/scroll-tech/go-ethereum/common" +) + +type missingHeader struct { + headerNum uint64 + difficulty uint64 + stateRoot common.Hash + extraData []byte +} + +type Reader struct { + file *os.File + reader *bufio.Reader + sortedVanities map[int][32]byte + lastReadHeader *missingHeader +} + +func NewReader(filePath string) (*Reader, error) { + f, err := os.Open(filePath) + if err != nil { + return nil, fmt.Errorf("failed to open file: %w", err) + } + + r := &Reader{ + file: f, + reader: bufio.NewReader(f), + } + + if err = r.initialize(); err != nil { + if err = f.Close(); err != nil { + return nil, fmt.Errorf("failed to close file after initialization error: %w", err) + } + return nil, fmt.Errorf("failed to initialize reader: %w", err) + } + + return r, nil +} + +func (r *Reader) initialize() error { + // reset the reader and last read header + if _, err := r.file.Seek(0, io.SeekStart); err != nil { + return fmt.Errorf("failed to seek to start: %w", err) + } + r.reader = bufio.NewReader(r.file) + r.lastReadHeader = nil + + // read the count of unique vanities + vanityCount, err := r.reader.ReadByte() + if err != nil { + return err + } + + // read the unique vanities + r.sortedVanities = make(map[int][32]byte) + for i := uint8(0); i < vanityCount; i++ { + var vanity [32]byte + if _, err = r.reader.Read(vanity[:]); err != nil { + return err + } + r.sortedVanities[int(i)] = vanity + } + + return nil +} + +func (r *Reader) Read(headerNum uint64) (difficulty uint64, stateRoot common.Hash, extraData []byte, err error) { + if r.lastReadHeader != nil && headerNum < r.lastReadHeader.headerNum { + if err = r.initialize(); err != nil { + return 0, common.Hash{}, nil, fmt.Errorf("failed to reinitialize reader due to requested header number being lower than last read header: %w", err) + } + } + + if r.lastReadHeader == nil { + if err = r.ReadNext(); err != nil { + return 0, common.Hash{}, nil, err + } + } + + if headerNum > r.lastReadHeader.headerNum { + // skip the headers until the requested header number + for i := r.lastReadHeader.headerNum; i < headerNum; i++ { + if err = r.ReadNext(); err != nil { + return 0, common.Hash{}, nil, err + } + } + } + + if headerNum == r.lastReadHeader.headerNum { + return r.lastReadHeader.difficulty, r.lastReadHeader.stateRoot, r.lastReadHeader.extraData, nil + } + + return 0, common.Hash{}, nil, fmt.Errorf("error reading header number %d: last read header number is %d", headerNum, r.lastReadHeader.headerNum) +} + +func (r *Reader) ReadNext() (err error) { + // read the bitmask + bitmaskByte, err := r.reader.ReadByte() + if err != nil { + return fmt.Errorf("failed to read bitmask: %v", err) + } + + bits := newBitMaskFromBytes(bitmaskByte) + + // read the vanity index + vanityIndex, err := r.reader.ReadByte() + if err != nil { + return fmt.Errorf("failed to read vanity index: %v", err) + } + + stateRoot := make([]byte, common.HashLength) + if _, err := io.ReadFull(r.reader, stateRoot); err != nil { + return fmt.Errorf("failed to read state root: %v", err) + } + + seal := make([]byte, bits.sealLen()) + if _, err = io.ReadFull(r.reader, seal); err != nil { + return fmt.Errorf("failed to read seal: %v", err) + } + + // construct the extraData field + vanity := r.sortedVanities[int(vanityIndex)] + var b bytes.Buffer + b.Write(vanity[:]) + b.Write(seal) + + // we don't have the header number, so we'll just increment the last read header number + // we assume that the headers are written in order, starting from 0 + if r.lastReadHeader == nil { + r.lastReadHeader = &missingHeader{ + headerNum: 0, + difficulty: uint64(bits.difficulty()), + stateRoot: common.BytesToHash(stateRoot), + extraData: b.Bytes(), + } + } else { + r.lastReadHeader.headerNum++ + r.lastReadHeader.difficulty = uint64(bits.difficulty()) + r.lastReadHeader.stateRoot = common.BytesToHash(stateRoot) + r.lastReadHeader.extraData = b.Bytes() + } + + return nil +} + +func (r *Reader) Close() error { + return r.file.Close() +} + +// bitMask is a bitmask that encodes the following information: +// +// bit 6: 0 if difficulty is 2, 1 if difficulty is 1 +// bit 7: 0 if seal length is 65, 1 if seal length is 85 +type bitMask struct { + b uint8 +} + +func newBitMaskFromBytes(b uint8) bitMask { + return bitMask{b} +} + +func (b bitMask) difficulty() int { + val := (b.b >> 6) & 0x01 + if val == 0 { + return 2 + } else { + return 1 + } +} + +func (b bitMask) sealLen() int { + val := (b.b >> 7) & 0x01 + if val == 0 { + return 65 + } else { + return 85 + } +} diff --git a/rollup/missing_header_fields/reader_test.go b/rollup/missing_header_fields/reader_test.go new file mode 100644 index 000000000000..1835c11ecba4 --- /dev/null +++ b/rollup/missing_header_fields/reader_test.go @@ -0,0 +1,74 @@ +package missing_header_fields + +import ( + "testing" + + "github.com/stretchr/testify/require" + + "github.com/scroll-tech/go-ethereum/common" +) + +type header struct { + number uint64 + difficulty uint64 + stateRoot common.Hash + extra []byte +} + +var expectedMissingHeaders = []header{ + {0, 1, common.HexToHash("0x20695989e9038823e35f0e88fbc44659ffdbfa1fe89fbeb2689b43f15fa64cb5"), common.FromHex("000000000000000000000000000000000000000000000000000000000000000048c3f81f3d998b6652900e1c3183736c238fe4290000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000")}, + {1, 2, common.HexToHash("0x20695989e9038823e35f0e88fbc44659ffdbfa1fe89fbeb2689b43f15fa64cb5"), common.FromHex("d88304031d846765746888676f312e31392e31856c696e7578000000000000001982b5c754257988f9486b158a33709645735e8e965912c508aee9b0513cc2f22fe13f0835ce1e11abe666c9dba6a1259612b812783cc457e5b34b025980635501")}, + {2, 2, common.HexToHash("0x11787ec3c17489215d0b13d594db83be55736f3933625aac0e1bba2812d49ffe"), common.FromHex("d88304031d846765746888676f312e31392e31856c696e757800000000000000237c933578bf062f86a30cdc71b0e946f0f685711e0e9cceeb1c953ed816d2694347e1e59625545c4040f2604b75448ccb5360fdcb378741331c1d4c0d342a7e01")}, + {3, 2, common.HexToHash("0x2b4cff60622970fe15dfe6e7a8c8bf9619aea5790b57b9bc34811dd6670c16bf"), common.FromHex("d88304031d846765746888676f312e31392e31856c696e75780000000000000012388a2df0f522f96e67564d38be64b5ca7fb37ef9b3f88de875d08653871407584b180917a47dc4abec60bf8da462c617328b9d2da8c4bb9978e018b44ec07401")}, + {4, 2, common.HexToHash("0x032c5535bc0684d25cd74a6bdc9f15052c9444ed8bfe5e9e4791d83f52b8dac1"), common.FromHex("d88304031d846765746888676f312e31392e31856c696e75780000000000000091e57e01b8ed1b433b2bd04e272f9eaf986f3fa728c8fc2b4112352101d24ba76ff1ee64e9a1f8a47c4c49e362e318b2b4767088514f72a7ba9bb7a45b4b447700")}, + {5, 2, common.HexToHash("0x0e13888dc93db27a55825d45b27d2b7d367ac9b6a302bd3480177253320f815a"), common.FromHex("d88304031d846765746888676f312e31392e31856c696e7578000000000000007ab6b6bd8d52c9beffe935e1bc805d9d4ad62d54485104e80943537d380d6f425ad055ad510c498d1e6efc2aa7e7cc7e1b6166f8421e94b13e291196cba1934a00")}, + {6, 2, common.HexToHash("0x29fee2024813a75d49d4670716e0f706eac9d72f091f64db1f802d346a1f2a58"), common.FromHex("d88304031d846765746888676f312e31392e31856c696e7578000000000000008f7175ed80593d395069afed2d970e505b076e15642e1f6e3bcb2f589a2a47fa5841868b80fcc70f18cd52de027bdb5ab881fdefc5ebf0d8034cb35e926e89f300")}, + {7, 2, common.HexToHash("0x1d9ce9ddef4ab5c0d120715cab5b4eeef6ae8b94addb7a8bdfcb6c8f76f63e34"), common.FromHex("d88304031d846765746888676f312e31392e31856c696e75780000000000000006224d2201ba60083743844ce0c2ec4b0ab3e79b69f64eae9a10055fe704380c6410c4f5119cb834f43705c1a785758170a868a38e536432e3a5a5805c83b13801")}, + {8, 2, common.HexToHash("0x1c9f95b06e3e65cdd0dae9d2b20de7760d9b6c768b05d043e0d0f493112d32f2"), common.FromHex("d88304031d846765746888676f312e31392e31856c696e757800000000000000b5c1f5c8aa79582f4b9a66ae8a59561d6c357deaefc7353ea6ace017e76e4b367118e0fd55b9f4cd0235ee1f14222e9b558156b6253e84f71d8048e13643af4801")}, + {9, 2, common.HexToHash("0x19c478ece75320a6c4a3d1cecfd219f130a8b377e47b76adaa3eb574396bacb1"), common.FromHex("d88304031d846765746888676f312e31392e31856c696e757800000000000000dff93471464bf856b2f633ac16b54c3ff88219a8c83067495ff9f16035c91fb56de8f6914eb4cbd8fbe8a54854e32d697a81408e20cdaa52fed9689d21f7ad0201")}, + {10, 2, common.HexToHash("0x00a85e26b58e8294115b097a808ba9840326981efb8c4abb054eb40494e5bc0a"), common.FromHex("d88304031d846765746888676f312e31392e31856c696e7578000000000000003c55c63554686f48d9e6dd78b8a7849152e33f169f843e623aa604f2c777eec9539d301fe5f1bea84e5cb7c40e74b723e7700b95eab08bc441d65092b40b548d01")}, +} + +func TestReader_Read(t *testing.T) { + expectedVanities := map[int][32]byte{ + 0: [32]byte(common.FromHex("0000000000000000000000000000000000000000000000000000000000000000")), + 1: [32]byte(common.FromHex("0xd88304031d846765746888676f312e31392e31856c696e757800000000000000")), + } + + reader, err := NewReader("testdata/missing-headers.bin") + require.NoError(t, err) + + require.Len(t, reader.sortedVanities, len(expectedVanities)) + for i, expectedVanity := range expectedVanities { + require.Equal(t, expectedVanity, reader.sortedVanities[i]) + } + + readAndAssertHeader(t, reader, expectedMissingHeaders, 0) + readAndAssertHeader(t, reader, expectedMissingHeaders, 0) + readAndAssertHeader(t, reader, expectedMissingHeaders, 1) + readAndAssertHeader(t, reader, expectedMissingHeaders, 6) + + // reading previous headers resets the file reader + readAndAssertHeader(t, reader, expectedMissingHeaders, 5) + + readAndAssertHeader(t, reader, expectedMissingHeaders, 8) + readAndAssertHeader(t, reader, expectedMissingHeaders, 8) + + // reading previous headers resets the file reader + readAndAssertHeader(t, reader, expectedMissingHeaders, 6) + + readAndAssertHeader(t, reader, expectedMissingHeaders, 9) + readAndAssertHeader(t, reader, expectedMissingHeaders, 10) + + // no data anymore + _, _, _, err = reader.Read(11) + require.Error(t, err) +} + +func readAndAssertHeader(t *testing.T, reader *Reader, expectedHeaders []header, headerNum uint64) { + difficulty, stateRoot, extra, err := reader.Read(headerNum) + require.NoError(t, err) + require.Equalf(t, expectedHeaders[headerNum].difficulty, difficulty, "expected difficulty %d, got %d", expectedHeaders[headerNum].difficulty, difficulty) + require.Equalf(t, expectedHeaders[headerNum].stateRoot, stateRoot, "expected state root %s, got %s", expectedHeaders[headerNum].stateRoot.Hex(), stateRoot.Hex()) + require.Equal(t, expectedHeaders[headerNum].extra, extra) +} diff --git a/rollup/missing_header_fields/testdata/missing-headers.bin b/rollup/missing_header_fields/testdata/missing-headers.bin new file mode 100644 index 000000000000..96068b3f606f Binary files /dev/null and b/rollup/missing_header_fields/testdata/missing-headers.bin differ diff --git a/rollup/rollup_sync_service/rollup_sync_service_test.go b/rollup/rollup_sync_service/rollup_sync_service_test.go index e6f6d7339697..68c53f6ac8f5 100644 --- a/rollup/rollup_sync_service/rollup_sync_service_test.go +++ b/rollup/rollup_sync_service/rollup_sync_service_test.go @@ -21,6 +21,7 @@ import ( "github.com/scroll-tech/go-ethereum/rollup/da_syncer" "github.com/scroll-tech/go-ethereum/rollup/da_syncer/da" "github.com/scroll-tech/go-ethereum/rollup/l1" + "github.com/scroll-tech/go-ethereum/rollup/missing_header_fields" ) func TestGetCommittedBatchMetaCodecV0(t *testing.T) { @@ -185,7 +186,7 @@ func (m mockEntryWithBlocks) Event() l1.RollupEvent { panic("implement me") } -func (m mockEntryWithBlocks) Blocks() ([]*da.PartialBlock, error) { +func (m mockEntryWithBlocks) Blocks(_ *missing_header_fields.Manager) ([]*da.PartialBlock, error) { panic("implement me") }