diff --git a/.env.example b/.env.example index d206ee7..ff59290 100644 --- a/.env.example +++ b/.env.example @@ -1,11 +1,37 @@ -# Mempool Rebroadcaster +# -------------------------------------------- +# MEMPOOL REBROADCASTER +# -------------------------------------------- + GETH_MEMPOOL_ENDPOINT=http://localhost:8546 RETH_MEMPOOL_ENDPOINT=http://localhost:8547 -# Flashblocks Archiver +# -------------------------------------------- +# FLASHBLOCKS ARCHIVER +# -------------------------------------------- DATABASE_URL=postgresql://postgres:password@localhost:5432/flashblocks_archiver -FLASHBLOCKS_WEBSOCKET_URLS=ws://one.example.com/ws,ws://two.example.com/ws DATABASE_MAX_CONNECTIONS=10 +DATABASE_CONNECT_TIMEOUT_SECONDS=30 + +FLASHBLOCKS_WEBSOCKET_URLS=wss://builder1.example.com/ws,wss://builder2.example.com/ws +FLASHBLOCKS_RECONNECT_DELAY_SECONDS=5 + +ARCHIVER_BUFFER_SIZE=1000 +ARCHIVER_BATCH_SIZE=100 +ARCHIVER_FLUSH_INTERVAL_SECONDS=5 + +RETENTION_ENABLED=true +RETENTION_PERIOD_DAYS=30 +RETENTION_ARCHIVE_INTERVAL_HOURS=6 +RETENTION_BLOCK_RANGE_SIZE=21600 + +S3_BUCKET_NAME=my-flashblocks-archive-bucket +S3_REGION=us-east-1 +S3_KEY_PREFIX=flashblocks/ + +# AWS_ACCESS_KEY_ID=AKIA... +# AWS_SECRET_ACCESS_KEY=xxx... -# Service selection for generic Docker commands +# -------------------------------------------- +# GENERIC +# -------------------------------------------- SERVICE=flashblocks-archiver \ No newline at end of file diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 97a044b..ac461e2 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -28,19 +28,6 @@ jobs: test: name: Test runs-on: ubuntu-latest - services: - postgres: - image: postgres:15 - env: - POSTGRES_PASSWORD: password - POSTGRES_DB: test_db - options: >- - --health-cmd pg_isready - --health-interval 10s - --health-timeout 5s - --health-retries 5 - ports: - - 5432:5432 steps: - name: Harden the runner (Audit all outbound calls) uses: step-security/harden-runner@002fdce3c6a235733a90a27c80493a3241e56863 # v2.12.1 @@ -51,8 +38,6 @@ jobs: - uses: dtolnay/rust-toolchain@4305c38b25d97ef35a8ad1f985ccf2d2242004f2 # stable - name: Run tests run: cargo test --workspace --all-features - env: - DATABASE_URL: postgresql://postgres:password@localhost:5432/test_db lint: name: Lint diff --git a/Cargo.lock b/Cargo.lock index 0d86425..b223859 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -59,6 +59,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5a15f179cd60c4584b8a8c596927aadc462e27f2ca70c04e0071964a73ba7a75" dependencies = [ "cfg-if", + "const-random", "getrandom 0.3.3", "once_cell", "version_check", @@ -301,7 +302,7 @@ checksum = "883dee3b4020fcb5667ee627b4f401e899dad82bf37b246620339dd980720ed9" dependencies = [ "alloy-primitives", "alloy-sol-types", - "http", + "http 1.3.1", "serde", "serde_json", "thiserror 2.0.12", @@ -702,7 +703,7 @@ dependencies = [ "async-trait", "auto_impl", "either", - "elliptic-curve", + "elliptic-curve 0.13.8", "k256", "thiserror 2.0.12", ] @@ -863,8 +864,8 @@ dependencies = [ "alloy-pubsub", "alloy-transport", "futures", - "http", - "rustls", + "http 1.3.1", + "rustls 0.23.31", "serde_json", "tokio", "tokio-tungstenite", @@ -1304,6 +1305,214 @@ dependencies = [ "serde", ] +[[package]] +name = "arrow" +version = "56.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c26b57282a08ae92f727497805122fec964c6245cfa0e13f0e75452eaf3bc41f" +dependencies = [ + "arrow-arith", + "arrow-array", + "arrow-buffer", + "arrow-cast", + "arrow-csv", + "arrow-data", + "arrow-ipc", + "arrow-json", + "arrow-ord", + "arrow-row", + "arrow-schema", + "arrow-select", + "arrow-string", +] + +[[package]] +name = "arrow-arith" +version = "56.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cebf38ca279120ff522f4954b81a39527425b6e9f615e6b72842f4de1ffe02b8" +dependencies = [ + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-schema", + "chrono", + "num", +] + +[[package]] +name = "arrow-array" +version = "56.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "744109142cdf8e7b02795e240e20756c2a782ac9180d4992802954a8f871c0de" +dependencies = [ + "ahash", + "arrow-buffer", + "arrow-data", + "arrow-schema", + "chrono", + "half", + "hashbrown 0.15.4", + "num", +] + +[[package]] +name = "arrow-buffer" +version = "56.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "601bb103c4c374bcd1f62c66bcea67b42a2ee91a690486c37d4c180236f11ccc" +dependencies = [ + "bytes", + "half", + "num", +] + +[[package]] +name = "arrow-cast" +version = "56.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eed61d9d73eda8df9e3014843def37af3050b5080a9acbe108f045a316d5a0be" +dependencies = [ + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-schema", + "arrow-select", + "atoi", + "base64 0.22.1", + "chrono", + "half", + "lexical-core", + "num", + "ryu", +] + +[[package]] +name = "arrow-csv" +version = "56.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fa95b96ce0c06b4d33ac958370db8c0d31e88e54f9d6e08b0353d18374d9f991" +dependencies = [ + "arrow-array", + "arrow-cast", + "arrow-schema", + "chrono", + "csv", + "csv-core", + "regex", +] + +[[package]] +name = "arrow-data" +version = "56.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "43407f2c6ba2367f64d85d4603d6fb9c4b92ed79d2ffd21021b37efa96523e12" +dependencies = [ + "arrow-buffer", + "arrow-schema", + "half", + "num", +] + +[[package]] +name = "arrow-ipc" +version = "56.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e4b0487c4d2ad121cbc42c4db204f1509f8618e589bc77e635e9c40b502e3b90" +dependencies = [ + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-schema", + "arrow-select", + "flatbuffers", +] + +[[package]] +name = "arrow-json" +version = "56.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "26d747573390905905a2dc4c5a61a96163fe2750457f90a04ee2a88680758c79" +dependencies = [ + "arrow-array", + "arrow-buffer", + "arrow-cast", + "arrow-data", + "arrow-schema", + "chrono", + "half", + "indexmap 2.10.0", + "lexical-core", + "memchr", + "num", + "serde", + "serde_json", + "simdutf8", +] + +[[package]] +name = "arrow-ord" +version = "56.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c142a147dceb59d057bad82400f1693847c80dca870d008bf7b91caf902810ae" +dependencies = [ + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-schema", + "arrow-select", +] + +[[package]] +name = "arrow-row" +version = "56.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dac6620667fccdab4204689ca173bd84a15de6bb6b756c3a8764d4d7d0c2fc04" +dependencies = [ + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-schema", + "half", +] + +[[package]] +name = "arrow-schema" +version = "56.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dfa93af9ff2bb80de539e6eb2c1c8764abd0f4b73ffb0d7c82bf1f9868785e66" + +[[package]] +name = "arrow-select" +version = "56.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "be8b2e0052cd20d36d64f32640b68a5ab54d805d24a473baee5d52017c85536c" +dependencies = [ + "ahash", + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-schema", + "num", +] + +[[package]] +name = "arrow-string" +version = "56.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2155e26e17f053c8975c546fc70cf19c00542f9abf43c23a88a46ef7204204f" +dependencies = [ + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-schema", + "arrow-select", + "memchr", + "num", + "regex", + "regex-syntax 0.8.5", +] + [[package]] name = "asn1_der" version = "0.7.6" @@ -1423,6 +1632,48 @@ version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c08606f8c3cbf4ce6ec8e28fb0014a2c086708fe954eaa885384a6165172e7e8" +[[package]] +name = "aws-config" +version = "1.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c478f5b10ce55c9a33f87ca3404ca92768b144fc1bfdede7c0121214a8283a25" +dependencies = [ + "aws-credential-types", + "aws-runtime", + "aws-sdk-sso", + "aws-sdk-ssooidc", + "aws-sdk-sts", + "aws-smithy-async", + "aws-smithy-http", + "aws-smithy-json", + "aws-smithy-runtime", + "aws-smithy-runtime-api", + "aws-smithy-types", + "aws-types", + "bytes", + "fastrand", + "hex", + "http 1.3.1", + "ring", + "time", + "tokio", + "tracing", + "url", + "zeroize", +] + +[[package]] +name = "aws-credential-types" +version = "1.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1541072f81945fa1251f8795ef6c92c4282d74d59f88498ae7d4bf00f0ebdad9" +dependencies = [ + "aws-smithy-async", + "aws-smithy-runtime-api", + "aws-smithy-types", + "zeroize", +] + [[package]] name = "aws-lc-rs" version = "1.13.3" @@ -1446,6 +1697,371 @@ dependencies = [ "fs_extra", ] +[[package]] +name = "aws-runtime" +version = "1.5.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c034a1bc1d70e16e7f4e4caf7e9f7693e4c9c24cd91cf17c2a0b21abaebc7c8b" +dependencies = [ + "aws-credential-types", + "aws-sigv4", + "aws-smithy-async", + "aws-smithy-eventstream", + "aws-smithy-http", + "aws-smithy-runtime", + "aws-smithy-runtime-api", + "aws-smithy-types", + "aws-types", + "bytes", + "fastrand", + "http 0.2.12", + "http-body 0.4.6", + "percent-encoding", + "pin-project-lite", + "tracing", + "uuid", +] + +[[package]] +name = "aws-sdk-s3" +version = "1.103.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "af040a86ae4378b7ed2f62c83b36be1848709bbbf5757ec850d0e08596a26be9" +dependencies = [ + "aws-credential-types", + "aws-runtime", + "aws-sigv4", + "aws-smithy-async", + "aws-smithy-checksums", + "aws-smithy-eventstream", + "aws-smithy-http", + "aws-smithy-json", + "aws-smithy-runtime", + "aws-smithy-runtime-api", + "aws-smithy-types", + "aws-smithy-xml", + "aws-types", + "bytes", + "fastrand", + "hex", + "hmac", + "http 0.2.12", + "http 1.3.1", + "http-body 0.4.6", + "lru 0.12.5", + "percent-encoding", + "regex-lite", + "sha2 0.10.9", + "tracing", + "url", +] + +[[package]] +name = "aws-sdk-sso" +version = "1.81.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "79ede098271e3471036c46957cba2ba30888f53bda2515bf04b560614a30a36e" +dependencies = [ + "aws-credential-types", + "aws-runtime", + "aws-smithy-async", + "aws-smithy-http", + "aws-smithy-json", + "aws-smithy-runtime", + "aws-smithy-runtime-api", + "aws-smithy-types", + "aws-types", + "bytes", + "fastrand", + "http 0.2.12", + "regex-lite", + "tracing", +] + +[[package]] +name = "aws-sdk-ssooidc" +version = "1.82.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "43326f724ba2cc957e6f3deac0ca1621a3e5d4146f5970c24c8a108dac33070f" +dependencies = [ + "aws-credential-types", + "aws-runtime", + "aws-smithy-async", + "aws-smithy-http", + "aws-smithy-json", + "aws-smithy-runtime", + "aws-smithy-runtime-api", + "aws-smithy-types", + "aws-types", + "bytes", + "fastrand", + "http 0.2.12", + "regex-lite", + "tracing", +] + +[[package]] +name = "aws-sdk-sts" +version = "1.83.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a5468593c47efc31fdbe6c902d1a5fde8d9c82f78a3f8ccfe907b1e9434748cb" +dependencies = [ + "aws-credential-types", + "aws-runtime", + "aws-smithy-async", + "aws-smithy-http", + "aws-smithy-json", + "aws-smithy-query", + "aws-smithy-runtime", + "aws-smithy-runtime-api", + "aws-smithy-types", + "aws-smithy-xml", + "aws-types", + "fastrand", + "http 0.2.12", + "regex-lite", + "tracing", +] + +[[package]] +name = "aws-sigv4" +version = "1.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "084c34162187d39e3740cb635acd73c4e3a551a36146ad6fe8883c929c9f876c" +dependencies = [ + "aws-credential-types", + "aws-smithy-eventstream", + "aws-smithy-http", + "aws-smithy-runtime-api", + "aws-smithy-types", + "bytes", + "crypto-bigint 0.5.5", + "form_urlencoded", + "hex", + "hmac", + "http 0.2.12", + "http 1.3.1", + "p256 0.11.1", + "percent-encoding", + "ring", + "sha2 0.10.9", + "subtle", + "time", + "tracing", + "zeroize", +] + +[[package]] +name = "aws-smithy-async" +version = "1.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e190749ea56f8c42bf15dd76c65e14f8f765233e6df9b0506d9d934ebef867c" +dependencies = [ + "futures-util", + "pin-project-lite", + "tokio", +] + +[[package]] +name = "aws-smithy-checksums" +version = "0.63.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4dbef71cd3cf607deb5c407df52f7e589e6849b296874ee448977efbb6d0832b" +dependencies = [ + "aws-smithy-http", + "aws-smithy-types", + "bytes", + "crc-fast", + "hex", + "http 0.2.12", + "http-body 0.4.6", + "md-5", + "pin-project-lite", + "sha1", + "sha2 0.10.9", + "tracing", +] + +[[package]] +name = "aws-smithy-eventstream" +version = "0.60.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "182b03393e8c677347fb5705a04a9392695d47d20ef0a2f8cfe28c8e6b9b9778" +dependencies = [ + "aws-smithy-types", + "bytes", + "crc32fast", +] + +[[package]] +name = "aws-smithy-http" +version = "0.62.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7c4dacf2d38996cf729f55e7a762b30918229917eca115de45dfa8dfb97796c9" +dependencies = [ + "aws-smithy-eventstream", + "aws-smithy-runtime-api", + "aws-smithy-types", + "bytes", + "bytes-utils", + "futures-core", + "http 0.2.12", + "http 1.3.1", + "http-body 0.4.6", + "percent-encoding", + "pin-project-lite", + "pin-utils", + "tracing", +] + +[[package]] +name = "aws-smithy-http-client" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4fdbad9bd9dbcc6c5e68c311a841b54b70def3ca3b674c42fbebb265980539f8" +dependencies = [ + "aws-smithy-async", + "aws-smithy-runtime-api", + "aws-smithy-types", + "h2 0.3.27", + "h2 0.4.12", + "http 0.2.12", + "http 1.3.1", + "http-body 0.4.6", + "hyper 0.14.32", + "hyper 1.6.0", + "hyper-rustls 0.24.2", + "hyper-rustls 0.27.7", + "hyper-util", + "pin-project-lite", + "rustls 0.21.12", + "rustls 0.23.31", + "rustls-native-certs 0.8.1", + "rustls-pki-types", + "tokio", + "tokio-rustls 0.26.2", + "tower 0.5.2", + "tracing", +] + +[[package]] +name = "aws-smithy-json" +version = "0.61.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a16e040799d29c17412943bdbf488fd75db04112d0c0d4b9290bacf5ae0014b9" +dependencies = [ + "aws-smithy-types", +] + +[[package]] +name = "aws-smithy-observability" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9364d5989ac4dd918e5cc4c4bdcc61c9be17dcd2586ea7f69e348fc7c6cab393" +dependencies = [ + "aws-smithy-runtime-api", +] + +[[package]] +name = "aws-smithy-query" +version = "0.60.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2fbd61ceb3fe8a1cb7352e42689cec5335833cd9f94103a61e98f9bb61c64bb" +dependencies = [ + "aws-smithy-types", + "urlencoding", +] + +[[package]] +name = "aws-smithy-runtime" +version = "1.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a3d57c8b53a72d15c8e190475743acf34e4996685e346a3448dd54ef696fc6e0" +dependencies = [ + "aws-smithy-async", + "aws-smithy-http", + "aws-smithy-http-client", + "aws-smithy-observability", + "aws-smithy-runtime-api", + "aws-smithy-types", + "bytes", + "fastrand", + "http 0.2.12", + "http 1.3.1", + "http-body 0.4.6", + "http-body 1.0.1", + "pin-project-lite", + "pin-utils", + "tokio", + "tracing", +] + +[[package]] +name = "aws-smithy-runtime-api" +version = "1.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "07f5e0fc8a6b3f2303f331b94504bbf754d85488f402d6f1dd7a6080f99afe56" +dependencies = [ + "aws-smithy-async", + "aws-smithy-types", + "bytes", + "http 0.2.12", + "http 1.3.1", + "pin-project-lite", + "tokio", + "tracing", + "zeroize", +] + +[[package]] +name = "aws-smithy-types" +version = "1.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d498595448e43de7f4296b7b7a18a8a02c61ec9349128c80a368f7c3b4ab11a8" +dependencies = [ + "base64-simd", + "bytes", + "bytes-utils", + "futures-core", + "http 0.2.12", + "http 1.3.1", + "http-body 0.4.6", + "http-body 1.0.1", + "http-body-util", + "itoa", + "num-integer", + "pin-project-lite", + "pin-utils", + "ryu", + "serde", + "time", + "tokio", + "tokio-util", +] + +[[package]] +name = "aws-smithy-xml" +version = "0.60.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3db87b96cb1b16c024980f133968d52882ca0daaee3a086c6decc500f6c99728" +dependencies = [ + "xmlparser", +] + +[[package]] +name = "aws-types" +version = "1.3.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b069d19bf01e46298eaedd7c6f283fe565a59263e53eebec945f3e6398f42390" +dependencies = [ + "aws-credential-types", + "aws-smithy-async", + "aws-smithy-runtime-api", + "aws-smithy-types", + "rustc_version 0.4.1", + "tracing", +] + [[package]] name = "axum" version = "0.7.9" @@ -1456,8 +2072,8 @@ dependencies = [ "axum-core", "bytes", "futures-util", - "http", - "http-body", + "http 1.3.1", + "http-body 1.0.1", "http-body-util", "itoa", "matchit", @@ -1482,8 +2098,8 @@ dependencies = [ "async-trait", "bytes", "futures-util", - "http", - "http-body", + "http 1.3.1", + "http-body 1.0.1", "http-body-util", "mime", "pin-project-lite", @@ -1530,6 +2146,12 @@ version = "0.2.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4cbbc9d0964165b47557570cce6c952866c2678457aca742aafc9fb771d30270" +[[package]] +name = "base16ct" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "349a06037c7bf932dd7e7d1f653678b2038b9ad46a74102f1fc7bd7872678cce" + [[package]] name = "base16ct" version = "0.2.0" @@ -1548,6 +2170,16 @@ version = "0.22.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" +[[package]] +name = "base64-simd" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "339abbe78e73178762e23bea9dfd08e697eb3f3301cd4be981c0f78ba5859195" +dependencies = [ + "outref", + "vsimd", +] + [[package]] name = "base64ct" version = "1.8.0" @@ -1879,18 +2511,18 @@ dependencies = [ "futures-util", "hex", "home", - "http", + "http 1.3.1", "http-body-util", - "hyper", + "hyper 1.6.0", "hyper-named-pipe", - "hyper-rustls", + "hyper-rustls 0.27.7", "hyper-util", "hyperlocal", "log", "pin-project-lite", - "rustls", - "rustls-native-certs", - "rustls-pemfile", + "rustls 0.23.31", + "rustls-native-certs 0.8.1", + "rustls-pemfile 2.2.0", "rustls-pki-types", "serde", "serde_derive", @@ -2009,6 +2641,16 @@ dependencies = [ "serde", ] +[[package]] +name = "bytes-utils" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7dafe3a8757b027e2be6e4e5601ed563c55989fcf1546e933c66c8eb3a058d35" +dependencies = [ + "bytes", + "either", +] + [[package]] name = "c-kzg" version = "2.1.1" @@ -2337,6 +2979,26 @@ version = "0.9.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2459377285ad874054d797f3ccebf984978aa39129f6eafde5cdc8315b612f8" +[[package]] +name = "const-random" +version = "0.1.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87e00182fe74b066627d63b85fd550ac2998d4b0bd86bfed477a0ae4c7c71359" +dependencies = [ + "const-random-macro", +] + +[[package]] +name = "const-random-macro" +version = "0.1.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f9d839f2a20b0aee515dc581a6172f2321f96cab76c1a38a4c584a194955390e" +dependencies = [ + "getrandom 0.2.16", + "once_cell", + "tiny-keccak", +] + [[package]] name = "const_format" version = "0.2.34" @@ -2431,6 +3093,19 @@ version = "2.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "19d374276b40fb8bbdee95aef7c7fa6b5316ec764510eb64b8dd0e2ed0d7e7f5" +[[package]] +name = "crc-fast" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6bf62af4cc77d8fe1c22dde4e721d87f2f54056139d8c412e1366b740305f56f" +dependencies = [ + "crc", + "digest 0.10.7", + "libc", + "rand 0.9.2", + "regex", +] + [[package]] name = "crc32fast" version = "1.5.0" @@ -2520,6 +3195,18 @@ version = "0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "460fbee9c2c2f33933d720630a6a0bac33ba7053db5344fac858d4b8952d77d5" +[[package]] +name = "crypto-bigint" +version = "0.4.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ef2b4b23cddf68b89b8f8069890e8c270d54e2d5fe1b143820234805e4cb17ef" +dependencies = [ + "generic-array", + "rand_core 0.6.4", + "subtle", + "zeroize", +] + [[package]] name = "crypto-bigint" version = "0.5.5" @@ -2543,6 +3230,27 @@ dependencies = [ "typenum", ] +[[package]] +name = "csv" +version = "1.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "acdc4883a9c96732e4733212c01447ebd805833b7275a73ca3ee080fd77afdaf" +dependencies = [ + "csv-core", + "itoa", + "ryu", + "serde", +] + +[[package]] +name = "csv-core" +version = "0.1.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d02f3b0da4c6504f86e9cd789d8dbafab48c2321be74e9987593de5a894d93d" +dependencies = [ + "memchr", +] + [[package]] name = "ctr" version = "0.9.2" @@ -2684,6 +3392,16 @@ dependencies = [ "tokio-util", ] +[[package]] +name = "der" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1a467a65c5e759bce6e65eaf91cc29f466cdc57cb65777bd646872a8a1fd4de" +dependencies = [ + "const-oid", + "zeroize", +] + [[package]] name = "der" version = "0.7.10" @@ -2846,7 +3564,7 @@ dependencies = [ "libc", "option-ext", "redox_users 0.5.2", - "windows-sys 0.59.0", + "windows-sys 0.60.2", ] [[package]] @@ -2939,19 +3657,31 @@ version = "1.0.19" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1c7a8fb8a9fbf66c1f703fe16184d10ca0ee9d23be5b4436400408ba54a95005" +[[package]] +name = "ecdsa" +version = "0.14.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "413301934810f597c1d19ca71c8710e99a3f1ba28a0d2ebc01551a2daeea3c5c" +dependencies = [ + "der 0.6.1", + "elliptic-curve 0.12.3", + "rfc6979 0.3.1", + "signature 1.6.4", +] + [[package]] name = "ecdsa" version = "0.16.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ee27f32b5c5292967d2d4a9d7f1e0b0aed2c15daded5a60300e4abb9d8020bca" dependencies = [ - "der", + "der 0.7.10", "digest 0.10.7", - "elliptic-curve", - "rfc6979", + "elliptic-curve 0.13.8", + "rfc6979 0.4.0", "serdect", - "signature", - "spki", + "signature 2.2.0", + "spki 0.7.3", ] [[package]] @@ -2960,8 +3690,8 @@ version = "2.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "115531babc129696a58c64a4fef0a8bf9e9698629fb97e9e40767d235cfbcd53" dependencies = [ - "pkcs8", - "signature", + "pkcs8 0.10.2", + "signature 2.2.0", ] [[package]] @@ -3000,21 +3730,41 @@ dependencies = [ "serde", ] +[[package]] +name = "elliptic-curve" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7bb888ab5300a19b8e5bceef25ac745ad065f3c9f7efc6de1b91958110891d3" +dependencies = [ + "base16ct 0.1.1", + "crypto-bigint 0.4.9", + "der 0.6.1", + "digest 0.10.7", + "ff 0.12.1", + "generic-array", + "group 0.12.1", + "pkcs8 0.9.0", + "rand_core 0.6.4", + "sec1 0.3.0", + "subtle", + "zeroize", +] + [[package]] name = "elliptic-curve" version = "0.13.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b5e6043086bf7973472e0c7dff2142ea0b680d30e18d9cc40f267efbf222bd47" dependencies = [ - "base16ct", - "crypto-bigint", + "base16ct 0.2.0", + "crypto-bigint 0.5.5", "digest 0.10.7", - "ff", + "ff 0.13.1", "generic-array", - "group", - "pkcs8", + "group 0.13.0", + "pkcs8 0.10.2", "rand_core 0.6.4", - "sec1", + "sec1 0.7.3", "serdect", "subtle", "zeroize", @@ -3091,7 +3841,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "778e2ac28f6c47af28e4907f13ffd1e1ddbd400980a9abd7c8df189bf578a5ad" dependencies = [ "libc", - "windows-sys 0.59.0", + "windows-sys 0.60.2", ] [[package]] @@ -3240,6 +3990,16 @@ dependencies = [ "thiserror 1.0.69", ] +[[package]] +name = "ff" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d013fc25338cc558c5c2cfbad646908fb23591e2404481826742b651c9af7160" +dependencies = [ + "rand_core 0.6.4", + "subtle", +] + [[package]] name = "ff" version = "0.13.1" @@ -3296,6 +4056,9 @@ dependencies = [ "alloy-rpc-types-eth", "alloy-trie", "anyhow", + "arrow", + "aws-config", + "aws-sdk-s3", "brotli", "chrono", "clap", @@ -3307,6 +4070,7 @@ dependencies = [ "op-alloy-rpc-jsonrpsee", "op-alloy-rpc-types", "op-alloy-rpc-types-engine", + "parquet", "reth", "reth-cli-util", "reth-e2e-test-utils", @@ -3324,7 +4088,7 @@ dependencies = [ "reth-rpc-eth-api", "reth-tracing", "rollup-boost", - "rustls", + "rustls 0.23.31", "serde", "serde_json", "sqlx", @@ -3340,6 +4104,16 @@ dependencies = [ "uuid", ] +[[package]] +name = "flatbuffers" +version = "25.2.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1045398c1bfd89168b5fd3f1fc11f6e70b34f6f66300c87d44d3de849463abf1" +dependencies = [ + "bitflags 2.9.1", + "rustc_version 0.4.1", +] + [[package]] name = "flate2" version = "1.1.2" @@ -3347,6 +4121,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4a3d7db9596fecd151c5f638c0ee5d5bd487b6e0ea232e5dc96d5250f6f94b1d" dependencies = [ "crc32fast", + "libz-rs-sys", "miniz_oxide", ] @@ -3632,7 +4407,7 @@ dependencies = [ "futures-core", "futures-sink", "gloo-utils", - "http", + "http 1.3.1", "js-sys", "pin-project", "serde", @@ -3678,17 +4453,47 @@ dependencies = [ "windows-sys 0.59.0", ] +[[package]] +name = "group" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5dfbfb3a6cfbd390d5c9564ab283a0349b9b9fcd46a706c1eb10e0db70bfbac7" +dependencies = [ + "ff 0.12.1", + "rand_core 0.6.4", + "subtle", +] + [[package]] name = "group" version = "0.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f0f9ef7462f7c099f518d754361858f86d8a07af53ba9af0fe635bbccb151a63" dependencies = [ - "ff", + "ff 0.13.1", "rand_core 0.6.4", "subtle", ] +[[package]] +name = "h2" +version = "0.3.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0beca50380b1fc32983fc1cb4587bfa4bb9e78fc259aad4a0032d2080309222d" +dependencies = [ + "bytes", + "fnv", + "futures-core", + "futures-sink", + "futures-util", + "http 0.2.12", + "indexmap 2.10.0", + "slab", + "tokio", + "tokio-util", + "tracing", +] + [[package]] name = "h2" version = "0.4.12" @@ -3700,7 +4505,7 @@ dependencies = [ "fnv", "futures-core", "futures-sink", - "http", + "http 1.3.1", "indexmap 2.10.0", "slab", "tokio", @@ -3708,6 +4513,17 @@ dependencies = [ "tracing", ] +[[package]] +name = "half" +version = "2.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "459196ed295495a68f7d7fe1d84f6c4b7ff0e21fe3017b2f283c6fac3ad803c9" +dependencies = [ + "cfg-if", + "crunchy", + "num-traits", +] + [[package]] name = "hash-db" version = "0.15.2" @@ -3880,15 +4696,37 @@ dependencies = [ "windows-sys 0.59.0", ] +[[package]] +name = "http" +version = "0.2.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "601cbb57e577e2f5ef5be8e7b83f0f63994f25aa94d673e54a92d5c516d101f1" +dependencies = [ + "bytes", + "fnv", + "itoa", +] + [[package]] name = "http" version = "1.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f4a85d31aea989eead29a3aaf9e1115a180df8282431156e533de47660892565" +checksum = "f4a85d31aea989eead29a3aaf9e1115a180df8282431156e533de47660892565" +dependencies = [ + "bytes", + "fnv", + "itoa", +] + +[[package]] +name = "http-body" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ceab25649e9960c0311ea418d17bee82c0dcec1bd053b5f9a66e265a693bed2" dependencies = [ "bytes", - "fnv", - "itoa", + "http 0.2.12", + "pin-project-lite", ] [[package]] @@ -3898,7 +4736,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1efedce1fb8e6913f23e0c92de8e62cd5b772a67e7b3946df930a62566c93184" dependencies = [ "bytes", - "http", + "http 1.3.1", ] [[package]] @@ -3909,8 +4747,8 @@ checksum = "b021d93e26becf5dc7e1b75b1bed1fd93124b374ceb73f43d4d4eafec896a64a" dependencies = [ "bytes", "futures-core", - "http", - "http-body", + "http 1.3.1", + "http-body 1.0.1", "pin-project-lite", ] @@ -3954,6 +4792,30 @@ dependencies = [ "serde", ] +[[package]] +name = "hyper" +version = "0.14.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41dfc780fdec9373c01bae43289ea34c972e40ee3c9f6b3c8801a35f35586ce7" +dependencies = [ + "bytes", + "futures-channel", + "futures-core", + "futures-util", + "h2 0.3.27", + "http 0.2.12", + "http-body 0.4.6", + "httparse", + "httpdate", + "itoa", + "pin-project-lite", + "socket2", + "tokio", + "tower-service", + "tracing", + "want", +] + [[package]] name = "hyper" version = "1.6.0" @@ -3963,9 +4825,9 @@ dependencies = [ "bytes", "futures-channel", "futures-util", - "h2", - "http", - "http-body", + "h2 0.4.12", + "http 1.3.1", + "http-body 1.0.1", "httparse", "httpdate", "itoa", @@ -3982,7 +4844,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "73b7d8abf35697b81a825e386fc151e0d503e8cb5fcb93cc8669c376dfd6f278" dependencies = [ "hex", - "hyper", + "hyper 1.6.0", "hyper-util", "pin-project-lite", "tokio", @@ -3990,21 +4852,37 @@ dependencies = [ "winapi", ] +[[package]] +name = "hyper-rustls" +version = "0.24.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec3efd23720e2049821a693cbc7e65ea87c72f1c58ff2f9522ff332b1491e590" +dependencies = [ + "futures-util", + "http 0.2.12", + "hyper 0.14.32", + "log", + "rustls 0.21.12", + "rustls-native-certs 0.6.3", + "tokio", + "tokio-rustls 0.24.1", +] + [[package]] name = "hyper-rustls" version = "0.27.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e3c93eb611681b207e1fe55d5a71ecf91572ec8a6705cdb6857f7d8d5242cf58" dependencies = [ - "http", - "hyper", + "http 1.3.1", + "hyper 1.6.0", "hyper-util", "log", - "rustls", - "rustls-native-certs", + "rustls 0.23.31", + "rustls-native-certs 0.8.1", "rustls-pki-types", "tokio", - "tokio-rustls", + "tokio-rustls 0.26.2", "tower-service", "webpki-roots 1.0.2", ] @@ -4015,7 +4893,7 @@ version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2b90d566bffbce6a75bd8b09a05aa8c2cb1fabb6cb348f8840c9e4c90a0d83b0" dependencies = [ - "hyper", + "hyper 1.6.0", "hyper-util", "pin-project-lite", "tokio", @@ -4030,7 +4908,7 @@ checksum = "70206fc6890eaca9fde8a0bf71caa2ddfc9fe045ac9e5c70df101a7dbde866e0" dependencies = [ "bytes", "http-body-util", - "hyper", + "hyper 1.6.0", "hyper-util", "native-tls", "tokio", @@ -4040,18 +4918,18 @@ dependencies = [ [[package]] name = "hyper-util" -version = "0.1.15" +version = "0.1.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f66d5bd4c6f02bf0542fad85d626775bab9258cf795a4256dcaf3161114d1df" +checksum = "8d9b05277c7e8da2c93a568989bb6207bef0112e8d17df7a6eda4a3cf143bc5e" dependencies = [ "base64 0.22.1", "bytes", "futures-channel", "futures-core", "futures-util", - "http", - "http-body", - "hyper", + "http 1.3.1", + "http-body 1.0.1", + "hyper 1.6.0", "ipnet", "libc", "percent-encoding", @@ -4072,7 +4950,7 @@ checksum = "986c5ce3b994526b3cd75578e62554abd09f0899d6206de48b3e96ab34ccc8c7" dependencies = [ "hex", "http-body-util", - "hyper", + "hyper 1.6.0", "hyper-util", "pin-project-lite", "tokio", @@ -4461,6 +5339,12 @@ dependencies = [ "syn 2.0.104", ] +[[package]] +name = "integer-encoding" +version = "3.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8bb03732005da905c88227371639bf1ad885cc712789c011c31c5fb3ab3ccf02" + [[package]] name = "interprocess" version = "2.2.3" @@ -4647,16 +5531,16 @@ dependencies = [ "futures-channel", "futures-util", "gloo-net", - "http", + "http 1.3.1", "jsonrpsee-core 0.25.1 (registry+https://github.com/rust-lang/crates.io-index)", "pin-project", - "rustls", + "rustls 0.23.31", "rustls-pki-types", "rustls-platform-verifier", "soketto", "thiserror 2.0.12", "tokio", - "tokio-rustls", + "tokio-rustls 0.26.2", "tokio-util", "tracing", "url", @@ -4672,8 +5556,8 @@ dependencies = [ "bytes", "futures-timer", "futures-util", - "http", - "http-body", + "http 1.3.1", + "http-body 1.0.1", "http-body-util", "jsonrpsee-types 0.25.1 (registry+https://github.com/rust-lang/crates.io-index)", "parking_lot", @@ -4698,8 +5582,8 @@ dependencies = [ "async-trait", "bytes", "futures-util", - "http", - "http-body", + "http 1.3.1", + "http-body 1.0.1", "http-body-util", "jsonrpsee-types 0.25.1 (git+https://github.com/paritytech/jsonrpsee?rev=f04afa740e55db60dce20d9839758792f035ffff)", "parking_lot", @@ -4721,13 +5605,13 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6962d2bd295f75e97dd328891e58fce166894b974c1f7ce2e7597f02eeceb791" dependencies = [ "base64 0.22.1", - "http-body", - "hyper", - "hyper-rustls", + "http-body 1.0.1", + "hyper 1.6.0", + "hyper-rustls 0.27.7", "hyper-util", "jsonrpsee-core 0.25.1 (registry+https://github.com/rust-lang/crates.io-index)", "jsonrpsee-types 0.25.1 (registry+https://github.com/rust-lang/crates.io-index)", - "rustls", + "rustls 0.23.31", "rustls-platform-verifier", "serde", "serde_json", @@ -4743,13 +5627,13 @@ version = "0.25.1" source = "git+https://github.com/paritytech/jsonrpsee?rev=f04afa740e55db60dce20d9839758792f035ffff#f04afa740e55db60dce20d9839758792f035ffff" dependencies = [ "base64 0.22.1", - "http-body", - "hyper", - "hyper-rustls", + "http-body 1.0.1", + "hyper 1.6.0", + "hyper-rustls 0.27.7", "hyper-util", "jsonrpsee-core 0.25.1 (git+https://github.com/paritytech/jsonrpsee?rev=f04afa740e55db60dce20d9839758792f035ffff)", "jsonrpsee-types 0.25.1 (git+https://github.com/paritytech/jsonrpsee?rev=f04afa740e55db60dce20d9839758792f035ffff)", - "rustls", + "rustls 0.23.31", "rustls-platform-verifier", "serde", "serde_json", @@ -4791,10 +5675,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d38b0bcf407ac68d241f90e2d46041e6a06988f97fe1721fb80b91c42584fae6" dependencies = [ "futures-util", - "http", - "http-body", + "http 1.3.1", + "http-body 1.0.1", "http-body-util", - "hyper", + "hyper 1.6.0", "hyper-util", "jsonrpsee-core 0.25.1 (registry+https://github.com/rust-lang/crates.io-index)", "jsonrpsee-types 0.25.1 (registry+https://github.com/rust-lang/crates.io-index)", @@ -4817,10 +5701,10 @@ version = "0.25.1" source = "git+https://github.com/paritytech/jsonrpsee?rev=f04afa740e55db60dce20d9839758792f035ffff#f04afa740e55db60dce20d9839758792f035ffff" dependencies = [ "futures-util", - "http", - "http-body", + "http 1.3.1", + "http-body 1.0.1", "http-body-util", - "hyper", + "hyper 1.6.0", "hyper-util", "jsonrpsee-core 0.25.1 (git+https://github.com/paritytech/jsonrpsee?rev=f04afa740e55db60dce20d9839758792f035ffff)", "jsonrpsee-types 0.25.1 (git+https://github.com/paritytech/jsonrpsee?rev=f04afa740e55db60dce20d9839758792f035ffff)", @@ -4843,7 +5727,7 @@ version = "0.25.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "66df7256371c45621b3b7d2fb23aea923d577616b9c0e9c0b950a6ea5c2be0ca" dependencies = [ - "http", + "http 1.3.1", "serde", "serde_json", "thiserror 2.0.12", @@ -4854,7 +5738,7 @@ name = "jsonrpsee-types" version = "0.25.1" source = "git+https://github.com/paritytech/jsonrpsee?rev=f04afa740e55db60dce20d9839758792f035ffff#f04afa740e55db60dce20d9839758792f035ffff" dependencies = [ - "http", + "http 1.3.1", "serde", "serde_json", "thiserror 2.0.12", @@ -4878,7 +5762,7 @@ version = "0.25.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2da2694c9ff271a9d3ebfe520f6b36820e85133a51be77a3cb549fd615095261" dependencies = [ - "http", + "http 1.3.1", "jsonrpsee-client-transport", "jsonrpsee-core 0.25.1 (registry+https://github.com/rust-lang/crates.io-index)", "jsonrpsee-types 0.25.1 (registry+https://github.com/rust-lang/crates.io-index)", @@ -4908,12 +5792,12 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f6e3919bbaa2945715f0bb6d3934a173d1e9a59ac23767fbaaef277265a7411b" dependencies = [ "cfg-if", - "ecdsa", - "elliptic-curve", + "ecdsa 0.16.9", + "elliptic-curve 0.13.8", "once_cell", "serdect", "sha2 0.10.9", - "signature", + "signature 2.2.0", ] [[package]] @@ -4970,6 +5854,70 @@ version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "830d08ce1d1d941e6b30645f1a0eb5643013d835ce3779a5fc208261dbe10f55" +[[package]] +name = "lexical-core" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b765c31809609075565a70b4b71402281283aeda7ecaf4818ac14a7b2ade8958" +dependencies = [ + "lexical-parse-float", + "lexical-parse-integer", + "lexical-util", + "lexical-write-float", + "lexical-write-integer", +] + +[[package]] +name = "lexical-parse-float" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "de6f9cb01fb0b08060209a057c048fcbab8717b4c1ecd2eac66ebfe39a65b0f2" +dependencies = [ + "lexical-parse-integer", + "lexical-util", + "static_assertions", +] + +[[package]] +name = "lexical-parse-integer" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72207aae22fc0a121ba7b6d479e42cbfea549af1479c3f3a4f12c70dd66df12e" +dependencies = [ + "lexical-util", + "static_assertions", +] + +[[package]] +name = "lexical-util" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a82e24bf537fd24c177ffbbdc6ebcc8d54732c35b50a3f28cc3f4e4c949a0b3" +dependencies = [ + "static_assertions", +] + +[[package]] +name = "lexical-write-float" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c5afc668a27f460fb45a81a757b6bf2f43c2d7e30cb5a2dcd3abf294c78d62bd" +dependencies = [ + "lexical-util", + "lexical-write-integer", + "static_assertions", +] + +[[package]] +name = "lexical-write-integer" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "629ddff1a914a836fb245616a7888b62903aae58fa771e1d83943035efa0f978" +dependencies = [ + "lexical-util", + "static_assertions", +] + [[package]] name = "libc" version = "0.2.174" @@ -4995,7 +5943,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "07033963ba89ebaf1584d767badaa2e8fcec21aedea6b8c0346d487d49c28667" dependencies = [ "cfg-if", - "windows-targets 0.52.6", + "windows-targets 0.53.3", ] [[package]] @@ -5101,6 +6049,15 @@ dependencies = [ "vcpkg", ] +[[package]] +name = "libz-rs-sys" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "172a788537a2221661b480fee8dc5f96c580eb34fa88764d3205dc356c7e4221" +dependencies = [ + "zlib-rs", +] + [[package]] name = "libz-sys" version = "1.1.22" @@ -5231,6 +6188,9 @@ name = "lz4_flex" version = "0.11.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "08ab2867e3eeeca90e844d1940eab391c9dc5228783db2ed999acbc0a9ed375a" +dependencies = [ + "twox-hash", +] [[package]] name = "mach2" @@ -5359,8 +6319,8 @@ checksum = "dd7399781913e5393588a8d8c6a2867bf85fb38eaf2502fdce465aad2dc6f034" dependencies = [ "base64 0.22.1", "http-body-util", - "hyper", - "hyper-rustls", + "hyper 1.6.0", + "hyper-rustls 0.27.7", "hyper-util", "indexmap 2.10.0", "ipnet", @@ -5400,7 +6360,7 @@ dependencies = [ "hashbrown 0.15.4", "indexmap 2.10.0", "metrics", - "ordered-float", + "ordered-float 4.6.0", "quanta", "radix_trie", "rand 0.9.2", @@ -5989,7 +6949,7 @@ checksum = "a8863faf2910030d139fb48715ad5ff2f35029fc5f244f6d5f689ddcf4d26253" dependencies = [ "async-trait", "bytes", - "http", + "http 1.3.1", "opentelemetry", "reqwest", "tracing", @@ -6003,7 +6963,7 @@ checksum = "5bef114c6d41bea83d6dc60eb41720eedd0261a67af57b66dd2b84ac46c01d91" dependencies = [ "async-trait", "futures-core", - "http", + "http 1.3.1", "opentelemetry", "opentelemetry-http", "opentelemetry-proto", @@ -6059,6 +7019,15 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "04744f49eae99ab78e0d5c0b603ab218f515ea8cfe5a456d7629ad883a3b6e7d" +[[package]] +name = "ordered-float" +version = "2.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68f19d67e5a2795c94e73e0bb1cc1a7edeb2e28efd39e2e1c9b7a40c1108b11c" +dependencies = [ + "num-traits", +] + [[package]] name = "ordered-float" version = "4.6.0" @@ -6068,20 +7037,37 @@ dependencies = [ "num-traits", ] +[[package]] +name = "outref" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a80800c0488c3a21695ea981a54918fbb37abf04f4d0720c453632255e2ff0e" + [[package]] name = "overload" version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b15813163c1d831bf4a13c3610c05c0d03b39feb07f7e09fa234dac9b15aaf39" +[[package]] +name = "p256" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "51f44edd08f51e2ade572f141051021c5af22677e42b7dd28a88155151c33594" +dependencies = [ + "ecdsa 0.14.8", + "elliptic-curve 0.12.3", + "sha2 0.10.9", +] + [[package]] name = "p256" version = "0.13.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c9863ad85fa8f4460f9c48cb909d38a0d689dba1f6f6988a5e3e0d31071bcd4b" dependencies = [ - "ecdsa", - "elliptic-curve", + "ecdsa 0.16.9", + "elliptic-curve 0.13.8", "primeorder", "sha2 0.10.9", ] @@ -6155,6 +7141,39 @@ dependencies = [ "windows-targets 0.52.6", ] +[[package]] +name = "parquet" +version = "56.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "89b56b41d1bd36aae415e42f91cae70ee75cf6cba74416b14dce3e958d5990ec" +dependencies = [ + "ahash", + "arrow-array", + "arrow-buffer", + "arrow-cast", + "arrow-data", + "arrow-ipc", + "arrow-schema", + "arrow-select", + "base64 0.22.1", + "brotli", + "bytes", + "chrono", + "flate2", + "half", + "hashbrown 0.15.4", + "lz4_flex", + "num", + "num-bigint", + "paste", + "seq-macro", + "simdutf8", + "snap", + "thrift", + "twox-hash", + "zstd", +] + [[package]] name = "parse-display" version = "0.9.1" @@ -6323,9 +7342,19 @@ version = "0.7.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c8ffb9f10fa047879315e6625af03c164b16962a5368d724ed16323b68ace47f" dependencies = [ - "der", - "pkcs8", - "spki", + "der 0.7.10", + "pkcs8 0.10.2", + "spki 0.7.3", +] + +[[package]] +name = "pkcs8" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9eca2c590a5f85da82668fa685c09ce2888b9430e83299debf1f34b65fd4a4ba" +dependencies = [ + "der 0.6.1", + "spki 0.6.0", ] [[package]] @@ -6334,8 +7363,8 @@ version = "0.10.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f950b2377845cebe5cf8b5165cb3cc1a5e0fa5cfa3e1f7f55707d8fd82e0a7b7" dependencies = [ - "der", - "spki", + "der 0.7.10", + "spki 0.7.3", ] [[package]] @@ -6427,7 +7456,7 @@ version = "0.13.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "353e1ca18966c16d9deb1c69278edbc5f194139612772bd9537af60ac231e1e6" dependencies = [ - "elliptic-curve", + "elliptic-curve 0.13.8", ] [[package]] @@ -6623,7 +7652,7 @@ dependencies = [ "quinn-proto", "quinn-udp", "rustc-hash 2.1.1", - "rustls", + "rustls 0.23.31", "socket2", "thiserror 2.0.12", "tokio", @@ -6643,7 +7672,7 @@ dependencies = [ "rand 0.9.2", "ring", "rustc-hash 2.1.1", - "rustls", + "rustls 0.23.31", "rustls-pki-types", "slab", "thiserror 2.0.12", @@ -6925,6 +7954,12 @@ dependencies = [ "regex-syntax 0.8.5", ] +[[package]] +name = "regex-lite" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53a49587ad06b26609c52e423de037e7f57f20d53535d66e08c695f347df952a" + [[package]] name = "regex-syntax" version = "0.6.29" @@ -6958,11 +7993,11 @@ dependencies = [ "futures-channel", "futures-core", "futures-util", - "http", - "http-body", + "http 1.3.1", + "http-body 1.0.1", "http-body-util", - "hyper", - "hyper-rustls", + "hyper 1.6.0", + "hyper-rustls 0.27.7", "hyper-tls", "hyper-util", "js-sys", @@ -6971,8 +8006,8 @@ dependencies = [ "percent-encoding", "pin-project-lite", "quinn", - "rustls", - "rustls-native-certs", + "rustls 0.23.31", + "rustls-native-certs 0.8.1", "rustls-pki-types", "serde", "serde_json", @@ -6980,7 +8015,7 @@ dependencies = [ "sync_wrapper", "tokio", "tokio-native-tls", - "tokio-rustls", + "tokio-rustls 0.26.2", "tokio-util", "tower 0.5.2", "tower-http", @@ -8647,7 +9682,7 @@ version = "1.6.0" source = "git+https://github.com/paradigmxyz/reth?tag=v1.6.0#d8451e54e7267f9f1634118d6d279b2216f7e2bb" dependencies = [ "eyre", - "http", + "http 1.3.1", "jsonrpsee-server 0.25.1 (registry+https://github.com/rust-lang/crates.io-index)", "metrics", "metrics-exporter-prometheus", @@ -9318,9 +10353,9 @@ dependencies = [ "async-trait", "derive_more", "futures", - "http", - "http-body", - "hyper", + "http 1.3.1", + "http-body 1.0.1", + "hyper 1.6.0", "jsonrpsee 0.25.1 (registry+https://github.com/rust-lang/crates.io-index)", "jsonrpsee-types 0.25.1 (registry+https://github.com/rust-lang/crates.io-index)", "jsonwebtoken", @@ -9400,7 +10435,7 @@ source = "git+https://github.com/paradigmxyz/reth?tag=v1.6.0#d8451e54e7267f9f163 dependencies = [ "alloy-network", "alloy-provider", - "http", + "http 1.3.1", "jsonrpsee 0.25.1 (registry+https://github.com/rust-lang/crates.io-index)", "metrics", "pin-project", @@ -9580,7 +10615,7 @@ version = "1.6.0" source = "git+https://github.com/paradigmxyz/reth?tag=v1.6.0#d8451e54e7267f9f1634118d6d279b2216f7e2bb" dependencies = [ "alloy-rpc-types-engine", - "http", + "http 1.3.1", "jsonrpsee-http-client 0.25.1 (registry+https://github.com/rust-lang/crates.io-index)", "pin-project", "tower 0.5.2", @@ -10202,7 +11237,7 @@ dependencies = [ "k256", "libsecp256k1", "once_cell", - "p256", + "p256 0.13.2", "revm-primitives", "ripemd", "rug", @@ -10234,6 +11269,17 @@ dependencies = [ "serde", ] +[[package]] +name = "rfc6979" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7743f17af12fa0b03b803ba12cd6a8d9483a587e89c69445e3909655c0b9fabb" +dependencies = [ + "crypto-bigint 0.4.9", + "hmac", + "zeroize", +] + [[package]] name = "rfc6979" version = "0.4.0" @@ -10346,10 +11392,10 @@ dependencies = [ "dotenvy", "eyre", "futures", - "http", + "http 1.3.1", "http-body-util", - "hyper", - "hyper-rustls", + "hyper 1.6.0", + "hyper-rustls 0.27.7", "hyper-util", "jsonrpsee 0.25.1 (git+https://github.com/paritytech/jsonrpsee?rev=f04afa740e55db60dce20d9839758792f035ffff)", "metrics", @@ -10364,7 +11410,7 @@ dependencies = [ "parking_lot", "paste", "reth-optimism-payload-builder", - "rustls", + "rustls 0.23.31", "serde", "serde_json", "sha2 0.10.9", @@ -10401,10 +11447,10 @@ dependencies = [ "num-integer", "num-traits", "pkcs1", - "pkcs8", + "pkcs8 0.10.2", "rand_core 0.6.4", - "signature", - "spki", + "signature 2.2.0", + "spki 0.7.3", "subtle", "zeroize", ] @@ -10523,25 +11569,49 @@ dependencies = [ "errno", "libc", "linux-raw-sys 0.9.4", - "windows-sys 0.59.0", + "windows-sys 0.60.2", +] + +[[package]] +name = "rustls" +version = "0.21.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f56a14d1f48b391359b22f731fd4bd7e43c97f3c50eee276f3aa09c94784d3e" +dependencies = [ + "log", + "ring", + "rustls-webpki 0.101.7", + "sct", ] [[package]] name = "rustls" -version = "0.23.29" +version = "0.23.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2491382039b29b9b11ff08b76ff6c97cf287671dbb74f0be44bda389fffe9bd1" +checksum = "c0ebcbd2f03de0fc1122ad9bb24b127a5a6cd51d72604a3f3c50ac459762b6cc" dependencies = [ "aws-lc-rs", "log", "once_cell", "ring", "rustls-pki-types", - "rustls-webpki", + "rustls-webpki 0.103.4", "subtle", "zeroize", ] +[[package]] +name = "rustls-native-certs" +version = "0.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a9aace74cb666635c918e9c12bc0d348266037aa8eb599b5cba565709a8dff00" +dependencies = [ + "openssl-probe", + "rustls-pemfile 1.0.4", + "schannel", + "security-framework 2.11.1", +] + [[package]] name = "rustls-native-certs" version = "0.8.1" @@ -10554,6 +11624,15 @@ dependencies = [ "security-framework 3.3.0", ] +[[package]] +name = "rustls-pemfile" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1c74cae0a4cf6ccbbf5f359f08efdf8ee7e1dc532573bf0db71968cb56b1448c" +dependencies = [ + "base64 0.21.7", +] + [[package]] name = "rustls-pemfile" version = "2.2.0" @@ -10584,10 +11663,10 @@ dependencies = [ "jni", "log", "once_cell", - "rustls", - "rustls-native-certs", + "rustls 0.23.31", + "rustls-native-certs 0.8.1", "rustls-platform-verifier-android", - "rustls-webpki", + "rustls-webpki 0.103.4", "security-framework 3.3.0", "security-framework-sys", "webpki-root-certs 0.26.11", @@ -10600,6 +11679,16 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f87165f0995f63a9fbeea62b64d10b4d9d8e78ec6d7d51fb2125fda7bb36788f" +[[package]] +name = "rustls-webpki" +version = "0.101.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b6275d1ee7a1cd780b64aca7726599a1dbc893b1e64144529e55c3c2f745765" +dependencies = [ + "ring", + "untrusted", +] + [[package]] name = "rustls-webpki" version = "0.103.4" @@ -10707,16 +11796,40 @@ version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" +[[package]] +name = "sct" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da046153aa2352493d6cb7da4b6e5c0c057d8a1d0a9aa8560baffdd945acd414" +dependencies = [ + "ring", + "untrusted", +] + +[[package]] +name = "sec1" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3be24c1842290c45df0a7bf069e0c268a747ad05a192f2fd7dcfdbc1cba40928" +dependencies = [ + "base16ct 0.1.1", + "der 0.6.1", + "generic-array", + "pkcs8 0.9.0", + "subtle", + "zeroize", +] + [[package]] name = "sec1" version = "0.7.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d3e97a565f76233a6003f9f5c54be1d9c5bdfa3eccfb189469f11ec4901c47dc" dependencies = [ - "base16ct", - "der", + "base16ct 0.2.0", + "der 0.7.10", "generic-array", - "pkcs8", + "pkcs8 0.10.2", "serdect", "subtle", "zeroize", @@ -10838,6 +11951,12 @@ version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cd0b0ec5f1c1ca621c432a25813d8d60c88abe6d3e08a3eb9cf37d97a0fe3d73" +[[package]] +name = "seq-macro" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1bc711410fbe7399f390ca1c3b60ad0f53f80e95c5eb935e52268a0e2cd49acc" + [[package]] name = "serde" version = "1.0.219" @@ -10941,7 +12060,7 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a84f14a19e9a014bb9f4512488d9829a68e04ecabffb0f9904cd1ace94598177" dependencies = [ - "base16ct", + "base16ct 0.2.0", "serde", ] @@ -11054,6 +12173,16 @@ dependencies = [ "libc", ] +[[package]] +name = "signature" +version = "1.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "74233d3b3b2f6d4b006dc19dee745e73e2a6bfb6f93607cd3b02bd5b00797d7c" +dependencies = [ + "digest 0.10.7", + "rand_core 0.6.4", +] + [[package]] name = "signature" version = "2.2.0" @@ -11064,6 +12193,12 @@ dependencies = [ "rand_core 0.6.4", ] +[[package]] +name = "simdutf8" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3a9fe34e3e7a50316060351f37187a3f546bce95496156754b601a5fa71b76e" + [[package]] name = "simple_asn1" version = "0.6.3" @@ -11144,7 +12279,7 @@ dependencies = [ "base64 0.22.1", "bytes", "futures", - "http", + "http 1.3.1", "httparse", "log", "rand 0.8.5", @@ -11160,6 +12295,16 @@ dependencies = [ "lock_api", ] +[[package]] +name = "spki" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67cf02bbac7a337dc36e4f5a693db6c21e7863f45070f7064577eb4367a3212b" +dependencies = [ + "base64ct", + "der 0.6.1", +] + [[package]] name = "spki" version = "0.7.3" @@ -11167,7 +12312,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d91ed6c858b01f942cd56b37a94b3e0a1798290327d1236e4d9cf4eaca44d29d" dependencies = [ "base64ct", - "der", + "der 0.7.10", ] [[package]] @@ -11711,6 +12856,17 @@ dependencies = [ "num_cpus", ] +[[package]] +name = "thrift" +version = "0.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7e54bc85fc7faa8bc175c4bab5b92ba8d9a3ce893d0e9f42cc455c8ab16a9e09" +dependencies = [ + "byteorder", + "integer-encoding", + "ordered-float 2.10.1", +] + [[package]] name = "tikv-jemalloc-ctl" version = "0.6.0" @@ -11861,13 +13017,23 @@ dependencies = [ "tokio", ] +[[package]] +name = "tokio-rustls" +version = "0.24.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c28327cf380ac148141087fbfb9de9d7bd4e84ab5d2c28fbc911d753de8a7081" +dependencies = [ + "rustls 0.21.12", + "tokio", +] + [[package]] name = "tokio-rustls" version = "0.26.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8e727b36a1a0e8b74c376ac2211e40c2c8af09fb4013c60d910495810f008e9b" dependencies = [ - "rustls", + "rustls 0.23.31", "tokio", ] @@ -11920,12 +13086,12 @@ dependencies = [ "futures-util", "log", "native-tls", - "rustls", - "rustls-native-certs", + "rustls 0.23.31", + "rustls-native-certs 0.8.1", "rustls-pki-types", "tokio", "tokio-native-tls", - "tokio-rustls", + "tokio-rustls 0.26.2", "tungstenite", "webpki-roots 0.26.11", ] @@ -11997,11 +13163,11 @@ dependencies = [ "axum", "base64 0.22.1", "bytes", - "h2", - "http", - "http-body", + "h2 0.4.12", + "http 1.3.1", + "http-body 1.0.1", "http-body-util", - "hyper", + "hyper 1.6.0", "hyper-timeout", "hyper-util", "percent-encoding", @@ -12068,8 +13234,8 @@ dependencies = [ "bytes", "futures-core", "futures-util", - "http", - "http-body", + "http 1.3.1", + "http-body 1.0.1", "http-body-util", "http-range-header", "httpdate", @@ -12301,18 +13467,24 @@ checksum = "4793cb5e56680ecbb1d843515b23b6de9a75eb04b66643e256a396d43be33c13" dependencies = [ "bytes", "data-encoding", - "http", + "http 1.3.1", "httparse", "log", "native-tls", "rand 0.9.2", - "rustls", + "rustls 0.23.31", "rustls-pki-types", "sha1", "thiserror 2.0.12", "utf-8", ] +[[package]] +name = "twox-hash" +version = "2.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b907da542cbced5261bd3256de1b3a1bf340a3d37f93425a07362a1d687de56" + [[package]] name = "typenum" version = "1.18.0" @@ -12457,6 +13629,12 @@ dependencies = [ "serde", ] +[[package]] +name = "urlencoding" +version = "2.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "daf8dba3b7eb870caf1ddeed7bc9d2a049f3cfdfae7cb521b087cc33ae4c49da" + [[package]] name = "utf-8" version = "0.7.6" @@ -12563,6 +13741,12 @@ dependencies = [ "syn 2.0.104", ] +[[package]] +name = "vsimd" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c3082ca00d5a5ef149bb8b555a72ae84c9c59f7250f013ac822ac2e49b19c64" + [[package]] name = "wait-timeout" version = "0.2.1" @@ -13436,6 +14620,12 @@ dependencies = [ "rustix 1.0.8", ] +[[package]] +name = "xmlparser" +version = "0.13.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "66fee0b777b0f5ac1c69bb06d361268faafa61cd4682ae064a171c16c433e9e4" + [[package]] name = "yansi" version = "1.0.1" @@ -13606,6 +14796,12 @@ dependencies = [ "syn 2.0.104", ] +[[package]] +name = "zlib-rs" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "626bd9fa9734751fc50d6060752170984d7053f5a39061f524cda68023d4db8a" + [[package]] name = "zstd" version = "0.13.3" @@ -13617,18 +14813,18 @@ dependencies = [ [[package]] name = "zstd-safe" -version = "7.2.4" +version = "7.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f49c4d5f0abb602a93fb8736af2a4f4dd9512e36f7f570d66e65ff867ed3b9d" +checksum = "54a3ab4db68cea366acc5c897c7b4d4d1b8994a9cd6e6f841f8964566a419059" dependencies = [ "zstd-sys", ] [[package]] name = "zstd-sys" -version = "2.0.15+zstd.1.5.7" +version = "2.0.13+zstd.1.5.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eb81183ddd97d0c74cedf1d50d85c8d08c1b8b68ee863bdee9e706eedba1a237" +checksum = "38ff0f21cfee8f97d94cef41359e0c89aa6113028ab0291aa8ca0038995a95aa" dependencies = [ "cc", "pkg-config", diff --git a/Cargo.toml b/Cargo.toml index e350029..63d6be0 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -12,6 +12,29 @@ serde_json = "1.0" metrics = "0.24.1" metrics-derive = "0.1" +# Shared dependencies used by multiple crates +uuid = { version = "1.18", features = ["v4", "serde"] } +chrono = { version = "0.4", features = ["serde"] } +anyhow = "1.0" +sqlx = { version = "0.8", features = [ + "runtime-tokio-native-tls", + "postgres", + "uuid", + "chrono", + "json", +] } +url = { version = "2.5", features = ["serde"] } +futures-util = "0.3" +bytes = "1.10" + +# AWS SDK +aws-config = { version = "1.8", features = ["behavior-version-latest"] } +aws-sdk-s3 = { version = "1.103", features = ["behavior-version-latest"] } + +# Arrow/Parquet +arrow = "56.1.0" +parquet = "56.1.0" + # alloy alloy-primitives = { version = "1.2.0", default-features = false, features = [ "map-foldhash", diff --git a/crates/flashblocks-archiver/Cargo.toml b/crates/flashblocks-archiver/Cargo.toml index b7b8638..f07284f 100644 --- a/crates/flashblocks-archiver/Cargo.toml +++ b/crates/flashblocks-archiver/Cargo.toml @@ -12,42 +12,41 @@ serde = { workspace = true } serde_json = { workspace = true } metrics = { workspace = true } metrics-derive = { workspace = true } +uuid = { workspace = true } +chrono = { workspace = true } +anyhow = { workspace = true } +sqlx = { workspace = true } +url = { workspace = true } +futures-util = { workspace = true } tokio-tungstenite = { version = "0.26.2", features = ["native-tls"] } -sqlx = { version = "0.8.6", features = [ - "runtime-tokio-native-tls", - "postgres", - "uuid", - "chrono", - "json", -] } -uuid = { version = "1.6.1", features = ["v4", "serde"] } -chrono = { version = "0.4", features = ["serde"] } -anyhow = "1.0" -futures-util = "0.3" -url = { version = "2.4", features = ["serde"] } brotli = "8.0.1" +# Retention dependencies +aws-sdk-s3 = { workspace = true } +aws-config = { workspace = true } +arrow = { workspace = true } +parquet = { workspace = true } +tempfile = "3.8" + # alloy -alloy-primitives = { version = "1.2.0", default-features = false, features = [ - "map-foldhash", -] } -alloy-genesis = { version = "1.0.23", default-features = false } -alloy-eips = { version = "1.0.23", default-features = false } -alloy-rpc-types = { version = "1.0.23", default-features = false } -alloy-rpc-types-engine = { version = "1.0.23", default-features = false } -alloy-rpc-types-eth = { version = "1.0.23" } -alloy-consensus = { version = "1.0.23" } -alloy-trie = { version = "0.9.0", default-features = false } -alloy-provider = { version = "1.0.23" } -alloy-hardforks = { version = "0.2.12" } -alloy-rpc-client = { version = "1.0.23" } +alloy-primitives = { workspace = true } +alloy-genesis = { workspace = true } +alloy-eips = { workspace = true } +alloy-rpc-types = { workspace = true } +alloy-rpc-types-engine = { workspace = true } +alloy-rpc-types-eth = { workspace = true } +alloy-consensus = { workspace = true } +alloy-trie = { workspace = true } +alloy-provider = { workspace = true } +alloy-hardforks = { workspace = true } +alloy-rpc-client = { workspace = true } # op-alloy -op-alloy-rpc-types = { version = "0.18.12", default-features = false } -op-alloy-rpc-types-engine = { version = "0.18.12", default-features = false } -op-alloy-rpc-jsonrpsee = { version = "0.18.12", default-features = false } -op-alloy-network = { version = "0.18.12", default-features = false } -op-alloy-consensus = { version = "0.18.12", default-features = false } +op-alloy-rpc-types = { workspace = true } +op-alloy-rpc-types-engine = { workspace = true } +op-alloy-rpc-jsonrpsee = { workspace = true } +op-alloy-network = { workspace = true } +op-alloy-consensus = { workspace = true } # reth @@ -80,6 +79,8 @@ path = "src/main.rs" [dev-dependencies] testcontainers = "0.23" -testcontainers-modules = { version = "0.11", features = ["postgres"] } -tempfile = "3.8" +testcontainers-modules = { version = "0.11", features = [ + "postgres", + "localstack", +] } tokio-test = "0.4" diff --git a/crates/flashblocks-archiver/migrations/001_initial_schema.sql b/crates/flashblocks-archiver/migrations/001_initial_schema.sql index 3e8aa8c..7798d92 100644 --- a/crates/flashblocks-archiver/migrations/001_initial_schema.sql +++ b/crates/flashblocks-archiver/migrations/001_initial_schema.sql @@ -19,7 +19,6 @@ CREATE TABLE IF NOT EXISTS flashblocks ( block_number BIGINT NOT NULL, received_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), - raw_message JSONB NOT NULL, UNIQUE(builder_id, payload_id, flashblock_index) ); @@ -46,6 +45,27 @@ CREATE TABLE IF NOT EXISTS transactions ( ); +-- Archival job tracking tables +CREATE TABLE IF NOT EXISTS archival_jobs ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + start_block BIGINT NOT NULL, + end_block BIGINT NOT NULL, + status TEXT NOT NULL DEFAULT 'pending', + s3_path TEXT, + archived_count BIGINT DEFAULT 0, + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + completed_at TIMESTAMPTZ, + error_message TEXT, + UNIQUE(start_block, end_block) +); + +CREATE TABLE IF NOT EXISTS archival_state ( + table_name TEXT PRIMARY KEY, + last_archived_block BIGINT NOT NULL, + updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW() +); + -- Indexes for better query performance CREATE INDEX IF NOT EXISTS idx_flashblocks_builder_block ON flashblocks(builder_id, block_number); CREATE INDEX IF NOT EXISTS idx_flashblocks_received_at ON flashblocks(received_at); @@ -55,4 +75,8 @@ CREATE INDEX IF NOT EXISTS idx_flashblocks_block_number ON flashblocks(block_num CREATE INDEX IF NOT EXISTS idx_transactions_flashblock_id ON transactions(flashblock_id); CREATE INDEX IF NOT EXISTS idx_transactions_builder_block ON transactions(builder_id, block_number); CREATE INDEX IF NOT EXISTS idx_transactions_payload_id ON transactions(payload_id); -CREATE INDEX IF NOT EXISTS idx_transactions_tx_hash ON transactions(tx_hash); \ No newline at end of file +CREATE INDEX IF NOT EXISTS idx_transactions_tx_hash ON transactions(tx_hash); + +-- Archival table indexes +CREATE INDEX IF NOT EXISTS idx_archival_jobs_status ON archival_jobs(status); +CREATE INDEX IF NOT EXISTS idx_archival_jobs_blocks ON archival_jobs(start_block, end_block); \ No newline at end of file diff --git a/crates/flashblocks-archiver/src/archiver.rs b/crates/flashblocks-archiver/src/archiver.rs index 89d9841..ca6ae71 100644 --- a/crates/flashblocks-archiver/src/archiver.rs +++ b/crates/flashblocks-archiver/src/archiver.rs @@ -1,5 +1,7 @@ +use crate::coordinator::ArchivalCoordinator; use crate::database::Database; use crate::metrics::Metrics; +use crate::s3::S3Manager; use crate::websocket::WebSocketPool; use crate::{cli::FlashblocksArchiverArgs, FlashblockMessage}; use anyhow::Result; @@ -15,6 +17,7 @@ pub struct FlashblocksArchiver { database: Database, builder_ids: HashMap, metrics: Metrics, + retention_coordinator: Option, } impl FlashblocksArchiver { @@ -32,11 +35,38 @@ impl FlashblocksArchiver { builder_ids.insert(builder_config.name.clone(), builder_id); } + let metrics = Metrics::default(); + + let retention_coordinator = if args.retention_enabled { + info!(message = "Initializing retention coordinator", bucket = %args.s3_bucket_name); + + let s3_manager = S3Manager::new( + args.s3_bucket_name.clone(), + args.s3_region.clone(), + args.s3_key_prefix.clone(), + ) + .await?; + + let coordinator = ArchivalCoordinator::new( + database.clone(), + s3_manager, + args.retention_blocks, + args.block_range_size, + metrics.clone(), + ); + + Some(coordinator) + } else { + info!(message = "Data retention disabled"); + None + }; + Ok(Self { args, database, builder_ids, - metrics: Metrics::default(), + metrics, + retention_coordinator, }) } @@ -44,7 +74,8 @@ impl FlashblocksArchiver { let builders = self.args.parse_builders()?; info!( message = "Starting FlashblocksArchiver", - builders_count = builders.len() + builders_count = builders.len(), + retention_enabled = self.retention_coordinator.is_some() ); if builders.is_empty() { @@ -58,6 +89,21 @@ impl FlashblocksArchiver { let mut batch = Vec::with_capacity(self.args.batch_size); let mut flush_interval = interval(Duration::from_secs(self.args.flush_interval_seconds)); + let mut retention_interval = if self.retention_coordinator.is_some() { + info!( + message = "Retention background task enabled", + interval_hours = self.args.archive_interval_hours, + retention_blocks = self.args.retention_blocks, + block_range_size = self.args.block_range_size + ); + + Some(interval(Duration::from_secs( + self.args.archive_interval_hours * 3600, + ))) + } else { + None + }; + info!(message = "FlashblocksArchiver started, listening for flashblock messages"); loop { @@ -98,6 +144,25 @@ impl FlashblocksArchiver { } } } + + // Run retention archival process periodically + _ = async { + match retention_interval.as_mut() { + Some(interval) => { + interval.tick().await; + } + None => { + std::future::pending::<()>().await; + } + } + }, if retention_interval.is_some() => { + if let Some(ref coordinator) = self.retention_coordinator { + info!(message = "Running scheduled retention archival cycle"); + if let Err(e) = coordinator.run_archival_cycle().await { + error!(message = "Retention archival cycle failed", error = %e); + } + } + } } } diff --git a/crates/flashblocks-archiver/src/cli.rs b/crates/flashblocks-archiver/src/cli.rs index 39296e8..82eb2d0 100644 --- a/crates/flashblocks-archiver/src/cli.rs +++ b/crates/flashblocks-archiver/src/cli.rs @@ -4,7 +4,9 @@ use url::Url; #[derive(Parser, Debug, Clone)] #[command(name = "flashblocks-archiver")] -#[command(about = "Archives flashblock messages from multiple builders to PostgreSQL")] +#[command( + about = "Archives flashblock messages from multiple builders to PostgreSQL, and periodically dumps Parquet files to S3" +)] pub struct FlashblocksArchiverArgs { #[arg( long, @@ -69,6 +71,62 @@ pub struct FlashblocksArchiverArgs { help = "Flush interval in seconds" )] pub flush_interval_seconds: u64, + + // Retention settings + #[arg( + long, + env = "RETENTION_ENABLED", + default_value = "true", + help = "Enable data retention/archival" + )] + pub retention_enabled: bool, + + #[arg( + long, + env = "RETENTION_BLOCKS", + default_value = "1296000", + help = "Number of blocks to keep data in PostgreSQL before archiving to S3 (43200 * 30 = 30 days on Base)" + )] + pub retention_blocks: u64, + + #[arg( + long, + env = "RETENTION_ARCHIVE_INTERVAL_HOURS", + default_value = "6", + help = "How often to run archival process in hours" + )] + pub archive_interval_hours: u64, + + #[arg( + long, + env = "RETENTION_BLOCK_RANGE_SIZE", + default_value = "21600", + help = "Number of blocks to archive in each job (21600 = 6 hours on Base)" + )] + pub block_range_size: u64, + + #[arg( + long, + env = "S3_BUCKET_NAME", + help = "S3 bucket name for archival storage" + )] + pub s3_bucket_name: String, + + #[arg( + long, + env = "S3_REGION", + default_value = "us-east-1", + help = "S3 region" + )] + pub s3_region: String, + + #[arg( + long, + env = "S3_KEY_PREFIX", + default_value = "flashblocks/", + help = "S3 key prefix for archived files" + )] + pub s3_key_prefix: String, } #[derive(Debug, Clone)] diff --git a/crates/flashblocks-archiver/src/coordinator.rs b/crates/flashblocks-archiver/src/coordinator.rs new file mode 100644 index 0000000..e158b4f --- /dev/null +++ b/crates/flashblocks-archiver/src/coordinator.rs @@ -0,0 +1,409 @@ +use crate::database::Database; +use crate::metrics::Metrics; +use crate::parquet::ParquetWriter; +use crate::s3::S3Manager; +use crate::types::{ArchivalJob, ArchivalStatus}; +use anyhow::Result; +use std::time::Instant; +use tempfile::TempDir; +use tracing::{error, info, warn}; +use uuid::Uuid; + +#[derive(Debug)] +pub struct ArchivalCoordinator { + database: Database, + s3_manager: S3Manager, + retention_blocks: u64, + block_range_size: u64, + metrics: Metrics, +} + +impl ArchivalCoordinator { + pub fn new( + database: Database, + s3_manager: S3Manager, + retention_blocks: u64, + block_range_size: u64, + metrics: Metrics, + ) -> Self { + Self { + database, + s3_manager, + retention_blocks, + block_range_size, + metrics, + } + } + + pub async fn run_archival_cycle(&self) -> Result<()> { + let cycle_start = Instant::now(); + info!(message = "Starting archival cycle"); + + let result = async { + self.create_archival_jobs().await?; + self.process_pending_jobs().await?; + Ok(()) + } + .await; + + match result { + Ok(()) => { + self.metrics.archival_cycles_completed.increment(1); + info!(message = "Archival cycle completed"); + } + Err(e) => { + self.metrics.archival_cycles_failed.increment(1); + error!(message = "Archival cycle failed", error = %e); + return Err(e); + } + } + + self.metrics + .archival_cycle_duration + .record(cycle_start.elapsed().as_secs_f64()); + + // Update pending jobs gauge + let pending_jobs = self.database.get_num_pending_archival_jobs().await?; + self.metrics.archival_jobs_pending.set(pending_jobs as f64); + + Ok(()) + } + + async fn create_archival_jobs(&self) -> Result<()> { + let latest_block = match self.database.get_global_latest_block_number().await? { + Some(block) => block, + None => { + info!(message = "No data found for archival"); + return Ok(()); + } + }; + + let retention_cutoff = if latest_block > self.retention_blocks { + latest_block - self.retention_blocks + } else { + info!( + message = "Not enough blocks for archival", + latest_block = latest_block, + retention_blocks = self.retention_blocks + ); + return Ok(()); + }; + + let oldest_block = match self.database.get_oldest_block_number().await? { + Some(block) => block, + None => { + info!(message = "No data found for archival"); + return Ok(()); + } + }; + + if oldest_block >= retention_cutoff { + info!( + message = "No blocks need archival", + oldest_block = oldest_block, + retention_cutoff = retention_cutoff + ); + return Ok(()); + } + + info!( + message = "Creating archival jobs", + oldest_block = oldest_block, + retention_cutoff = retention_cutoff, + latest_block = latest_block, + retention_blocks = self.retention_blocks + ); + + // Create block-aligned archival jobs for ranges that are older than retention cutoff + // Align to block_range_size boundaries to prevent overlapping jobs from multiple instances + let start_aligned_block = (oldest_block / self.block_range_size) * self.block_range_size; + let end_aligned_block = + ((retention_cutoff - 1) / self.block_range_size) * self.block_range_size; + + let mut current_block = start_aligned_block; + while current_block <= end_aligned_block { + let end_block = std::cmp::min( + current_block + self.block_range_size - 1, + retention_cutoff, // Don't go past the retention cutoff + ); + + // Only create job if this range has data to archive + let count = self + .database + .count_flashblocks_in_range(current_block, end_block) + .await?; + + if count > 0 { + // Use idempotent job creation - handles concurrent instances gracefully + match self + .database + .create_archival_job_idempotent(current_block, end_block) + .await? + { + Some(job_id) => { + self.metrics.archival_jobs_created.increment(1); + + info!( + message = "Created archival job", + job_id = %job_id, + start_block = current_block, + end_block = end_block, + flashblock_count = count + ); + } + None => { + // Job already exists (created by another instance) + info!( + message = "Archival job already exists", + start_block = current_block, + end_block = end_block, + flashblock_count = count + ); + } + } + } + + // Move to next block-aligned range + current_block += self.block_range_size; + } + + Ok(()) + } + + async fn process_pending_jobs(&self) -> Result<()> { + let pending_jobs = self.database.get_pending_archival_jobs(10).await?; // Process up to 10 jobs at once + + for job in pending_jobs { + if let Err(e) = self.process_single_job(job).await { + error!(message = "Failed to process archival job", error = %e); + } + } + + Ok(()) + } + + async fn process_single_job(&self, job: ArchivalJob) -> Result<()> { + let job_start = Instant::now(); + let job_id = job.id; + let start_block = job.start_block as u64; + let end_block = job.end_block as u64; + + info!( + message = "Processing archival job", + job_id = %job_id, + start_block = start_block, + end_block = end_block + ); + + // Try to acquire lock for this job + let lock_acquired = self.database.acquire_archival_lock(job_id).await?; + + if !lock_acquired { + info!( + message = "Could not acquire lock for job, skipping", + job_id = %job_id + ); + return Ok(()); + } + + // Update job status to processing + self.database + .update_archival_job_status(job_id, ArchivalStatus::Processing, None, None, None) + .await?; + + let result = self + .archive_block_range(job_id, start_block, end_block) + .await; + + // Always release the lock + if let Err(e) = self.database.release_archival_lock(job_id).await { + error!(message = "Failed to release archival lock", job_id = %job_id, error = %e); + } + + match result { + Ok((s3_path, archived_count)) => { + self.database + .update_archival_job_status( + job_id, + ArchivalStatus::Completed, + Some(&s3_path), + Some(archived_count), + None, + ) + .await?; + + self.metrics.archival_jobs_completed.increment(1); + self.metrics + .flashblocks_archived_count + .record(archived_count as f64); + self.metrics + .archival_job_duration + .record(job_start.elapsed().as_secs_f64()); + + info!( + message = "Successfully completed archival job", + job_id = %job_id, + s3_path = %s3_path, + archived_count = archived_count + ); + } + Err(e) => { + self.database + .update_archival_job_status( + job_id, + ArchivalStatus::Failed, + None, + None, + Some(&e.to_string()), + ) + .await?; + + self.metrics.archival_jobs_failed.increment(1); + + error!( + message = "Archival job failed", + job_id = %job_id, + error = %e + ); + } + } + + Ok(()) + } + + async fn archive_block_range( + &self, + _job_id: Uuid, + start_block: u64, + end_block: u64, + ) -> Result<(String, i64)> { + info!( + message = "Archiving block range", + start_block = start_block, + end_block = end_block + ); + + // Create temporary directory for parquet file + let temp_dir = TempDir::new()?; + let archive_key = self.s3_manager.generate_archive_key(start_block, end_block); + let temp_file_path = temp_dir.path().join(&archive_key); + + // Check if file already exists in S3 + if self.s3_manager.file_exists(&archive_key).await? { + warn!( + message = "Archive file already exists in S3", + s3_key = %archive_key + ); + return Ok((archive_key, 0)); + } + + let chunk_size = 1000; + let mut offset = 0; + let mut all_data = Vec::new(); + let mut total_count = 0i64; + + loop { + let chunk = self + .database + .get_flashblocks_with_transactions(start_block, end_block, chunk_size, offset) + .await?; + + if chunk.is_empty() { + break; + } + + let chunk_len = chunk.len(); + total_count += chunk_len as i64; + all_data.extend(chunk); + offset += chunk_size; + + info!( + message = "Loaded chunk for archival", + chunk_size = chunk_len, + total_loaded = all_data.len() + ); + } + + if all_data.is_empty() { + info!( + message = "No data found for block range", + start_block = start_block, + end_block = end_block + ); + return Ok((archive_key, 0)); + } + + let parquet_start = Instant::now(); + let parquet_path = temp_file_path.to_str().unwrap(); + + let rows_written = match ParquetWriter::write_to_file(parquet_path, all_data) { + Ok(rows) => { + self.metrics + .parquet_creation_duration + .record(parquet_start.elapsed().as_secs_f64()); + self.metrics.parquet_rows_written.record(rows as f64); + + // Record file size + if let Ok(metadata) = std::fs::metadata(parquet_path) { + self.metrics + .parquet_file_size_bytes + .record(metadata.len() as f64); + } + + rows + } + Err(e) => { + self.metrics.parquet_creation_errors.increment(1); + return Err(e); + } + }; + + info!( + message = "Created Parquet file", + file_path = %parquet_path, + rows_written = rows_written + ); + + let s3_start = Instant::now(); + let file_size = std::fs::metadata(parquet_path)?.len(); + + let s3_key = match self + .s3_manager + .upload_file(parquet_path, &archive_key) + .await + { + Ok(key) => { + self.metrics + .s3_upload_duration + .record(s3_start.elapsed().as_secs_f64()); + self.metrics.s3_upload_size_bytes.record(file_size as f64); + self.metrics.s3_uploads_completed.increment(1); + self.metrics.total_data_archived_bytes.increment(file_size); + key + } + Err(e) => { + self.metrics.s3_uploads_failed.increment(1); + return Err(e); + } + }; + + let (deleted_flashblocks, deleted_transactions) = self + .database + .delete_archived_data(start_block, end_block) + .await?; + + self.metrics + .transactions_archived_count + .record(deleted_transactions as f64); + + // std::fs::remove_file(parquet_path)?; + + info!( + message = "Deleted archived data from database", + deleted_flashblocks = deleted_flashblocks, + deleted_transactions = deleted_transactions + ); + + Ok((s3_key, total_count)) + } +} diff --git a/crates/flashblocks-archiver/src/database.rs b/crates/flashblocks-archiver/src/database.rs index 4e896d7..7faec12 100644 --- a/crates/flashblocks-archiver/src/database.rs +++ b/crates/flashblocks-archiver/src/database.rs @@ -1,11 +1,11 @@ -use crate::types::{Flashblock, FlashblockMessage}; +use crate::types::{ArchivalJob, ArchivalStatus, Flashblock, FlashblockMessage, Transaction}; use alloy_primitives::keccak256; use anyhow::Result; use sqlx::PgPool; use tracing::info; use uuid::Uuid; -#[derive(Debug)] +#[derive(Debug, Clone)] pub struct Database { pool: PgPool, } @@ -60,19 +60,16 @@ impl Database { builder_id: Uuid, payload: &FlashblockMessage, ) -> Result { - let raw_message = serde_json::to_value(payload)?; - let flashblock_id = sqlx::query_scalar::<_, Uuid>( r#" INSERT INTO flashblocks ( - builder_id, payload_id, flashblock_index, block_number, raw_message + builder_id, payload_id, flashblock_index, block_number ) VALUES ( - $1, $2, $3, $4, $5 + $1, $2, $3, $4 ) ON CONFLICT (builder_id, payload_id, flashblock_index) DO UPDATE SET block_number = EXCLUDED.block_number, - raw_message = EXCLUDED.raw_message, received_at = NOW() RETURNING id "#, @@ -81,7 +78,6 @@ impl Database { .bind(payload.payload_id.to_string()) .bind(payload.index as i64) .bind(payload.metadata.block_number as i64) - .bind(raw_message) .fetch_one(&self.pool) .await?; @@ -104,7 +100,7 @@ impl Database { } #[allow(clippy::too_many_arguments)] - async fn store_transaction( + pub async fn store_transaction( &self, flashblock_id: Uuid, builder_id: Uuid, @@ -166,4 +162,241 @@ impl Database { Ok(result.0.map(|b| b as u64)) } + + // Archival methods + + /// Creates an archival job, returning Ok(Some(job_id)) if created or Ok(None) if job already exists + /// This handles the case where multiple instances try to create the same block-aligned job + pub async fn create_archival_job_idempotent( + &self, + start_block: u64, + end_block: u64, + ) -> Result> { + match sqlx::query_scalar::<_, Uuid>( + r#" + INSERT INTO archival_jobs (start_block, end_block, status) + VALUES ($1, $2, $3) + ON CONFLICT (start_block, end_block) DO NOTHING + RETURNING id + "#, + ) + .bind(start_block as i64) + .bind(end_block as i64) + .bind(ArchivalStatus::Pending.to_string()) + .fetch_optional(&self.pool) + .await? + { + Some(job_id) => Ok(Some(job_id)), + None => { + // Job already exists, try to get its ID + match sqlx::query_scalar::<_, Uuid>( + "SELECT id FROM archival_jobs WHERE start_block = $1 AND end_block = $2", + ) + .bind(start_block as i64) + .bind(end_block as i64) + .fetch_optional(&self.pool) + .await? + { + Some(_existing_id) => Ok(None), // Job exists but wasn't created by us + None => Ok(None), // Shouldn't happen but handle gracefully + } + } + } + } + + pub async fn get_pending_archival_jobs(&self, limit: i64) -> Result> { + let jobs = sqlx::query_as::<_, ArchivalJob>( + r#" + SELECT * FROM archival_jobs + WHERE status = 'pending' + ORDER BY created_at ASC + LIMIT $1 + "#, + ) + .bind(limit) + .fetch_all(&self.pool) + .await?; + + Ok(jobs) + } + + pub async fn get_archival_jobs(&self, limit: i64) -> Result> { + let jobs = sqlx::query_as::<_, ArchivalJob>( + "SELECT * FROM archival_jobs ORDER BY created_at ASC LIMIT $1", + ) + .bind(limit) + .fetch_all(&self.pool) + .await?; + + Ok(jobs) + } + + pub async fn get_num_pending_archival_jobs(&self) -> Result { + let count = sqlx::query_scalar::<_, i64>( + "SELECT COUNT(*) FROM archival_jobs WHERE status = 'pending'", + ) + .fetch_one(&self.pool) + .await?; + + Ok(count) + } + + pub async fn update_archival_job_status( + &self, + job_id: Uuid, + status: ArchivalStatus, + s3_path: Option<&str>, + archived_count: Option, + error_message: Option<&str>, + ) -> Result<()> { + // Simple approach - update all fields + sqlx::query( + r#" + UPDATE archival_jobs + SET status = $2, + updated_at = NOW(), + completed_at = CASE WHEN $2 = 'completed' THEN NOW() ELSE completed_at END, + s3_path = COALESCE($3, s3_path), + archived_count = COALESCE($4, archived_count), + error_message = COALESCE($5, error_message) + WHERE id = $1 + "#, + ) + .bind(job_id) + .bind(status.to_string()) + .bind(s3_path) + .bind(archived_count) + .bind(error_message) + .execute(&self.pool) + .await?; + + Ok(()) + } + + pub async fn acquire_archival_lock(&self, job_id: Uuid) -> Result { + let result = sqlx::query_scalar::<_, bool>("SELECT pg_try_advisory_lock($1)") + .bind(job_id.as_u128() as i64) + .fetch_one(&self.pool) + .await?; + + Ok(result) + } + + pub async fn release_archival_lock(&self, job_id: Uuid) -> Result<()> { + sqlx::query("SELECT pg_advisory_unlock($1)") + .bind(job_id.as_u128() as i64) + .execute(&self.pool) + .await?; + + Ok(()) + } + + pub async fn get_flashblocks_with_transactions( + &self, + start_block: u64, + end_block: u64, + limit: usize, + offset: usize, + ) -> Result)>> { + let flashblocks = sqlx::query_as::<_, Flashblock>( + r#" + SELECT * FROM flashblocks + WHERE block_number >= $1 AND block_number <= $2 + ORDER BY block_number ASC, flashblock_index ASC + LIMIT $3 OFFSET $4 + "#, + ) + .bind(start_block as i64) + .bind(end_block as i64) + .bind(limit as i64) + .bind(offset as i64) + .fetch_all(&self.pool) + .await?; + + let mut results = Vec::new(); + for flashblock in flashblocks { + let transactions = sqlx::query_as::<_, Transaction>( + "SELECT * FROM transactions WHERE flashblock_id = $1 ORDER BY tx_index ASC", + ) + .bind(flashblock.id) + .fetch_all(&self.pool) + .await?; + + results.push((flashblock, transactions)); + } + + Ok(results) + } + + pub async fn delete_archived_data( + &self, + start_block: u64, + end_block: u64, + ) -> Result<(i64, i64)> { + let mut tx = self.pool.begin().await?; + + let transaction_result = sqlx::query( + r#" + DELETE FROM transactions + WHERE block_number >= $1 AND block_number <= $2 + "#, + ) + .bind(start_block as i64) + .bind(end_block as i64) + .execute(&mut *tx) + .await?; + + let transaction_count = transaction_result.rows_affected() as i64; + + let flashblock_result = sqlx::query( + r#" + DELETE FROM flashblocks + WHERE block_number >= $1 AND block_number <= $2 + "#, + ) + .bind(start_block as i64) + .bind(end_block as i64) + .execute(&mut *tx) + .await?; + + let flashblock_count = flashblock_result.rows_affected() as i64; + + tx.commit().await?; + + Ok((flashblock_count, transaction_count)) + } + + pub async fn get_oldest_block_number(&self) -> Result> { + let result: (Option,) = + sqlx::query_as("SELECT MIN(block_number) as min_block FROM flashblocks") + .fetch_one(&self.pool) + .await?; + + Ok(result.0.map(|b| b as u64)) + } + + pub async fn get_global_latest_block_number(&self) -> Result> { + let result: (Option,) = + sqlx::query_as("SELECT MAX(block_number) as max_block FROM flashblocks") + .fetch_one(&self.pool) + .await?; + + Ok(result.0.map(|b| b as u64)) + } + + pub async fn count_flashblocks_in_range( + &self, + start_block: u64, + end_block: u64, + ) -> Result { + let count = sqlx::query_scalar::<_, i64>( + "SELECT COUNT(*) FROM flashblocks WHERE block_number >= $1 AND block_number <= $2", + ) + .bind(start_block as i64) + .bind(end_block as i64) + .fetch_one(&self.pool) + .await?; + + Ok(count) + } } diff --git a/crates/flashblocks-archiver/src/lib.rs b/crates/flashblocks-archiver/src/lib.rs index 71574a6..e95e762 100644 --- a/crates/flashblocks-archiver/src/lib.rs +++ b/crates/flashblocks-archiver/src/lib.rs @@ -1,15 +1,16 @@ -mod metrics; - pub mod archiver; pub mod cli; +pub mod coordinator; pub mod database; +pub mod metrics; +pub mod parquet; +pub mod s3; pub mod types; pub mod websocket; #[cfg(test)] mod tests; -pub use archiver::*; -pub use cli::*; -pub use database::Database; -pub use types::*; +pub use archiver::FlashblocksArchiver; +pub use cli::FlashblocksArchiverArgs; +pub use types::FlashblockMessage; diff --git a/crates/flashblocks-archiver/src/metrics.rs b/crates/flashblocks-archiver/src/metrics.rs index c4d13a1..51978fd 100644 --- a/crates/flashblocks-archiver/src/metrics.rs +++ b/crates/flashblocks-archiver/src/metrics.rs @@ -1,5 +1,6 @@ -use metrics::{Counter, Histogram}; +use metrics::{Counter, Gauge, Histogram}; use metrics_derive::Metrics; + #[derive(Metrics, Clone)] #[metrics(scope = "flashblocks_archiver")] pub struct Metrics { @@ -8,4 +9,61 @@ pub struct Metrics { #[metric(describe = "Count of errors when flushing a batch of flashblocks")] pub flush_batch_error: Counter, + + #[metric(describe = "Time taken to complete an archival cycle")] + pub archival_cycle_duration: Histogram, + + #[metric(describe = "Number of archival cycles completed successfully")] + pub archival_cycles_completed: Counter, + + #[metric(describe = "Number of archival cycles that failed")] + pub archival_cycles_failed: Counter, + + #[metric(describe = "Number of archival jobs created")] + pub archival_jobs_created: Counter, + + #[metric(describe = "Number of archival jobs completed successfully")] + pub archival_jobs_completed: Counter, + + #[metric(describe = "Number of archival jobs that failed")] + pub archival_jobs_failed: Counter, + + #[metric(describe = "Number of pending archival jobs")] + pub archival_jobs_pending: Gauge, + + #[metric(describe = "Time taken to process a single archival job")] + pub archival_job_duration: Histogram, + + #[metric(describe = "Number of flashblocks archived per job")] + pub flashblocks_archived_count: Histogram, + + #[metric(describe = "Number of transactions archived per job")] + pub transactions_archived_count: Histogram, + + #[metric(describe = "Time taken to create a parquet file")] + pub parquet_creation_duration: Histogram, + + #[metric(describe = "Size of parquet files created in bytes")] + pub parquet_file_size_bytes: Histogram, + + #[metric(describe = "Number of rows written to parquet files")] + pub parquet_rows_written: Histogram, + + #[metric(describe = "Number of parquet file creation errors")] + pub parquet_creation_errors: Counter, + + #[metric(describe = "Time taken to upload files to S3")] + pub s3_upload_duration: Histogram, + + #[metric(describe = "Size of files uploaded to S3 in bytes")] + pub s3_upload_size_bytes: Histogram, + + #[metric(describe = "Number of successful S3 uploads")] + pub s3_uploads_completed: Counter, + + #[metric(describe = "Number of failed S3 uploads")] + pub s3_uploads_failed: Counter, + + #[metric(describe = "Total size of data archived in bytes")] + pub total_data_archived_bytes: Counter, } diff --git a/crates/flashblocks-archiver/src/parquet.rs b/crates/flashblocks-archiver/src/parquet.rs new file mode 100644 index 0000000..9388797 --- /dev/null +++ b/crates/flashblocks-archiver/src/parquet.rs @@ -0,0 +1,187 @@ +use crate::types::{Flashblock, Transaction}; +use anyhow::Result; +use arrow::array::{ + ArrayRef, BinaryBuilder, Int32Builder, Int64Builder, ListBuilder, StringBuilder, StructBuilder, +}; +use arrow::datatypes::{DataType, Field, Schema}; +use arrow::record_batch::RecordBatch; +use parquet::arrow::ArrowWriter; +use parquet::file::properties::WriterProperties; +use std::fs::File; +use std::sync::Arc; + +pub struct ParquetWriter; + +impl ParquetWriter { + pub fn write_to_file( + file_path: &str, + data: Vec<(Flashblock, Vec)>, + ) -> Result { + let schema = Self::create_schema(); + let record_batch = Self::create_record_batch(schema.clone(), data)?; + + let file = File::create(file_path)?; + let props = WriterProperties::builder() + .set_compression(parquet::basic::Compression::SNAPPY) + .build(); + + let mut writer = ArrowWriter::try_new(file, schema, Some(props))?; + writer.write(&record_batch)?; + writer.close()?; + + Ok(record_batch.num_rows() as u64) + } + + fn create_schema() -> Arc { + // Transaction struct fields + let transaction_fields = vec![ + Field::new("tx_data", DataType::Binary, false), + Field::new("tx_hash", DataType::Utf8, false), + Field::new("tx_index", DataType::Int32, false), + Field::new("created_at", DataType::Int64, false), // Unix timestamp + ]; + + let transaction_struct = DataType::Struct(transaction_fields.into()); + let transactions_list = + DataType::List(Arc::new(Field::new("item", transaction_struct, false))); + + let fields = vec![ + Field::new("id", DataType::Utf8, false), // UUID as string + Field::new("builder_id", DataType::Utf8, false), // UUID as string + Field::new("payload_id", DataType::Utf8, false), + Field::new("flashblock_index", DataType::Int64, false), + Field::new("block_number", DataType::Int64, false), + Field::new("received_at", DataType::Int64, false), // Unix timestamp + Field::new("transactions", transactions_list, false), + ]; + + Arc::new(Schema::new(fields)) + } + + fn create_record_batch( + schema: Arc, + data: Vec<(Flashblock, Vec)>, + ) -> Result { + let _num_rows = data.len(); + + // Flashblock field builders + let mut id_builder = StringBuilder::new(); + let mut builder_id_builder = StringBuilder::new(); + let mut payload_id_builder = StringBuilder::new(); + let mut flashblock_index_builder = Int64Builder::new(); + let mut block_number_builder = Int64Builder::new(); + let mut received_at_builder = Int64Builder::new(); + + // Transaction list builder - use the same field definition as in schema + let transaction_struct_fields = vec![ + Field::new("tx_data", DataType::Binary, false), + Field::new("tx_hash", DataType::Utf8, false), + Field::new("tx_index", DataType::Int32, false), + Field::new("created_at", DataType::Int64, false), + ]; + + let transaction_struct = DataType::Struct(transaction_struct_fields.clone().into()); + let list_field = Field::new("item", transaction_struct, false); + + let struct_builder = StructBuilder::new( + transaction_struct_fields, + vec![ + Box::new(BinaryBuilder::new()), + Box::new(StringBuilder::new()), + Box::new(Int32Builder::new()), + Box::new(Int64Builder::new()), + ], + ); + + let mut transactions_list_builder = ListBuilder::new(struct_builder).with_field(list_field); + + for (flashblock, transactions) in data { + // Add flashblock fields + id_builder.append_value(flashblock.id.to_string()); + builder_id_builder.append_value(flashblock.builder_id.to_string()); + payload_id_builder.append_value(&flashblock.payload_id); + flashblock_index_builder.append_value(flashblock.flashblock_index); + block_number_builder.append_value(flashblock.block_number); + received_at_builder.append_value(flashblock.received_at.timestamp()); + + // Add transactions to the list + for transaction in transactions { + let struct_builder = transactions_list_builder.values(); + struct_builder.append(true); + + // Add fields one at a time to avoid multiple mutable borrows + struct_builder + .field_builder::(0) + .unwrap() + .append_value(&transaction.tx_data); + struct_builder + .field_builder::(1) + .unwrap() + .append_value(&transaction.tx_hash); + struct_builder + .field_builder::(2) + .unwrap() + .append_value(transaction.tx_index); + struct_builder + .field_builder::(3) + .unwrap() + .append_value(transaction.created_at.timestamp()); + } + transactions_list_builder.append(true); + } + + let arrays: Vec = vec![ + Arc::new(id_builder.finish()), + Arc::new(builder_id_builder.finish()), + Arc::new(payload_id_builder.finish()), + Arc::new(flashblock_index_builder.finish()), + Arc::new(block_number_builder.finish()), + Arc::new(received_at_builder.finish()), + Arc::new(transactions_list_builder.finish()), + ]; + + Ok(RecordBatch::try_new(schema, arrays)?) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use chrono::Utc; + use tempfile::NamedTempFile; + use uuid::Uuid; + + #[test] + fn test_archive_write() -> Result<()> { + let temp_file = NamedTempFile::new()?; + let file_path = temp_file.path().to_str().unwrap(); + + let flashblock = Flashblock { + id: Uuid::new_v4(), + builder_id: Uuid::new_v4(), + payload_id: "test_payload".to_string(), + flashblock_index: 1, + block_number: 100, + received_at: Utc::now(), + }; + + let transactions = vec![Transaction { + id: Uuid::new_v4(), + flashblock_id: flashblock.id, + builder_id: flashblock.builder_id, + payload_id: flashblock.payload_id.clone(), + flashblock_index: flashblock.flashblock_index, + block_number: flashblock.block_number, + tx_data: vec![1, 2, 3, 4], + tx_hash: "0x1234".to_string(), + tx_index: 0, + created_at: Utc::now(), + }]; + + let data = vec![(flashblock, transactions)]; + let row_count = ParquetWriter::write_to_file(file_path, data)?; + + assert_eq!(row_count, 1); + Ok(()) + } +} diff --git a/crates/flashblocks-archiver/src/s3.rs b/crates/flashblocks-archiver/src/s3.rs new file mode 100644 index 0000000..ee8fa78 --- /dev/null +++ b/crates/flashblocks-archiver/src/s3.rs @@ -0,0 +1,209 @@ +use anyhow::Result; +use aws_config::{BehaviorVersion, Region}; +use aws_sdk_s3::{self as s3, operation::head_object::HeadObjectError}; +use std::path::Path; +use tracing::{error, info}; + +#[derive(Debug, Clone)] +pub struct S3Manager { + client: s3::Client, + bucket: String, + key_prefix: String, +} + +impl S3Manager { + pub async fn new(bucket: String, region: String, key_prefix: String) -> Result { + let config = aws_config::defaults(BehaviorVersion::latest()) + .region(Region::new(region)) + .load() + .await; + + let client = s3::Client::new(&config); + + Ok(Self { + client, + bucket, + key_prefix, + }) + } + + #[cfg(test)] + pub async fn new_with_endpoint( + bucket: String, + key_prefix: String, + endpoint_url: String, + ) -> Result { + let config = aws_sdk_s3::config::Builder::default() + .behavior_version(BehaviorVersion::latest()) + .region(Region::new("us-east-1")) + .credentials_provider(s3::config::Credentials::new( + "fake", "fake", None, None, "test", + )) + .endpoint_url(endpoint_url) + .build(); + + let client = s3::Client::from_conf(config); + + client.create_bucket().bucket(bucket.clone()).send().await?; + info!("Created bucket {}", bucket); + + Ok(Self { + client, + bucket, + key_prefix, + }) + } + + pub async fn upload_file(&self, local_path: &str, key_suffix: &str) -> Result { + let key = format!("{}{}", self.key_prefix, key_suffix); + + info!( + message = "Uploading file to S3", + local_path = %local_path, + bucket = %self.bucket, + key = %key + ); + + let file_size = std::fs::metadata(local_path)?.len(); + let body = s3::primitives::ByteStream::from_path(Path::new(local_path)).await?; + + let request = self + .client + .put_object() + .bucket(&self.bucket) + .key(&key) + .body(body) + .content_type("application/octet-stream") + .metadata("file_size", file_size.to_string()); + + match request.send().await { + Ok(_) => { + info!( + message = "Successfully uploaded file to S3", + bucket = %self.bucket, + key = %key, + file_size = file_size + ); + Ok(key) + } + Err(e) => { + error!( + message = "Failed to upload file to S3", + bucket = %self.bucket, + key = %key, + error = %e + ); + Err(anyhow::anyhow!("S3 upload failed: {}", e)) + } + } + } + + pub async fn delete_file(&self, key: &str) -> Result<()> { + info!( + message = "Deleting file from S3", + bucket = %self.bucket, + key = %key + ); + + let request = self.client.delete_object().bucket(&self.bucket).key(key); + + match request.send().await { + Ok(_) => { + info!( + message = "Successfully deleted file from S3", + bucket = %self.bucket, + key = %key + ); + Ok(()) + } + Err(e) => { + error!( + message = "Failed to delete file from S3", + bucket = %self.bucket, + key = %key, + error = %e + ); + Err(anyhow::anyhow!("S3 delete failed: {}", e)) + } + } + } + + pub async fn file_exists(&self, key: &str) -> Result { + match self + .client + .head_object() + .bucket(&self.bucket) + .key(key) + .send() + .await + { + Ok(_) => Ok(true), + Err(e) => match e.into_service_error() { + HeadObjectError::NotFound(_) => Ok(false), + err => Err(anyhow::anyhow!("S3 head_object failed: {:?}", err)), + }, + } + } + + pub async fn list_files(&self) -> Result> { + let mut keys = Vec::new(); + let mut continuation_token = None; + + loop { + let mut request = self + .client + .list_objects_v2() + .bucket(&self.bucket) + .prefix(&self.key_prefix); + + if let Some(token) = &continuation_token { + request = request.continuation_token(token); + } + + let response = request.send().await?; + + for object in response.contents() { + if let Some(key) = object.key() { + keys.push(key.to_string()); + } + } + + if response.is_truncated() == Some(true) { + continuation_token = response.next_continuation_token().map(|s| s.to_string()); + } else { + break; + } + } + + Ok(keys) + } + + pub fn generate_archive_key(&self, start_block: u64, end_block: u64) -> String { + format!("flashblocks_archive_{}_{}.parquet", start_block, end_block) + } + + pub fn bucket(&self) -> &str { + &self.bucket + } + + pub fn key_prefix(&self) -> &str { + &self.key_prefix + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_generate_archive_key() { + let s3_manager = S3Manager { + client: s3::Client::new(&aws_config::SdkConfig::builder().build()), + bucket: "test-bucket".to_string(), + key_prefix: "flashblocks/".to_string(), + }; + + let key = s3_manager.generate_archive_key(1000, 2000); + assert_eq!(key, "flashblocks_archive_1000_2000.parquet"); + } +} diff --git a/crates/flashblocks-archiver/src/tests/common.rs b/crates/flashblocks-archiver/src/tests/common.rs index d111c33..e3b0330 100644 --- a/crates/flashblocks-archiver/src/tests/common.rs +++ b/crates/flashblocks-archiver/src/tests/common.rs @@ -1,5 +1,5 @@ use crate::database::Database; -use testcontainers_modules::postgres::Postgres; +use testcontainers_modules::{localstack::LocalStack, postgres::Postgres}; use uuid::Uuid; pub struct PostgresTestContainer { @@ -8,6 +8,11 @@ pub struct PostgresTestContainer { pub database_url: String, } +pub struct LocalStackTestContainer { + pub _container: testcontainers::ContainerAsync, + pub s3_url: String, +} + impl PostgresTestContainer { pub async fn new(db_name: &str) -> anyhow::Result { use testcontainers::{runners::AsyncRunner, ImageExt}; @@ -41,3 +46,20 @@ impl PostgresTestContainer { self.database.get_or_create_builder(url, name).await } } + +impl LocalStackTestContainer { + pub async fn new() -> anyhow::Result { + use testcontainers::{runners::AsyncRunner, ImageExt}; + + let localstack = LocalStack::default().with_env_var("SERVICES", "s3"); + let container = localstack.start().await?; + + let host_port = container.get_host_port_ipv4(4566).await?; + let endpoint_url = format!("http://s3.localhost.localstack.cloud:{host_port}"); + + Ok(Self { + _container: container, + s3_url: endpoint_url, + }) + } +} diff --git a/crates/flashblocks-archiver/src/tests/integration_tests.rs b/crates/flashblocks-archiver/src/tests/integration_tests.rs index 4093b09..3221e3a 100644 --- a/crates/flashblocks-archiver/src/tests/integration_tests.rs +++ b/crates/flashblocks-archiver/src/tests/integration_tests.rs @@ -1,6 +1,9 @@ use crate::{ - cli::BuilderConfig, tests::common::PostgresTestContainer, types::Metadata, - websocket::WebSocketManager, Database, FlashblockMessage, + cli::BuilderConfig, + database::Database, + tests::common::PostgresTestContainer, + types::{FlashblockMessage, Metadata}, + websocket::WebSocketManager, }; use alloy_primitives::{ map::foldhash::{HashMap, HashMapExt}, diff --git a/crates/flashblocks-archiver/src/tests/mod.rs b/crates/flashblocks-archiver/src/tests/mod.rs index bb752a8..ad2b14d 100644 --- a/crates/flashblocks-archiver/src/tests/mod.rs +++ b/crates/flashblocks-archiver/src/tests/mod.rs @@ -1,4 +1,5 @@ #[cfg(test)] mod common; mod integration_tests; +mod retention_integration_tests; mod sepolia_integration_tests; diff --git a/crates/flashblocks-archiver/src/tests/retention_integration_tests.rs b/crates/flashblocks-archiver/src/tests/retention_integration_tests.rs new file mode 100644 index 0000000..1d61986 --- /dev/null +++ b/crates/flashblocks-archiver/src/tests/retention_integration_tests.rs @@ -0,0 +1,265 @@ +use crate::{ + coordinator::ArchivalCoordinator, + database::Database, + metrics::Metrics, + s3::S3Manager, + tests::common::{LocalStackTestContainer, PostgresTestContainer}, + types::{FlashblockMessage, Metadata}, +}; +use alloy_primitives::map::foldhash::{HashMap, HashMapExt}; +use reth_optimism_primitives::OpReceipt; +use rollup_boost::{ExecutionPayloadBaseV1, ExecutionPayloadFlashblockDeltaV1}; +use tracing::info; +use uuid::Uuid; + +struct RetentionTestSetup { + postgres: PostgresTestContainer, + _localstack: LocalStackTestContainer, + builder_id: Uuid, + s3_manager: S3Manager, +} + +impl RetentionTestSetup { + async fn new() -> anyhow::Result { + let postgres = PostgresTestContainer::new("test_retention").await?; + let localstack = LocalStackTestContainer::new().await?; + let builder_id = postgres + .create_test_builder("ws://test-builder.example.com", Some("test_builder")) + .await?; + + let s3_manager = S3Manager::new_with_endpoint( + "test-bucket".to_string(), + "flashblocks/".to_string(), + localstack.s3_url.clone(), + ) + .await?; + + Ok(Self { + postgres, + _localstack: localstack, + builder_id, + s3_manager, + }) + } + + fn database(&self) -> &Database { + &self.postgres.database + } + + fn s3_manager(&self) -> &S3Manager { + &self.s3_manager + } + + fn create_test_flashblock_message(&self, index: u64, block_number: u64) -> FlashblockMessage { + use alloy_primitives::{Address, Bloom, Bytes, B256, U256}; + use alloy_rpc_types_engine::PayloadId; + + let base = if index == 0 { + Some(ExecutionPayloadBaseV1 { + parent_beacon_block_root: B256::from([2; 32]), + parent_hash: B256::from([3; 32]), + fee_recipient: Address::from([4; 20]), + prev_randao: B256::from([5; 32]), + block_number, + gas_limit: 30_000_000, + timestamp: 1700000000, + extra_data: Bytes::from(vec![6, 7, 8]), + base_fee_per_gas: U256::from(20_000_000_000u64), + }) + } else { + None + }; + + FlashblockMessage { + payload_id: PayloadId::new([((block_number + index) % 256) as u8; 8]), + index, + base, + diff: ExecutionPayloadFlashblockDeltaV1 { + state_root: B256::from([9; 32]), + receipts_root: B256::from([10; 32]), + logs_bloom: Bloom::default(), + gas_used: 21000, + block_hash: B256::from([11; 32]), + transactions: vec![Bytes::from(vec![0x02, 0x01, 0x00])], + withdrawals: vec![], + withdrawals_root: B256::ZERO, + }, + metadata: Metadata { + receipts: HashMap::::new(), + new_account_balances: HashMap::::new(), + block_number, + }, + } + } + + async fn create_test_flashblocks( + &self, + num_blocks: usize, + start_block: i64, + ) -> anyhow::Result> { + let mut flashblock_ids = Vec::new(); + + for i in 0..num_blocks { + for j in 0..11 { + let message = + self.create_test_flashblock_message(j as u64, (start_block + i as i64) as u64); + let flashblock_id = self + .database() + .store_flashblock(self.builder_id, &message) + .await?; + flashblock_ids.push(flashblock_id); + } + } + + Ok(flashblock_ids) + } +} + +#[tokio::test] +async fn test_full_archival_cycle_e2e() -> anyhow::Result<()> { + let _ = tracing_subscriber::fmt() + .with_env_filter("flashblocks_archiver=debug") + .try_init(); + + let setup = RetentionTestSetup::new().await?; + + let _flashblock_ids = setup.create_test_flashblocks(50, 1000).await?; + + let initial_count = setup + .database() + .count_flashblocks_in_range(1000, 1050) + .await?; + assert_eq!(initial_count, 550); + + let coordinator = ArchivalCoordinator::new( + setup.database().clone(), + setup.s3_manager().clone(), + 40, + 5, + Metrics::default(), + ); + + // Run the archival cycle + coordinator.run_archival_cycle().await?; + + // Verify archival jobs were created for blocks 1000-1010 + // Should create 2 jobs: 1000-1004, 1005-1009 + let jobs = setup.database().get_archival_jobs(10).await?; + assert!( + jobs.len() == 2, + "Expected 2 archival jobs, got {}", + jobs.len() + ); + assert_eq!(jobs[0].start_block, 1000); + assert_eq!(jobs[0].end_block, 1004); + assert_eq!(jobs[1].start_block, 1005); + assert_eq!(jobs[1].end_block, 1009); + + // Verify that blocks 1011-1050 (latest 40 blocks ie 440 flashblocks) are still in the database + let retained_count = setup + .database() + .count_flashblocks_in_range(1010, 1050) + .await?; + assert_eq!( + retained_count, 440, + "Latest 40 blocks (440 flashblocks) should be retained" + ); + + // Verify that 10 blocks were archived + let files = setup.s3_manager().list_files().await?; + info!("Uploaded files: {:?}", files); + + Ok(()) +} + +#[tokio::test] +async fn test_block_aligned_job_creation() -> anyhow::Result<()> { + let _ = tracing_subscriber::fmt() + .with_env_filter("flashblocks_archiver=debug") + .try_init(); + let setup = RetentionTestSetup::new().await?; + + // Create data from blocks 1050-1200 (151 blocks) + let _flashblock_ids = setup.create_test_flashblocks(151, 1050).await?; + + // Create coordinator with block_range_size = 50 and retention = 100 blocks + // Latest blocks are 1150-1200 (51 blocks), so retention cutoff is at 1101 + // Should create jobs for blocks 1050-1100 (aligned ranges) + let coordinator = ArchivalCoordinator::new( + setup.database().clone(), + setup.s3_manager().clone(), + 100, // retention_blocks - keep latest 100 blocks (1101-1200) + 50, // block_range_size - creates aligned ranges of 50 blocks + Metrics::default(), + ); + + coordinator.run_archival_cycle().await?; + + let jobs = setup.database().get_archival_jobs(10).await?; + + // Should create exactly one job for the aligned range 1050-1099 + // (1000-1049 is empty, 1050-1099 has data, 1100+ is retained) + assert_eq!(jobs.len(), 1, "Expected 1 aligned job"); + + let job = &jobs[0]; + assert_eq!( + job.start_block, 1050, + "Job should start at aligned boundary" + ); + assert_eq!(job.end_block, 1099, "Job should end at aligned boundary"); + + // Test idempotent creation - running again shouldn't create duplicate jobs + coordinator.run_archival_cycle().await?; + + let jobs_after = setup.database().get_archival_jobs(10).await?; + assert_eq!(jobs_after.len(), 1, "No duplicate jobs should be created"); + assert_eq!(jobs_after[0].id, job.id, "Same job should exist"); + + Ok(()) +} + +#[tokio::test] +async fn test_concurrent_job_creation_safety() -> anyhow::Result<()> { + let _ = tracing_subscriber::fmt() + .with_env_filter("flashblocks_archiver=debug") + .try_init(); + let setup = RetentionTestSetup::new().await?; + + // Create test data + let _flashblock_ids = setup.create_test_flashblocks(100, 2000).await?; + + // Test that multiple calls to create_archival_job_idempotent are safe + let database = setup.database().clone(); + + // Try to create the same job range concurrently + let job1_result = database.create_archival_job_idempotent(2000, 2049).await?; + let job2_result = database.create_archival_job_idempotent(2000, 2049).await?; + + // Exactly one should succeed, the other should return None + match (job1_result, job2_result) { + (Some(_), None) | (None, Some(_)) => { + // One succeeded, one was duplicate - this is correct + } + (Some(_), Some(_)) => { + panic!("Both job creations succeeded - unique constraint not working"); + } + (None, None) => { + panic!("Both job creations failed - unexpected"); + } + } + + // Verify only one job exists in the database + let jobs = setup.database().get_pending_archival_jobs(10).await?; + let matching_jobs: Vec<_> = jobs + .iter() + .filter(|j| j.start_block == 2000 && j.end_block == 2049) + .collect(); + + assert_eq!( + matching_jobs.len(), + 1, + "Exactly one job should exist for the range" + ); + + Ok(()) +} diff --git a/crates/flashblocks-archiver/src/tests/sepolia_integration_tests.rs b/crates/flashblocks-archiver/src/tests/sepolia_integration_tests.rs index 4fe018d..987fe3a 100644 --- a/crates/flashblocks-archiver/src/tests/sepolia_integration_tests.rs +++ b/crates/flashblocks-archiver/src/tests/sepolia_integration_tests.rs @@ -28,6 +28,13 @@ impl SepoliaTestSetup { buffer_size: 100, batch_size: 10, flush_interval_seconds: 1, + retention_enabled: false, + retention_blocks: 1296000, + archive_interval_hours: 6, + block_range_size: 21600, + s3_bucket_name: "bucket".to_string(), + s3_region: "us-east-1".to_string(), + s3_key_prefix: "flashblocks/".to_string(), }; Ok(Self { postgres, args }) @@ -133,6 +140,60 @@ async fn test_sepolia_data_integrity() -> anyhow::Result<()> { count = flashblocks_count ); + // Calculate storage size for flashblocks table + let flashblocks_size: Option = + sqlx::query_scalar("SELECT pg_total_relation_size('flashblocks')") + .fetch_one(setup.postgres.database.get_pool()) + .await?; + + // Calculate storage size for transactions table + let transactions_size: Option = + sqlx::query_scalar("SELECT pg_total_relation_size('transactions')") + .fetch_one(setup.postgres.database.get_pool()) + .await?; + + // Get transaction count + let transactions_count = sqlx::query_scalar::<_, i64>("SELECT COUNT(*) FROM transactions") + .fetch_one(setup.postgres.database.get_pool()) + .await?; + + let flashblocks_size_bytes = flashblocks_size.unwrap_or(0); + let transactions_size_bytes = transactions_size.unwrap_or(0); + let total_size_bytes = flashblocks_size_bytes + transactions_size_bytes; + + // Calculate average sizes + let avg_flashblock_size = if flashblocks_count > 0 { + flashblocks_size_bytes / flashblocks_count + } else { + 0 + }; + let avg_transaction_size = if transactions_count > 0 { + transactions_size_bytes / transactions_count + } else { + 0 + }; + let avg_total_size_per_flashblock = if flashblocks_count > 0 { + total_size_bytes / flashblocks_count + } else { + 0 + }; + + info!( + message = "Database storage analysis", + flashblocks_count = flashblocks_count, + transactions_count = transactions_count, + flashblocks_table_size_bytes = flashblocks_size_bytes, + flashblocks_table_size_kb = flashblocks_size_bytes / 1024, + transactions_table_size_bytes = transactions_size_bytes, + transactions_table_size_kb = transactions_size_bytes / 1024, + total_size_bytes = total_size_bytes, + total_size_kb = total_size_bytes / 1024, + avg_flashblock_size_bytes = avg_flashblock_size, + avg_transaction_size_bytes = avg_transaction_size, + avg_total_size_per_flashblock_bytes = avg_total_size_per_flashblock, + avg_total_size_per_flashblock_kb = avg_total_size_per_flashblock / 1024 + ); + // Validate relationships between tables let orphaned_transactions = sqlx::query_scalar::<_, i64>( "SELECT COUNT(*) FROM transactions t @@ -218,6 +279,13 @@ async fn test_websocket_error_handling() -> anyhow::Result<()> { buffer_size: 10, batch_size: 5, flush_interval_seconds: 1, + retention_enabled: false, + retention_blocks: 1296000, + archive_interval_hours: 6, + block_range_size: 21600, + s3_bucket_name: "test-bucket".to_string(), + s3_region: "us-east-1".to_string(), + s3_key_prefix: "flashblocks/".to_string(), }; // This should fail to create the archiver due to database connection failure diff --git a/crates/flashblocks-archiver/src/types.rs b/crates/flashblocks-archiver/src/types.rs index 4a6c3d0..6468a0d 100644 --- a/crates/flashblocks-archiver/src/types.rs +++ b/crates/flashblocks-archiver/src/types.rs @@ -1,10 +1,10 @@ use alloy_primitives::{map::foldhash::HashMap, Address, B256, U256}; use alloy_rpc_types_engine::PayloadId; +use anyhow; use chrono::{DateTime, Utc}; use reth_optimism_primitives::OpReceipt; use rollup_boost::{ExecutionPayloadBaseV1, ExecutionPayloadFlashblockDeltaV1}; use serde::{Deserialize, Serialize}; -use serde_json::Value; use uuid::Uuid; #[derive(Debug, Clone, Serialize, Deserialize)] @@ -41,10 +41,9 @@ pub struct Flashblock { pub flashblock_index: i64, pub block_number: i64, pub received_at: DateTime, - pub raw_message: Value, } -#[derive(Debug, sqlx::FromRow)] +#[derive(Debug, sqlx::FromRow, Serialize, Deserialize)] pub struct Transaction { pub id: Uuid, pub flashblock_id: Uuid, @@ -53,6 +52,55 @@ pub struct Transaction { pub flashblock_index: i64, pub block_number: i64, pub tx_data: Vec, + pub tx_hash: String, pub tx_index: i32, pub created_at: DateTime, } + +// Archival types +#[derive(Debug, sqlx::FromRow)] +pub struct ArchivalJob { + pub id: Uuid, + pub start_block: i64, + pub end_block: i64, + pub status: String, + pub s3_path: Option, + pub archived_count: i64, + pub created_at: DateTime, + pub updated_at: DateTime, + pub completed_at: Option>, + pub error_message: Option, +} + +#[derive(Debug, Clone, PartialEq)] +pub enum ArchivalStatus { + Pending, + Processing, + Completed, + Failed, +} + +impl std::fmt::Display for ArchivalStatus { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + ArchivalStatus::Pending => write!(f, "pending"), + ArchivalStatus::Processing => write!(f, "processing"), + ArchivalStatus::Completed => write!(f, "completed"), + ArchivalStatus::Failed => write!(f, "failed"), + } + } +} + +impl std::str::FromStr for ArchivalStatus { + type Err = anyhow::Error; + + fn from_str(s: &str) -> Result { + match s { + "pending" => Ok(ArchivalStatus::Pending), + "processing" => Ok(ArchivalStatus::Processing), + "completed" => Ok(ArchivalStatus::Completed), + "failed" => Ok(ArchivalStatus::Failed), + _ => Err(anyhow::anyhow!("Invalid archival status: {}", s)), + } + } +} diff --git a/crates/flashblocks-archiver/src/websocket.rs b/crates/flashblocks-archiver/src/websocket.rs index 6fa79a7..4a1f191 100644 --- a/crates/flashblocks-archiver/src/websocket.rs +++ b/crates/flashblocks-archiver/src/websocket.rs @@ -1,4 +1,4 @@ -use crate::{cli::BuilderConfig, FlashblockMessage, Metadata}; +use crate::{cli::BuilderConfig, types::Metadata, FlashblockMessage}; use anyhow::{anyhow, Result}; use futures_util::StreamExt; use rollup_boost::FlashblocksPayloadV1;