From 67b46d35ac9b84bfa307bc0eb2bf1d7fffacd1e0 Mon Sep 17 00:00:00 2001 From: George Date: Mon, 6 Oct 2025 14:01:21 -0700 Subject: [PATCH 1/5] Add support for load testing --- cmd/stellar-rpc/internal/config/main.go | 92 ++++++++++++---------- cmd/stellar-rpc/internal/config/options.go | 6 ++ cmd/stellar-rpc/internal/daemon/daemon.go | 13 ++- go.mod | 3 +- go.sum | 4 +- 5 files changed, 73 insertions(+), 45 deletions(-) diff --git a/cmd/stellar-rpc/internal/config/main.go b/cmd/stellar-rpc/internal/config/main.go index dd6028be..24409f5a 100644 --- a/cmd/stellar-rpc/internal/config/main.go +++ b/cmd/stellar-rpc/internal/config/main.go @@ -25,31 +25,37 @@ type Config struct { CaptiveCoreHTTPQueryThreadPoolSize uint16 CaptiveCoreHTTPQuerySnapshotLedgers uint16 - Endpoint string - AdminEndpoint string - CheckpointFrequency uint32 - CoreRequestTimeout time.Duration - DefaultEventsLimit uint - DefaultTransactionsLimit uint - DefaultLedgersLimit uint - FriendbotURL string - HistoryArchiveURLs []string - HistoryArchiveUserAgent string - IngestionTimeout time.Duration - LogFormat LogFormat - LogLevel logrus.Level - MaxEventsLimit uint - MaxTransactionsLimit uint - MaxLedgersLimit uint - MaxHealthyLedgerLatency time.Duration - NetworkPassphrase string - PreflightWorkerCount uint - PreflightWorkerQueueSize uint - PreflightEnableDebug bool - SQLiteDBPath string - HistoryRetentionWindow uint32 - SorobanFeeStatsLedgerRetentionWindow uint32 - ClassicFeeStatsLedgerRetentionWindow uint32 + Endpoint string + NetworkPassphrase string + SQLiteDBPath string + HistoryArchiveURLs []string + AdminEndpoint string + HistoryArchiveUserAgent string + FriendbotURL string + CheckpointFrequency uint32 + + CoreRequestTimeout time.Duration + IngestionTimeout time.Duration + + LogFormat LogFormat + LogLevel logrus.Level + + PreflightWorkerCount uint + PreflightWorkerQueueSize uint + PreflightEnableDebug bool + + HistoryRetentionWindow uint32 + SorobanFeeStatsLedgerRetentionWindow uint32 + ClassicFeeStatsLedgerRetentionWindow uint32 + + DefaultEventsLimit uint + DefaultTransactionsLimit uint + DefaultLedgersLimit uint + MaxEventsLimit uint + MaxTransactionsLimit uint + MaxLedgersLimit uint + MaxHealthyLedgerLatency time.Duration + RequestBacklogGlobalQueueLimit uint RequestBacklogGetHealthQueueLimit uint RequestBacklogGetEventsQueueLimit uint @@ -64,22 +70,26 @@ type Config struct { RequestBacklogSimulateTransactionQueueLimit uint RequestBacklogGetFeeStatsTransactionQueueLimit uint RequestExecutionWarningThreshold time.Duration - MaxRequestExecutionDuration time.Duration - MaxGetHealthExecutionDuration time.Duration - MaxGetEventsExecutionDuration time.Duration - MaxGetNetworkExecutionDuration time.Duration - MaxGetVersionInfoExecutionDuration time.Duration - MaxGetLatestLedgerExecutionDuration time.Duration - MaxGetLedgerEntriesExecutionDuration time.Duration - MaxGetTransactionExecutionDuration time.Duration - MaxGetTransactionsExecutionDuration time.Duration - MaxGetLedgersExecutionDuration time.Duration - MaxSendTransactionExecutionDuration time.Duration - MaxSimulateTransactionExecutionDuration time.Duration - MaxGetFeeStatsExecutionDuration time.Duration - ServeLedgersFromDatastore bool - BufferedStorageBackendConfig ledgerbackend.BufferedStorageBackendConfig - DataStoreConfig datastore.DataStoreConfig + + MaxRequestExecutionDuration time.Duration + MaxGetHealthExecutionDuration time.Duration + MaxGetEventsExecutionDuration time.Duration + MaxGetNetworkExecutionDuration time.Duration + MaxGetVersionInfoExecutionDuration time.Duration + MaxGetLatestLedgerExecutionDuration time.Duration + MaxGetLedgerEntriesExecutionDuration time.Duration + MaxGetTransactionExecutionDuration time.Duration + MaxGetTransactionsExecutionDuration time.Duration + MaxGetLedgersExecutionDuration time.Duration + MaxSendTransactionExecutionDuration time.Duration + MaxSimulateTransactionExecutionDuration time.Duration + MaxGetFeeStatsExecutionDuration time.Duration + + ServeLedgersFromDatastore bool + BufferedStorageBackendConfig ledgerbackend.BufferedStorageBackendConfig + DataStoreConfig datastore.DataStoreConfig + + LoadTestFile string // We memoize these, so they bind to pflags correctly optionsCache *Options diff --git a/cmd/stellar-rpc/internal/config/options.go b/cmd/stellar-rpc/internal/config/options.go index 09c1ac7c..3d4df20b 100644 --- a/cmd/stellar-rpc/internal/config/options.go +++ b/cmd/stellar-rpc/internal/config/options.go @@ -569,6 +569,12 @@ func (cfg *Config) options() Options { return toml.LoadBytes(tomlBytes) }, }, + { + TomlKey: strutils.KebabToConstantCase("load-test"), + ConfigKey: &cfg.LoadTestFile, + Usage: "", + DefaultValue: "", + }, } return *cfg.optionsCache } diff --git a/cmd/stellar-rpc/internal/daemon/daemon.go b/cmd/stellar-rpc/internal/daemon/daemon.go index 63fc8120..14d885d8 100644 --- a/cmd/stellar-rpc/internal/daemon/daemon.go +++ b/cmd/stellar-rpc/internal/daemon/daemon.go @@ -21,6 +21,7 @@ import ( "github.com/stellar/go/clients/stellarcore" "github.com/stellar/go/historyarchive" "github.com/stellar/go/ingest/ledgerbackend" + "github.com/stellar/go/ingest/loadtest" "github.com/stellar/go/support/datastore" supporthttp "github.com/stellar/go/support/http" supportlog "github.com/stellar/go/support/log" @@ -292,6 +293,16 @@ func createIngestService(cfg *config.Config, logger *supportlog.Entry, daemon *D logger.WithError(err).Error("could not run ingestion. Retrying") } + var backend ledgerbackend.LedgerBackend = daemon.core + if cfg.LoadTestFile != "" { + backend = loadtest.NewLedgerBackend(loadtest.LedgerBackendConfig{ + NetworkPassphrase: cfg.NetworkPassphrase, + LedgersFilePath: cfg.LoadTestFile, + LedgerBackend: daemon.core, + LedgerCloseDuration: time.Second * 5, + }) + } + return ingest.NewService(ingest.Config{ Logger: logger, DB: db.NewReadWriter( @@ -304,7 +315,7 @@ func createIngestService(cfg *config.Config, logger *supportlog.Entry, daemon *D ), NetworkPassPhrase: cfg.NetworkPassphrase, Archive: *historyArchive, - LedgerBackend: daemon.core, + LedgerBackend: backend, Timeout: cfg.IngestionTimeout, OnIngestionRetry: onIngestionRetry, Daemon: daemon, diff --git a/go.mod b/go.mod index 868fed9d..0d6ebf84 100644 --- a/go.mod +++ b/go.mod @@ -19,7 +19,7 @@ require ( github.com/sirupsen/logrus v1.9.3 github.com/spf13/cobra v1.7.0 github.com/spf13/pflag v1.0.5 - github.com/stellar/go v0.0.0-20250818235326-815d6a25c539 + github.com/stellar/go v0.0.0-20251003192800-0e7db2ccfd7d github.com/stretchr/testify v1.9.0 ) @@ -48,6 +48,7 @@ require ( github.com/gorilla/handlers v1.5.2 // indirect github.com/gorilla/mux v1.8.1 // indirect github.com/pkg/xattr v0.4.9 // indirect + github.com/xdrpp/goxdr v0.1.1 // indirect ) require ( diff --git a/go.sum b/go.sum index ad47a387..56a9af3b 100644 --- a/go.sum +++ b/go.sum @@ -393,8 +393,8 @@ github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/spf13/viper v1.17.0 h1:I5txKw7MJasPL/BrfkbA0Jyo/oELqVmux4pR/UxOMfI= github.com/spf13/viper v1.17.0/go.mod h1:BmMMMLQXSbcHK6KAOiFLz0l5JHrU89OdIRHvsk0+yVI= -github.com/stellar/go v0.0.0-20250818235326-815d6a25c539 h1:B4UgruJPJ9mdvKPKoYltToLjz4cYZrTj6VYlaTiCX5o= -github.com/stellar/go v0.0.0-20250818235326-815d6a25c539/go.mod h1:ac8hwpljbFXC3Sf9nGfqBXXEvAEdnNRqQHGqP7QN8oY= +github.com/stellar/go v0.0.0-20251003192800-0e7db2ccfd7d h1:qkPPmOc/dTmEX/wcbbuqMVWRJ9ESq6OoybjgjHd8/+M= +github.com/stellar/go v0.0.0-20251003192800-0e7db2ccfd7d/go.mod h1:ac8hwpljbFXC3Sf9nGfqBXXEvAEdnNRqQHGqP7QN8oY= github.com/stellar/go-xdr v0.0.0-20231122183749-b53fb00bcac2 h1:OzCVd0SV5qE3ZcDeSFCmOWLZfEWZ3Oe8KtmSOYKEVWE= github.com/stellar/go-xdr v0.0.0-20231122183749-b53fb00bcac2/go.mod h1:yoxyU/M8nl9LKeWIoBrbDPQ7Cy+4jxRcWcOayZ4BMps= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= From eb8b1a42aa5733ec1ac3ab1bb1273bd9c79b08a4 Mon Sep 17 00:00:00 2001 From: George Date: Mon, 6 Oct 2025 14:51:14 -0700 Subject: [PATCH 2/5] Add logging line and change configuration file --- cmd/stellar-rpc/internal/config/options.go | 2 +- cmd/stellar-rpc/internal/daemon/daemon.go | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/cmd/stellar-rpc/internal/config/options.go b/cmd/stellar-rpc/internal/config/options.go index 3d4df20b..4059e04e 100644 --- a/cmd/stellar-rpc/internal/config/options.go +++ b/cmd/stellar-rpc/internal/config/options.go @@ -570,7 +570,7 @@ func (cfg *Config) options() Options { }, }, { - TomlKey: strutils.KebabToConstantCase("load-test"), + TomlKey: strutils.KebabToConstantCase("load-test-file"), ConfigKey: &cfg.LoadTestFile, Usage: "", DefaultValue: "", diff --git a/cmd/stellar-rpc/internal/daemon/daemon.go b/cmd/stellar-rpc/internal/daemon/daemon.go index 14d885d8..150e1d25 100644 --- a/cmd/stellar-rpc/internal/daemon/daemon.go +++ b/cmd/stellar-rpc/internal/daemon/daemon.go @@ -295,6 +295,10 @@ func createIngestService(cfg *config.Config, logger *supportlog.Entry, daemon *D var backend ledgerbackend.LedgerBackend = daemon.core if cfg.LoadTestFile != "" { + daemon.Logger(). + WithField("path", cfg.LoadTestFile). + Warnf("Ingestion will run with load testing") + backend = loadtest.NewLedgerBackend(loadtest.LedgerBackendConfig{ NetworkPassphrase: cfg.NetworkPassphrase, LedgersFilePath: cfg.LoadTestFile, From 9131f2342cb8d0184c40d7a27997d4c0ba88d524 Mon Sep 17 00:00:00 2001 From: George Date: Tue, 7 Oct 2025 14:48:28 -0700 Subject: [PATCH 3/5] Don't use the core backend during load testing --- .dockerignore | 1 + cmd/stellar-rpc/docker/Dockerfile | 2 +- cmd/stellar-rpc/internal/daemon/daemon.go | 3 +-- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.dockerignore b/.dockerignore index 7af275d4..be3f035d 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,2 +1,3 @@ target/ .soroban/ +.cargo/ diff --git a/cmd/stellar-rpc/docker/Dockerfile b/cmd/stellar-rpc/docker/Dockerfile index 1c870fc2..d2585a84 100644 --- a/cmd/stellar-rpc/docker/Dockerfile +++ b/cmd/stellar-rpc/docker/Dockerfile @@ -1,4 +1,4 @@ -FROM golang:1.24-bullseye as build +FROM golang:1.24-bullseye AS build ARG RUST_TOOLCHAIN_VERSION=stable ARG REPOSITORY_VERSION diff --git a/cmd/stellar-rpc/internal/daemon/daemon.go b/cmd/stellar-rpc/internal/daemon/daemon.go index 150e1d25..1082b301 100644 --- a/cmd/stellar-rpc/internal/daemon/daemon.go +++ b/cmd/stellar-rpc/internal/daemon/daemon.go @@ -302,8 +302,7 @@ func createIngestService(cfg *config.Config, logger *supportlog.Entry, daemon *D backend = loadtest.NewLedgerBackend(loadtest.LedgerBackendConfig{ NetworkPassphrase: cfg.NetworkPassphrase, LedgersFilePath: cfg.LoadTestFile, - LedgerBackend: daemon.core, - LedgerCloseDuration: time.Second * 5, + LedgerCloseDuration: time.Second * 5000, }) } From ad40837376fa82d55de07071405f2ebd2f9c56b2 Mon Sep 17 00:00:00 2001 From: George Date: Tue, 7 Oct 2025 15:12:25 -0700 Subject: [PATCH 4/5] Add usage information --- cmd/stellar-rpc/internal/config/options.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/stellar-rpc/internal/config/options.go b/cmd/stellar-rpc/internal/config/options.go index 4059e04e..734e1476 100644 --- a/cmd/stellar-rpc/internal/config/options.go +++ b/cmd/stellar-rpc/internal/config/options.go @@ -572,7 +572,7 @@ func (cfg *Config) options() Options { { TomlKey: strutils.KebabToConstantCase("load-test-file"), ConfigKey: &cfg.LoadTestFile, - Usage: "", + Usage: "Perform ingestion load testing with the given .xdr.zstd bundle of ledgers instead of the live network. WARNING: This will be destructive to your database.", DefaultValue: "", }, } From 3ce1c3b21387807a4f720ba91191c684f51a8422 Mon Sep 17 00:00:00 2001 From: George Date: Tue, 14 Oct 2025 09:08:18 -0700 Subject: [PATCH 5/5] Add configurable ability to merge and control close time frequency --- cmd/stellar-rpc/internal/config/main.go | 4 +++- cmd/stellar-rpc/internal/config/options.go | 14 +++++++++++++- cmd/stellar-rpc/internal/daemon/daemon.go | 15 ++++++++++++--- 3 files changed, 28 insertions(+), 5 deletions(-) diff --git a/cmd/stellar-rpc/internal/config/main.go b/cmd/stellar-rpc/internal/config/main.go index 24409f5a..cd253378 100644 --- a/cmd/stellar-rpc/internal/config/main.go +++ b/cmd/stellar-rpc/internal/config/main.go @@ -89,7 +89,9 @@ type Config struct { BufferedStorageBackendConfig ledgerbackend.BufferedStorageBackendConfig DataStoreConfig datastore.DataStoreConfig - LoadTestFile string + LoadTestFile string + LoadTestMergingEnabled bool + LoadTestFrequency time.Duration // We memoize these, so they bind to pflags correctly optionsCache *Options diff --git a/cmd/stellar-rpc/internal/config/options.go b/cmd/stellar-rpc/internal/config/options.go index 734e1476..2570faf8 100644 --- a/cmd/stellar-rpc/internal/config/options.go +++ b/cmd/stellar-rpc/internal/config/options.go @@ -572,9 +572,21 @@ func (cfg *Config) options() Options { { TomlKey: strutils.KebabToConstantCase("load-test-file"), ConfigKey: &cfg.LoadTestFile, - Usage: "Perform ingestion load testing with the given .xdr.zstd bundle of ledgers instead of the live network. WARNING: This will be destructive to your database.", + Usage: "Perform ingestion load testing with the given .xdr.zstd bundle of ledgers. WARNING: This will be destructive to your database.", DefaultValue: "", }, + { + TomlKey: strutils.KebabToConstantCase("load-test-merging"), + ConfigKey: &cfg.LoadTestMergingEnabled, + Usage: "Merge load testing ledgers (LOAD_TEST_FILE) with live ingestion.", + DefaultValue: false, + }, + { + TomlKey: strutils.KebabToConstantCase("load-test-frequency"), + ConfigKey: &cfg.LoadTestFrequency, + Usage: "Ingest a ledger every duration (seconds)", + DefaultValue: time.Second * 2, + }, } return *cfg.optionsCache } diff --git a/cmd/stellar-rpc/internal/daemon/daemon.go b/cmd/stellar-rpc/internal/daemon/daemon.go index 1082b301..fa33f3fc 100644 --- a/cmd/stellar-rpc/internal/daemon/daemon.go +++ b/cmd/stellar-rpc/internal/daemon/daemon.go @@ -299,11 +299,20 @@ func createIngestService(cfg *config.Config, logger *supportlog.Entry, daemon *D WithField("path", cfg.LoadTestFile). Warnf("Ingestion will run with load testing") - backend = loadtest.NewLedgerBackend(loadtest.LedgerBackendConfig{ + config := loadtest.LedgerBackendConfig{ NetworkPassphrase: cfg.NetworkPassphrase, LedgersFilePath: cfg.LoadTestFile, - LedgerCloseDuration: time.Second * 5000, - }) + LedgerCloseDuration: cfg.LoadTestFrequency, + } + + if cfg.LoadTestMergingEnabled { + daemon.Logger(). + WithField("path", cfg.LoadTestFile). + Warnf("Load testing will merge with live ingestion") + config.LedgerBackend = daemon.core + } + + backend = loadtest.NewLedgerBackend(config) } return ingest.NewService(ingest.Config{