diff --git a/apps/evm/go.mod b/apps/evm/go.mod index 39bcd6988..19b857993 100644 --- a/apps/evm/go.mod +++ b/apps/evm/go.mod @@ -4,6 +4,7 @@ go 1.25.6 replace ( github.com/evstack/ev-node => ../../ + github.com/evstack/ev-node/core => ../../core github.com/evstack/ev-node/execution/evm => ../../execution/evm ) diff --git a/apps/evm/go.sum b/apps/evm/go.sum index 5ea0339d4..58c363be5 100644 --- a/apps/evm/go.sum +++ b/apps/evm/go.sum @@ -411,8 +411,6 @@ github.com/ethereum/go-ethereum v1.16.8 h1:LLLfkZWijhR5m6yrAXbdlTeXoqontH+Ga2f9i github.com/ethereum/go-ethereum v1.16.8/go.mod h1:Fs6QebQbavneQTYcA39PEKv2+zIjX7rPUZ14DER46wk= github.com/ethereum/go-verkle v0.2.2 h1:I2W0WjnrFUIzzVPwm8ykY+7pL2d4VhlsePn4j7cnFk8= github.com/ethereum/go-verkle v0.2.2/go.mod h1:M3b90YRnzqKyyzBEWJGqj8Qff4IDeXnzFw0P9bFw3uk= -github.com/evstack/ev-node/core v1.0.0-rc.1 h1:Dic2PMUMAYUl5JW6DkDj6HXDEWYzorVJQuuUJOV0FjE= -github.com/evstack/ev-node/core v1.0.0-rc.1/go.mod h1:n2w/LhYQTPsi48m6lMj16YiIqsaQw6gxwjyJvR+B3sY= github.com/fatih/color v1.10.0/go.mod h1:ELkj/draVOlAH/xkhN6mQ50Qd0MPOk5AAr3maGEBuJM= github.com/fatih/color v1.13.0/go.mod h1:kLAiJbzzSOZDVNGyDpeOxJ47H46qBXwg5ILebYFFOfk= github.com/fatih/color v1.15.0/go.mod h1:0h5ZqXfHYED7Bhv2ZJamyIOUej9KtShiJESRwBDUSsw= diff --git a/apps/grpc/go.mod b/apps/grpc/go.mod index b547d3f5c..73531392f 100644 --- a/apps/grpc/go.mod +++ b/apps/grpc/go.mod @@ -4,6 +4,7 @@ go 1.25.6 replace ( github.com/evstack/ev-node => ../../ + github.com/evstack/ev-node/core => ../../core github.com/evstack/ev-node/execution/grpc => ../../execution/grpc ) diff --git a/apps/grpc/go.sum b/apps/grpc/go.sum index fb175ac5c..d1fd875f0 100644 --- a/apps/grpc/go.sum +++ b/apps/grpc/go.sum @@ -367,8 +367,6 @@ github.com/envoyproxy/protoc-gen-validate v0.10.0/go.mod h1:DRjgyB0I43LtJapqN6Ni github.com/envoyproxy/protoc-gen-validate v0.10.1/go.mod h1:DRjgyB0I43LtJapqN6NiRwroiAU2PaFuvk/vjgh61ss= github.com/envoyproxy/protoc-gen-validate v1.0.1/go.mod h1:0vj8bNkYbSTNS2PIyH87KZaeN4x9zpL9Qt8fQC7d+vs= github.com/envoyproxy/protoc-gen-validate v1.0.2/go.mod h1:GpiZQP3dDbg4JouG/NNS7QWXpgx6x8QiMKdmN72jogE= -github.com/evstack/ev-node/core v1.0.0-rc.1 h1:Dic2PMUMAYUl5JW6DkDj6HXDEWYzorVJQuuUJOV0FjE= -github.com/evstack/ev-node/core v1.0.0-rc.1/go.mod h1:n2w/LhYQTPsi48m6lMj16YiIqsaQw6gxwjyJvR+B3sY= github.com/fatih/color v1.10.0/go.mod h1:ELkj/draVOlAH/xkhN6mQ50Qd0MPOk5AAr3maGEBuJM= github.com/fatih/color v1.13.0/go.mod h1:kLAiJbzzSOZDVNGyDpeOxJ47H46qBXwg5ILebYFFOfk= github.com/fatih/color v1.15.0/go.mod h1:0h5ZqXfHYED7Bhv2ZJamyIOUej9KtShiJESRwBDUSsw= diff --git a/apps/testapp/go.mod b/apps/testapp/go.mod index cc0bf4721..4c57d790c 100644 --- a/apps/testapp/go.mod +++ b/apps/testapp/go.mod @@ -2,6 +2,11 @@ module github.com/evstack/ev-node/apps/testapp go 1.25.6 +replace ( + github.com/evstack/ev-node => ../../. + github.com/evstack/ev-node/core => ../../core +) + require ( github.com/evstack/ev-node v1.0.0-rc.2 github.com/evstack/ev-node/core v1.0.0-rc.1 diff --git a/apps/testapp/go.sum b/apps/testapp/go.sum index 0645d7939..d1fd875f0 100644 --- a/apps/testapp/go.sum +++ b/apps/testapp/go.sum @@ -367,10 +367,6 @@ github.com/envoyproxy/protoc-gen-validate v0.10.0/go.mod h1:DRjgyB0I43LtJapqN6Ni github.com/envoyproxy/protoc-gen-validate v0.10.1/go.mod h1:DRjgyB0I43LtJapqN6NiRwroiAU2PaFuvk/vjgh61ss= github.com/envoyproxy/protoc-gen-validate v1.0.1/go.mod h1:0vj8bNkYbSTNS2PIyH87KZaeN4x9zpL9Qt8fQC7d+vs= github.com/envoyproxy/protoc-gen-validate v1.0.2/go.mod h1:GpiZQP3dDbg4JouG/NNS7QWXpgx6x8QiMKdmN72jogE= -github.com/evstack/ev-node v1.0.0-rc.2 h1:gUQzLTkCj6D751exm/FIR/yw2aXWiW2aEREEwtxMvw0= -github.com/evstack/ev-node v1.0.0-rc.2/go.mod h1:Qa2nN1D6PJQRU2tiarv6X5Der5OZg/+2QGY/K2mA760= -github.com/evstack/ev-node/core v1.0.0-rc.1 h1:Dic2PMUMAYUl5JW6DkDj6HXDEWYzorVJQuuUJOV0FjE= -github.com/evstack/ev-node/core v1.0.0-rc.1/go.mod h1:n2w/LhYQTPsi48m6lMj16YiIqsaQw6gxwjyJvR+B3sY= github.com/fatih/color v1.10.0/go.mod h1:ELkj/draVOlAH/xkhN6mQ50Qd0MPOk5AAr3maGEBuJM= github.com/fatih/color v1.13.0/go.mod h1:kLAiJbzzSOZDVNGyDpeOxJ47H46qBXwg5ILebYFFOfk= github.com/fatih/color v1.15.0/go.mod h1:0h5ZqXfHYED7Bhv2ZJamyIOUej9KtShiJESRwBDUSsw= @@ -586,8 +582,9 @@ github.com/hashicorp/go-msgpack v0.5.5/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iP github.com/hashicorp/go-msgpack/v2 v2.1.2 h1:4Ee8FTp834e+ewB71RDrQ0VKpyFdrKOjvYtnQ/ltVj0= github.com/hashicorp/go-msgpack/v2 v2.1.2/go.mod h1:upybraOAblm4S7rx0+jeNy+CWWhzywQsSRV5033mMu4= github.com/hashicorp/go-retryablehttp v0.5.3/go.mod h1:9B5zBasrRhHXnJnui7y6sL7es7NDiJgTc6Er0maI1Xs= -github.com/hashicorp/go-uuid v1.0.0 h1:RS8zrF7PhGwyNPOtxSClXXj9HA8feRnJzgnI1RJCSnM= github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= +github.com/hashicorp/go-uuid v1.0.2 h1:cfejS+Tpcp13yd5nYHWDI6qVCny6wyX2Mt5SGur2IGE= +github.com/hashicorp/go-uuid v1.0.2/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v1.0.2 h1:dV3g9Z/unq5DpblPpw+Oqcv4dU/1omnb4Ok8iPY6p1c= diff --git a/block/internal/submitting/submitter.go b/block/internal/submitting/submitter.go index 1c5b034c1..6b1a2971f 100644 --- a/block/internal/submitting/submitter.go +++ b/block/internal/submitting/submitter.go @@ -364,6 +364,44 @@ func (s *Submitter) processDAInclusionLoop() { // This can only be performed after the height has been persisted to store s.cache.DeleteHeight(nextHeight) } + + // Run height-based pruning if enabled. + if s.config.Node.PruningEnabled && s.config.Node.PruningKeepRecent > 0 && s.config.Node.PruningInterval > 0 { + currentDAIncluded = s.GetDAIncludedHeight() + + var lastPruned uint64 + if bz, err := s.store.GetMetadata(s.ctx, store.LastPrunedBlockHeightKey); err == nil && len(bz) == 8 { + lastPruned = binary.LittleEndian.Uint64(bz) + } + + storeHeight, err := s.store.Height(s.ctx) + if err != nil { + s.logger.Error().Err(err).Msg("failed to get store height for pruning") + continue + } + if storeHeight <= lastPruned+s.config.Node.PruningInterval { + continue + } + + // Never prune blocks that are not DA included + upperBound := min(storeHeight, currentDAIncluded) + if upperBound <= s.config.Node.PruningKeepRecent { + // Not enough fully included blocks to prune + continue + } + + targetHeight := upperBound - s.config.Node.PruningKeepRecent + + if err := s.store.PruneBlocks(s.ctx, targetHeight); err != nil { + s.logger.Error().Err(err).Uint64("target_height", targetHeight).Msg("failed to prune old block data") + } + + if pruner, ok := s.exec.(coreexecutor.ExecPruner); ok { + if err := pruner.PruneExec(s.ctx, targetHeight); err != nil { + s.logger.Error().Err(err).Uint64("target_height", targetHeight).Msg("failed to prune execution metadata") + } + } + } } } } diff --git a/core/execution/execution.go b/core/execution/execution.go index 276da369e..f3ebe1da9 100644 --- a/core/execution/execution.go +++ b/core/execution/execution.go @@ -161,3 +161,13 @@ type Rollbackable interface { // Rollback resets the execution layer head to the specified height. Rollback(ctx context.Context, targetHeight uint64) error } + +// ExecPruner is an optional interface that execution clients can implement +// to support height-based pruning of their execution metadata. +type ExecPruner interface { + // PruneExec should delete execution metadata for all heights up to and + // including the given height. Implementations should be idempotent and track + // their own progress so that repeated calls with the same or decreasing + // heights are cheap no-ops. + PruneExec(ctx context.Context, height uint64) error +} diff --git a/execution/evm/execution.go b/execution/evm/execution.go index 064c2c749..014ff1638 100644 --- a/execution/evm/execution.go +++ b/execution/evm/execution.go @@ -65,6 +65,10 @@ var _ execution.HeightProvider = (*EngineClient)(nil) // Ensure EngineClient implements the execution.Rollbackable interface var _ execution.Rollbackable = (*EngineClient)(nil) +// Ensure EngineClient implements optional pruning interface when used with +// ev-node's height-based pruning. +var _ execution.ExecPruner = (*EngineClient)(nil) + // validatePayloadStatus checks the payload status and returns appropriate errors. // It implements the Engine API specification's status handling: // - VALID: Operation succeeded, return nil @@ -265,6 +269,13 @@ func NewEngineExecutionClient( }, nil } +// PruneExec implements execution.ExecPruner by delegating to the +// underlying EVMStore. It is safe to call this multiple times with the same +// or increasing heights; the store tracks its own last-pruned height. +func (c *EngineClient) PruneExec(ctx context.Context, height uint64) error { + return c.store.PruneExec(ctx, height) +} + // SetLogger allows callers to attach a structured logger. func (c *EngineClient) SetLogger(l zerolog.Logger) { c.logger = l diff --git a/execution/evm/go.mod b/execution/evm/go.mod index f4c560a7b..c0e639989 100644 --- a/execution/evm/go.mod +++ b/execution/evm/go.mod @@ -91,7 +91,7 @@ require ( go.opentelemetry.io/proto/otlp v1.9.0 // indirect go.yaml.in/yaml/v3 v3.0.4 // indirect golang.org/x/crypto v0.47.0 // indirect - golang.org/x/exp v0.0.0-20251209150349-8475f28825e9 // indirect + golang.org/x/exp v0.0.0-20260112195511-716be5621a96 // indirect golang.org/x/net v0.49.0 // indirect golang.org/x/sync v0.19.0 // indirect golang.org/x/sys v0.40.0 // indirect @@ -103,3 +103,8 @@ require ( gopkg.in/yaml.v3 v3.0.1 // indirect lukechampine.com/blake3 v1.4.1 // indirect ) + +replace ( + github.com/evstack/ev-node => ../../ + github.com/evstack/ev-node/core => ../../core +) diff --git a/execution/evm/go.sum b/execution/evm/go.sum index 93b03d7e0..3294e40da 100644 --- a/execution/evm/go.sum +++ b/execution/evm/go.sum @@ -64,6 +64,8 @@ github.com/dgraph-io/badger/v4 v4.5.1 h1:7DCIXrQjo1LKmM96YD+hLVJ2EEsyyoWxJfpdd56 github.com/dgraph-io/badger/v4 v4.5.1/go.mod h1:qn3Be0j3TfV4kPbVoK0arXCD1/nr1ftth6sbL5jxdoA= github.com/dgraph-io/ristretto/v2 v2.1.0 h1:59LjpOJLNDULHh8MC4UaegN52lC4JnO2dITsie/Pa8I= github.com/dgraph-io/ristretto/v2 v2.1.0/go.mod h1:uejeqfYXpUomfse0+lO+13ATz4TypQYLJZzBSAemuB4= +github.com/dunglas/httpsfv v1.1.0 h1:Jw76nAyKWKZKFrpMMcL76y35tOpYHqQPzHQiwDvpe54= +github.com/dunglas/httpsfv v1.1.0/go.mod h1:zID2mqw9mFsnt7YC3vYQ9/cjq30q41W+1AnDwH8TiMg= github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= github.com/emicklei/dot v1.6.2 h1:08GN+DD79cy/tzN6uLCT84+2Wk9u+wvqP+Hkx/dIR8A= @@ -76,10 +78,6 @@ github.com/ethereum/go-ethereum v1.16.8 h1:LLLfkZWijhR5m6yrAXbdlTeXoqontH+Ga2f9i github.com/ethereum/go-ethereum v1.16.8/go.mod h1:Fs6QebQbavneQTYcA39PEKv2+zIjX7rPUZ14DER46wk= github.com/ethereum/go-verkle v0.2.2 h1:I2W0WjnrFUIzzVPwm8ykY+7pL2d4VhlsePn4j7cnFk8= github.com/ethereum/go-verkle v0.2.2/go.mod h1:M3b90YRnzqKyyzBEWJGqj8Qff4IDeXnzFw0P9bFw3uk= -github.com/evstack/ev-node v1.0.0-rc.2 h1:gUQzLTkCj6D751exm/FIR/yw2aXWiW2aEREEwtxMvw0= -github.com/evstack/ev-node v1.0.0-rc.2/go.mod h1:Qa2nN1D6PJQRU2tiarv6X5Der5OZg/+2QGY/K2mA760= -github.com/evstack/ev-node/core v1.0.0-rc.1 h1:Dic2PMUMAYUl5JW6DkDj6HXDEWYzorVJQuuUJOV0FjE= -github.com/evstack/ev-node/core v1.0.0-rc.1/go.mod h1:n2w/LhYQTPsi48m6lMj16YiIqsaQw6gxwjyJvR+B3sY= github.com/ferranbt/fastssz v0.1.4 h1:OCDB+dYDEQDvAgtAGnTSidK1Pe2tW3nFV40XyMkTeDY= github.com/ferranbt/fastssz v0.1.4/go.mod h1:Ea3+oeoRGGLGm5shYAeDgu6PGUlcvQhE2fILyD9+tGg= github.com/filecoin-project/go-clock v0.1.0 h1:SFbYIM75M8NnFm1yMHhN9Ahy3W5bEZV9gd6MPfXbKVU= @@ -135,8 +133,8 @@ github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6 github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golang/snappy v1.0.0 h1:Oy607GVXHs7RtbggtPBnr2RmDArIsAefDwvrdWvRhGs= github.com/golang/snappy v1.0.0/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= -github.com/google/flatbuffers v24.12.23+incompatible h1:ubBKR94NR4pXUCY/MUsRVzd9umNW7ht7EG9hHfS9FX8= -github.com/google/flatbuffers v24.12.23+incompatible/go.mod h1:1AeVuKshWv4vARoZatz6mlQ0JxURH0Kv5+zNeJKJCa8= +github.com/google/flatbuffers v25.1.24+incompatible h1:4wPqL3K7GzBd1CwyhSd3usxLKOaJN/AC6puCca6Jm7o= +github.com/google/flatbuffers v25.1.24+incompatible/go.mod h1:1AeVuKshWv4vARoZatz6mlQ0JxURH0Kv5+zNeJKJCa8= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= @@ -180,8 +178,8 @@ github.com/ipfs/go-detect-race v0.0.1 h1:qX/xay2W3E4Q1U7d9lNs1sU9nvguX0a7319XbyQ github.com/ipfs/go-detect-race v0.0.1/go.mod h1:8BNT7shDZPo99Q74BpGMK+4D8Mn4j46UU0LZ723meps= github.com/ipfs/go-ds-badger4 v0.1.8 h1:frNczf5CjCVm62RJ5mW5tD/oLQY/9IKAUpKviRV9QAI= github.com/ipfs/go-ds-badger4 v0.1.8/go.mod h1:FdqSLA5TMsyqooENB/Hf4xzYE/iH0z/ErLD6ogtfMrA= -github.com/ipfs/go-log/v2 v2.9.0 h1:l4b06AwVXwldIzbVPZy5z7sKp9lHFTX0KWfTBCtHaOk= -github.com/ipfs/go-log/v2 v2.9.0/go.mod h1:UhIYAwMV7Nb4ZmihUxfIRM2Istw/y9cAk3xaK+4Zs2c= +github.com/ipfs/go-log/v2 v2.9.1 h1:3JXwHWU31dsCpvQ+7asz6/QsFJHqFr4gLgQ0FWteujk= +github.com/ipfs/go-log/v2 v2.9.1/go.mod h1:evFx7sBiohUN3AG12mXlZBw5hacBQld3ZPHrowlJYoo= github.com/ipld/go-ipld-prime v0.21.0 h1:n4JmcpOlPDIxBcY037SVfpd1G+Sj1nKZah0m6QH9C2E= github.com/ipld/go-ipld-prime v0.21.0/go.mod h1:3RLqy//ERg/y5oShXXdx5YIp50cFGOanyMctpPjsvxQ= github.com/jackpal/go-nat-pmp v1.0.2 h1:KzKSgb7qkJvOUTqYl9/Hg/me3pWgBmERKrTGD7BdWus= @@ -208,12 +206,12 @@ github.com/libp2p/go-cidranger v1.1.0 h1:ewPN8EZ0dd1LSnrtuwd4709PXVcITVeuwbag38y github.com/libp2p/go-cidranger v1.1.0/go.mod h1:KWZTfSr+r9qEo9OkI9/SIEeAtw+NNoU0dXIXt15Okic= github.com/libp2p/go-flow-metrics v0.3.0 h1:q31zcHUvHnwDO0SHaukewPYgwOBSxtt830uJtUx6784= github.com/libp2p/go-flow-metrics v0.3.0/go.mod h1:nuhlreIwEguM1IvHAew3ij7A8BMlyHQJ279ao24eZZo= -github.com/libp2p/go-libp2p v0.46.0 h1:0T2yvIKpZ3DVYCuPOFxPD1layhRU486pj9rSlGWYnDM= -github.com/libp2p/go-libp2p v0.46.0/go.mod h1:TbIDnpDjBLa7isdgYpbxozIVPBTmM/7qKOJP4SFySrQ= +github.com/libp2p/go-libp2p v0.47.0 h1:qQpBjSCWNQFF0hjBbKirMXE9RHLtSuzTDkTfr1rw0yc= +github.com/libp2p/go-libp2p v0.47.0/go.mod h1:s8HPh7mMV933OtXzONaGFseCg/BE//m1V34p3x4EUOY= github.com/libp2p/go-libp2p-asn-util v0.4.1 h1:xqL7++IKD9TBFMgnLPZR6/6iYhawHKHl950SO9L6n94= github.com/libp2p/go-libp2p-asn-util v0.4.1/go.mod h1:d/NI6XZ9qxw67b4e+NgpQexCIiFYJjErASrYW4PFDN8= -github.com/libp2p/go-libp2p-kad-dht v0.37.0 h1:V1IkFzK9taNS1UNAx260foulcBPH+watAUFjNo2qMUY= -github.com/libp2p/go-libp2p-kad-dht v0.37.0/go.mod h1:o4FPa1ea++UVAMJ1c+kyjUmj3CKm9+ZCyzQb4uutCFM= +github.com/libp2p/go-libp2p-kad-dht v0.37.1 h1:jtX8bQIXVCs6/allskNB4m5n95Xvwav7wHAhopGZfS0= +github.com/libp2p/go-libp2p-kad-dht v0.37.1/go.mod h1:Uwokdh232k9Y1uMy2yJOK5zb7hpMHn4P8uWS4s9i05Q= github.com/libp2p/go-libp2p-kbucket v0.8.0 h1:QAK7RzKJpYe+EuSEATAaaHYMYLkPDGC18m9jxPLnU8s= github.com/libp2p/go-libp2p-kbucket v0.8.0/go.mod h1:JMlxqcEyKwO6ox716eyC0hmiduSWZZl6JY93mGaaqc4= github.com/libp2p/go-libp2p-pubsub v0.15.0 h1:cG7Cng2BT82WttmPFMi50gDNV+58K626m/wR00vGL1o= @@ -224,8 +222,8 @@ github.com/libp2p/go-libp2p-routing-helpers v0.7.5 h1:HdwZj9NKovMx0vqq6YNPTh6aaN github.com/libp2p/go-libp2p-routing-helpers v0.7.5/go.mod h1:3YaxrwP0OBPDD7my3D0KxfR89FlcX/IEbxDEDfAmj98= github.com/libp2p/go-msgio v0.3.0 h1:mf3Z8B1xcFN314sWX+2vOTShIE0Mmn2TXn3YCUQGNj0= github.com/libp2p/go-msgio v0.3.0/go.mod h1:nyRM819GmVaF9LX3l03RMh10QdOroF++NBbxAb0mmDM= -github.com/libp2p/go-netroute v0.3.0 h1:nqPCXHmeNmgTJnktosJ/sIef9hvwYCrsLxXmfNks/oc= -github.com/libp2p/go-netroute v0.3.0/go.mod h1:Nkd5ShYgSMS5MUKy/MU2T57xFoOKvvLR92Lic48LEyA= +github.com/libp2p/go-netroute v0.4.0 h1:sZZx9hyANYUx9PZyqcgE/E1GUG3iEtTZHUEvdtXT7/Q= +github.com/libp2p/go-netroute v0.4.0/go.mod h1:Nkd5ShYgSMS5MUKy/MU2T57xFoOKvvLR92Lic48LEyA= github.com/libp2p/go-reuseport v0.4.0 h1:nR5KU7hD0WxXCJbmw7r2rhRYruNRl2koHw8fQscQm2s= github.com/libp2p/go-reuseport v0.4.0/go.mod h1:ZtI03j/wO5hZVDFo2jKywN6bYKWLOy8Se6DrI2E1cLU= github.com/libp2p/go-yamux/v5 v5.0.1 h1:f0WoX/bEF2E8SbE4c/k1Mo+/9z0O4oC/hWEA+nfYRSg= @@ -354,10 +352,10 @@ github.com/prysmaticlabs/gohashtree v0.0.4-beta h1:H/EbCuXPeTV3lpKeXGPpEV9gsUpkq github.com/prysmaticlabs/gohashtree v0.0.4-beta/go.mod h1:BFdtALS+Ffhg3lGQIHv9HDWuHS8cTvHZzrHWxwOtGOs= github.com/quic-go/qpack v0.6.0 h1:g7W+BMYynC1LbYLSqRt8PBg5Tgwxn214ZZR34VIOjz8= github.com/quic-go/qpack v0.6.0/go.mod h1:lUpLKChi8njB4ty2bFLX2x4gzDqXwUpaO1DP9qMDZII= -github.com/quic-go/quic-go v0.57.1 h1:25KAAR9QR8KZrCZRThWMKVAwGoiHIrNbT72ULHTuI10= -github.com/quic-go/quic-go v0.57.1/go.mod h1:ly4QBAjHA2VhdnxhojRsCUOeJwKYg+taDlos92xb1+s= -github.com/quic-go/webtransport-go v0.9.0 h1:jgys+7/wm6JarGDrW+lD/r9BGqBAmqY/ssklE09bA70= -github.com/quic-go/webtransport-go v0.9.0/go.mod h1:4FUYIiUc75XSsF6HShcLeXXYZJ9AGwo/xh3L8M/P1ao= +github.com/quic-go/quic-go v0.59.0 h1:OLJkp1Mlm/aS7dpKgTc6cnpynnD2Xg7C1pwL6vy/SAw= +github.com/quic-go/quic-go v0.59.0/go.mod h1:upnsH4Ju1YkqpLXC305eW3yDZ4NfnNbmQRCMWS58IKU= +github.com/quic-go/webtransport-go v0.10.0 h1:LqXXPOXuETY5Xe8ITdGisBzTYmUOy5eSj+9n4hLTjHI= +github.com/quic-go/webtransport-go v0.10.0/go.mod h1:LeGIXr5BQKE3UsynwVBeQrU1TPrbh73MGoC6jd+V7ow= github.com/rivo/uniseg v0.2.0 h1:S1pD9weZBuJdFmowNwbpi7BJ8TNftyUImj/0WQi72jY= github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ= @@ -455,11 +453,11 @@ golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.47.0 h1:V6e3FRj+n4dbpw86FJ8Fv7XVOql7TEwpHapKoMJ/GO8= golang.org/x/crypto v0.47.0/go.mod h1:ff3Y9VzzKbwSSEzWqJsJVBnWmRwRSHt/6Op5n9bQc4A= -golang.org/x/exp v0.0.0-20251209150349-8475f28825e9 h1:MDfG8Cvcqlt9XXrmEiD4epKn7VJHZO84hejP9Jmp0MM= -golang.org/x/exp v0.0.0-20251209150349-8475f28825e9/go.mod h1:EPRbTFwzwjXj9NpYyyrvenVh9Y+GFeEvMNh7Xuz7xgU= +golang.org/x/exp v0.0.0-20260112195511-716be5621a96 h1:Z/6YuSHTLOHfNFdb8zVZomZr7cqNgTJvA8+Qz75D8gU= +golang.org/x/exp v0.0.0-20260112195511-716be5621a96/go.mod h1:nzimsREAkjBCIEFtHiYkrJyT+2uy9YZJB7H1k68CXZU= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.31.0 h1:HaW9xtz0+kOcWKwli0ZXy79Ix+UW/vOfmWI5QVd2tgI= -golang.org/x/mod v0.31.0/go.mod h1:43JraMp9cGx1Rx3AqioxrbrhNsLl2l/iNAvuBkrezpg= +golang.org/x/mod v0.32.0 h1:9F4d3PHLljb6x//jOyokMv3eX+YDeepZSEo3mFJy93c= +golang.org/x/mod v0.32.0/go.mod h1:SgipZ/3h2Ci89DlEtEXWUk/HteuRin+HHhN+WbNhguU= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= @@ -500,8 +498,8 @@ golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.40.0 h1:DBZZqJ2Rkml6QMQsZywtnjnnGvHza6BTfYFWY9kjEWQ= golang.org/x/sys v0.40.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= -golang.org/x/telemetry v0.0.0-20251203150158-8fff8a5912fc h1:bH6xUXay0AIFMElXG2rQ4uiE+7ncwtiOdPfYK1NK2XA= -golang.org/x/telemetry v0.0.0-20251203150158-8fff8a5912fc/go.mod h1:hKdjCMrbv9skySur+Nek8Hd0uJ0GuxJIoIX2payrIdQ= +golang.org/x/telemetry v0.0.0-20260109210033-bd525da824e2 h1:O1cMQHRfwNpDfDJerqRoE2oD+AFlyid87D40L/OkkJo= +golang.org/x/telemetry v0.0.0-20260109210033-bd525da824e2/go.mod h1:b7fPSJ0pKZ3ccUh8gnTONJxhn3c/PS6tyzQvyqw4iA8= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -515,15 +513,15 @@ golang.org/x/time v0.12.0/go.mod h1:CDIdPxbZBQxdj6cxyCIdrNogrJKMJ7pr37NYpMcMDSg= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.40.0 h1:yLkxfA+Qnul4cs9QA3KnlFu0lVmd8JJfoq+E41uSutA= -golang.org/x/tools v0.40.0/go.mod h1:Ik/tzLRlbscWpqqMRjyWYDisX8bG13FrdXp3o4Sr9lc= +golang.org/x/tools v0.41.0 h1:a9b8iMweWG+S0OBnlU36rzLp20z1Rp10w+IY2czHTQc= +golang.org/x/tools v0.41.0/go.mod h1:XSY6eDqxVNiYgezAVqqCeihT4j1U2CCsqvH3WhQpnlg= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20220517211312-f3a8303e98df/go.mod h1:K8+ghG5WaK9qNqU5K3HdILfMLy1f3aNYFI/wnl100a8= -gonum.org/v1/gonum v0.16.0 h1:5+ul4Swaf3ESvrOnidPp4GZbzf0mxVQpDCYUQE7OJfk= -gonum.org/v1/gonum v0.16.0/go.mod h1:fef3am4MQ93R2HHpKnLk4/Tbh/s0+wqD5nfa6Pnwy4E= +gonum.org/v1/gonum v0.17.0 h1:VbpOemQlsSMrYmn7T2OUvQ4dqxQXU+ouZFQsZOx50z4= +gonum.org/v1/gonum v0.17.0/go.mod h1:El3tOrEuMpv2UdMrbNlKEh9vd86bmQ6vqIcDwxEOc1E= google.golang.org/genproto/googleapis/api v0.0.0-20260128011058-8636f8732409 h1:merA0rdPeUV3YIIfHHcH4qBkiQAc1nfCKSI7lB4cV2M= google.golang.org/genproto/googleapis/api v0.0.0-20260128011058-8636f8732409/go.mod h1:fl8J1IvUjCilwZzQowmw2b7HQB2eAuYBabMXzWurF+I= google.golang.org/genproto/googleapis/rpc v0.0.0-20260128011058-8636f8732409 h1:H86B94AW+VfJWDqFeEbBPhEtHzJwJfTbgE2lZa54ZAQ= diff --git a/execution/evm/store.go b/execution/evm/store.go index af731ee7c..21fe5008a 100644 --- a/execution/evm/store.go +++ b/execution/evm/store.go @@ -15,6 +15,11 @@ import ( // Store prefix for execution/evm data - keeps it isolated from other ev-node data const evmStorePrefix = "evm/" +// lastPrunedExecMetaKey is the datastore key used to track the highest +// execution height for which ExecMeta has been pruned. All ExecMeta entries +// for heights <= this value are considered pruned. +const lastPrunedExecMetaKey = evmStorePrefix + "last-pruned-execmeta-height" + // ExecMeta stages const ( ExecStageStarted = "started" @@ -140,6 +145,54 @@ func (s *EVMStore) SaveExecMeta(ctx context.Context, meta *ExecMeta) error { return nil } +// PruneExec removes ExecMeta entries up to and including the given height. +// It is safe to call this multiple times with the same or increasing heights; +// previously pruned ranges will be skipped based on the last-pruned marker. +func (s *EVMStore) PruneExec(ctx context.Context, height uint64) error { + // Load last pruned height, if any. + var lastPruned uint64 + data, err := s.db.Get(ctx, ds.NewKey(lastPrunedExecMetaKey)) + if err != nil { + if !errors.Is(err, ds.ErrNotFound) { + return fmt.Errorf("failed to get last pruned execmeta height: %w", err) + } + } else if len(data) == 8 { + lastPruned = binary.BigEndian.Uint64(data) + } + + // Nothing new to prune. + if height <= lastPruned { + return nil + } + + batch, err := s.db.Batch(ctx) + if err != nil { + return fmt.Errorf("failed to create batch for execmeta pruning: %w", err) + } + + for h := lastPruned + 1; h <= height; h++ { + key := execMetaKey(h) + if err := batch.Delete(ctx, key); err != nil { + if !errors.Is(err, ds.ErrNotFound) { + return fmt.Errorf("failed to delete exec meta at height %d: %w", h, err) + } + } + } + + // Persist updated last pruned height. + buf := make([]byte, 8) + binary.BigEndian.PutUint64(buf, height) + if err := batch.Put(ctx, ds.NewKey(lastPrunedExecMetaKey), buf); err != nil { + return fmt.Errorf("failed to update last pruned execmeta height: %w", err) + } + + if err := batch.Commit(ctx); err != nil { + return fmt.Errorf("failed to commit execmeta pruning batch: %w", err) + } + + return nil +} + // Sync ensures all pending writes are flushed to disk. func (s *EVMStore) Sync(ctx context.Context) error { return s.db.Sync(ctx, ds.NewKey(evmStorePrefix)) diff --git a/execution/evm/store_test.go b/execution/evm/store_test.go new file mode 100644 index 000000000..d3067fc1a --- /dev/null +++ b/execution/evm/store_test.go @@ -0,0 +1,99 @@ +package evm + +import ( + "context" + "encoding/binary" + "testing" + + ds "github.com/ipfs/go-datastore" + dssync "github.com/ipfs/go-datastore/sync" + "github.com/stretchr/testify/require" +) + +// newTestDatastore creates an in-memory datastore for testing. +func newTestDatastore(t *testing.T) ds.Batching { + t.Helper() + // Wrap the in-memory MapDatastore to satisfy the Batching interface. + return dssync.MutexWrap(ds.NewMapDatastore()) +} + +func TestPruneExec_PrunesUpToTargetHeight(t *testing.T) { + t.Parallel() + + ctx := context.Background() + db := newTestDatastore(t) + store := NewEVMStore(db) + + // Seed ExecMeta entries at heights 1..5 + for h := uint64(1); h <= 5; h++ { + meta := &ExecMeta{Height: h} + require.NoError(t, store.SaveExecMeta(ctx, meta)) + } + + // Sanity: all heights should be present + for h := uint64(1); h <= 5; h++ { + meta, err := store.GetExecMeta(ctx, h) + require.NoError(t, err) + require.NotNil(t, meta) + require.Equal(t, h, meta.Height) + } + + // Prune up to height 3 + require.NoError(t, store.PruneExec(ctx, 3)) + + // Heights 1..3 should be gone + for h := uint64(1); h <= 3; h++ { + meta, err := store.GetExecMeta(ctx, h) + require.NoError(t, err) + require.Nil(t, meta) + } + + // Heights 4..5 should remain + for h := uint64(4); h <= 5; h++ { + meta, err := store.GetExecMeta(ctx, h) + require.NoError(t, err) + require.NotNil(t, meta) + } + + // Re-pruning with the same height should be a no-op + require.NoError(t, store.PruneExec(ctx, 3)) +} + +func TestPruneExec_TracksLastPrunedHeight(t *testing.T) { + t.Parallel() + + ctx := context.Background() + db := newTestDatastore(t) + store := NewEVMStore(db) + + // Seed ExecMeta entries at heights 1..5 + for h := uint64(1); h <= 5; h++ { + meta := &ExecMeta{Height: h} + require.NoError(t, store.SaveExecMeta(ctx, meta)) + } + + // First prune up to 2 + require.NoError(t, store.PruneExec(ctx, 2)) + + // Then prune up to 4; heights 3..4 should be deleted in this run + require.NoError(t, store.PruneExec(ctx, 4)) + + // Verify all heights 1..4 are gone, 5 remains + for h := uint64(1); h <= 4; h++ { + meta, err := store.GetExecMeta(ctx, h) + require.NoError(t, err) + require.Nil(t, meta) + } + + meta, err := store.GetExecMeta(ctx, 5) + require.NoError(t, err) + require.NotNil(t, meta) + require.Equal(t, uint64(5), meta.Height) + + // Ensure last-pruned marker is set to 4 + raw, err := db.Get(ctx, ds.NewKey(lastPrunedExecMetaKey)) + require.NoError(t, err) + require.Len(t, raw, 8) + last := binary.BigEndian.Uint64(raw) + require.Equal(t, uint64(4), last) +} diff --git a/go.mod b/go.mod index d07ea7a7e..8f10b14e5 100644 --- a/go.mod +++ b/go.mod @@ -195,3 +195,6 @@ replace ( google.golang.org/genproto/googleapis/api => google.golang.org/genproto/googleapis/api v0.0.0-20240213162025-012b6fc9bca9 google.golang.org/genproto/googleapis/rpc => google.golang.org/genproto/googleapis/rpc v0.0.0-20240213162025-012b6fc9bca9 ) + +// use local core module during development/CI +replace github.com/evstack/ev-node/core => ./core diff --git a/go.sum b/go.sum index fb175ac5c..d1fd875f0 100644 --- a/go.sum +++ b/go.sum @@ -367,8 +367,6 @@ github.com/envoyproxy/protoc-gen-validate v0.10.0/go.mod h1:DRjgyB0I43LtJapqN6Ni github.com/envoyproxy/protoc-gen-validate v0.10.1/go.mod h1:DRjgyB0I43LtJapqN6NiRwroiAU2PaFuvk/vjgh61ss= github.com/envoyproxy/protoc-gen-validate v1.0.1/go.mod h1:0vj8bNkYbSTNS2PIyH87KZaeN4x9zpL9Qt8fQC7d+vs= github.com/envoyproxy/protoc-gen-validate v1.0.2/go.mod h1:GpiZQP3dDbg4JouG/NNS7QWXpgx6x8QiMKdmN72jogE= -github.com/evstack/ev-node/core v1.0.0-rc.1 h1:Dic2PMUMAYUl5JW6DkDj6HXDEWYzorVJQuuUJOV0FjE= -github.com/evstack/ev-node/core v1.0.0-rc.1/go.mod h1:n2w/LhYQTPsi48m6lMj16YiIqsaQw6gxwjyJvR+B3sY= github.com/fatih/color v1.10.0/go.mod h1:ELkj/draVOlAH/xkhN6mQ50Qd0MPOk5AAr3maGEBuJM= github.com/fatih/color v1.13.0/go.mod h1:kLAiJbzzSOZDVNGyDpeOxJ47H46qBXwg5ILebYFFOfk= github.com/fatih/color v1.15.0/go.mod h1:0h5ZqXfHYED7Bhv2ZJamyIOUej9KtShiJESRwBDUSsw= diff --git a/pkg/config/config.go b/pkg/config/config.go index e03a277ce..f42bfb666 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -261,6 +261,13 @@ type NodeConfig struct { // Readiness / health configuration ReadinessWindowSeconds uint64 `mapstructure:"readiness_window_seconds" yaml:"readiness_window_seconds" comment:"Time window in seconds used to calculate ReadinessMaxBlocksBehind based on block time. Default: 15 seconds."` ReadinessMaxBlocksBehind uint64 `mapstructure:"readiness_max_blocks_behind" yaml:"readiness_max_blocks_behind" comment:"How many blocks behind best-known head the node can be and still be considered ready. 0 means must be exactly at head."` + + // Pruning configuration + // When enabled, the node will periodically prune old block data (headers, data, + // signatures, and hash index) from the local store while keeping recent history. + PruningEnabled bool `mapstructure:"pruning_enabled" yaml:"pruning_enabled" comment:"Enable height-based pruning of stored block data. When disabled, all blocks are kept (archive mode)."` + PruningKeepRecent uint64 `mapstructure:"pruning_keep_recent" yaml:"pruning_keep_recent" comment:"Number of most recent blocks to retain when pruning is enabled. Must be > 0 when pruning is enabled; set pruning_enabled=false to keep all blocks (archive mode)."` + PruningInterval uint64 `mapstructure:"pruning_interval" yaml:"pruning_interval" comment:"Run pruning every N blocks. Must be >= 1 when pruning is enabled."` } // LogConfig contains all logging configuration parameters @@ -376,6 +383,21 @@ func (c *Config) Validate() error { if err := c.Raft.Validate(); err != nil { return err } + + // Validate pruning configuration + if c.Node.PruningEnabled { + // When pruning is enabled, pruning_interval must be >= 1 + if c.Node.PruningInterval == 0 { + return fmt.Errorf("pruning_interval must be >= 1 when pruning is enabled") + } + + // When pruning is enabled, keeping 0 blocks is contradictory; use pruning_enabled=false + // for archive mode instead. + if c.Node.PruningKeepRecent == 0 { + return fmt.Errorf("pruning_keep_recent must be > 0 when pruning is enabled; use pruning_enabled=false to keep all blocks") + } + } + return nil } @@ -436,6 +458,10 @@ func AddFlags(cmd *cobra.Command) { cmd.Flags().Uint64(FlagReadinessWindowSeconds, def.Node.ReadinessWindowSeconds, "time window in seconds for calculating readiness threshold based on block time (default: 15s)") cmd.Flags().Uint64(FlagReadinessMaxBlocksBehind, def.Node.ReadinessMaxBlocksBehind, "how many blocks behind best-known head the node can be and still be considered ready (0 = must be at head)") cmd.Flags().Duration(FlagScrapeInterval, def.Node.ScrapeInterval.Duration, "interval at which the reaper polls the execution layer for new transactions") + // Pruning configuration flags + cmd.Flags().Bool(FlagPrefixEvnode+"node.pruning_enabled", def.Node.PruningEnabled, "enable height-based pruning of stored block data (headers, data, signatures, index)") + cmd.Flags().Uint64(FlagPrefixEvnode+"node.pruning_keep_recent", def.Node.PruningKeepRecent, "number of most recent blocks to retain when pruning is enabled (must be > 0; disable pruning to keep all blocks)") + cmd.Flags().Uint64(FlagPrefixEvnode+"node.pruning_interval", def.Node.PruningInterval, "run pruning every N blocks (must be >= 1 when pruning is enabled)") // Data Availability configuration flags cmd.Flags().String(FlagDAAddress, def.DA.Address, "DA address (host:port)") diff --git a/pkg/config/config_test.go b/pkg/config/config_test.go index 1834e1b40..9189aa57e 100644 --- a/pkg/config/config_test.go +++ b/pkg/config/config_test.go @@ -112,7 +112,7 @@ func TestAddFlags(t *testing.T) { assertFlagValue(t, flags, FlagRPCEnableDAVisualization, DefaultConfig().RPC.EnableDAVisualization) // Count the number of flags we're explicitly checking - expectedFlagCount := 63 // Update this number if you add more flag checks above + expectedFlagCount := 66 // Update this number if you add more flag checks above // Get the actual number of flags (both regular and persistent) actualFlagCount := 0 diff --git a/pkg/config/defaults.go b/pkg/config/defaults.go index 0de2f4bc2..ee2cbfbee 100644 --- a/pkg/config/defaults.go +++ b/pkg/config/defaults.go @@ -69,6 +69,9 @@ func DefaultConfig() Config { ReadinessWindowSeconds: defaultReadinessWindowSeconds, ReadinessMaxBlocksBehind: calculateReadinessMaxBlocksBehind(defaultBlockTime.Duration, defaultReadinessWindowSeconds), ScrapeInterval: DurationWrapper{1 * time.Second}, + PruningEnabled: false, + PruningKeepRecent: 0, + PruningInterval: 0, }, DA: DAConfig{ Address: "http://localhost:7980", diff --git a/pkg/store/cached_store.go b/pkg/store/cached_store.go index e59d4e28c..3fdabc9e5 100644 --- a/pkg/store/cached_store.go +++ b/pkg/store/cached_store.go @@ -157,6 +157,19 @@ func (cs *CachedStore) Rollback(ctx context.Context, height uint64, aggregator b return nil } +// PruneBlocks wraps the underlying store's PruneBlocks and invalidates caches. +// After pruning historical block data from disk, any cached entries for pruned +// heights must not be served, so we conservatively clear the entire cache. +func (cs *CachedStore) PruneBlocks(ctx context.Context, height uint64) error { + if err := cs.Store.PruneBlocks(ctx, height); err != nil { + return err + } + + // Invalidate cache for pruned heights + cs.InvalidateRange(1, height) + return nil +} + // Close closes the underlying store. func (cs *CachedStore) Close() error { cs.ClearCache() diff --git a/pkg/store/keys.go b/pkg/store/keys.go index dd989c0e8..ff96fd895 100644 --- a/pkg/store/keys.go +++ b/pkg/store/keys.go @@ -25,6 +25,12 @@ const ( // LastSubmittedHeaderHeightKey is the key used for persisting the last submitted header height in store. LastSubmittedHeaderHeightKey = "last-submitted-header-height" + // LastPrunedBlockHeightKey is the metadata key used for persisting the last + // pruned block height in the store. All block data (header, data, + // signature, and hash index) for heights <= this value are considered + // pruned and may be missing from the store. + LastPrunedBlockHeightKey = "last-pruned-block-height" + headerPrefix = "h" dataPrefix = "d" signaturePrefix = "c" diff --git a/pkg/store/store.go b/pkg/store/store.go index a821e6a8d..fa8a2bfd5 100644 --- a/pkg/store/store.go +++ b/pkg/store/store.go @@ -296,6 +296,99 @@ func (s *DefaultStore) Rollback(ctx context.Context, height uint64, aggregator b return nil } +// PruneBlocks removes block data (header, data, signature, and hash index) +// up to and including the given height from the store. It does not modify +// the current chain height or any state snapshots. +// +// This method is intended for long-term storage reduction and is safe to +// call repeatedly with the same or increasing heights. +func (s *DefaultStore) PruneBlocks(ctx context.Context, height uint64) error { + batch, err := s.db.Batch(ctx) + if err != nil { + return fmt.Errorf("failed to create a new batch for pruning: %w", err) + } + + // Track the last successfully pruned height so we can resume across restarts. + var lastPruned uint64 + meta, err := s.GetMetadata(ctx, LastPrunedBlockHeightKey) + if err != nil { + if !errors.Is(err, ds.ErrNotFound) { + return fmt.Errorf("failed to get last pruned height: %w", err) + } + } else if len(meta) == heightLength { + lastPruned, err = decodeHeight(meta) + if err != nil { + return fmt.Errorf("failed to decode last pruned height: %w", err) + } + } + + // Nothing new to prune. + if height <= lastPruned { + return nil + } + + // Delete block data for heights in (lastPruned, height]. + for h := lastPruned + 1; h <= height; h++ { + // Get header blob to compute the hash index key. If header is already + // missing (e.g. due to previous partial pruning), just skip this height. + headerBlob, err := s.db.Get(ctx, ds.NewKey(getHeaderKey(h))) + if err != nil { + if errors.Is(err, ds.ErrNotFound) { + continue + } + return fmt.Errorf("failed to get header at height %d during pruning: %w", h, err) + } + + if err := batch.Delete(ctx, ds.NewKey(getHeaderKey(h))); err != nil { + if !errors.Is(err, ds.ErrNotFound) { + return fmt.Errorf("failed to delete header at height %d during pruning: %w", h, err) + } + } + + if err := batch.Delete(ctx, ds.NewKey(getDataKey(h))); err != nil { + if !errors.Is(err, ds.ErrNotFound) { + return fmt.Errorf("failed to delete data at height %d during pruning: %w", h, err) + } + } + + if err := batch.Delete(ctx, ds.NewKey(getSignatureKey(h))); err != nil { + if !errors.Is(err, ds.ErrNotFound) { + return fmt.Errorf("failed to delete signature at height %d during pruning: %w", h, err) + } + } + + // Delete per-height DA metadata associated with this height, if any. + if err := batch.Delete(ctx, ds.NewKey(getMetaKey(GetHeightToDAHeightHeaderKey(h)))); err != nil { + if !errors.Is(err, ds.ErrNotFound) { + return fmt.Errorf("failed to delete header DA height metadata at height %d during pruning: %w", h, err) + } + } + if err := batch.Delete(ctx, ds.NewKey(getMetaKey(GetHeightToDAHeightDataKey(h)))); err != nil { + if !errors.Is(err, ds.ErrNotFound) { + return fmt.Errorf("failed to delete data DA height metadata at height %d during pruning: %w", h, err) + } + } + + headerHash := sha256.Sum256(headerBlob) + if err := batch.Delete(ctx, ds.NewKey(getIndexKey(headerHash[:]))); err != nil { + if !errors.Is(err, ds.ErrNotFound) { + return fmt.Errorf("failed to delete index for height %d during pruning: %w", h, err) + } + } + } + + // Persist the updated last pruned height. + if err := batch.Put(ctx, ds.NewKey(getMetaKey(LastPrunedBlockHeightKey)), encodeHeight(height)); err != nil { + return fmt.Errorf("failed to update last pruned height: %w", err) + } + + if err := batch.Commit(ctx); err != nil { + return fmt.Errorf("failed to commit pruning batch: %w", err) + } + + return nil +} + const heightLength = 8 func encodeHeight(height uint64) []byte { diff --git a/pkg/store/store_adapter.go b/pkg/store/store_adapter.go index ecd4da7ab..6731cfe47 100644 --- a/pkg/store/store_adapter.go +++ b/pkg/store/store_adapter.go @@ -128,6 +128,7 @@ func (hs *heightSub) notifyUpTo(h uint64) { // a block, it writes to the underlying store, and subsequent reads will come from the store. type StoreAdapter[H EntityWithDAHint[H]] struct { getter StoreGetter[H] + store Store genesisInitialHeight uint64 // heightSub tracks the current height and allows waiting for specific heights. @@ -150,9 +151,9 @@ type StoreAdapter[H EntityWithDAHint[H]] struct { onDeleteFn func(context.Context, uint64) error } -// NewStoreAdapter creates a new StoreAdapter wrapping the given store getter. +// NewStoreAdapter creates a new StoreAdapter wrapping the given store getter and backing Store. // The genesis is used to determine the initial height for efficient Tail lookups. -func NewStoreAdapter[H EntityWithDAHint[H]](getter StoreGetter[H], gen genesis.Genesis) *StoreAdapter[H] { +func NewStoreAdapter[H EntityWithDAHint[H]](getter StoreGetter[H], store Store, gen genesis.Genesis) *StoreAdapter[H] { // Create LRU cache for pending items - ignore error as size is constant and valid pendingCache, _ := lru.New[uint64, H](defaultPendingCacheSize) daHintsCache, _ := lru.New[uint64, uint64](defaultPendingCacheSize) @@ -165,6 +166,7 @@ func NewStoreAdapter[H EntityWithDAHint[H]](getter StoreGetter[H], gen genesis.G adapter := &StoreAdapter[H]{ getter: getter, + store: store, genesisInitialHeight: max(gen.InitialHeight, 1), pending: pendingCache, daHints: daHintsCache, @@ -251,7 +253,6 @@ func (a *StoreAdapter[H]) Head(ctx context.Context, _ ...header.HeadOption[H]) ( // Tail returns the lowest item in the store. // For ev-node, this is typically the genesis/initial height. // If pruning has occurred, it walks up from initialHeight to find the first available item. -// TODO(@julienrbrt): Optimize this when pruning is enabled. func (a *StoreAdapter[H]) Tail(ctx context.Context) (H, error) { var zero H @@ -265,19 +266,30 @@ func (a *StoreAdapter[H]) Tail(ctx context.Context) (H, error) { height = h } - // Try genesisInitialHeight first (most common case - no pruning) - item, err := a.getter.GetByHeight(ctx, a.genesisInitialHeight) + // Determine the first candidate tail height. By default, this is the + // genesis initial height, but if pruning metadata is available we can + // skip directly past fully-pruned ranges. + startHeight := a.genesisInitialHeight + if meta, err := a.store.GetMetadata(ctx, LastPrunedBlockHeightKey); err == nil && len(meta) == heightLength { + if lastPruned, err := decodeHeight(meta); err == nil { + if candidate := lastPruned + 1; candidate > startHeight { + startHeight = candidate + } + } + } + + item, err := a.getter.GetByHeight(ctx, startHeight) if err == nil { return item, nil } - // Check pending for genesisInitialHeight - if pendingItem, ok := a.pending.Peek(a.genesisInitialHeight); ok { + // Check pending for the start height + if pendingItem, ok := a.pending.Peek(startHeight); ok { return pendingItem, nil } - // Walk up from genesisInitialHeight to find the first available item (pruning case) - for h := a.genesisInitialHeight + 1; h <= height; h++ { + // Walk up from startHeight to find the first available item + for h := startHeight + 1; h <= height; h++ { item, err = a.getter.GetByHeight(ctx, h) if err == nil { return item, nil @@ -718,11 +730,11 @@ type DataStoreAdapter = StoreAdapter[*types.P2PData] // NewHeaderStoreAdapter creates a new StoreAdapter for headers. // The genesis is used to determine the initial height for efficient Tail lookups. func NewHeaderStoreAdapter(store Store, gen genesis.Genesis) *HeaderStoreAdapter { - return NewStoreAdapter(NewHeaderStoreGetter(store), gen) + return NewStoreAdapter(NewHeaderStoreGetter(store), store, gen) } // NewDataStoreAdapter creates a new StoreAdapter for data. // The genesis is used to determine the initial height for efficient Tail lookups. func NewDataStoreAdapter(store Store, gen genesis.Genesis) *DataStoreAdapter { - return NewStoreAdapter(NewDataStoreGetter(store), gen) + return NewStoreAdapter(NewDataStoreGetter(store), store, gen) } diff --git a/pkg/store/store_test.go b/pkg/store/store_test.go index 8636a0ad0..5ba5eecfa 100644 --- a/pkg/store/store_test.go +++ b/pkg/store/store_test.go @@ -734,6 +734,85 @@ func TestRollback(t *testing.T) { require.Equal(rollbackToHeight, state.LastBlockHeight) } +func TestPruneBlocks_RemovesOldBlockDataOnly(t *testing.T) { + t.Parallel() + + ctx := context.Background() + ds, err := NewTestInMemoryKVStore() + require.NoError(t, err) + + s := New(ds).(*DefaultStore) + + // create and store a few blocks with headers, data, signatures, state, and per-height DA metadata + batch, err := s.NewBatch(ctx) + require.NoError(t, err) + + var lastState types.State + for h := uint64(1); h <= 5; h++ { + header := &types.SignedHeader{Header: types.Header{BaseHeader: types.BaseHeader{Height: h}}} + data := &types.Data{} + sig := types.Signature([]byte{byte(h)}) + + require.NoError(t, batch.SaveBlockData(header, data, &sig)) + + // fake state snapshot per height + lastState = types.State{LastBlockHeight: h} + require.NoError(t, batch.UpdateState(lastState)) + + // store fake DA metadata per height + hDaKey := GetHeightToDAHeightHeaderKey(h) + dDaKey := GetHeightToDAHeightDataKey(h) + bz := make([]byte, 8) + binary.LittleEndian.PutUint64(bz, h+100) // arbitrary DA height + require.NoError(t, s.SetMetadata(ctx, hDaKey, bz)) + require.NoError(t, s.SetMetadata(ctx, dDaKey, bz)) + } + require.NoError(t, batch.SetHeight(5)) + require.NoError(t, batch.Commit()) + + // prune everything up to height 3 + require.NoError(t, s.PruneBlocks(ctx, 3)) + + // old block data should be gone + for h := uint64(1); h <= 3; h++ { + _, _, err := s.GetBlockData(ctx, h) + assert.Error(t, err, "expected block data at height %d to be pruned", h) + } + + // recent block data should remain + for h := uint64(4); h <= 5; h++ { + _, _, err := s.GetBlockData(ctx, h) + assert.NoError(t, err, "expected block data at height %d to be kept", h) + } + + // state snapshots are not pruned by PruneBlocks + for h := uint64(1); h <= 5; h++ { + st, err := s.GetStateAtHeight(ctx, h) + assert.NoError(t, err, "expected state at height %d to remain", h) + assert.Equal(t, h, st.LastBlockHeight) + } + + // per-height DA metadata for pruned heights should be gone + for h := uint64(1); h <= 3; h++ { + hDaKey := GetHeightToDAHeightHeaderKey(h) + dDaKey := GetHeightToDAHeightDataKey(h) + _, err := s.GetMetadata(ctx, hDaKey) + assert.Error(t, err, "expected header DA metadata at height %d to be pruned", h) + _, err = s.GetMetadata(ctx, dDaKey) + assert.Error(t, err, "expected data DA metadata at height %d to be pruned", h) + } + + // per-height DA metadata for unpruned heights should remain + for h := uint64(4); h <= 5; h++ { + hDaKey := GetHeightToDAHeightHeaderKey(h) + dDaKey := GetHeightToDAHeightDataKey(h) + _, err := s.GetMetadata(ctx, hDaKey) + assert.NoError(t, err, "expected header DA metadata at height %d to remain", h) + _, err = s.GetMetadata(ctx, dDaKey) + assert.NoError(t, err, "expected data DA metadata at height %d to remain", h) + } +} + // TestRollbackToSameHeight verifies that rollback to same height is a no-op func TestRollbackToSameHeight(t *testing.T) { t.Parallel() diff --git a/pkg/store/tracing.go b/pkg/store/tracing.go index 04baf748f..004e0bd38 100644 --- a/pkg/store/tracing.go +++ b/pkg/store/tracing.go @@ -230,6 +230,22 @@ func (t *tracedStore) Rollback(ctx context.Context, height uint64, aggregator bo return nil } +func (t *tracedStore) PruneBlocks(ctx context.Context, height uint64) error { + ctx, span := t.tracer.Start(ctx, "Store.PruneBlocks", + trace.WithAttributes(attribute.Int64("height", int64(height))), + ) + defer span.End() + + err := t.inner.PruneBlocks(ctx, height) + if err != nil { + span.RecordError(err) + span.SetStatus(codes.Error, err.Error()) + return err + } + + return nil +} + func (t *tracedStore) Close() error { return t.inner.Close() } diff --git a/pkg/store/tracing_test.go b/pkg/store/tracing_test.go index 5c273f661..6d7e520e1 100644 --- a/pkg/store/tracing_test.go +++ b/pkg/store/tracing_test.go @@ -116,6 +116,12 @@ func (m *tracingMockStore) Rollback(ctx context.Context, height uint64, aggregat return nil } +func (m *tracingMockStore) PruneBlocks(ctx context.Context, height uint64) error { + // For tracing tests we don't need pruning behavior; just satisfy the Store + // interface. Specific pruning behavior is tested separately in store_test.go. + return nil +} + func (m *tracingMockStore) Close() error { return nil } diff --git a/pkg/store/types.go b/pkg/store/types.go index fa1ecdc92..fe36e0be5 100644 --- a/pkg/store/types.go +++ b/pkg/store/types.go @@ -30,9 +30,15 @@ type Batch interface { } // Store is minimal interface for storing and retrieving blocks, commits and state. +// +// It is composed from three concerns: +// - Reader: read access to blocks, state, and metadata +// - Rollback: consensus rollback logic (used for chain reorgs / recovery) +// - Pruner: long-term height-based pruning of historical block data type Store interface { - Rollback Reader + Rollback + Pruner // SetMetadata saves arbitrary value in the store. // @@ -78,3 +84,14 @@ type Rollback interface { // Aggregator is used to determine if the rollback is performed on the aggregator node. Rollback(ctx context.Context, height uint64, aggregator bool) error } + +// Pruner provides long-term, height-based pruning of historical block data. +// +// Implementations SHOULD be idempotent and safe to call multiple times for +// the same or increasing target heights. +type Pruner interface { + // PruneBlocks removes block data (header, data, signature, and hash index) + // up to and including the given height from the store, without modifying + // state snapshots or the current chain height. + PruneBlocks(ctx context.Context, height uint64) error +} diff --git a/test/mocks/store.go b/test/mocks/store.go index 79b3ac6f5..44f81c1d6 100644 --- a/test/mocks/store.go +++ b/test/mocks/store.go @@ -39,6 +39,14 @@ func (_m *MockStore) EXPECT() *MockStore_Expecter { return &MockStore_Expecter{mock: &_m.Mock} } +// PruneBlocks provides a mock implementation for the Store's pruning method. +// Tests using MockStore currently do not exercise pruning behavior, so this +// method simply satisfies the interface and can be extended with expectations +// later if needed. +func (_mock *MockStore) PruneBlocks(ctx context.Context, height uint64) error { + return nil +} + // Close provides a mock function for the type MockStore func (_mock *MockStore) Close() error { ret := _mock.Called()