From 227350acf18de0a7f86b28186df58420be045dd1 Mon Sep 17 00:00:00 2001 From: Raneet Debnath Date: Tue, 10 Dec 2024 17:08:37 +0530 Subject: [PATCH 1/6] cmd,store: rewind the latest block when executing rollback --- cmd/tendermint/commands/rollback.go | 1 + store/store.go | 25 +++++++++++++++++++++++++ 2 files changed, 26 insertions(+) diff --git a/cmd/tendermint/commands/rollback.go b/cmd/tendermint/commands/rollback.go index 5b933a7a7c5..e750d7cb379 100644 --- a/cmd/tendermint/commands/rollback.go +++ b/cmd/tendermint/commands/rollback.go @@ -153,5 +153,6 @@ func rollbackState(blockStoreDB, stateDB db.DB) (int64, []byte, error) { // saving the state state.SaveState(stateDB, rolledBackState) + blockStore.RemoveLatestBlock() return rolledBackState.LastBlockHeight, rolledBackState.AppHash, nil } diff --git a/store/store.go b/store/store.go index d8ec2f022d0..f7ab95f1e2e 100644 --- a/store/store.go +++ b/store/store.go @@ -188,6 +188,31 @@ func (bs *BlockStore) SaveBlock(block *types.Block, blockParts *types.PartSet, s bs.db.SetSync(nil, nil) } +// RemoveLatestBlock deletes the latest block from the store +// and rewinds the latest height by 1. +func (bs *BlockStore) RemoveLatestBlock() { + bs.mtx.RLock() + height := bs.Height() + bs.mtx.RUnlock() + + blockMeta := bs.LoadBlockMeta(height) + // delete block parts + for i := 0; i < int(blockMeta.BlockID.PartsHeader.Total); i++ { + bs.db.Delete(calcBlockPartKey(height, i)) + } + + bs.db.Delete(calcBlockMetaKey(height)) + bs.db.Delete(calcBlockCommitKey(height)) + bs.db.Delete(calcSeenCommitKey(height)) + + // rollback height + bs.mtx.Lock() + bs.height = height - 1 + bs.mtx.Unlock() + + bs.db.SetSync(nil, nil) +} + func (bs *BlockStore) saveBlockPart(height int64, index int, part *types.Part) { if height != bs.Height()+1 { panic(fmt.Sprintf("BlockStore can only save contiguous blocks. Wanted %v, got %v", bs.Height()+1, height)) From 0d70f7a7c94661e840ba115ba1beadf26aca070c Mon Sep 17 00:00:00 2001 From: Raneet Debnath Date: Tue, 10 Dec 2024 17:26:28 +0530 Subject: [PATCH 2/6] cmd: close stores upon completion --- cmd/tendermint/commands/rollback.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/cmd/tendermint/commands/rollback.go b/cmd/tendermint/commands/rollback.go index e750d7cb379..84c30448257 100644 --- a/cmd/tendermint/commands/rollback.go +++ b/cmd/tendermint/commands/rollback.go @@ -44,6 +44,10 @@ application. func RollbackState(config *cfg.Config) (int64, []byte, error) { // use the parsed config to load the block and state store blocksdb, statedb, err := loadStateAndBlockStore(config) + defer func() { + blocksdb.Close() + statedb.Close() + }() if err != nil { return -1, nil, err } From 85e09eecd5e4a0da4388ab882afd4a6ef178d79f Mon Sep 17 00:00:00 2001 From: Raneet Debnath Date: Tue, 10 Dec 2024 18:19:55 +0530 Subject: [PATCH 3/6] store: remove db.setsync --- store/store.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/store/store.go b/store/store.go index f7ab95f1e2e..5779ab40ebe 100644 --- a/store/store.go +++ b/store/store.go @@ -210,7 +210,7 @@ func (bs *BlockStore) RemoveLatestBlock() { bs.height = height - 1 bs.mtx.Unlock() - bs.db.SetSync(nil, nil) + // bs.db.SetSync(nil, nil) } func (bs *BlockStore) saveBlockPart(height int64, index int, part *types.Part) { From 80c5bb86e478095d83f7290877bec1d9fd44daa7 Mon Sep 17 00:00:00 2001 From: Raneet Debnath Date: Tue, 10 Dec 2024 18:32:13 +0530 Subject: [PATCH 4/6] store: use batch --- store/store.go | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/store/store.go b/store/store.go index 5779ab40ebe..49e94a71dbf 100644 --- a/store/store.go +++ b/store/store.go @@ -191,6 +191,9 @@ func (bs *BlockStore) SaveBlock(block *types.Block, blockParts *types.PartSet, s // RemoveLatestBlock deletes the latest block from the store // and rewinds the latest height by 1. func (bs *BlockStore) RemoveLatestBlock() { + batch := bs.db.NewBatch() + defer batch.Close() + bs.mtx.RLock() height := bs.Height() bs.mtx.RUnlock() @@ -198,19 +201,19 @@ func (bs *BlockStore) RemoveLatestBlock() { blockMeta := bs.LoadBlockMeta(height) // delete block parts for i := 0; i < int(blockMeta.BlockID.PartsHeader.Total); i++ { - bs.db.Delete(calcBlockPartKey(height, i)) + batch.Delete(calcBlockPartKey(height, i)) } - bs.db.Delete(calcBlockMetaKey(height)) - bs.db.Delete(calcBlockCommitKey(height)) - bs.db.Delete(calcSeenCommitKey(height)) + batch.Delete(calcBlockMetaKey(height)) + batch.Delete(calcBlockCommitKey(height)) + batch.Delete(calcSeenCommitKey(height)) // rollback height bs.mtx.Lock() bs.height = height - 1 bs.mtx.Unlock() - // bs.db.SetSync(nil, nil) + batch.WriteSync() } func (bs *BlockStore) saveBlockPart(height int64, index int, part *types.Part) { From 6d022b3fe2ce3b6d006cafc990ecd9ccc0b244db Mon Sep 17 00:00:00 2001 From: Raneet Debnath Date: Wed, 11 Dec 2024 10:53:43 +0530 Subject: [PATCH 5/6] cmd: add force flag --- cmd/tendermint/commands/rollback.go | 29 ++++++++++++++++++----------- 1 file changed, 18 insertions(+), 11 deletions(-) diff --git a/cmd/tendermint/commands/rollback.go b/cmd/tendermint/commands/rollback.go index 84c30448257..88ddcf818e2 100644 --- a/cmd/tendermint/commands/rollback.go +++ b/cmd/tendermint/commands/rollback.go @@ -5,6 +5,7 @@ import ( "path/filepath" "github.com/spf13/cobra" + "github.com/spf13/viper" cfg "github.com/tendermint/tendermint/config" "github.com/tendermint/tendermint/libs/os" "github.com/tendermint/tendermint/state" @@ -13,21 +14,24 @@ import ( db "github.com/tendermint/tm-db" ) +const flagForce = "force" + func MakeRollbackStateCommand() *cobra.Command { - confg := cfg.DefaultConfig() - return &cobra.Command{ + config := cfg.DefaultConfig() + cmd := &cobra.Command{ Use: "rollback", Short: "rollback tendermint state by one height", Long: ` A state rollback is performed to recover from an incorrect application state transition, when Tendermint has persisted an incorrect app hash and is thus unable to make progress. Rollback overwrites a state at height n with the state at height n - 1. -The application should also roll back to height n - 1. No blocks are removed, so upon -restarting Tendermint the transactions in block n will be re-executed against the -application. +The application should also roll back to height n - 1. Upon using the --force flag, tendermint +will rollback the state and delete the block n from the blockstore. This is useful when the +application returned a bad state upon processing the block n - 1. `, RunE: func(cmd *cobra.Command, args []string) error { - height, hash, err := RollbackState(confg) + forceRollback := viper.GetBool(flagForce) + height, hash, err := RollbackState(config, forceRollback) if err != nil { return fmt.Errorf("failed to rollback state: %w", err) } @@ -36,12 +40,15 @@ application. return nil }, } + + cmd.Flags().Bool(flagForce, false, "force rollback") + return cmd } // RollbackState takes the state at the current height n and overwrites it with the state // at height n - 1. Note state here refers to tendermint state not application state. // Returns the latest state height and app hash alongside an error if there was one. -func RollbackState(config *cfg.Config) (int64, []byte, error) { +func RollbackState(config *cfg.Config, forceRollback bool) (int64, []byte, error) { // use the parsed config to load the block and state store blocksdb, statedb, err := loadStateAndBlockStore(config) defer func() { @@ -52,7 +59,7 @@ func RollbackState(config *cfg.Config) (int64, []byte, error) { return -1, nil, err } // rollback the last state - return rollbackState(blocksdb, statedb) + return rollbackState(blocksdb, statedb, forceRollback) } func loadStateAndBlockStore(cfg *cfg.Config) (db.DB, db.DB, error) { @@ -75,17 +82,17 @@ func loadStateAndBlockStore(cfg *cfg.Config) (db.DB, db.DB, error) { return blockStoreDB, stateDB, nil } -func rollbackState(blockStoreDB, stateDB db.DB) (int64, []byte, error) { +func rollbackState(blockStoreDB, stateDB db.DB, forceRollback bool) (int64, []byte, error) { blockStore := store.NewBlockStore(blockStoreDB) invalidState := state.LoadState(stateDB) height := blockStore.Height() // skip - if height == invalidState.LastBlockHeight+1 { + if height == invalidState.LastBlockHeight+1 && !forceRollback { return invalidState.LastBlockHeight, invalidState.AppHash, nil } - if height != invalidState.LastBlockHeight { + if height != invalidState.LastBlockHeight && !forceRollback { return -1, nil, fmt.Errorf("statestore height (%d) is not one below or equal to blockstore height (%d)", invalidState.LastBlockHeight, height) } From c89e1767f476e5dd65e970413e9d6c95251e6528 Mon Sep 17 00:00:00 2001 From: Raneet Debnath Date: Thu, 12 Dec 2024 11:33:46 +0530 Subject: [PATCH 6/6] cmd,store: rollback BlockStoreStateJSON --- cmd/tendermint/commands/rollback.go | 2 +- store/store.go | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/cmd/tendermint/commands/rollback.go b/cmd/tendermint/commands/rollback.go index 88ddcf818e2..a9324fe9c21 100644 --- a/cmd/tendermint/commands/rollback.go +++ b/cmd/tendermint/commands/rollback.go @@ -87,7 +87,7 @@ func rollbackState(blockStoreDB, stateDB db.DB, forceRollback bool) (int64, []by invalidState := state.LoadState(stateDB) height := blockStore.Height() - // skip + // skip rollback if the state is already at the last block height if height == invalidState.LastBlockHeight+1 && !forceRollback { return invalidState.LastBlockHeight, invalidState.AppHash, nil } diff --git a/store/store.go b/store/store.go index 49e94a71dbf..92c834f3844 100644 --- a/store/store.go +++ b/store/store.go @@ -208,6 +208,9 @@ func (bs *BlockStore) RemoveLatestBlock() { batch.Delete(calcBlockCommitKey(height)) batch.Delete(calcSeenCommitKey(height)) + // rollback BlockStoreStateJSON + BlockStoreStateJSON{Height: height - 1}.Save(bs.db) + // rollback height bs.mtx.Lock() bs.height = height - 1