Return SYNCING instead of INVALID to op-node when a forkchoice update…

… requests progression to a tip which fails a state transition due to TBC full node missing a full block or header.

Correctly identify missing TBC full node header in TBCBlocksAvailableToHeader

core/blockchain.go

@@ -1331,7 +1331,7 @@ func (bc *BlockChain) applyHvmHeaderConsensusUpdate(header *types.Header) error
 		// Proactively check for any missing full blocks between full TBC's current indexed height and the
 		// new canonical tip that resulted from adding the new headers, and trigger a fetch from peers for
 		// any blocks that we will need in the future when the full TBC node's indexers are advanced.
-		// (bool, *[]wire.BlockHeader, *chainhash.Hash, error)
+
 		// Convert tbcd.BlockHeader (contains position and cumulative diff. info) to wire.BlockHeader
 		cbhWire, err := cbh.Wire()
 		if err != nil {
@@ -3768,7 +3768,7 @@ func (bc *BlockChain) insertChain(chain types.Blocks, setHead bool) (int, error)
 							"inserting block %s @ %d", block.Hash().String(), block.NumberU64()))
 					}
 
-					if errors.Is(err, consensus.ErrFullTBCMissingFullBTCBlock) {
+					if errors.Is(err, consensus.ErrFullTBCMissingFullBTCBlock) || errors.Is(err, consensus.ErrFullTBCMissingBTCHeader) {
 						// This might recover, add this block as future block for later processing
 						bc.futureBlocks.Add(block.Hash(), block)
 						return it.index, err

core/vm/contracts.go

@@ -457,7 +457,7 @@ func TBCBlocksAvailableToHeader(ctx context.Context, endingHeader *wire.BlockHea
 
 	missingFullBlocks := make([]wire.BlockHeader, 0)
 
-	log.Debug(fmt.Sprintf("TBCBlocksAvailableToHeader called with endingHeader=%s, UTXOs synced to: "+
+	log.Info(fmt.Sprintf("TBCBlocksAvailableToHeader called with endingHeader=%s, UTXOs synced to: "+
 		"%s and Txs synced to: %s", endingHeader.BlockHash().String(), utxoSync.Hash.String(), txSync.Hash.String()))
 
 	// When both indexers are at the same header, this will be that header.
@@ -478,8 +478,10 @@ func TBCBlocksAvailableToHeader(ctx context.Context, endingHeader *wire.BlockHea
 	targetHH, err := hashHeightForHeader(ctx, endingHeader)
 	if err != nil {
 		if errors.As(err, &database.ErrNotFound) {
+			endingHeaderHash := endingHeader.BlockHash()
+			log.Warn(fmt.Sprintf("Header %s not found", endingHeaderHash.String()), "err", err)
 			// TBC full node does not know about the ending header
-			return false, nil, &targetHH.Hash, nil
+			return false, nil, &endingHeaderHash, nil
 		}
 		return false, nil, nil, err
 	}

eth/catalyst/api.go

@@ -20,6 +20,7 @@ package catalyst
 import (
 	"errors"
 	"fmt"
+	"github.com/ethereum/go-ethereum/consensus"
 	"sync"
 	"time"
 
@@ -627,12 +628,25 @@ func (api *ConsensusAPI) newPayload(params engine.ExecutableData, versionedHashe
 	if err := api.eth.BlockChain().InsertBlockWithoutSetHead(block); err != nil {
 		log.Warn("NewPayloadV1: inserting block failed", "error", err)
 
-		api.invalidLock.Lock()
-		api.invalidBlocksHits[block.Hash()] = 1
-		api.invalidTipsets[block.Hash()] = block.Header()
-		api.invalidLock.Unlock()
+		if errors.Is(err, consensus.ErrFullTBCMissingFullBTCBlock) {
+			log.Info("error is full TBC missing full BTC block, delaying payload import")
+			// TBC is missing a full block which is a failure which could be recovered in future.
+			// Delay the payload import,
+			return api.delayPayloadImport(block)
+		} else if errors.Is(err, consensus.ErrFullTBCMissingBTCHeader) {
+			log.Info("error is full TBC missing a BTC header, delaying payload import")
+			// TBC is missing a block header which is a failure which could be recovered in future.
+			// Delay the payload import,
+			return api.delayPayloadImport(block)
+		} else {
+			log.Warn("InsertBlockWithoutSetHead failed but error is not full TBC missing full block or header!", "err", err)
+			api.invalidLock.Lock()
+			api.invalidBlocksHits[block.Hash()] = 1
+			api.invalidTipsets[block.Hash()] = block.Header()
+			api.invalidLock.Unlock()
 
-		return api.invalid(err, parent.Header()), nil
+			return api.invalid(err, parent.Header()), nil
+		}
 	}
 	// We've accepted a valid payload from the beacon client. Mark the local
 	// chain transitions to notify other subsystems (e.g. downloader) of the