feat(btc): Duplicate coinbase transaction handling (#7)

* feat(btc): tx_hash can duplicated in block v1

Co-authored-by: Gaze <dev@gaze.network>

* feat(btc): duplicate tx  will use same txin/txout from previous tx

Co-authored-by: Gaze <dev@gaze.network>

* feat(btc): prevent revert block v1 data

if you really want to revert the data before the block version 2, you should reset the database and reindex the data instead.

Co-authored-by: Gaze <dev@gaze.network>

* doc(btc): update list duplicate tx hash

Co-authored-by: Gaze <dev@gaze.network>

* doc(btc): update docs

Co-authored-by: Gaze <dev@gaze.network>

* fix(btc): use last v1 block instead

Co-authored-by: Gaze <dev@gaze.network>

---------

Co-authored-by: Gaze <gazenw@users.noreply.github.com>

modules/bitcoin/constants.go

@@ -1,6 +1,8 @@
 package bitcoin
 
 import (
+	"github.com/Cleverse/go-utilities/utils"
+	"github.com/btcsuite/btcd/chaincfg/chainhash"
 	"github.com/gaze-network/indexer-network/common"
 	"github.com/gaze-network/indexer-network/core/types"
 )
@@ -10,8 +12,15 @@ const (
 	DBVersion = 1
 )
 
-// DefaultCurrentBlockHeight is the default value for the current block height for first time indexing
-var defaultCurrentBlock = types.BlockHeader{
-	Hash:   common.ZeroHash,
-	Height: -1,
-}
+var (
+	// defaultCurrentBlockHeight is the default value for the current block height for first time indexing
+	defaultCurrentBlock = types.BlockHeader{
+		Hash:   common.ZeroHash,
+		Height: -1,
+	}
+
+	lastV1Block = types.BlockHeader{
+		Hash:   *utils.Must(chainhash.NewHashFromStr("00000000000001aa077d7aa84c532a4d69bdbff519609d1da0835261b7a74eb6")),
+		Height: 227835,
+	}
+)

modules/bitcoin/database/postgresql/migrations/000001_initialize_table.down.sql

@@ -2,8 +2,8 @@ BEGIN;
 
 -- DROP INDEX
 DROP INDEX IF EXISTS bitcoin_blocks_block_hash_idx;
+DROP INDEX IF EXISTS bitcoin_transactions_tx_hash_idx;
 DROP INDEX IF EXISTS bitcoin_transactions_block_hash_idx;
-DROP INDEX IF EXISTS bitcoin_transactions_block_height_idx;
 DROP INDEX IF EXISTS bitcoin_transaction_txouts_pkscript_idx;
 DROP INDEX IF EXISTS bitcoin_transaction_txins_prevout_idx;
 

modules/bitcoin/database/postgresql/migrations/000001_initialize_table.up.sql

@@ -32,16 +32,17 @@ CREATE TABLE IF NOT EXISTS "bitcoin_blocks" (
 CREATE INDEX IF NOT EXISTS bitcoin_blocks_block_hash_idx ON "bitcoin_blocks" USING HASH ("block_hash");
 
 CREATE TABLE IF NOT EXISTS "bitcoin_transactions" (
-	"tx_hash" TEXT NOT NULL PRIMARY KEY,
+	"tx_hash" TEXT NOT NULL, -- can't use as primary key because block v1 has duplicate tx hashes (coinbase tx). See: https://github.com/bitcoin/bitcoin/commit/a206b0ea12eb4606b93323268fc81a4f1f952531
 	"version" INT NOT NULL,
 	"locktime" BIGINT NOT NULL,
 	"block_height" INT NOT NULL,
 	"block_hash" TEXT NOT NULL,
-	"idx" INT NOT NULL
+	"idx" INT NOT NULL,
+	PRIMARY KEY ("block_height", "idx")
 );
 
-CREATE INDEX IF NOT EXISTS bitcoin_transactions_block_height_idx ON "bitcoin_transactions" USING BTREE ("block_height");
-CREATE INDEX IF NOT EXISTS bitcoin_transactions_block_hash_idx ON "bitcoin_transactions" USING BTREE ("block_hash");
+CREATE INDEX IF NOT EXISTS bitcoin_transactions_tx_hash_idx ON "bitcoin_transactions" USING HASH ("tx_hash");
+CREATE INDEX IF NOT EXISTS bitcoin_transactions_block_hash_idx ON "bitcoin_transactions" USING HASH ("block_hash");
 
 CREATE TABLE IF NOT EXISTS "bitcoin_transaction_txouts" (
 	"tx_hash" TEXT NOT NULL,

modules/bitcoin/processor.go

@@ -1,9 +1,7 @@
 package bitcoin
 
 import (
-	"cmp"
 	"context"
-	"slices"
 
 	"github.com/cockroachdb/errors"
 	"github.com/gaze-network/indexer-network/common/errs"
@@ -39,27 +37,10 @@ func (p *Processor) Process(ctx context.Context, inputs []*types.Block) error {
 		return nil
 	}
 
-	// Sort ASC by block height
-	slices.SortFunc(inputs, func(t1, t2 *types.Block) int {
-		return cmp.Compare(t1.Header.Height, t2.Header.Height)
-	})
-
-	latestBlock, err := p.CurrentBlock(ctx)
+	// Process the given blocks before inserting to the database
+	inputs, err := p.process(ctx, inputs)
 	if err != nil {
-		return errors.Wrap(err, "failed to get latest indexed block header")
-	}
-
-	// check if the given blocks are continue from the latest indexed block
-	// return an error to prevent inserting out-of-order blocks or duplicate blocks
-	if inputs[0].Header.Height != latestBlock.Height+1 {
-		return errors.New("given blocks are not continue from the latest indexed block")
-	}
-
-	// check if the given blocks are in sequence and not missing any block
-	for i := 1; i < len(inputs); i++ {
-		if inputs[i].Header.Height != inputs[i-1].Header.Height+1 {
-			return errors.New("given blocks are not in sequence")
-		}
+		return errors.WithStack(err)
 	}
 
 	// Insert blocks
@@ -90,6 +71,12 @@ func (p *Processor) GetIndexedBlock(ctx context.Context, height int64) (types.Bl
 }
 
 func (p *Processor) RevertData(ctx context.Context, from int64) error {
+	// to prevent remove txin/txout of duplicated coinbase transaction in the blocks 91842 and 91880
+	// if you really want to revert the data before the block `227835`, you should reset the database and reindex the data instead.
+	if from <= lastV1Block.Height {
+		return errors.Wrapf(errs.InvalidArgument, "can't revert data before block version 2, height: %d", lastV1Block.Height)
+	}
+
 	if err := p.bitcoinDg.RevertBlocks(ctx, from); err != nil {
 		return errors.WithStack(err)
 	}

modules/bitcoin/processor_process.go

@@ -0,0 +1,91 @@
+package bitcoin
+
+import (
+	"cmp"
+	"context"
+	"slices"
+
+	"github.com/cockroachdb/errors"
+	"github.com/gaze-network/indexer-network/core/types"
+)
+
+// process is a processing rules for the given blocks before inserting to the database
+//
+// this function will modify the given data directly.
+func (p *Processor) process(ctx context.Context, blocks []*types.Block) ([]*types.Block, error) {
+	if len(blocks) == 0 {
+		return blocks, nil
+	}
+
+	// Sort ASC by block height
+	slices.SortFunc(blocks, func(t1, t2 *types.Block) int {
+		return cmp.Compare(t1.Header.Height, t2.Header.Height)
+	})
+
+	if !p.isContinueFromLatestIndexedBlock(ctx, blocks[0]) {
+		return nil, errors.New("given blocks are not continue from the latest indexed block")
+	}
+
+	if !p.isBlocksSequential(blocks) {
+		return nil, errors.New("given blocks are not in sequence")
+	}
+
+	p.removeDuplicateCoinbaseTxInputsOutputs(blocks)
+
+	return blocks, nil
+}
+
+// check if the given blocks are continue from the latest indexed block
+// to prevent inserting out-of-order blocks or duplicate blocks
+func (p *Processor) isBlocksSequential(blocks []*types.Block) bool {
+	if len(blocks) == 0 {
+		return true
+	}
+
+	for i, block := range blocks {
+		if i == 0 {
+			continue
+		}
+
+		if block.Header.Height != blocks[i-1].Header.Height+1 {
+			return false
+		}
+	}
+
+	return true
+}
+
+// check if the given blocks are continue from the latest indexed block
+// to prevent inserting out-of-order blocks or duplicate blocks
+func (p *Processor) isContinueFromLatestIndexedBlock(ctx context.Context, block *types.Block) bool {
+	latestBlock, err := p.CurrentBlock(ctx)
+	if err != nil {
+		return false
+	}
+
+	return block.Header.Height == latestBlock.Height+1
+}
+
+// there 2 coinbase transaction that are duplicated in the blocks 91842 and 91880.
+// if the given block version is v1 and height is `91842` or `91880`,
+// then remove transaction inputs/outputs to prevent duplicate txin/txout error when inserting to the database.
+//
+// Theses duplicated coinbase transactions are having the same transaction input(from coinbase)/output and
+// utxo from these 2 duplicated coinbase txs can redeem only once), so, it's safe to remove them and can
+// use inputs/outputs from the previous block.
+//
+// Duplicate Coinbase Transactions:
+//   - `454279874213763724535987336644243549a273058910332236515429488599` in blocks 91812, 91842
+//   - `e3bf3d07d4b0375638d5f1db5255fe07ba2c4cb067cd81b84ee974b6585fb468` in blocks 91722, 91880
+//
+// This function will modify the given data directly.
+func (p *Processor) removeDuplicateCoinbaseTxInputsOutputs(blocks []*types.Block) {
+	for _, block := range blocks {
+		header := block.Header
+		if header.Version == 1 && (header.Height == 91842 || header.Height == 91880) {
+			// remove transaction inputs/outputs from coinbase transaction (first transaction)
+			block.Transactions[0].TxIn = nil
+			block.Transactions[0].TxOut = nil
+		}
+	}
+}

modules/bitcoin/processor_process_test.go

@@ -0,0 +1,144 @@
+package bitcoin
+
+import (
+	"fmt"
+	"testing"
+
+	"github.com/gaze-network/indexer-network/core/types"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestDuplicateCoinbaseTxHashHandling(t *testing.T) {
+	processor := Processor{}
+	generator := func() []*types.Block {
+		return []*types.Block{
+			{
+				Header: types.BlockHeader{Height: 91842, Version: 1},
+				Transactions: []*types.Transaction{
+					{
+						TxIn:  []*types.TxIn{{}, {}, {}, {}},
+						TxOut: []*types.TxOut{{}, {}, {}, {}},
+					},
+					{
+						TxIn:  []*types.TxIn{{}, {}, {}, {}},
+						TxOut: []*types.TxOut{{}, {}, {}, {}},
+					},
+				},
+			},
+			{
+				Header: types.BlockHeader{Height: 91880, Version: 1},
+				Transactions: []*types.Transaction{
+					{
+						TxIn:  []*types.TxIn{{}, {}, {}, {}},
+						TxOut: []*types.TxOut{{}, {}, {}, {}},
+					},
+					{
+						TxIn:  []*types.TxIn{{}, {}, {}, {}},
+						TxOut: []*types.TxOut{{}, {}, {}, {}},
+					},
+				},
+			},
+		}
+	}
+
+	t.Run("all_duplicated_txs", func(t *testing.T) {
+		blocks := generator()
+		processor.removeDuplicateCoinbaseTxInputsOutputs(blocks)
+
+		assert.Len(t, blocks, 2, "should not remove any blocks")
+		for _, block := range blocks {
+			assert.Len(t, block.Transactions, 2, "should not remove any transactions")
+			assert.Len(t, block.Transactions[0].TxIn, 0, "should remove tx inputs from coinbase transaction")
+			assert.Len(t, block.Transactions[0].TxOut, 0, "should remove tx outputs from coinbase transaction")
+		}
+	})
+
+	t.Run("not_duplicated_txs", func(t *testing.T) {
+		blocks := []*types.Block{
+			{
+				Header: types.BlockHeader{Height: 91812, Version: 1},
+				Transactions: []*types.Transaction{
+					{
+						TxIn:  []*types.TxIn{{}, {}, {}, {}},
+						TxOut: []*types.TxOut{{}, {}, {}, {}},
+					},
+					{
+						TxIn:  []*types.TxIn{{}, {}, {}, {}},
+						TxOut: []*types.TxOut{{}, {}, {}, {}},
+					},
+				},
+			},
+			{
+				Header: types.BlockHeader{Height: 91722, Version: 1},
+				Transactions: []*types.Transaction{
+					{
+						TxIn:  []*types.TxIn{{}, {}, {}, {}},
+						TxOut: []*types.TxOut{{}, {}, {}, {}},
+					},
+					{
+						TxIn:  []*types.TxIn{{}, {}, {}, {}},
+						TxOut: []*types.TxOut{{}, {}, {}, {}},
+					},
+				},
+			},
+		}
+		processor.removeDuplicateCoinbaseTxInputsOutputs(blocks)
+
+		assert.Len(t, blocks, 2, "should not remove any blocks")
+		for _, block := range blocks {
+			assert.Len(t, block.Transactions, 2, "should not remove any transactions")
+			assert.Len(t, block.Transactions[0].TxIn, 4, "should not remove tx inputs from coinbase transaction")
+			assert.Len(t, block.Transactions[0].TxOut, 4, "should not remove tx outputs from coinbase transaction")
+		}
+	})
+
+	t.Run("mixed", func(t *testing.T) {
+		blocks := []*types.Block{
+			{
+				Header: types.BlockHeader{Height: 91812, Version: 1},
+				Transactions: []*types.Transaction{
+					{
+						TxIn:  []*types.TxIn{{}, {}, {}, {}},
+						TxOut: []*types.TxOut{{}, {}, {}, {}},
+					},
+					{
+						TxIn:  []*types.TxIn{{}, {}, {}, {}},
+						TxOut: []*types.TxOut{{}, {}, {}, {}},
+					},
+				},
+			},
+		}
+		blocks = append(blocks, generator()...)
+		blocks = append(blocks, &types.Block{
+			Header: types.BlockHeader{Height: 91722, Version: 1},
+			Transactions: []*types.Transaction{
+				{
+					TxIn:  []*types.TxIn{{}, {}, {}, {}},
+					TxOut: []*types.TxOut{{}, {}, {}, {}},
+				},
+				{
+					TxIn:  []*types.TxIn{{}, {}, {}, {}},
+					TxOut: []*types.TxOut{{}, {}, {}, {}},
+				},
+			},
+		})
+		processor.removeDuplicateCoinbaseTxInputsOutputs(blocks)
+
+		assert.Len(t, blocks, 4, "should not remove any blocks")
+
+		// only 2nd and 3rd blocks should be modified
+		for i, block := range blocks {
+			t.Run(fmt.Sprint(i), func(t *testing.T) {
+				if i == 1 || i == 2 {
+					assert.Len(t, block.Transactions, 2, "should not remove any transactions")
+					assert.Len(t, block.Transactions[0].TxIn, 0, "should remove tx inputs from coinbase transaction")
+					assert.Len(t, block.Transactions[0].TxOut, 0, "should remove tx outputs from coinbase transaction")
+				} else {
+					assert.Len(t, block.Transactions, 2, "should not remove any transactions")
+					assert.Lenf(t, block.Transactions[0].TxIn, 4, "should not remove tx inputs from coinbase transaction")
+					assert.Len(t, block.Transactions[0].TxOut, 4, "should not remove tx outputs from coinbase transaction")
+				}
+			})
+		}
+	})
+}