Files
gaze-brc20-indexer/modules/bitcoin/repository/postgres/mappers.go
gazenw fcdecd4046 feat: v0.1.0 release (#13)
* fix: don't remove first block

* fix: make etching_terms nullable

* fix: fix panic if empty pkscript

* chore: change testnet starting block

* feat: more logs

* fix: extract tapscript bug

* feat: more logs

* fix: switch pk to block height

* chore: remove redundant log

* fix: repo

* fix: not found error

* fix: golangci-lint

* feat: add etching tx hash to rune entries

* feat: stop main if indexer failed

* fix: check balance after populating current balance

* fix: sql ambiguous column

* feat: add tx hash and out index in tx output

* fix: actually use transactions to write db

* fix: create rune entry states only during flushes

* fix: mint cap reached off by one

* fix: debug log unsafe

* feat: prevent processing of txs before activation height

* feat: add rune number to rune entry

* feat: include new rune entries in event hash and flushing

* refactor(config): separate init and get config func

Co-authored-by: Gaze <dev@gaze.network>

* feat: remove annoying log

Co-authored-by: Gaze <dev@gaze.network>

* feat: mod tidy

Co-authored-by: Gaze <dev@gaze.network>

* refactor: move main to root

Co-authored-by: Gaze <dev@gaze.network>

* feat(cli): create cli commands

Co-authored-by: Gaze <dev@gaze.network>

* refactor: move main logic to command

Co-authored-by: Gaze <dev@gaze.network>

* doc: remove unused desc

Co-authored-by: Gaze <dev@gaze.network>

* refactor: test structure in runestone_test.go

* fix: edict flaws were ignored

* feat: more tests

* refactor(cli): add local flag

Co-authored-by: Gaze <dev@gaze.network>

* feat: set symbol limit to utf8.MaxRune

* refactor(cli): flags for each module

Co-authored-by: Gaze <dev@gaze.network>

* feat(cli): support db selection

Co-authored-by: Gaze <dev@gaze.network>

* fix: remove temp code

Co-authored-by: Gaze <dev@gaze.network>

* fix: get data from cache in processor first, then dg

* feat(cli): add version command

Co-authored-by: Gaze <dev@gaze.network>

* doc(cli): add refactor plan

Co-authored-by: Gaze <dev@gaze.network>

* refactor(cli): rename files

Co-authored-by: Gaze <dev@gaze.network>

* feat: add main.go

Co-authored-by: Gaze <dev@gaze.network>

* feat: more tests

* feat: add overflow err

* feat: finish runestone tests

* refactor(cli): separate protocol config and cli flag

Co-authored-by: Gaze <dev@gaze.network>

* chore(btc): update example config

Co-authored-by: Gaze <dev@gaze.network>

* feat(btc): add get block header to datasource interface

Co-authored-by: Gaze <dev@gaze.network>

* feat(btc): reorg handling

Co-authored-by: Gaze <dev@gaze.network>

* fix: interface

Co-authored-by: Gaze <dev@gaze.network>

* fix: rename postgres config key

* fix: migrated runes indexer integration to new cli

* fix: commit every block

* feat(btc): add revert data query

Co-authored-by: Gaze <dev@gaze.network>

* feat(btc): add revert data to processor

Co-authored-by: Gaze <dev@gaze.network>

* feat: implement public errors

* fix: use errs in api

* refactor: move api and usecase outside of internal

* feat: add custom opcode check for datapush

* fix: break if input utxo is not P2TR

* fix: zero len destination case

* fix: get the rest of transaction data in GetTransaction

* refactor: create subscription utils tools

Co-authored-by: Gaze <dev@gaze.network>

* feat(btc): add btc_database from datasource

Co-authored-by: Gaze <dev@gaze.network>

* doc(btc): add note

Co-authored-by: Gaze <dev@gaze.network>

* wip(btc): imple prepare range func

Co-authored-by: Gaze <dev@gaze.network>

* feat(btc): add pg queries for datasource

Co-authored-by: Gaze <dev@gaze.network>

* feat(btc): update queries

Co-authored-by: Gaze <dev@gaze.network>

* feat(btc): implement repo for get blocks

Co-authored-by: Gaze <dev@gaze.network>

* feat(btc): update dg

Co-authored-by: Gaze <dev@gaze.network>

* fix(btc): return nil if errors

Co-authored-by: Gaze <dev@gaze.network>

* feat(btc): update fetch async for db datasource

Co-authored-by: Gaze <dev@gaze.network>

* feat(btc): add get block header from db for reorg handling

Co-authored-by: Gaze <dev@gaze.network>

* feat(btc): add todo notes

Co-authored-by: Gaze <dev@gaze.network>

* feat: implement get tx by hash

* fix: rename func

* fix: rename func

* fix: rename func

* fix: fix get transaction by hash

* feat: integrate bitcoin client db to main

* fix: reduce chunk size

* fix: stop main if bitcoin indexer failed

* fix: stop main if runes indexer failed

* fix: move stop() inside goroutine

* chore: add log

* fix: duplicate rune entry number

* feat(btc): add witness utils

Co-authored-by: Gaze <dev@gaze.network>

* feat(btc): witness datamodel parsing

Co-authored-by: Gaze <dev@gaze.network>

* fix(btc): invalid table name

Co-authored-by: Gaze <dev@gaze.network>

* fix(btc): remove uniqte index for hash

Co-authored-by: Gaze <dev@gaze.network>

* doc: add todo

Co-authored-by: Gaze <dev@gaze.network>

* feat(logger): remove error verbose

Co-authored-by: Gaze <dev@gaze.network>

* feat: support postgresql db

Co-authored-by: Gaze <dev@gaze.network>

* feat(btc): add err notfound

Co-authored-by: Gaze <dev@gaze.network>

* fix: invalid pgx version

Co-authored-by: Gaze <dev@gaze.network>

* fix: invalid indexer flow

Co-authored-by: Gaze <dev@gaze.network>

* feat: refactor runes api

* feat: implement http server

* fix: mount runes api

* fix: error handler

* fix: first empty state error

Co-authored-by: Gaze <dev@gaze.network>

* fix: off by one confirmation

* ci: ignore RollBack error

* fix: change WithPublicMessage to be prefix

* feat: bump cstream version

Co-authored-by: Gaze <dev@gaze.network>

* feat(btc): nullable pkscript

Co-authored-by: Gaze <dev@gaze.network>

* feat(btc): change rollback style

Co-authored-by: Gaze <dev@gaze.network>

* refactor: move runes out of internal

* feat: rename id field to runeId in rune transaction

* feat(btc): update index

Co-authored-by: Gaze <dev@gaze.network>

* feat(btc): add default current block

Co-authored-by: Gaze <dev@gaze.network>

* doc: add note

Co-authored-by: Gaze <dev@gaze.network>

* fix(btc): use int64 to store sequence

Co-authored-by: Gaze <dev@gaze.network>

* fix(btc): upgrade data type for numbers

Co-authored-by: Gaze <dev@gaze.network>

* feat(btc):  upgrade data type for idx

Co-authored-by: Gaze <dev@gaze.network>

* feat(btc): get indexed block impl

Co-authored-by: Gaze <dev@gaze.network>

* feat(btc): add common.ZeroHash

Co-authored-by: Gaze <dev@gaze.network>

* feat: add chainparam

* feat: implement get transactions

* fix: wrong condition for non-OP_RETURN output

* feat(btc): add verify indexer states

Co-authored-by: Gaze <dev@gaze.network>

* refactor: sorting code

Co-authored-by: Gaze <dev@gaze.network>

* feat: fix interface

* feat(btc): update chuunk size

Co-authored-by: Gaze <dev@gaze.network>

* feat: add rune_etched column in rune transaction

* fix: missing field in create

* feat: add runeEtched in get transactions

* feat: implement get token info

* feat: add holders count in token info

* feat: implement get holders

* fix: return a new repository when beginning a new tx

* fix: rename type

* feat: add pkscript to outpoint balance

* feat: implement get utxos by address api

* fix: spend outpoint bug

* feat: implement get balances by address batch

* feat: sort balances result by amount

* ci: create Dockerfile

Co-authored-by: Gaze <dev@gaze.network>

* ci: add arg run

Co-authored-by: Gaze <dev@gaze.network>

* perf: add automaxprocs

Co-authored-by: Gaze <dev@gaze.network>

* chore: add performance logging

Co-authored-by: Gaze <dev@gaze.network>

* chore: add performance logger for debyug

Co-authored-by: Gaze <dev@gaze.network>

* fix: empty etched at

* fix: revert data sequentially

* fix: remove unused funcs

* fix: main.go

* feat: add flag --api-only to run cmd

* fix: create index

* fix: don't add zero mint to unallocated

* fix: ignore zero burn amount

* feat(reorg): add reorg detail

Co-authored-by: Gaze <dev@gaze.network>

* fix: wrong index type

* feat: implement reporting client to report runes blocks

* feat: implement report node

* feat(runes): add latest block api

Co-authored-by: Gaze <dev@gaze.network>

* feat(btc): use logger warn

Co-authored-by: Gaze <dev@gaze.network>

* fix(btc): txout aren't revert if it's have to revert spent

Co-authored-by: Gaze <dev@gaze.network>

* fix: annoying error when unsubscribe fetcher

Co-authored-by: Gaze <dev@gaze.network>

* refactor(btc): readable code

Co-authored-by: Gaze <dev@gaze.network>

* fix(indexer): fix subscription closed before process when success fetch

Co-authored-by: Gaze <dev@gaze.network>

* fix: remove module enum

* fix: increase max reorg limit

* feat: add starting height for runes mainnet

* fix(btc): fix `with` modified same row twice

Co-authored-by: Gaze <dev@gaze.network>

* fix(runes): handling latest block not found

Co-authored-by: Gaze <dev@gaze.network>

* feat: add decimals in get transactions

* fix: wrong condition

* feat: add more index

* feat: implement get transactions by pkscript

* feat: allow query by rune id too

* feat: more comments

* perf(btc): bitcoin indexer performance optimization (#4)

* feat(btc): not null to witness

Co-authored-by: Gaze <dev@gaze.network>

* perf(btc): add batch insert txin

Co-authored-by: Gaze <dev@gaze.network>

* perf(btc): batch insert txout

Co-authored-by: Gaze <dev@gaze.network>

* perf(btc): batch insert transaction

Co-authored-by: Gaze <dev@gaze.network>

* feat(btc): remove old queries

Co-authored-by: Gaze <dev@gaze.network>

* fix(btc): typo

Co-authored-by: Gaze <dev@gaze.network>

* perf(btc): batch insert blocks (#5)

Co-authored-by: Gaze <gazenw@users.noreply.github.com>

---------

Co-authored-by: Gaze <gazenw@users.noreply.github.com>

* feat(btc): Duplicate coinbase transaction handling (#7)

* feat(btc): tx_hash can duplicated in block v1

Co-authored-by: Gaze <dev@gaze.network>

* feat(btc): duplicate tx  will use same txin/txout from previous tx

Co-authored-by: Gaze <dev@gaze.network>

* feat(btc): prevent revert block v1 data

if you really want to revert the data before the block version 2, you should reset the database and reindex the data instead.

Co-authored-by: Gaze <dev@gaze.network>

* doc(btc): update list duplicate tx hash

Co-authored-by: Gaze <dev@gaze.network>

* doc(btc): update docs

Co-authored-by: Gaze <dev@gaze.network>

* fix(btc): use last v1 block instead

Co-authored-by: Gaze <dev@gaze.network>

---------

Co-authored-by: Gaze <gazenw@users.noreply.github.com>

* feat: add ping handler

* fix: type

Co-authored-by: Gaze <dev@gaze.network>

* doc: add refactor note

Co-authored-by: Gaze <dev@gaze.network>

* ci: add golang linter and test runner gh action

* ci: use go-test-action@v0

* ci: annotate test result

* ci: update running flag

* fix: try to fix malformed import path

* feat: add mock test

* ci: remove annotation ci

* ci: add annotate test result

* chore: remove unused

* feat: try testify

* feat: remove test

* ci: add go test on macos, windows and go latest version

* ci: test building

* feat: remove mock code

* ci: add sqlc diff checker action (#10)

* feat: Graceful shutdown (#8)

* feat: add shutdown function for indexer

Co-authored-by: Gaze <dev@gaze.network>

* feat: add force shutdown

Co-authored-by: Gaze <dev@gaze.network>

* revert

Co-authored-by: Gaze <dev@gaze.network>

* feat(btc): remove unused

Co-authored-by: Gaze <dev@gaze.network>

* style: go fmt

Co-authored-by: Gaze <dev@gaze.network>

* feat: separate context for worker and application

* feat: increase force shutdown timeout

Co-authored-by: Gaze <dev@gaze.network>

* feat(btc): update logging

Co-authored-by: Gaze <dev@gaze.network>

* feat(btc): update shutdown function

Co-authored-by: Gaze <dev@gaze.network>

* feat: remove wg for shutdown

Co-authored-by: Gaze <dev@gaze.network>

* feat: refactor shutdown flow

Co-authored-by: Gaze <dev@gaze.network>

* feat: update shutdown flow

Co-authored-by: Gaze <dev@gaze.network>

* feat: update maming

Co-authored-by: Gaze <dev@gaze.network>

* feat: update force shutdown logic

Co-authored-by: Gaze <dev@gaze.network>

---------

Co-authored-by: Gaze <gazenw@users.noreply.github.com>

* feat: check reporting config name

* fix: use db config in bitcoin module for runes datasource

* Add migrate commands (#2)

* feat: add migrate up

* feat: add down migration

* fix: example

* feat: change description

* fix: hardcode migration source directory

* Update README.md for public release. (#11)

* feat: initial draft for README.md

* fix: remove some sections

* feat: add block reporting to first description

* fix: reduce redundancy

* feat: update README.md

* Update README.md

* feat: update README.md

* fix: update config.yaml in README

* fix: remove redundant words

* fix: change default datasource

* fix: config.yaml comments

* feat: update README.md

* refactor(logger): format logging (#12)

* feat(logger): format main logger

* feat(logger): use duration ms for gcp output

* refactor(logger): bitcoin node logger

* refactor(logger): indexer logger

* refactor(logger): fix cmd logger

* refactor(logger): logger in config pacakge

* refactor(logger): set pgx error log level debug

* refactor(logger): btcclient datasource

* refactor: processor name

* refactor(logger): runese logger

* refactor(logger): update logger

* fix(runes): wrong btc db datasource

* refactor(logger): remove unnecessary debug log

* refactor: update logger in indexer

* fix(logger): deadlock in load()

* fix: remove unused

---------

Co-authored-by: Gaze <gazenw@users.noreply.github.com>

* feat(btc): remove unused func

* fix: fix golangci-lint error

* fix(pg): update logger level

* doc: update config example

* feat: go mod tidy

* doc: update readme

* fix: panic cause didn't handle error

* doc: update example config

* doc: update example config in readme

* feat(logger): only log error stacktrace when debug mode is on

* feat(reporting): handling invalid config error

* feat(pg): handling invalid config error

* fix: panic in get_token_info

---------

Co-authored-by: Gaze <gazenw@users.noreply.github.com>
Co-authored-by: Planxnx <thanee@cleverse.com>
Co-authored-by: Thanee Charattrakool <37617738+Planxnx@users.noreply.github.com>
2024-04-29 15:16:10 +07:00

198 lines
7.0 KiB
Go

package postgres
import (
"cmp"
"encoding/hex"
"slices"
"github.com/btcsuite/btcd/chaincfg/chainhash"
"github.com/cockroachdb/errors"
"github.com/gaze-network/indexer-network/common/errs"
"github.com/gaze-network/indexer-network/core/types"
"github.com/gaze-network/indexer-network/modules/bitcoin/repository/postgres/gen"
"github.com/gaze-network/indexer-network/pkg/btcutils"
"github.com/jackc/pgx/v5/pgtype"
)
func mapBlockHeaderModelToType(src gen.BitcoinBlock) (types.BlockHeader, error) {
hash, err := chainhash.NewHashFromStr(src.BlockHash)
if err != nil {
return types.BlockHeader{}, errors.Join(errors.Wrap(err, "failed to parse block hash"), errs.InternalError)
}
prevHash, err := chainhash.NewHashFromStr(src.PrevBlockHash)
if err != nil {
return types.BlockHeader{}, errors.Join(errors.Wrap(err, "failed to parse prev block hash"), errs.InternalError)
}
merkleRoot, err := chainhash.NewHashFromStr(src.MerkleRoot)
if err != nil {
return types.BlockHeader{}, errors.Join(errors.Wrap(err, "failed to parse merkle root"), errs.InternalError)
}
return types.BlockHeader{
Hash: *hash,
Height: int64(src.BlockHeight),
Version: src.Version,
PrevBlock: *prevHash,
MerkleRoot: *merkleRoot,
Timestamp: src.Timestamp.Time,
Bits: uint32(src.Bits),
Nonce: uint32(src.Nonce),
}, nil
}
func mapBlocksTypeToParams(src []*types.Block) (gen.BatchInsertBlocksParams, gen.BatchInsertTransactionsParams, gen.BatchInsertTransactionTxOutsParams, gen.BatchInsertTransactionTxInsParams) {
blocks := gen.BatchInsertBlocksParams{
BlockHeightArr: make([]int32, 0, len(src)),
BlockHashArr: make([]string, 0, len(src)),
VersionArr: make([]int32, 0, len(src)),
MerkleRootArr: make([]string, 0, len(src)),
PrevBlockHashArr: make([]string, 0, len(src)),
TimestampArr: make([]pgtype.Timestamptz, 0, len(src)),
BitsArr: make([]int64, 0, len(src)),
NonceArr: make([]int64, 0, len(src)),
}
txs := gen.BatchInsertTransactionsParams{
TxHashArr: []string{},
VersionArr: []int32{},
LocktimeArr: []int64{},
BlockHeightArr: []int32{},
BlockHashArr: []string{},
IdxArr: []int32{},
}
txouts := gen.BatchInsertTransactionTxOutsParams{
TxHashArr: []string{},
TxIdxArr: []int32{},
PkscriptArr: []string{},
ValueArr: []int64{},
}
txins := gen.BatchInsertTransactionTxInsParams{
PrevoutTxHashArr: []string{},
PrevoutTxIdxArr: []int32{},
TxHashArr: []string{},
TxIdxArr: []int32{},
ScriptsigArr: []string{},
WitnessArr: []string{},
SequenceArr: []int32{},
}
for _, block := range src {
blockHash := block.Header.Hash.String()
// Batch insert blocks
blocks.BlockHeightArr = append(blocks.BlockHeightArr, int32(block.Header.Height))
blocks.BlockHashArr = append(blocks.BlockHashArr, blockHash)
blocks.VersionArr = append(blocks.VersionArr, block.Header.Version)
blocks.MerkleRootArr = append(blocks.MerkleRootArr, block.Header.MerkleRoot.String())
blocks.PrevBlockHashArr = append(blocks.PrevBlockHashArr, block.Header.PrevBlock.String())
blocks.TimestampArr = append(blocks.TimestampArr, pgtype.Timestamptz{
Time: block.Header.Timestamp,
Valid: true,
})
blocks.BitsArr = append(blocks.BitsArr, int64(block.Header.Bits))
blocks.NonceArr = append(blocks.NonceArr, int64(block.Header.Nonce))
for txIdx, srcTx := range block.Transactions {
txHash := srcTx.TxHash.String()
// Batch insert transactions
txs.TxHashArr = append(txs.TxHashArr, txHash)
txs.VersionArr = append(txs.VersionArr, srcTx.Version)
txs.LocktimeArr = append(txs.LocktimeArr, int64(srcTx.LockTime))
txs.BlockHeightArr = append(txs.BlockHeightArr, int32(block.Header.Height))
txs.BlockHashArr = append(txs.BlockHashArr, blockHash)
txs.IdxArr = append(txs.IdxArr, int32(txIdx))
// Batch insert txins
for idx, txin := range srcTx.TxIn {
var witness string
if len(txin.Witness) > 0 {
witness = btcutils.WitnessToString(txin.Witness)
}
txins.TxHashArr = append(txins.TxHashArr, txHash)
txins.TxIdxArr = append(txins.TxIdxArr, int32(idx))
txins.PrevoutTxHashArr = append(txins.PrevoutTxHashArr, txin.PreviousOutTxHash.String())
txins.PrevoutTxIdxArr = append(txins.PrevoutTxIdxArr, int32(txin.PreviousOutIndex))
txins.ScriptsigArr = append(txins.ScriptsigArr, hex.EncodeToString(txin.SignatureScript))
txins.WitnessArr = append(txins.WitnessArr, witness)
txins.SequenceArr = append(txins.SequenceArr, int32(txin.Sequence))
}
// Batch insert txouts
for idx, txout := range srcTx.TxOut {
txouts.TxHashArr = append(txouts.TxHashArr, txHash)
txouts.TxIdxArr = append(txouts.TxIdxArr, int32(idx))
txouts.PkscriptArr = append(txouts.PkscriptArr, hex.EncodeToString(txout.PkScript))
txouts.ValueArr = append(txouts.ValueArr, txout.Value)
}
}
}
return blocks, txs, txouts, txins
}
func mapTransactionModelToType(src gen.BitcoinTransaction, txInModel []gen.BitcoinTransactionTxin, txOutModels []gen.BitcoinTransactionTxout) (types.Transaction, error) {
blockHash, err := chainhash.NewHashFromStr(src.BlockHash)
if err != nil {
return types.Transaction{}, errors.Wrap(err, "failed to parse block hash")
}
txHash, err := chainhash.NewHashFromStr(src.TxHash)
if err != nil {
return types.Transaction{}, errors.Wrap(err, "failed to parse tx hash")
}
// Sort txins and txouts by index (Asc)
slices.SortFunc(txOutModels, func(i, j gen.BitcoinTransactionTxout) int {
return cmp.Compare(i.TxIdx, j.TxIdx)
})
slices.SortFunc(txInModel, func(i, j gen.BitcoinTransactionTxin) int {
return cmp.Compare(i.TxIdx, j.TxIdx)
})
txIns := make([]*types.TxIn, 0, len(txInModel))
txOuts := make([]*types.TxOut, 0, len(txOutModels))
for _, txInModel := range txInModel {
scriptsig, err := hex.DecodeString(txInModel.Scriptsig)
if err != nil {
return types.Transaction{}, errors.Wrap(err, "failed to decode scriptsig")
}
prevoutTxHash, err := chainhash.NewHashFromStr(txInModel.PrevoutTxHash)
if err != nil {
return types.Transaction{}, errors.Wrap(err, "failed to parse prevout tx hash")
}
witness, err := btcutils.WitnessFromString(txInModel.Witness)
if err != nil {
return types.Transaction{}, errors.Wrap(err, "failed to parse witness from hex string")
}
txIns = append(txIns, &types.TxIn{
SignatureScript: scriptsig,
Witness: witness,
Sequence: uint32(txInModel.Sequence),
PreviousOutIndex: uint32(txInModel.PrevoutTxIdx),
PreviousOutTxHash: *prevoutTxHash,
})
}
for _, txOutModel := range txOutModels {
pkscript, err := hex.DecodeString(txOutModel.Pkscript)
if err != nil {
return types.Transaction{}, errors.Wrap(err, "failed to decode pkscript")
}
txOuts = append(txOuts, &types.TxOut{
PkScript: pkscript,
Value: txOutModel.Value,
})
}
return types.Transaction{
BlockHeight: int64(src.BlockHeight),
BlockHash: *blockHash,
Index: uint32(src.Idx),
TxHash: *txHash,
Version: src.Version,
LockTime: uint32(src.Locktime),
TxIn: txIns,
TxOut: txOuts,
}, nil
}