fix: handle case where input rune id is not found

* feat: add with request headers config

* feat: add with fields config

* feat: format request queries

README.md

@@ -2,7 +2,7 @@
 
 # Gaze Indexer
 
-Gaze Indexer is an open-source and modular indexing client for Bitcoin meta-protocols. It has support for Runes out of the box, with **Unified Consistent APIs** across fungible token protocols.
+Gaze Indexer is an open-source and modular indexing client for Bitcoin meta-protocols with **Unified Consistent APIs** across fungible token protocols.
 
 Gaze Indexer is built with **modularity** in mind, allowing users to run all modules in one monolithic instance with a single command, or as a distributed cluster of micro-services.
 

cmd/cmd_run.go

@@ -19,9 +19,9 @@ import (
 	"github.com/gaze-network/indexer-network/internal/config"
 	"github.com/gaze-network/indexer-network/modules/runes"
 	"github.com/gaze-network/indexer-network/pkg/automaxprocs"
-	"github.com/gaze-network/indexer-network/pkg/errorhandler"
 	"github.com/gaze-network/indexer-network/pkg/logger"
 	"github.com/gaze-network/indexer-network/pkg/logger/slogx"
+	"github.com/gaze-network/indexer-network/pkg/middleware/errorhandler"
 	"github.com/gaze-network/indexer-network/pkg/middleware/requestcontext"
 	"github.com/gaze-network/indexer-network/pkg/middleware/requestlogger"
 	"github.com/gaze-network/indexer-network/pkg/reportingclient"
@@ -136,8 +136,16 @@ func runHandler(cmd *cobra.Command, _ []string) error {
 	// Initialize HTTP server
 	do.Provide(injector, func(i do.Injector) (*fiber.App, error) {
 		app := fiber.New(fiber.Config{
-			AppName:      "Gaze Indexer",
-			ErrorHandler: errorhandler.NewHTTPErrorHandler(),
+			AppName: "Gaze Indexer",
+			ErrorHandler: func(c *fiber.Ctx, err error) error {
+				logger.ErrorContext(c.UserContext(), "Something went wrong, unhandled api error",
+					slogx.String("event", "api_unhandled_error"),
+					slogx.Error(err),
+				)
+				return errors.WithStack(c.Status(http.StatusInternalServerError).JSON(fiber.Map{
+					"error": "Internal Server Error",
+				}))
+			},
 		})
 		app.
 			Use(favicon.New()).
@@ -156,6 +164,7 @@ func runHandler(cmd *cobra.Command, _ []string) error {
 					logger.ErrorContext(c.UserContext(), "Something went wrong, panic in http handler", slogx.Any("panic", e), slog.String("stacktrace", string(buf)))
 				},
 			})).
+			Use(errorhandler.New()).
 			Use(compress.New(compress.Config{
 				Level: compress.LevelDefault,
 			}))

core/datasources/bitcoin_node.go

@@ -243,39 +243,32 @@ func (d *BitcoinNodeDatasource) prepareRange(fromHeight, toHeight int64) (start,
 }
 
 // GetTransaction fetch transaction from Bitcoin node
-func (d *BitcoinNodeDatasource) GetTransactionByHash(ctx context.Context, txHash chainhash.Hash) (*types.Transaction, error) {
+func (d *BitcoinNodeDatasource) GetRawTransactionAndHeightByTxHash(ctx context.Context, txHash chainhash.Hash) (*wire.MsgTx, int64, error) {
 	rawTxVerbose, err := d.btcclient.GetRawTransactionVerbose(&txHash)
 	if err != nil {
-		return nil, errors.Wrap(err, "failed to get raw transaction")
+		return nil, 0, errors.Wrap(err, "failed to get raw transaction")
 	}
 
 	blockHash, err := chainhash.NewHashFromStr(rawTxVerbose.BlockHash)
 	if err != nil {
-		return nil, errors.Wrap(err, "failed to parse block hash")
+		return nil, 0, errors.Wrap(err, "failed to parse block hash")
 	}
-	block, err := d.btcclient.GetBlockVerboseTx(blockHash)
+	block, err := d.btcclient.GetBlockVerbose(blockHash)
 	if err != nil {
-		return nil, errors.Wrap(err, "failed to get block header")
+		return nil, 0, errors.Wrap(err, "failed to get block header")
 	}
 
 	// parse tx
 	txBytes, err := hex.DecodeString(rawTxVerbose.Hex)
 	if err != nil {
-		return nil, errors.Wrap(err, "failed to decode transaction hex")
+		return nil, 0, errors.Wrap(err, "failed to decode transaction hex")
 	}
 	var msgTx wire.MsgTx
 	if err := msgTx.Deserialize(bytes.NewReader(txBytes)); err != nil {
-		return nil, errors.Wrap(err, "failed to deserialize transaction")
-	}
-	var txIndex uint32
-	for i, tx := range block.Tx {
-		if tx.Hex == rawTxVerbose.Hex {
-			txIndex = uint32(i)
-			break
-		}
+		return nil, 0, errors.Wrap(err, "failed to deserialize transaction")
 	}
 
-	return types.ParseMsgTx(&msgTx, block.Height, *blockHash, txIndex), nil
+	return &msgTx, block.Height, nil
 }
 
 // GetBlockHeader fetch block header from Bitcoin node

modules/runes/api/httphandler/get_holders.go

@@ -72,6 +72,9 @@ func (h *HttpHandler) GetHolders(ctx *fiber.Ctx) (err error) {
 
 	runeEntry, err := h.usecase.GetRuneEntryByRuneIdAndHeight(ctx.UserContext(), runeId, blockHeight)
 	if err != nil {
+		if errors.Is(err, errs.NotFound) {
+			return errs.NewPublicError("rune not found")
+		}
 		return errors.Wrap(err, "error during GetHoldersByHeight")
 	}
 	holdingBalances, err := h.usecase.GetBalancesByRuneId(ctx.UserContext(), runeId, blockHeight)

modules/runes/api/httphandler/get_token_info.go

@@ -99,6 +99,9 @@ func (h *HttpHandler) GetTokenInfo(ctx *fiber.Ctx) (err error) {
 
 	runeEntry, err := h.usecase.GetRuneEntryByRuneIdAndHeight(ctx.UserContext(), runeId, blockHeight)
 	if err != nil {
+		if errors.Is(err, errs.NotFound) {
+			return errs.NewPublicError("rune not found")
+		}
 		return errors.Wrap(err, "error during GetTokenInfoByHeight")
 	}
 	holdingBalances, err := h.usecase.GetBalancesByRuneId(ctx.UserContext(), runeId, blockHeight)

modules/runes/api/httphandler/get_transactions.go

@@ -2,6 +2,7 @@ package httphandler
 
 import (
 	"encoding/hex"
+	"fmt"
 	"slices"
 
 	"github.com/btcsuite/btcd/chaincfg/chainhash"
@@ -14,9 +15,11 @@ import (
 )
 
 type getTransactionsRequest struct {
-	Wallet      string `query:"wallet"`
-	Id          string `query:"id"`
-	BlockHeight uint64 `query:"blockHeight"`
+	Wallet string `query:"wallet"`
+	Id     string `query:"id"`
+
+	FromBlock int64 `query:"fromBlock"`
+	ToBlock   int64 `query:"toBlock"`
 }
 
 func (r getTransactionsRequest) Validate() error {
@@ -24,6 +27,12 @@ func (r getTransactionsRequest) Validate() error {
 	if r.Id != "" && !isRuneIdOrRuneName(r.Id) {
 		errList = append(errList, errors.New("'id' is not valid rune id or rune name"))
 	}
+	if r.FromBlock < -1 {
+		errList = append(errList, errors.Errorf("invalid fromBlock range"))
+	}
+	if r.ToBlock < -1 {
+		errList = append(errList, errors.Errorf("invalid toBlock range"))
+	}
 	return errs.WithPublicMessage(errors.Join(errList...), "validation error")
 }
 
@@ -125,17 +134,31 @@ func (h *HttpHandler) GetTransactions(ctx *fiber.Ctx) (err error) {
 		}
 	}
 
-	blockHeight := req.BlockHeight
-	// set blockHeight to the latest block height blockHeight, pkScript, and runeId are not provided
-	if blockHeight == 0 && pkScript == nil && runeId == (runes.RuneId{}) {
+	// default to latest block
+	if req.ToBlock == 0 {
+		req.ToBlock = -1
+	}
+
+	// get latest block height if block height is -1
+	if req.FromBlock == -1 || req.ToBlock == -1 {
 		blockHeader, err := h.usecase.GetLatestBlock(ctx.UserContext())
 		if err != nil {
 			return errors.Wrap(err, "error during GetLatestBlock")
 		}
-		blockHeight = uint64(blockHeader.Height)
+		if req.FromBlock == -1 {
+			req.FromBlock = blockHeader.Height
+		}
+		if req.ToBlock == -1 {
+			req.ToBlock = blockHeader.Height
+		}
 	}
 
-	txs, err := h.usecase.GetRuneTransactions(ctx.UserContext(), pkScript, runeId, blockHeight)
+	// validate block height range
+	if req.FromBlock > req.ToBlock {
+		return errs.NewPublicError(fmt.Sprintf("fromBlock must be less than or equal to toBlock, got fromBlock=%d, toBlock=%d", req.FromBlock, req.ToBlock))
+	}
+
+	txs, err := h.usecase.GetRuneTransactions(ctx.UserContext(), pkScript, runeId, uint64(req.FromBlock), uint64(req.ToBlock))
 	if err != nil {
 		return errors.Wrap(err, "error during GetRuneTransactions")
 	}

modules/runes/database/postgresql/queries/data.sql

@@ -55,8 +55,9 @@ SELECT * FROM runes_transactions
     OR runes_transactions.burns ? @rune_id
     OR (runes_transactions.rune_etched = TRUE AND runes_transactions.block_height = @rune_id_block_height AND runes_transactions.index = @rune_id_tx_index)
   ) AND (
-    @block_height::INT = 0 OR runes_transactions.block_height = @block_height::INT -- if @block_height > 0, apply block_height filter
-  );
+    @from_block <= runes_transactions.block_height AND runes_transactions.block_height <= @to_block
+  )
+ORDER BY runes_transactions.block_height DESC LIMIT 10000;
 
 -- name: CountRuneEntries :one
 SELECT COUNT(*) FROM runes_entries;

modules/runes/datagateway/runes.go

@@ -27,7 +27,7 @@ type RunesReaderDataGateway interface {
 	GetLatestBlock(ctx context.Context) (types.BlockHeader, error)
 	GetIndexedBlockByHeight(ctx context.Context, height int64) (*entity.IndexedBlock, error)
 	// GetRuneTransactions returns the runes transactions, filterable by pkScript, runeId and height. If pkScript, runeId or height is zero value, that filter is ignored.
-	GetRuneTransactions(ctx context.Context, pkScript []byte, runeId runes.RuneId, height uint64) ([]*entity.RuneTransaction, error)
+	GetRuneTransactions(ctx context.Context, pkScript []byte, runeId runes.RuneId, fromBlock, toBlock uint64) ([]*entity.RuneTransaction, error)
 
 	GetRunesBalancesAtOutPoint(ctx context.Context, outPoint wire.OutPoint) (map[runes.RuneId]*entity.OutPointBalance, error)
 	GetUnspentOutPointBalancesByPkScript(ctx context.Context, pkScript []byte, blockHeight uint64) ([]*entity.OutPointBalance, error)

modules/runes/processor_process.go

@@ -146,7 +146,7 @@ func (p *Processor) processTx(ctx context.Context, tx *types.Transaction, blockH
 					// find all non-OP_RETURN outputs
 					var destinations []int
 					for i, txOut := range tx.TxOut {
-						if txOut.IsOpReturn() {
+						if !txOut.IsOpReturn() {
 							destinations = append(destinations, i)
 						}
 					}
@@ -466,7 +466,7 @@ func (p *Processor) txCommitsToRune(ctx context.Context, tx *types.Transaction,
 			// It is impossible to verify that input utxo is a P2TR output with just the input.
 			// Need to verify with utxo's pk script.
 
-			prevTx, err := p.bitcoinClient.GetTransactionByHash(ctx, txIn.PreviousOutTxHash)
+			prevTx, blockHeight, err := p.bitcoinClient.GetRawTransactionAndHeightByTxHash(ctx, txIn.PreviousOutTxHash)
 			if err != nil && errors.Is(err, errs.NotFound) {
 				continue
 			}
@@ -479,7 +479,7 @@ func (p *Processor) txCommitsToRune(ctx context.Context, tx *types.Transaction,
 				break
 			}
 			// input must be mature enough
-			confirmations := tx.BlockHeight - prevTx.BlockHeight + 1
+			confirmations := tx.BlockHeight - blockHeight + 1
 			if confirmations < runes.RUNE_COMMIT_BLOCKS {
 				continue
 			}

modules/runes/repository/postgres/gen/data.sql.go

@@ -646,8 +646,9 @@ SELECT hash, runes_transactions.block_height, index, timestamp, inputs, outputs,
     OR runes_transactions.burns ? $5
     OR (runes_transactions.rune_etched = TRUE AND runes_transactions.block_height = $6 AND runes_transactions.index = $7)
   ) AND (
-    $8::INT = 0 OR runes_transactions.block_height = $8::INT -- if @block_height > 0, apply block_height filter
+    $8 <= runes_transactions.block_height AND runes_transactions.block_height <= $9
   )
+ORDER BY runes_transactions.block_height DESC LIMIT 10000
 `
 
 type GetRuneTransactionsParams struct {
@@ -658,7 +659,8 @@ type GetRuneTransactionsParams struct {
 	RuneID            []byte
 	RuneIDBlockHeight int32
 	RuneIDTxIndex     int32
-	BlockHeight       int32
+	FromBlock         int32
+	ToBlock           int32
 }
 
 type GetRuneTransactionsRow struct {
@@ -703,7 +705,8 @@ func (q *Queries) GetRuneTransactions(ctx context.Context, arg GetRuneTransactio
 		arg.RuneID,
 		arg.RuneIDBlockHeight,
 		arg.RuneIDTxIndex,
-		arg.BlockHeight,
+		arg.FromBlock,
+		arg.ToBlock,
 	)
 	if err != nil {
 		return nil, err

modules/runes/repository/postgres/runes.go

@@ -62,7 +62,7 @@ func (r *Repository) GetIndexedBlockByHeight(ctx context.Context, height int64)
 	return indexedBlock, nil
 }
 
-func (r *Repository) GetRuneTransactions(ctx context.Context, pkScript []byte, runeId runes.RuneId, height uint64) ([]*entity.RuneTransaction, error) {
+func (r *Repository) GetRuneTransactions(ctx context.Context, pkScript []byte, runeId runes.RuneId, fromBlock, toBlock uint64) ([]*entity.RuneTransaction, error) {
 	pkScriptParam := []byte(fmt.Sprintf(`[{"pkScript":"%s"}]`, hex.EncodeToString(pkScript)))
 	runeIdParam := []byte(fmt.Sprintf(`[{"runeId":"%s"}]`, runeId.String()))
 	rows, err := r.queries.GetRuneTransactions(ctx, gen.GetRuneTransactionsParams{
@@ -75,7 +75,8 @@ func (r *Repository) GetRuneTransactions(ctx context.Context, pkScript []byte, r
 		RuneIDBlockHeight: int32(runeId.BlockHeight),
 		RuneIDTxIndex:     int32(runeId.TxIndex),
 
-		BlockHeight: int32(height),
+		FromBlock: int32(fromBlock),
+		ToBlock:   int32(toBlock),
 	})
 	if err != nil {
 		return nil, errors.Wrap(err, "error during query")

modules/runes/usecase/get_transactions.go

@@ -8,8 +8,8 @@ import (
 	"github.com/gaze-network/indexer-network/modules/runes/runes"
 )
 
-func (u *Usecase) GetRuneTransactions(ctx context.Context, pkScript []byte, runeId runes.RuneId, height uint64) ([]*entity.RuneTransaction, error) {
-	txs, err := u.runesDg.GetRuneTransactions(ctx, pkScript, runeId, height)
+func (u *Usecase) GetRuneTransactions(ctx context.Context, pkScript []byte, runeId runes.RuneId, fromBlock, toBlock uint64) ([]*entity.RuneTransaction, error) {
+	txs, err := u.runesDg.GetRuneTransactions(ctx, pkScript, runeId, fromBlock, toBlock)
 	if err != nil {
 		return nil, errors.Wrap(err, "error during GetTransactionsByHeight")
 	}

pkg/btcclient/contract.go

@@ -4,9 +4,9 @@ import (
 	"context"
 
 	"github.com/btcsuite/btcd/chaincfg/chainhash"
-	"github.com/gaze-network/indexer-network/core/types"
+	"github.com/btcsuite/btcd/wire"
 )
 
 type Contract interface {
-	GetTransactionByHash(ctx context.Context, txHash chainhash.Hash) (*types.Transaction, error)
+	GetRawTransactionAndHeightByTxHash(ctx context.Context, txHash chainhash.Hash) (*wire.MsgTx, int64, error)
 }

pkg/btcutils/address.go

@@ -14,6 +14,11 @@ import (
 	"github.com/gaze-network/indexer-network/pkg/logger/slogx"
 )
 
+const (
+	// MaxSupportedPkScriptSize is the maximum supported size of a pkScript.
+	MaxSupportedPkScriptSize = 40
+)
+
 // IsAddress returns whether or not the passed string is a valid bitcoin address and valid supported type.
 //
 // NetParams is optional. If provided, we only check for that network,
@@ -50,11 +55,12 @@ func GetAddressType(address string, net *chaincfg.Params) (AddressType, error) {
 }
 
 type Address struct {
-	decoded      btcutil.Address
-	net          *chaincfg.Params
-	encoded      string
-	encodedType  AddressType
-	scriptPubKey []byte
+	decoded          btcutil.Address
+	net              *chaincfg.Params
+	encoded          string
+	encodedType      AddressType
+	scriptPubKey     [MaxSupportedPkScriptSize]byte
+	scriptPubKeySize int
 }
 
 // NewAddress creates a new address from the given address string.
@@ -87,12 +93,15 @@ func SafeNewAddress(address string, defaultNet ...*chaincfg.Params) (Address, er
 		return Address{}, errors.Wrap(err, "can't get script pubkey")
 	}
 
+	fixedPkScript := [MaxSupportedPkScriptSize]byte{}
+	copy(fixedPkScript[:], scriptPubkey)
 	return Address{
-		decoded:      decoded,
-		net:          net,
-		encoded:      decoded.EncodeAddress(),
-		encodedType:  addrType,
-		scriptPubKey: scriptPubkey,
+		decoded:          decoded,
+		net:              net,
+		encoded:          decoded.EncodeAddress(),
+		encodedType:      addrType,
+		scriptPubKey:     fixedPkScript,
+		scriptPubKeySize: len(scriptPubkey),
 	}, nil
 }
 
@@ -133,7 +142,7 @@ func (a Address) NetworkName() string {
 
 // ScriptPubKey or pubkey script
 func (a Address) ScriptPubKey() []byte {
-	return a.scriptPubKey
+	return a.scriptPubKey[:a.scriptPubKeySize]
 }
 
 // Equal return true if addresses are equal

pkg/btcutils/address_test.go

@@ -1,13 +1,16 @@
 package btcutils_test
 
 import (
+	"encoding/hex"
 	"encoding/json"
+	"errors"
 	"fmt"
 	"testing"
 
 	"github.com/btcsuite/btcd/chaincfg"
 	"github.com/gaze-network/indexer-network/pkg/btcutils"
 	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
 )
 
 func TestGetAddressType(t *testing.T) {
@@ -361,3 +364,86 @@ func TestAddressDecoding(t *testing.T) {
 		assert.Error(t, err)
 	})
 }
+
+func TestAddressPkScript(t *testing.T) {
+	anyErr := errors.New("any error")
+	type Spec struct {
+		Address          string
+		DefaultNet       *chaincfg.Params
+		ExpectedError    error
+		ExpectedPkScript string // hex encoded
+	}
+	specs := []Spec{
+		{
+			Address:          "some_invalid_address",
+			DefaultNet:       &chaincfg.MainNetParams,
+			ExpectedError:    anyErr,
+			ExpectedPkScript: "",
+		},
+		{
+			// P2WPKH
+			Address:          "bc1qdx72th7e3z8zc5wdrdxweswfcne974pjneyjln",
+			DefaultNet:       &chaincfg.MainNetParams,
+			ExpectedError:    nil,
+			ExpectedPkScript: "001469bca5dfd9888e2c51cd1b4cecc1c9c4f25f5432",
+		},
+		{
+			// P2WPKH
+			Address:          "bc1q7cj6gz6t3d28qg7kxhrc7h5t3h0re34fqqalga",
+			DefaultNet:       &chaincfg.MainNetParams,
+			ExpectedError:    nil,
+			ExpectedPkScript: "0014f625a40b4b8b547023d635c78f5e8b8dde3cc6a9",
+		},
+		{
+			// P2TR
+			Address:          "bc1pfd0zw2jwlpn4xckpr3dxpt7x0gw6wetuftxvrc4dt2qgn9azjuus65fug6",
+			DefaultNet:       &chaincfg.MainNetParams,
+			ExpectedError:    nil,
+			ExpectedPkScript: "51204b5e272a4ef8675362c11c5a60afc67a1da7657c4accc1e2ad5a808997a29739",
+		},
+		{
+			// P2TR
+			Address:          "bc1pxpumml545tqum5afarzlmnnez2npd35nvf0j0vnrp88nemqsn54qle05sm",
+			DefaultNet:       &chaincfg.MainNetParams,
+			ExpectedError:    nil,
+			ExpectedPkScript: "51203079bdfe95a2c1cdd3a9e8c5fdce7912a616c693625f27b26309cf3cec109d2a",
+		},
+		{
+			// P2SH
+			Address:          "3Ccte7SJz71tcssLPZy3TdWz5DTPeNRbPw",
+			DefaultNet:       &chaincfg.MainNetParams,
+			ExpectedError:    nil,
+			ExpectedPkScript: "a91477e1a3d54f545d83869ae3a6b28b071422801d7b87",
+		},
+		{
+			// P2PKH
+			Address:          "1KrRZSShVkdc8J71CtY4wdw46Rx3BRLKyH",
+			DefaultNet:       &chaincfg.MainNetParams,
+			ExpectedError:    nil,
+			ExpectedPkScript: "76a914cecb25b53809991c7beef2d27bc2be49e78c684388ac",
+		},
+		{
+			// P2WSH
+			Address:          "bc1qeklep85ntjz4605drds6aww9u0qr46qzrv5xswd35uhjuj8ahfcqgf6hak",
+			DefaultNet:       &chaincfg.MainNetParams,
+			ExpectedError:    nil,
+			ExpectedPkScript: "0020cdbf909e935c855d3e8d1b61aeb9c5e3c03ae8021b286839b1a72f2e48fdba70",
+		},
+	}
+
+	for _, spec := range specs {
+		t.Run(spec.Address, func(t *testing.T) {
+			addr, err := btcutils.SafeNewAddress(spec.Address, spec.DefaultNet)
+			if spec.ExpectedError != nil {
+				if errors.Is(spec.ExpectedError, anyErr) {
+					require.Error(t, err)
+				} else {
+					require.ErrorIs(t, err, spec.ExpectedError)
+				}
+				return
+			}
+			require.NoError(t, err)
+			assert.Equal(t, spec.ExpectedPkScript, hex.EncodeToString(addr.ScriptPubKey()))
+		})
+	}
+}

pkg/btcutils/btc.go

@@ -42,3 +42,17 @@ const (
 	AddressP2PKH   = txscript.PubKeyHashTy
 	AddressP2WSH   = txscript.WitnessV0ScriptHashTy
 )
+
+// IsSupportType returns true if the given tx/address type is supported.
+func IsSupportType(t txscript.ScriptClass) bool {
+	_, ok := supportedTypes[t]
+	return ok
+}
+
+var supportedTypes = map[txscript.ScriptClass]struct{}{
+	txscript.WitnessV0PubKeyHashTy: {},
+	txscript.WitnessV1TaprootTy:    {},
+	txscript.ScriptHashTy:          {},
+	txscript.PubKeyHashTy:          {},
+	txscript.WitnessV0ScriptHashTy: {},
+}

pkg/btcutils/pk_script.go

@@ -5,6 +5,7 @@ import (
 	"github.com/btcsuite/btcd/chaincfg"
 	"github.com/btcsuite/btcd/txscript"
 	"github.com/cockroachdb/errors"
+	"github.com/gaze-network/indexer-network/common/errs"
 )
 
 // NewPkScript creates a pubkey script(or witness program) from the given address string
@@ -36,19 +37,33 @@ func GetAddressTypeFromPkScript(pkScript []byte, defaultNet ...*chaincfg.Params)
 // ExtractAddressFromPkScript extracts address from the given pubkey script/script pubkey.
 // multi-signature script not supported
 func ExtractAddressFromPkScript(pkScript []byte, defaultNet ...*chaincfg.Params) (Address, error) {
+	if len(pkScript) == 0 {
+		return Address{}, errors.New("empty pkScript")
+	}
+	if pkScript[0] == txscript.OP_RETURN {
+		return Address{}, errors.Wrap(errs.NotSupported, "OP_RETURN script")
+	}
 	net := utils.DefaultOptional(defaultNet, &chaincfg.MainNetParams)
 	addrType, addrs, _, err := txscript.ExtractPkScriptAddrs(pkScript, net)
 	if err != nil {
 		return Address{}, errors.Wrap(err, "can't parse pkScript")
 	}
+	if !IsSupportType(addrType) {
+		return Address{}, errors.Wrapf(errs.NotSupported, "unsupported pkscript type %s", addrType)
+	}
 	if len(addrs) == 0 {
 		return Address{}, errors.New("can't extract address from pkScript")
 	}
+
+	fixedPkScript := [MaxSupportedPkScriptSize]byte{}
+	copy(fixedPkScript[:], pkScript)
+
 	return Address{
-		decoded:      addrs[0],
-		net:          net,
-		encoded:      addrs[0].EncodeAddress(),
-		encodedType:  addrType,
-		scriptPubKey: pkScript,
+		decoded:          addrs[0],
+		net:              net,
+		encoded:          addrs[0].EncodeAddress(),
+		encodedType:      addrType,
+		scriptPubKey:     fixedPkScript,
+		scriptPubKeySize: len(pkScript),
 	}, nil
 }

pkg/btcutils/pk_script_test.go

@@ -186,6 +186,18 @@ func TestGetAddressTypeFromPkScript(t *testing.T) {
 			ExpectedError:       nil,
 			ExpectedAddressType: btcutils.AddressP2WSH,
 		},
+		{
+			PubkeyScript: "0020cdbf909e935c855d3e8d1b61aeb9c5e3c03ae8021b286839b1a72f2e48fdba70",
+
+			ExpectedError:       nil,
+			ExpectedAddressType: btcutils.AddressP2WSH,
+		},
+		{
+			PubkeyScript: "6a5d0614c0a2331441",
+
+			ExpectedError:       nil,
+			ExpectedAddressType: txscript.NonStandardTy,
+		},
 	}
 
 	for _, spec := range specs {

pkg/btcutils/signature.go

@@ -1,17 +1,19 @@
 package btcutils
 
 import (
+	"github.com/Cleverse/go-utilities/utils"
 	verifier "github.com/bitonicnl/verify-signed-message/pkg"
+	"github.com/btcsuite/btcd/chaincfg"
 	"github.com/cockroachdb/errors"
-	"github.com/gaze-network/indexer-network/common"
 )
 
-func VerifySignature(address string, message string, sigBase64 string, network common.Network) error {
+func VerifySignature(address string, message string, sigBase64 string, defaultNet ...*chaincfg.Params) error {
+	net := utils.DefaultOptional(defaultNet, &chaincfg.MainNetParams)
 	_, err := verifier.VerifyWithChain(verifier.SignedMessage{
 		Address:   address,
 		Message:   message,
 		Signature: sigBase64,
-	}, network.ChainParams())
+	}, net)
 	if err != nil {
 		return errors.WithStack(err)
 	}

pkg/btcutils/signature_test.go

@@ -3,7 +3,7 @@ package btcutils
 import (
 	"testing"
 
-	"github.com/gaze-network/indexer-network/common"
+	"github.com/btcsuite/btcd/chaincfg"
 	"github.com/stretchr/testify/assert"
 )
 
@@ -12,7 +12,7 @@ func TestVerifySignature(t *testing.T) {
 		message := "Test123"
 		address := "18J72YSM9pKLvyXX1XAjFXA98zeEvxBYmw"
 		signature := "Gzhfsw0ItSrrTCChykFhPujeTyAcvVxiXwywxpHmkwFiKuUR2ETbaoFcocmcSshrtdIjfm8oXlJoTOLosZp3Yc8="
-		network := common.NetworkMainnet
+		network := &chaincfg.MainNetParams
 
 		err := VerifySignature(address, message, signature, network)
 		assert.NoError(t, err)
@@ -21,7 +21,7 @@ func TestVerifySignature(t *testing.T) {
 		address := "tb1qr97cuq4kvq7plfetmxnl6kls46xaka78n2288z"
 		message := "The outage comes at a time when bitcoin has been fast approaching new highs not seen since June 26, 2019."
 		signature := "H/bSByRH7BW1YydfZlEx9x/nt4EAx/4A691CFlK1URbPEU5tJnTIu4emuzkgZFwC0ptvKuCnyBThnyLDCqPqT10="
-		network := common.NetworkTestnet
+		network := &chaincfg.TestNet3Params
 
 		err := VerifySignature(address, message, signature, network)
 		assert.NoError(t, err)
@@ -31,7 +31,7 @@ func TestVerifySignature(t *testing.T) {
 		address := "tb1qp7y2ywgrv8a4t9h47yphtgj8w759rk6vgd9ran"
 		message := "The outage comes at a time when bitcoin has been fast approaching new highs not seen since June 26, 2019."
 		signature := "H/bSByRH7BW1YydfZlEx9x/nt4EAx/4A691CFlK1URbPEU5tJnTIu4emuzkgZFwC0ptvKuCnyBThnyLDCqPqT10="
-		network := common.NetworkTestnet
+		network := &chaincfg.TestNet3Params
 
 		err := VerifySignature(address, message, signature, network)
 		assert.Error(t, err)
@@ -41,7 +41,7 @@ func TestVerifySignature(t *testing.T) {
 		address := "tb1qr97cuq4kvq7plfetmxnl6kls46xaka78n2288z"
 		message := "The outage comes at a time when bitcoin has been fast approaching new highs not seen since June 26, 2019."
 		signature := "Gzhfsw0ItSrrTCChykFhPujeTyAcvVxiXwywxpHmkwFiKuUR2ETbaoFcocmcSshrtdIjfm8oXlJoTOLosZp3Yc8="
-		network := common.NetworkTestnet
+		network := &chaincfg.TestNet3Params
 
 		err := VerifySignature(address, message, signature, network)
 		assert.Error(t, err)
@@ -51,7 +51,17 @@ func TestVerifySignature(t *testing.T) {
 		address := "tb1qr97cuq4kvq7plfetmxnl6kls46xaka78n2288z"
 		message := "Hello World"
 		signature := "H/bSByRH7BW1YydfZlEx9x/nt4EAx/4A691CFlK1URbPEU5tJnTIu4emuzkgZFwC0ptvKuCnyBThnyLDCqPqT10="
-		network := common.NetworkTestnet
+		network := &chaincfg.TestNet3Params
+
+		err := VerifySignature(address, message, signature, network)
+		assert.Error(t, err)
+	}
+	{
+		// Missmatch network
+		address := "tb1qr97cuq4kvq7plfetmxnl6kls46xaka78n2288z"
+		message := "The outage comes at a time when bitcoin has been fast approaching new highs not seen since June 26, 2019."
+		signature := "H/bSByRH7BW1YydfZlEx9x/nt4EAx/4A691CFlK1URbPEU5tJnTIu4emuzkgZFwC0ptvKuCnyBThnyLDCqPqT10="
+		network := &chaincfg.MainNetParams
 
 		err := VerifySignature(address, message, signature, network)
 		assert.Error(t, err)

pkg/httpclient/httpclient.go

@@ -5,12 +5,11 @@ import (
 	"encoding/json"
 	"log/slog"
 	"net/url"
+	"path"
 	"strings"
 	"time"
 
-	"github.com/Cleverse/go-utilities/utils"
 	"github.com/cockroachdb/errors"
-	"github.com/gaze-network/indexer-network/common/errs"
 	"github.com/gaze-network/indexer-network/pkg/logger"
 	"github.com/valyala/fasthttp"
 )
@@ -24,13 +23,14 @@ type Config struct {
 }
 
 type Client struct {
-	baseURL string
+	baseURL *url.URL
 	Config
 }
 
 func New(baseURL string, config ...Config) (*Client, error) {
-	if _, err := url.Parse(baseURL); err != nil {
-		return nil, errors.Join(errs.InvalidArgument, errors.Wrap(err, "can't parse base url"))
+	parsedBaseURL, err := url.Parse(baseURL)
+	if err != nil {
+		return nil, errors.Wrap(err, "can't parse base url")
 	}
 	var cf Config
 	if len(config) > 0 {
@@ -40,7 +40,7 @@ func New(baseURL string, config ...Config) (*Client, error) {
 		cf.Headers = make(map[string]string)
 	}
 	return &Client{
-		baseURL: baseURL,
+		baseURL: parsedBaseURL,
 		Config:  cf,
 	}, nil
 }
@@ -60,11 +60,21 @@ type HttpResponse struct {
 }
 
 func (r *HttpResponse) UnmarshalBody(out any) error {
-	err := json.Unmarshal(r.Body(), out)
+	body, err := r.BodyUncompressed()
 	if err != nil {
-		return errors.Wrapf(err, "can't unmarshal json body from %v, %v", r.URL, string(r.Body()))
+		return errors.Wrapf(err, "can't uncompress body from %v", r.URL)
+	}
+	switch strings.ToLower(string(r.Header.ContentType())) {
+	case "application/json", "application/json; charset=utf-8":
+		if err := json.Unmarshal(body, out); err != nil {
+			return errors.Wrapf(err, "can't unmarshal json body from %s, %q", r.URL, string(body))
+		}
+		return nil
+	case "text/plain", "text/plain; charset=utf-8":
+		return errors.Errorf("can't unmarshal plain text %q", string(body))
+	default:
+		return errors.Errorf("unsupported content type: %s, contents: %v", r.Header.ContentType(), string(r.Body()))
 	}
-	return nil
 }
 
 func (h *Client) request(ctx context.Context, reqOptions RequestOptions) (*HttpResponse, error) {
@@ -77,9 +87,14 @@ func (h *Client) request(ctx context.Context, reqOptions RequestOptions) (*HttpR
 	for k, v := range reqOptions.Header {
 		req.Header.Set(k, v)
 	}
-	parsedUrl := utils.Must(url.Parse(h.baseURL)) // checked in httpclient.New
-	parsedUrl.Path = reqOptions.path
-	parsedUrl.RawQuery = reqOptions.Query.Encode()
+
+	parsedUrl := h.BaseURL()
+	parsedUrl.Path = path.Join(parsedUrl.Path, reqOptions.path)
+	baseQuery := parsedUrl.Query()
+	for k, v := range reqOptions.Query {
+		baseQuery[k] = v
+	}
+	parsedUrl.RawQuery = baseQuery.Encode()
 
 	// remove %20 from url (empty space)
 	url := strings.TrimSuffix(parsedUrl.String(), "%20")
@@ -111,6 +126,7 @@ func (h *Client) request(ctx context.Context, reqOptions RequestOptions) (*HttpR
 				logger = logger.With(
 					slog.Int("status_code", resp.StatusCode()),
 					slog.String("resp_content_type", string(resp.Header.ContentType())),
+					slog.String("resp_content_encoding", string(resp.Header.ContentEncoding())),
 					slog.Int("resp_content_length", len(resp.Body())),
 				)
 			}
@@ -134,6 +150,12 @@ func (h *Client) request(ctx context.Context, reqOptions RequestOptions) (*HttpR
 	return &httpResponse, nil
 }
 
+// BaseURL returns the cloned base URL of the client.
+func (h *Client) BaseURL() *url.URL {
+	u := *h.baseURL
+	return &u
+}
+
 func (h *Client) Do(ctx context.Context, method, path string, reqOptions RequestOptions) (*HttpResponse, error) {
 	reqOptions.path = path
 	reqOptions.method = method

pkg/logger/logger.go

@@ -119,10 +119,10 @@ type Config struct {
 	//  - Text (default)
 	//  - JSON
 	//  - GCP: Output format for Stackdriver Logging/Cloud Logging or others GCP services.
-	Output string `mapstructure:"output"`
+	Output string `mapstructure:"output" env:"OUTPUT" envDefault:"text"`
 
 	// Debug is enabled logger level debug. (default: false)
-	Debug bool `mapstructure:"debug"`
+	Debug bool `mapstructure:"debug" env:"DEBUG" envDefault:"false"`
 }
 
 var (

pkg/middleware/errorhandler/errorhandler.go

@@ -10,23 +10,28 @@ import (
 	"github.com/gofiber/fiber/v2"
 )
 
-func NewHTTPErrorHandler() func(ctx *fiber.Ctx, err error) error {
-	return func(ctx *fiber.Ctx, err error) error {
+// New setup error handler middleware
+func New() fiber.Handler {
+	return func(ctx *fiber.Ctx) error {
+		err := ctx.Next()
+		if err == nil {
+			return nil
+		}
 		if e := new(errs.PublicError); errors.As(err, &e) {
-			return errors.WithStack(ctx.Status(http.StatusBadRequest).JSON(map[string]any{
+			return errors.WithStack(ctx.Status(http.StatusBadRequest).JSON(fiber.Map{
 				"error": e.Message(),
 			}))
 		}
 		if e := new(fiber.Error); errors.As(err, &e) {
-			return errors.WithStack(ctx.Status(e.Code).SendString(e.Error()))
+			return errors.WithStack(ctx.Status(e.Code).JSON(fiber.Map{
+				"error": e.Error(),
+			}))
 		}
-
-		logger.ErrorContext(ctx.UserContext(), "Something went wrong, unhandled api error",
-			slogx.String("event", "api_unhandled_error"),
+		logger.ErrorContext(ctx.UserContext(), "Something went wrong, api error",
+			slogx.String("event", "api_error"),
 			slogx.Error(err),
 		)
-
-		return errors.WithStack(ctx.Status(http.StatusInternalServerError).JSON(map[string]any{
+		return errors.WithStack(ctx.Status(http.StatusInternalServerError).JSON(fiber.Map{
 			"error": "Internal Server Error",
 		}))
 	}

pkg/middleware/requestlogger/requestlogger.go

@@ -13,10 +13,13 @@ import (
 )
 
 type Config struct {
-	WithRequestHeader    bool     `env:"REQUEST_HEADER" envDefault:"false" mapstructure:"request_header"`
-	WithRequestQuery     bool     `env:"REQUEST_QUERY" envDefault:"false" mapstructure:"request_query"`
-	Disable              bool     `env:"DISABLE" envDefault:"false" mapstructure:"disable"` // Disable logger level `INFO`
-	HiddenRequestHeaders []string `env:"HIDDEN_REQUEST_HEADERS" mapstructure:"hidden_request_headers"`
+	AllRequestHeaders    bool                   `env:"REQUEST_HEADER" envDefault:"false" mapstructure:"request_header"`   // Log all request headers
+	AllResponseHeaders   bool                   `env:"RESPONSE_HEADER" envDefault:"false" mapstructure:"response_header"` // Log all response headers
+	AllRequestQueries    bool                   `env:"REQUEST_QUERY" envDefault:"false" mapstructure:"request_query"`     // Log all request queries
+	Disable              bool                   `env:"DISABLE" envDefault:"false" mapstructure:"disable"`                 // Disable logger level `INFO`
+	HiddenRequestHeaders []string               `env:"HIDDEN_REQUEST_HEADERS" mapstructure:"hidden_request_headers"`      // Hide specific headers from log
+	WithRequestHeaders   []string               `env:"WITH_REQUEST_HEADERS" mapstructure:"with_request_headers"`          // Add specific headers to log (higher priority than `HiddenRequestHeaders`)
+	With                 map[string]interface{} `env:"WITH" mapstructure:"with"`                                          // Additional fields to log
 }
 
 // New setup request context and information
@@ -25,6 +28,10 @@ func New(config Config) fiber.Handler {
 	for _, header := range config.HiddenRequestHeaders {
 		hiddenRequestHeaders[strings.TrimSpace(strings.ToLower(header))] = struct{}{}
 	}
+	withRequestHeaders := make(map[string]struct{}, len(config.WithRequestHeaders))
+	for _, header := range config.WithRequestHeaders {
+		withRequestHeaders[strings.TrimSpace(strings.ToLower(header))] = struct{}{}
+	}
 	return func(c *fiber.Ctx) error {
 		start := time.Now()
 
@@ -41,6 +48,11 @@ func New(config Config) fiber.Handler {
 			slog.String("latencyHuman", latency.String()),
 		}
 
+		// add `with` fields
+		for k, v := range config.With {
+			baseAttrs = append(baseAttrs, slog.Any(k, v))
+		}
+
 		// prep request attributes
 		requestAttributes := []slog.Attr{
 			slog.Time("time", start),
@@ -53,6 +65,7 @@ func New(config Config) fiber.Handler {
 			slog.Any("x-forwarded-for", c.IPs()),
 			slog.String("user-agent", string(c.Context().UserAgent())),
 			slog.Any("params", c.AllParams()),
+			slog.Any("query", c.Queries()),
 			slog.Int("length", len((c.Body()))),
 		}
 
@@ -63,23 +76,64 @@ func New(config Config) fiber.Handler {
 			slog.Int("length", len(c.Response().Body())),
 		}
 
-		// request query
-		if config.WithRequestQuery {
-			requestAttributes = append(requestAttributes, slog.String("query", string(c.Request().URI().QueryString())))
+		// request queries
+		if config.AllRequestQueries {
+			args := c.Request().URI().QueryArgs()
+			logAttrs := make([]any, 0, args.Len())
+			args.VisitAll(func(k, v []byte) {
+				logAttrs = append(logAttrs, slog.Any(string(k), string(v)))
+			})
+			requestAttributes = append(requestAttributes, slog.Group("queries", logAttrs...))
 		}
 
 		// request headers
-		if config.WithRequestHeader {
+		if config.AllRequestHeaders || len(config.WithRequestHeaders) > 0 {
 			kv := []any{}
 
 			for k, v := range c.GetReqHeaders() {
+				h := strings.ToLower(k)
+
+				// add headers for WithRequestHeaders
+				if _, found := withRequestHeaders[h]; found {
+					goto add
+				}
+
+				// skip hidden headers
+				if _, found := hiddenRequestHeaders[h]; found {
+					continue
+				}
+
+				// skip if not AllRequestHeaders
+				if !config.AllRequestHeaders {
+					continue
+				}
+
+			add:
+				val := any(v)
+				if len(v) == 1 {
+					val = v[0]
+				}
+				kv = append(kv, slog.Any(k, val))
+			}
+
+			requestAttributes = append(requestAttributes, slog.Group("headers", kv...))
+		}
+
+		if config.AllResponseHeaders {
+			kv := []any{}
+			for k, v := range c.GetRespHeaders() {
+				// skip hidden headers
 				if _, found := hiddenRequestHeaders[strings.ToLower(k)]; found {
 					continue
 				}
-				kv = append(kv, slog.Any(k, v))
-			}
 
-			requestAttributes = append(requestAttributes, slog.Group("header", kv...))
+				val := any(v)
+				if len(v) == 1 {
+					val = v[0]
+				}
+				kv = append(kv, slog.Any(k, val))
+			}
+			responseAttributes = append(responseAttributes, slog.Group("headers", kv...))
 		}
 
 		level := slog.LevelInfo