mirror of
https://github.com/alexgo-io/gaze-indexer.git
synced 2026-04-30 12:41:59 +08:00
wip: parse aws transaction to type transaction
This commit is contained in:
Planxnx
@@ -8,8 +8,10 @@ package datasources
|
||||
import (
|
||||
"cmp"
|
||||
"context"
|
||||
"encoding/hex"
|
||||
"fmt"
|
||||
"io"
|
||||
"math"
|
||||
"slices"
|
||||
"strconv"
|
||||
"strings"
|
||||
@@ -21,10 +23,13 @@ import (
|
||||
"github.com/aws/aws-sdk-go-v2/service/s3"
|
||||
s3types "github.com/aws/aws-sdk-go-v2/service/s3/types"
|
||||
"github.com/btcsuite/btcd/chaincfg/chainhash"
|
||||
"github.com/btcsuite/btcd/wire"
|
||||
"github.com/cockroachdb/errors"
|
||||
"github.com/gaze-network/indexer-network/common"
|
||||
"github.com/gaze-network/indexer-network/common/errs"
|
||||
"github.com/gaze-network/indexer-network/core/types"
|
||||
"github.com/gaze-network/indexer-network/internal/subscription"
|
||||
"github.com/gaze-network/indexer-network/pkg/btcutils"
|
||||
"github.com/gaze-network/indexer-network/pkg/logger"
|
||||
"github.com/gaze-network/indexer-network/pkg/logger/slogx"
|
||||
"github.com/gaze-network/indexer-network/pkg/parquetutils"
|
||||
@@ -290,7 +295,7 @@ func (d *AWSPublicDataDatasource) FetchAsync(ctx context.Context, from, to int64
|
||||
|
||||
txs := make([]*types.Transaction, 0, len(groupRawTxs[blockHeader.Height]))
|
||||
for _, rawTx := range groupRawTxs[rawBlock.Number] {
|
||||
tx, err := rawTx.ToTransaction()
|
||||
tx, err := rawTx.ToTransaction(rawBlock)
|
||||
if err != nil {
|
||||
logger.ErrorContext(ctx, "Failed to convert aws transaction to type transaction", slogx.Error(err))
|
||||
if err := subscription.SendError(ctx, errors.Wrap(err, "can't convert aws transaction to type transaction")); err != nil {
|
||||
@@ -451,52 +456,54 @@ type (
|
||||
LastModified string `parquet:"name=last_modified, type=INT96, repetitiontype=OPTIONAL"`
|
||||
}
|
||||
awsTransaction struct {
|
||||
Hash string `parquet:"name=hash, type=BYTE_ARRAY, convertedtype=UTF8, repetitiontype=OPTIONAL"`
|
||||
Version int64 `parquet:"name=version, type=INT64, repetitiontype=OPTIONAL"`
|
||||
Size int64 `parquet:"name=size, type=INT64, repetitiontype=OPTIONAL"`
|
||||
BlockHash string `parquet:"name=block_hash, type=BYTE_ARRAY, convertedtype=UTF8, repetitiontype=OPTIONAL"`
|
||||
BlockNumber int64 `parquet:"name=block_number, type=INT64, repetitiontype=OPTIONAL"`
|
||||
Index int64 `parquet:"name=index, type=INT64, repetitiontype=OPTIONAL"`
|
||||
Virtual_size int64 `parquet:"name=virtual_size, type=INT64, repetitiontype=OPTIONAL"`
|
||||
Lock_time int64 `parquet:"name=lock_time, type=INT64, repetitiontype=OPTIONAL"`
|
||||
Input_count int64 `parquet:"name=input_count, type=INT64, repetitiontype=OPTIONAL"`
|
||||
Output_count int64 `parquet:"name=output_count, type=INT64, repetitiontype=OPTIONAL"`
|
||||
Is_coinbase bool `parquet:"name=is_coinbase, type=BOOLEAN, repetitiontype=OPTIONAL"`
|
||||
Output_value float64 `parquet:"name=output_value, type=DOUBLE, repetitiontype=OPTIONAL"`
|
||||
Outputs []*struct {
|
||||
Address string `parquet:"name=address, type=BYTE_ARRAY, convertedtype=UTF8, repetitiontype=OPTIONAL"`
|
||||
Index int64 `parquet:"name=index, type=INT64, repetitiontype=OPTIONAL"`
|
||||
Required_signatures int64 `parquet:"name=required_signatures, type=INT64, repetitiontype=OPTIONAL"`
|
||||
Script_asm string `parquet:"name=script_asm, type=BYTE_ARRAY, convertedtype=UTF8, repetitiontype=OPTIONAL"`
|
||||
Script_hex string `parquet:"name=script_hex, type=BYTE_ARRAY, convertedtype=UTF8, repetitiontype=OPTIONAL"`
|
||||
Type string `parquet:"name=type, type=BYTE_ARRAY, convertedtype=UTF8, repetitiontype=OPTIONAL"`
|
||||
Value float64 `parquet:"name=value, type=DOUBLE, repetitiontype=OPTIONAL"`
|
||||
} `parquet:"name=outputs, type=LIST, repetitiontype=OPTIONAL, valuetype=STRUCT"`
|
||||
Block_timestamp string `parquet:"name=block_timestamp, type=INT96, repetitiontype=OPTIONAL"`
|
||||
Date string `parquet:"name=date, type=BYTE_ARRAY, convertedtype=UTF8, repetitiontype=OPTIONAL"`
|
||||
Last_modified string `parquet:"name=last_modified, type=INT96, repetitiontype=OPTIONAL"`
|
||||
Fee float64 `parquet:"name=fee, type=DOUBLE, repetitiontype=OPTIONAL"`
|
||||
Input_value float64 `parquet:"name=input_value, type=DOUBLE, repetitiontype=OPTIONAL"`
|
||||
Inputs []*struct {
|
||||
Address string `parquet:"name=address, type=BYTE_ARRAY, convertedtype=UTF8, repetitiontype=OPTIONAL"`
|
||||
Index int64 `parquet:"name=index, type=INT64, repetitiontype=OPTIONAL"`
|
||||
Required_signatures int64 `parquet:"name=required_signatures, type=INT64, repetitiontype=OPTIONAL"`
|
||||
Script_asm string `parquet:"name=script_asm, type=BYTE_ARRAY, convertedtype=UTF8, repetitiontype=OPTIONAL"`
|
||||
Script_hex string `parquet:"name=script_hex, type=BYTE_ARRAY, convertedtype=UTF8, repetitiontype=OPTIONAL"`
|
||||
Sequence int64 `parquet:"name=sequence, type=INT64, repetitiontype=OPTIONAL"`
|
||||
Spent_output_index int64 `parquet:"name=spent_output_index, type=INT64, repetitiontype=OPTIONAL"`
|
||||
Spent_transaction_hash string `parquet:"name=spent_transaction_hash, type=BYTE_ARRAY, convertedtype=UTF8, repetitiontype=OPTIONAL"`
|
||||
Txinwitness []*string `parquet:"name=txinwitness, type=LIST, repetitiontype=OPTIONAL, valuetype=BYTE_ARRAY, convertedtype=UTF8"`
|
||||
Type string `parquet:"name=type, type=BYTE_ARRAY, convertedtype=UTF8, repetitiontype=OPTIONAL"`
|
||||
Value float64 `parquet:"name=value, type=DOUBLE, repetitiontype=OPTIONAL"`
|
||||
} `parquet:"name=inputs, type=LIST, repetitiontype=OPTIONAL, valuetype=STRUCT"`
|
||||
Hash string `parquet:"name=hash, type=BYTE_ARRAY, convertedtype=UTF8, repetitiontype=OPTIONAL"`
|
||||
Version int64 `parquet:"name=version, type=INT64, repetitiontype=OPTIONAL"`
|
||||
Size int64 `parquet:"name=size, type=INT64, repetitiontype=OPTIONAL"`
|
||||
BlockHash string `parquet:"name=block_hash, type=BYTE_ARRAY, convertedtype=UTF8, repetitiontype=OPTIONAL"`
|
||||
BlockNumber int64 `parquet:"name=block_number, type=INT64, repetitiontype=OPTIONAL"`
|
||||
Index int64 `parquet:"name=index, type=INT64, repetitiontype=OPTIONAL"`
|
||||
VirtualSize int64 `parquet:"name=virtual_size, type=INT64, repetitiontype=OPTIONAL"`
|
||||
LockTime int64 `parquet:"name=lock_time, type=INT64, repetitiontype=OPTIONAL"`
|
||||
InputCount int64 `parquet:"name=input_count, type=INT64, repetitiontype=OPTIONAL"`
|
||||
OutputCount int64 `parquet:"name=output_count, type=INT64, repetitiontype=OPTIONAL"`
|
||||
IsCoinbase bool `parquet:"name=is_coinbase, type=BOOLEAN, repetitiontype=OPTIONAL"`
|
||||
OutputValue float64 `parquet:"name=output_value, type=DOUBLE, repetitiontype=OPTIONAL"`
|
||||
Outputs []*awsTxOutput `parquet:"name=outputs, type=LIST, repetitiontype=OPTIONAL, valuetype=STRUCT"`
|
||||
BlockTimestamp string `parquet:"name=block_timestamp, type=INT96, repetitiontype=OPTIONAL"`
|
||||
Date string `parquet:"name=date, type=BYTE_ARRAY, convertedtype=UTF8, repetitiontype=OPTIONAL"`
|
||||
LastModified string `parquet:"name=last_modified, type=INT96, repetitiontype=OPTIONAL"`
|
||||
Fee float64 `parquet:"name=fee, type=DOUBLE, repetitiontype=OPTIONAL"`
|
||||
InputValue float64 `parquet:"name=input_value, type=DOUBLE, repetitiontype=OPTIONAL"`
|
||||
Inputs []*awsTxInput `parquet:"name=inputs, type=LIST, repetitiontype=OPTIONAL, valuetype=STRUCT"`
|
||||
}
|
||||
awsTxInput struct {
|
||||
Address string `parquet:"name=address, type=BYTE_ARRAY, convertedtype=UTF8, repetitiontype=OPTIONAL"`
|
||||
Index int64 `parquet:"name=index, type=INT64, repetitiontype=OPTIONAL"`
|
||||
RequiredSignatures int64 `parquet:"name=required_signatures, type=INT64, repetitiontype=OPTIONAL"`
|
||||
ScriptAsm string `parquet:"name=script_asm, type=BYTE_ARRAY, convertedtype=UTF8, repetitiontype=OPTIONAL"`
|
||||
ScriptHex string `parquet:"name=script_hex, type=BYTE_ARRAY, convertedtype=UTF8, repetitiontype=OPTIONAL"`
|
||||
Sequence int64 `parquet:"name=sequence, type=INT64, repetitiontype=OPTIONAL"`
|
||||
SpentOutputIndex int64 `parquet:"name=spent_output_index, type=INT64, repetitiontype=OPTIONAL"`
|
||||
SpentTransactionHash string `parquet:"name=spent_transaction_hash, type=BYTE_ARRAY, convertedtype=UTF8, repetitiontype=OPTIONAL"`
|
||||
TxInWitness []*string `parquet:"name=txinwitness, type=LIST, repetitiontype=OPTIONAL, valuetype=BYTE_ARRAY, convertedtype=UTF8"`
|
||||
Type string `parquet:"name=type, type=BYTE_ARRAY, convertedtype=UTF8, repetitiontype=OPTIONAL"`
|
||||
Value float64 `parquet:"name=value, type=DOUBLE, repetitiontype=OPTIONAL"`
|
||||
}
|
||||
awsTxOutput struct {
|
||||
Address string `parquet:"name=address, type=BYTE_ARRAY, convertedtype=UTF8, repetitiontype=OPTIONAL"`
|
||||
Index int64 `parquet:"name=index, type=INT64, repetitiontype=OPTIONAL"`
|
||||
Required_signatures int64 `parquet:"name=required_signatures, type=INT64, repetitiontype=OPTIONAL"`
|
||||
Script_asm string `parquet:"name=script_asm, type=BYTE_ARRAY, convertedtype=UTF8, repetitiontype=OPTIONAL"`
|
||||
Script_hex string `parquet:"name=script_hex, type=BYTE_ARRAY, convertedtype=UTF8, repetitiontype=OPTIONAL"`
|
||||
Type string `parquet:"name=type, type=BYTE_ARRAY, convertedtype=UTF8, repetitiontype=OPTIONAL"`
|
||||
Value float64 `parquet:"name=value, type=DOUBLE, repetitiontype=OPTIONAL"`
|
||||
}
|
||||
)
|
||||
|
||||
func (a awsBlock) ToBlockHeader() (types.BlockHeader, error) {
|
||||
hash, err := chainhash.NewHashFromStr(a.Hash)
|
||||
if err != nil {
|
||||
return types.BlockHeader{}, errors.Wrap(err, "can't convert hash")
|
||||
return types.BlockHeader{}, errors.Wrap(err, "can't convert block hash")
|
||||
}
|
||||
prevBlockHash, err := chainhash.NewHashFromStr(a.PreviousBlockHash)
|
||||
if err != nil {
|
||||
@@ -524,16 +531,87 @@ func (a awsBlock) ToBlockHeader() (types.BlockHeader, error) {
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (a awsTransaction) ToTransaction() (*types.Transaction, error) {
|
||||
// TODO: implement this
|
||||
return &types.Transaction{
|
||||
BlockHeight: 0,
|
||||
BlockHash: [32]byte{},
|
||||
Index: 0,
|
||||
TxHash: [32]byte{},
|
||||
Version: 0,
|
||||
LockTime: 0,
|
||||
TxIn: []*types.TxIn{},
|
||||
TxOut: []*types.TxOut{},
|
||||
func (a awsTransaction) ToTransaction(block awsBlock) (*types.Transaction, error) {
|
||||
blockhash, err := chainhash.NewHashFromStr(block.Hash)
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, "can't convert block hash")
|
||||
}
|
||||
msgtx, err := a.MsgTx(block)
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, "can't convert aws tx to wire.msgtx")
|
||||
}
|
||||
return types.ParseMsgTx(msgtx, a.BlockNumber, *blockhash, uint32(a.Index)), nil
|
||||
}
|
||||
|
||||
func (a awsTransaction) MsgTx(block awsBlock) (*wire.MsgTx, error) {
|
||||
txIn := make([]*wire.TxIn, 0, len(a.Inputs))
|
||||
txOut := make([]*wire.TxOut, 0, len(a.Outputs))
|
||||
|
||||
// coinbase tx from AWS S3 has no inputs, so we need to add it manually
|
||||
if a.IsCoinbase {
|
||||
scriptsig, err := hex.DecodeString(block.CoinbaseParam)
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, "can't decode script hex")
|
||||
}
|
||||
|
||||
txIn = append(txIn, &wire.TxIn{
|
||||
PreviousOutPoint: wire.OutPoint{
|
||||
Hash: common.ZeroHash,
|
||||
Index: math.MaxUint32,
|
||||
},
|
||||
SignatureScript: scriptsig,
|
||||
Witness: btcutils.CoinbaseWitness,
|
||||
Sequence: 0,
|
||||
})
|
||||
}
|
||||
|
||||
for _, in := range a.Inputs {
|
||||
scriptsig, err := hex.DecodeString(in.ScriptHex)
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, "can't decode script hex")
|
||||
}
|
||||
|
||||
witness, err := btcutils.WitnessFromHex(lo.Map(in.TxInWitness, func(src *string, _ int) string {
|
||||
if src == nil {
|
||||
return ""
|
||||
}
|
||||
return *src
|
||||
}))
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, "can't convert witness")
|
||||
}
|
||||
|
||||
prevOutHash, err := chainhash.NewHashFromStr(in.SpentTransactionHash)
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, "can't convert prevout hash")
|
||||
}
|
||||
|
||||
txIn = append(txIn, &wire.TxIn{
|
||||
PreviousOutPoint: wire.OutPoint{
|
||||
Hash: *prevOutHash,
|
||||
Index: uint32(in.SpentOutputIndex),
|
||||
},
|
||||
SignatureScript: scriptsig,
|
||||
Witness: witness,
|
||||
Sequence: uint32(in.Sequence),
|
||||
})
|
||||
}
|
||||
|
||||
for _, out := range a.Outputs {
|
||||
scriptpubkey, err := hex.DecodeString(out.Script_hex)
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, "can't decode script hex")
|
||||
}
|
||||
txOut = append(txOut, &wire.TxOut{
|
||||
Value: btcutils.BitcoinToSatoshi(out.Value),
|
||||
PkScript: scriptpubkey,
|
||||
})
|
||||
}
|
||||
|
||||
return &wire.MsgTx{
|
||||
Version: int32(a.Version),
|
||||
TxIn: txIn,
|
||||
TxOut: txOut,
|
||||
LockTime: uint32(a.LockTime),
|
||||
}, nil
|
||||
}
|
||||
|
||||
2
go.mod
2
go.mod
@@ -20,7 +20,7 @@ require (
|
||||
github.com/planxnx/concurrent-stream v0.1.5
|
||||
github.com/samber/do/v2 v2.0.0-beta.7
|
||||
github.com/samber/lo v1.39.0
|
||||
github.com/shopspring/decimal v1.3.1
|
||||
github.com/shopspring/decimal v1.4.0
|
||||
github.com/spf13/cobra v1.8.0
|
||||
github.com/spf13/pflag v1.0.5
|
||||
github.com/spf13/viper v1.18.2
|
||||
|
||||
4
go.sum
4
go.sum
@@ -657,8 +657,8 @@ github.com/samber/lo v1.39.0/go.mod h1:+m/ZKRl6ClXCE2Lgf3MsQlWfh4bn1bz6CXEOxnEXn
|
||||
github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0=
|
||||
github.com/shopspring/decimal v0.0.0-20180709203117-cd690d0c9e24/go.mod h1:M+9NzErvs504Cn4c5DxATwIqPbtswREoFCre64PpcG4=
|
||||
github.com/shopspring/decimal v1.2.0/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o=
|
||||
github.com/shopspring/decimal v1.3.1 h1:2Usl1nmF/WZucqkFZhnfFYxxxu8LG21F6nPQBE5gKV8=
|
||||
github.com/shopspring/decimal v1.3.1/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o=
|
||||
github.com/shopspring/decimal v1.4.0 h1:bxl37RwXBklmTi0C79JfXCEBD1cqqHt0bbgBAGFp81k=
|
||||
github.com/shopspring/decimal v1.4.0/go.mod h1:gawqmDU56v4yIKSwfBSFip1HdCCXN8/+DMd9qYNcwME=
|
||||
github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q=
|
||||
github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE=
|
||||
github.com/sirupsen/logrus v1.9.0/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ=
|
||||
|
||||
21
pkg/btcutils/satoshi.go
Normal file
21
pkg/btcutils/satoshi.go
Normal file
@@ -0,0 +1,21 @@
|
||||
package btcutils
|
||||
|
||||
import "github.com/shopspring/decimal"
|
||||
|
||||
const (
|
||||
BitcoinDecimals = 8
|
||||
)
|
||||
|
||||
// satsUnit is 10^8
|
||||
var satsUnit = decimal.New(1, BitcoinDecimals)
|
||||
|
||||
// BitcoinToSatoshi converts a amount in Bitcoin format to Satoshi format.
|
||||
func BitcoinToSatoshi(v float64) int64 {
|
||||
amount := decimal.NewFromFloat(v)
|
||||
return amount.Mul(satsUnit).IntPart()
|
||||
}
|
||||
|
||||
// SatoshiToBitcoin converts a amount in Satoshi format to Bitcoin format.
|
||||
func SatoshiToBitcoin(v int64) float64 {
|
||||
return decimal.New(v, -BitcoinDecimals).InexactFloat64()
|
||||
}
|
||||
39
pkg/btcutils/satoshi_test.go
Normal file
39
pkg/btcutils/satoshi_test.go
Normal file
@@ -0,0 +1,39 @@
|
||||
package btcutils
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestSatoshiConversion(t *testing.T) {
|
||||
testcases := []struct {
|
||||
sats float64
|
||||
btc int64
|
||||
}{
|
||||
{2.29980951, 229980951},
|
||||
{1.29609085, 129609085},
|
||||
{1.2768897, 127688970},
|
||||
{0.62518296, 62518296},
|
||||
{0.29998462, 29998462},
|
||||
{0.1251, 12510000},
|
||||
{0.02016011, 2016011},
|
||||
{0.0198473, 1984730},
|
||||
{0.0051711, 517110},
|
||||
{0.0012, 120000},
|
||||
{7e-05, 7000},
|
||||
{3.835e-05, 3835},
|
||||
{1.962e-05, 1962},
|
||||
}
|
||||
for _, testcase := range testcases {
|
||||
t.Run(fmt.Sprintf("BtcToSats/%v", testcase.sats), func(t *testing.T) {
|
||||
require.NotEqual(t, testcase.btc, int64(testcase.sats*1e8), "Testcase value should have precision error")
|
||||
assert.Equal(t, testcase.btc, BitcoinToSatoshi(testcase.sats))
|
||||
})
|
||||
t.Run(fmt.Sprintf("SatsToBtc/%v", testcase.sats), func(t *testing.T) {
|
||||
assert.Equal(t, testcase.sats, SatoshiToBitcoin(testcase.btc))
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -4,6 +4,7 @@ import (
|
||||
"encoding/hex"
|
||||
"strings"
|
||||
|
||||
"github.com/Cleverse/go-utilities/utils"
|
||||
"github.com/btcsuite/btcd/wire"
|
||||
"github.com/cockroachdb/errors"
|
||||
)
|
||||
@@ -12,6 +13,9 @@ const (
|
||||
witnessSeparator = " "
|
||||
)
|
||||
|
||||
// CoinbaseWitness is the witness data for a coinbase transaction.
|
||||
var CoinbaseWitness = utils.Must(WitnessFromHex([]string{"0000000000000000000000000000000000000000000000000000000000000000"}))
|
||||
|
||||
// WitnessToHex formats the passed witness stack as a slice of hex-encoded strings.
|
||||
func WitnessToHex(witness wire.TxWitness) []string {
|
||||
if len(witness) == 0 {
|
||||
|
||||
Reference in New Issue
Block a user