From 806d27fb4653149b3ea2148230fc0c869c0f375d Mon Sep 17 00:00:00 2001 From: Gaze Date: Sat, 8 Jun 2024 00:23:51 +0700 Subject: [PATCH 01/14] fix: remove wrong incomplete field check --- modules/brc20/internal/ordinals/envelope.go | 5 ----- 1 file changed, 5 deletions(-) diff --git a/modules/brc20/internal/ordinals/envelope.go b/modules/brc20/internal/ordinals/envelope.go index 01855d4..72067c6 100644 --- a/modules/brc20/internal/ordinals/envelope.go +++ b/modules/brc20/internal/ordinals/envelope.go @@ -177,11 +177,6 @@ func envelopeFromTokenizer(tokenizer txscript.ScriptTokenizer, inputIndex int, o key := chunk[0] value := chunk[1] // key cannot be empty, as checked by bodyIndex above - // if key exceeds 1 byte, it would not match any tags - if len(key) > 1 { - incompleteField = true - continue - } tag := Tag(key[0]) fields[tag] = append(fields[tag], value) } From 99bdf49f0231ec97d5328e48a61a51a59d40f381 Mon Sep 17 00:00:00 2001 From: Gaze Date: Sun, 9 Jun 2024 14:49:13 +0700 Subject: [PATCH 02/14] feat: brc20 indexing logic --- .../000001_initialize_tables.up.sql | 16 +- .../database/postgresql/queries/data.sql | 41 +- modules/brc20/internal/datagateway/brc20.go | 11 + modules/brc20/internal/entity/balance.go | 11 + modules/brc20/internal/entity/event_deploy.go | 2 +- .../entity/event_inscribe_transfer.go | 2 +- modules/brc20/internal/entity/event_mint.go | 2 +- .../entity/event_transfer_transfer.go | 3 +- modules/brc20/internal/entity/flotsam.go | 1 + .../internal/entity/inscription_transfer.go | 8 +- .../internal/repository/postgres/brc20.go | 104 +++++ .../internal/repository/postgres/gen/batch.go | 12 +- .../repository/postgres/gen/data.sql.go | 186 +++++++- .../repository/postgres/gen/models.go | 4 + .../internal/repository/postgres/mapper.go | 42 +- modules/brc20/processor.go | 18 +- modules/brc20/processor_brc20.go | 403 ++++++++++++++++-- modules/brc20/processor_inscription.go | 81 +++- 18 files changed, 884 insertions(+), 63 deletions(-) create mode 100644 modules/brc20/internal/entity/balance.go diff --git a/modules/brc20/database/postgresql/migrations/000001_initialize_tables.up.sql b/modules/brc20/database/postgresql/migrations/000001_initialize_tables.up.sql index 86659cd..01e5faf 100644 --- a/modules/brc20/database/postgresql/migrations/000001_initialize_tables.up.sql +++ b/modules/brc20/database/postgresql/migrations/000001_initialize_tables.up.sql @@ -50,10 +50,8 @@ CREATE TABLE IF NOT EXISTS "brc20_tick_entry_states" ( PRIMARY KEY ("tick", "block_height") ); -CREATE SEQUENCE IF NOT EXISTS brc20_event_id_seq; - CREATE TABLE IF NOT EXISTS "brc20_event_deploys" ( - "id" BIGINT PRIMARY KEY DEFAULT nextval('brc20_event_id_seq'), + "id" BIGINT PRIMARY KEY NOT NULL, "inscription_id" TEXT NOT NULL, "inscription_number" BIGINT NOT NULL, "tick" TEXT NOT NULL, -- lowercase of original_tick @@ -73,7 +71,7 @@ CREATE TABLE IF NOT EXISTS "brc20_event_deploys" ( CREATE INDEX IF NOT EXISTS brc20_event_deploys_block_height_idx ON "brc20_event_deploys" USING BTREE ("block_height"); CREATE TABLE IF NOT EXISTS "brc20_event_mints" ( - "id" BIGINT PRIMARY KEY DEFAULT nextval('brc20_event_id_seq'), + "id" BIGINT PRIMARY KEY NOT NULL, "inscription_id" TEXT NOT NULL, "inscription_number" BIGINT NOT NULL, "tick" TEXT NOT NULL, -- lowercase of original_tick @@ -91,7 +89,7 @@ CREATE TABLE IF NOT EXISTS "brc20_event_mints" ( CREATE INDEX IF NOT EXISTS brc20_event_mints_block_height_idx ON "brc20_event_mints" USING BTREE ("block_height"); CREATE TABLE IF NOT EXISTS "brc20_event_inscribe_transfers" ( - "id" BIGINT PRIMARY KEY DEFAULT nextval('brc20_event_id_seq'), + "id" BIGINT PRIMARY KEY NOT NULL, "inscription_id" TEXT NOT NULL, "inscription_number" BIGINT NOT NULL, "tick" TEXT NOT NULL, -- lowercase of original_tick @@ -108,9 +106,10 @@ CREATE TABLE IF NOT EXISTS "brc20_event_inscribe_transfers" ( "amount" DECIMAL NOT NULL ); CREATE INDEX IF NOT EXISTS brc20_event_inscribe_transfers_block_height_idx ON "brc20_event_inscribe_transfers" USING BTREE ("block_height"); +CREATE INDEX IF NOT EXISTS brc20_event_inscribe_transfers_inscription_id_idx ON "brc20_event_inscribe_transfers" USING BTREE ("inscription_id"); -- used for validating transfer transfer events CREATE TABLE IF NOT EXISTS "brc20_event_transfer_transfers" ( - "id" BIGINT PRIMARY KEY DEFAULT nextval('brc20_event_id_seq'), + "id" BIGINT PRIMARY KEY NOT NULL, "inscription_id" TEXT NOT NULL, "inscription_number" BIGINT NOT NULL, "tick" TEXT NOT NULL, -- lowercase of original_tick @@ -126,6 +125,7 @@ CREATE TABLE IF NOT EXISTS "brc20_event_transfer_transfers" ( "to_pkscript" TEXT NOT NULL, "to_satpoint" TEXT NOT NULL, "to_output_index" INT NOT NULL, + "spent_as_fee" BOOLEAN NOT NULL, "amount" DECIMAL NOT NULL ); CREATE INDEX IF NOT EXISTS brc20_event_transfer_transfers_block_height_idx ON "brc20_event_transfer_transfers" USING BTREE ("block_height"); @@ -156,6 +156,7 @@ CREATE TABLE IF NOT EXISTS "brc20_inscription_entries" ( "created_at" TIMESTAMP NOT NULL, "created_at_height" INT NOT NULL ); +CREATE INDEX IF NOT EXISTS brc20_inscription_entries_id_number_idx ON "brc20_inscription_entries" USING BTREE ("id", "number"); CREATE TABLE IF NOT EXISTS "brc20_inscription_entry_states" ( "id" TEXT NOT NULL, @@ -168,6 +169,8 @@ CREATE TABLE IF NOT EXISTS "brc20_inscription_transfers" ( "inscription_id" TEXT NOT NULL, "block_height" INT NOT NULL, "tx_index" INT NOT NULL, + "tx_hash" TEXT NOT NULL, + "from_input_index" INT NOT NULL, "old_satpoint_tx_hash" TEXT, "old_satpoint_out_idx" INT, "old_satpoint_offset" BIGINT, @@ -177,6 +180,7 @@ CREATE TABLE IF NOT EXISTS "brc20_inscription_transfers" ( "new_pkscript" TEXT NOT NULL, "new_output_value" BIGINT NOT NULL, "sent_as_fee" BOOLEAN NOT NULL, + "transfer_count" INT NOT NULL, PRIMARY KEY ("inscription_id", "block_height", "tx_index") ); CREATE INDEX IF NOT EXISTS brc20_inscription_transfers_block_height_tx_index_idx ON "brc20_inscription_transfers" USING BTREE ("block_height", "tx_index"); diff --git a/modules/brc20/database/postgresql/queries/data.sql b/modules/brc20/database/postgresql/queries/data.sql index bda94ee..14ca824 100644 --- a/modules/brc20/database/postgresql/queries/data.sql +++ b/modules/brc20/database/postgresql/queries/data.sql @@ -35,6 +35,43 @@ SELECT * FROM "brc20_tick_entries" LEFT JOIN "states" ON "brc20_tick_entries"."tick" = "states"."tick" WHERE "brc20_tick_entries"."tick" = ANY(@ticks::text[]); +-- name: GetInscriptionNumbersByIds :many +SELECT id, number FROM "brc20_inscription_entries" WHERE "id" = ANY(@inscription_ids::text[]); + +-- name: GetInscriptionParentsByIds :many +SELECT id, parents FROM "brc20_inscription_entries" WHERE "id" = ANY(@inscription_ids::text[]); + +-- name: GetLatestEventIds :one +WITH "latest_deploy_id" AS ( + SELECT "id" FROM "brc20_event_deploys" ORDER BY "id" DESC LIMIT 1 +), +"latest_mint_id" AS ( + SELECT "id" FROM "brc20_event_mints" ORDER BY "id" DESC LIMIT 1 +), +"latest_inscribe_transfer_id" AS ( + SELECT "id" FROM "brc20_event_inscribe_transfers" ORDER BY "id" DESC LIMIT 1 +), +"latest_transfer_transfer_id" AS ( + SELECT "id" FROM "brc20_event_transfer_transfers" ORDER BY "id" DESC LIMIT 1 +) +SELECT + (SELECT "id" FROM "latest_deploy_id") AS "event_deploy_id", + (SELECT "id" FROM "latest_mint_id") AS "event_mint_id", + (SELECT "id" FROM "latest_inscribe_transfer_id") AS "event_inscribe_transfer_id", + (SELECT "id" FROM "latest_transfer_transfer_id") AS "event_transfer_transfer_id"; + +-- name: GetBalancesBatchAtHeight :many +SELECT DISTINCT ON ("brc20_balances"."pkscript", "brc20_balances"."tick") "brc20_balances".* FROM "brc20_balances" + INNER JOIN ( + SELECT + unnest(@pkscript_arr::text[]) AS "pkscript", + unnest(@tick_arr::text[]) AS "tick" + ) "queries" ON "brc20_balances"."pkscript" = "queries"."pkscript" AND "brc20_balances"."tick" = "queries"."tick" AND "brc20_balances"."block_height" <= @block_height + ORDER BY "brc20_balances"."pkscript", "brc20_balances"."tick", "block_height" DESC; + +-- name: GetEventInscribeTransfersByInscriptionIds :many +SELECT * FROM "brc20_event_inscribe_transfers" WHERE "inscription_id" = ANY(@inscription_ids::text[]); + -- name: CreateIndexedBlock :exec INSERT INTO "brc20_indexed_blocks" ("height", "hash", "event_hash", "cumulative_event_hash") VALUES ($1, $2, $3, $4); @@ -54,7 +91,7 @@ INSERT INTO "brc20_inscription_entries" ("id", "number", "sequence_number", "del INSERT INTO "brc20_inscription_entry_states" ("id", "block_height", "transfer_count") VALUES ($1, $2, $3); -- name: CreateInscriptionTransfers :batchexec -INSERT INTO "brc20_inscription_transfers" ("inscription_id", "block_height", "tx_index", "old_satpoint_tx_hash", "old_satpoint_out_idx", "old_satpoint_offset", "new_satpoint_tx_hash", "new_satpoint_out_idx", "new_satpoint_offset", "new_pkscript", "new_output_value", "sent_as_fee") VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12); +INSERT INTO "brc20_inscription_transfers" ("inscription_id", "block_height", "tx_index", "tx_hash", "from_input_index", "old_satpoint_tx_hash", "old_satpoint_out_idx", "old_satpoint_offset", "new_satpoint_tx_hash", "new_satpoint_out_idx", "new_satpoint_offset", "new_pkscript", "new_output_value", "sent_as_fee", "transfer_count") VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15); -- name: CreateEventDeploys :batchexec INSERT INTO "brc20_event_deploys" ("inscription_id", "inscription_number", "tick", "original_tick", "tx_hash", "block_height", "tx_index", "timestamp", "pkscript", "satpoint", "total_supply", "decimals", "limit_per_mint", "is_self_mint") VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14); @@ -66,7 +103,7 @@ INSERT INTO "brc20_event_mints" ("inscription_id", "inscription_number", "tick", INSERT INTO "brc20_event_inscribe_transfers" ("inscription_id", "inscription_number", "tick", "original_tick", "tx_hash", "block_height", "tx_index", "timestamp", "pkscript", "satpoint", "output_index", "sats_amount", "amount") VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13); -- name: CreateEventTransferTransfers :batchexec -INSERT INTO "brc20_event_transfer_transfers" ("inscription_id", "inscription_number", "tick", "original_tick", "tx_hash", "block_height", "tx_index", "timestamp", "from_pkscript", "from_satpoint", "from_input_index", "to_pkscript", "to_satpoint", "to_output_index", "amount") VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15); +INSERT INTO "brc20_event_transfer_transfers" ("inscription_id", "inscription_number", "tick", "original_tick", "tx_hash", "block_height", "tx_index", "timestamp", "from_pkscript", "from_satpoint", "from_input_index", "to_pkscript", "to_satpoint", "to_output_index", "spent_as_fee", "amount") VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16); -- name: DeleteIndexedBlocksSinceHeight :exec DELETE FROM "brc20_indexed_blocks" WHERE "height" >= $1; diff --git a/modules/brc20/internal/datagateway/brc20.go b/modules/brc20/internal/datagateway/brc20.go index 1721ab9..9d0d429 100644 --- a/modules/brc20/internal/datagateway/brc20.go +++ b/modules/brc20/internal/datagateway/brc20.go @@ -28,7 +28,12 @@ type BRC20ReaderDataGateway interface { GetProcessorStats(ctx context.Context) (*entity.ProcessorStats, error) GetInscriptionTransfersInOutPoints(ctx context.Context, outPoints []wire.OutPoint) (map[ordinals.SatPoint][]*entity.InscriptionTransfer, error) GetInscriptionEntriesByIds(ctx context.Context, ids []ordinals.InscriptionId) (map[ordinals.InscriptionId]*ordinals.InscriptionEntry, error) + GetInscriptionNumbersByIds(ctx context.Context, ids []ordinals.InscriptionId) (map[ordinals.InscriptionId]int64, error) + GetInscriptionParentsByIds(ctx context.Context, ids []ordinals.InscriptionId) (map[ordinals.InscriptionId]ordinals.InscriptionId, error) + GetBalancesBatchAtHeight(ctx context.Context, blockHeight uint64, queries []GetBalancesBatchAtHeightQuery) (map[string]map[string]*entity.Balance, error) GetTickEntriesByTicks(ctx context.Context, ticks []string) (map[string]*entity.TickEntry, error) + GetEventInscribeTransfersByInscriptionIds(ctx context.Context, ids []ordinals.InscriptionId) (map[ordinals.InscriptionId]*entity.EventInscribeTransfer, error) + GetLatestEventId(ctx context.Context) (uint64, error) } type BRC20WriterDataGateway interface { @@ -58,3 +63,9 @@ type BRC20WriterDataGateway interface { DeleteInscriptionEntryStatesSinceHeight(ctx context.Context, height uint64) error DeleteInscriptionTransfersSinceHeight(ctx context.Context, height uint64) error } + +type GetBalancesBatchAtHeightQuery struct { + PkScriptHex string + Tick string + BlockHeight uint64 +} diff --git a/modules/brc20/internal/entity/balance.go b/modules/brc20/internal/entity/balance.go new file mode 100644 index 0000000..c959801 --- /dev/null +++ b/modules/brc20/internal/entity/balance.go @@ -0,0 +1,11 @@ +package entity + +import "github.com/shopspring/decimal" + +type Balance struct { + PkScript []byte + Tick string + BlockHeight uint64 + OverallBalance decimal.Decimal + AvailableBalance decimal.Decimal +} diff --git a/modules/brc20/internal/entity/event_deploy.go b/modules/brc20/internal/entity/event_deploy.go index 35cf6b7..55277d5 100644 --- a/modules/brc20/internal/entity/event_deploy.go +++ b/modules/brc20/internal/entity/event_deploy.go @@ -11,7 +11,7 @@ import ( type EventDeploy struct { Id uint64 InscriptionId ordinals.InscriptionId - InscriptionNumber uint64 + InscriptionNumber int64 Tick string OriginalTick string TxHash chainhash.Hash diff --git a/modules/brc20/internal/entity/event_inscribe_transfer.go b/modules/brc20/internal/entity/event_inscribe_transfer.go index 9c35eb3..34293cd 100644 --- a/modules/brc20/internal/entity/event_inscribe_transfer.go +++ b/modules/brc20/internal/entity/event_inscribe_transfer.go @@ -11,7 +11,7 @@ import ( type EventInscribeTransfer struct { Id uint64 InscriptionId ordinals.InscriptionId - InscriptionNumber uint64 + InscriptionNumber int64 Tick string OriginalTick string TxHash chainhash.Hash diff --git a/modules/brc20/internal/entity/event_mint.go b/modules/brc20/internal/entity/event_mint.go index 4854b0c..d358ed1 100644 --- a/modules/brc20/internal/entity/event_mint.go +++ b/modules/brc20/internal/entity/event_mint.go @@ -11,7 +11,7 @@ import ( type EventMint struct { Id uint64 InscriptionId ordinals.InscriptionId - InscriptionNumber uint64 + InscriptionNumber int64 Tick string OriginalTick string TxHash chainhash.Hash diff --git a/modules/brc20/internal/entity/event_transfer_transfer.go b/modules/brc20/internal/entity/event_transfer_transfer.go index f449254..2058184 100644 --- a/modules/brc20/internal/entity/event_transfer_transfer.go +++ b/modules/brc20/internal/entity/event_transfer_transfer.go @@ -11,7 +11,7 @@ import ( type EventTransferTransfer struct { Id uint64 InscriptionId ordinals.InscriptionId - InscriptionNumber uint64 + InscriptionNumber int64 Tick string OriginalTick string TxHash chainhash.Hash @@ -25,5 +25,6 @@ type EventTransferTransfer struct { ToPkScript []byte ToSatPoint ordinals.SatPoint ToOutputIndex uint32 + SpentAsFee bool Amount decimal.Decimal } diff --git a/modules/brc20/internal/entity/flotsam.go b/modules/brc20/internal/entity/flotsam.go index a658b32..384a80c 100644 --- a/modules/brc20/internal/entity/flotsam.go +++ b/modules/brc20/internal/entity/flotsam.go @@ -8,6 +8,7 @@ import ( type OriginOld struct { Content []byte OldSatPoint ordinals.SatPoint + InputIndex uint32 } type OriginNew struct { Inscription ordinals.Inscription diff --git a/modules/brc20/internal/entity/inscription_transfer.go b/modules/brc20/internal/entity/inscription_transfer.go index 6e1aa8b..6d63644 100644 --- a/modules/brc20/internal/entity/inscription_transfer.go +++ b/modules/brc20/internal/entity/inscription_transfer.go @@ -1,15 +1,21 @@ package entity -import "github.com/gaze-network/indexer-network/modules/brc20/internal/ordinals" +import ( + "github.com/btcsuite/btcd/chaincfg/chainhash" + "github.com/gaze-network/indexer-network/modules/brc20/internal/ordinals" +) type InscriptionTransfer struct { InscriptionId ordinals.InscriptionId BlockHeight uint64 TxIndex uint32 + TxHash chainhash.Hash Content []byte + FromInputIndex uint32 OldSatPoint ordinals.SatPoint NewSatPoint ordinals.SatPoint NewPkScript []byte NewOutputValue uint64 SentAsFee bool + TransferCount uint32 } diff --git a/modules/brc20/internal/repository/postgres/brc20.go b/modules/brc20/internal/repository/postgres/brc20.go index c39a5b4..21e3d53 100644 --- a/modules/brc20/internal/repository/postgres/brc20.go +++ b/modules/brc20/internal/repository/postgres/brc20.go @@ -110,6 +110,110 @@ func (r *Repository) GetInscriptionEntriesByIds(ctx context.Context, ids []ordin return result, nil } +func (r *Repository) GetInscriptionNumbersByIds(ctx context.Context, ids []ordinals.InscriptionId) (map[ordinals.InscriptionId]int64, error) { + idStrs := lo.Map(ids, func(id ordinals.InscriptionId, _ int) string { return id.String() }) + models, err := r.queries.GetInscriptionNumbersByIds(ctx, idStrs) + if err != nil { + return nil, errors.WithStack(err) + } + + result := make(map[ordinals.InscriptionId]int64) + for _, model := range models { + inscriptionId, err := ordinals.NewInscriptionIdFromString(model.Id) + if err != nil { + return nil, errors.Wrap(err, "failed to parse inscription id") + } + result[inscriptionId] = model.Number + } + return result, nil +} + +func (r *Repository) GetInscriptionParentsByIds(ctx context.Context, ids []ordinals.InscriptionId) (map[ordinals.InscriptionId]ordinals.InscriptionId, error) { + idStrs := lo.Map(ids, func(id ordinals.InscriptionId, _ int) string { return id.String() }) + models, err := r.queries.GetInscriptionParentsByIds(ctx, idStrs) + if err != nil { + return nil, errors.WithStack(err) + } + + result := make(map[ordinals.InscriptionId]ordinals.InscriptionId) + for _, model := range models { + if len(model.Parents) == 0 { + // no parent + continue + } + if len(model.Parents) > 1 { + // sanity check, should not happen since 0.14 ord supports only 1 parent + continue + } + inscriptionId, err := ordinals.NewInscriptionIdFromString(model.Id) + if err != nil { + return nil, errors.Wrap(err, "failed to parse inscription id") + } + parentId, err := ordinals.NewInscriptionIdFromString(model.Parents[0]) + if err != nil { + return nil, errors.Wrap(err, "failed to parse parent id") + } + result[inscriptionId] = parentId + } + return result, nil +} + +func (r *Repository) GetLatestEventId(ctx context.Context) (uint64, error) { + row, err := r.queries.GetLatestEventIds(ctx) + if err != nil { + return 0, errors.WithStack(err) + } + return uint64(max(row.EventDeployID, row.EventMintID, row.EventInscribeTransferID, row.EventTransferTransferID)), nil +} + +func (r *Repository) GetBalancesBatchAtHeight(ctx context.Context, blockHeight uint64, queries []datagateway.GetBalancesBatchAtHeightQuery) (map[string]map[string]*entity.Balance, error) { + pkScripts := make([]string, 0) + ticks := make([]string, 0) + for _, query := range queries { + pkScripts = append(pkScripts, query.PkScriptHex) + ticks = append(ticks, query.Tick) + } + models, err := r.queries.GetBalancesBatchAtHeight(ctx, gen.GetBalancesBatchAtHeightParams{ + PkscriptArr: pkScripts, + TickArr: ticks, + BlockHeight: int32(blockHeight), + }) + if err != nil { + return nil, errors.WithStack(err) + } + + result := make(map[string]map[string]*entity.Balance) + for _, model := range models { + balance, err := mapBalanceModelToType(model) + if err != nil { + return nil, errors.Wrap(err, "failed to parse balance model") + } + if _, ok := result[model.Pkscript]; !ok { + result[model.Pkscript] = make(map[string]*entity.Balance) + } + result[model.Pkscript][model.Tick] = &balance + } + return result, nil +} + +func (r *Repository) GetEventInscribeTransfersByInscriptionIds(ctx context.Context, ids []ordinals.InscriptionId) (map[ordinals.InscriptionId]*entity.EventInscribeTransfer, error) { + idStrs := lo.Map(ids, func(id ordinals.InscriptionId, _ int) string { return id.String() }) + models, err := r.queries.GetEventInscribeTransfersByInscriptionIds(ctx, idStrs) + if err != nil { + return nil, errors.WithStack(err) + } + + result := make(map[ordinals.InscriptionId]*entity.EventInscribeTransfer) + for _, model := range models { + event, err := mapEventInscribeTransferModelToType(model) + if err != nil { + return nil, errors.Wrap(err, "failed to parse event inscribe transfer model") + } + result[event.InscriptionId] = &event + } + return result, nil +} + func (r *Repository) GetTickEntriesByTicks(ctx context.Context, ticks []string) (map[string]*entity.TickEntry, error) { models, err := r.queries.GetTickEntriesByTicks(ctx, ticks) if err != nil { diff --git a/modules/brc20/internal/repository/postgres/gen/batch.go b/modules/brc20/internal/repository/postgres/gen/batch.go index 42f99c1..10fe598 100644 --- a/modules/brc20/internal/repository/postgres/gen/batch.go +++ b/modules/brc20/internal/repository/postgres/gen/batch.go @@ -231,7 +231,7 @@ func (b *CreateEventMintsBatchResults) Close() error { } const createEventTransferTransfers = `-- name: CreateEventTransferTransfers :batchexec -INSERT INTO "brc20_event_transfer_transfers" ("inscription_id", "inscription_number", "tick", "original_tick", "tx_hash", "block_height", "tx_index", "timestamp", "from_pkscript", "from_satpoint", "from_input_index", "to_pkscript", "to_satpoint", "to_output_index", "amount") VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15) +INSERT INTO "brc20_event_transfer_transfers" ("inscription_id", "inscription_number", "tick", "original_tick", "tx_hash", "block_height", "tx_index", "timestamp", "from_pkscript", "from_satpoint", "from_input_index", "to_pkscript", "to_satpoint", "to_output_index", "spent_as_fee", "amount") VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16) ` type CreateEventTransferTransfersBatchResults struct { @@ -255,6 +255,7 @@ type CreateEventTransferTransfersParams struct { ToPkscript string ToSatpoint string ToOutputIndex int32 + SpentAsFee bool Amount pgtype.Numeric } @@ -276,6 +277,7 @@ func (q *Queries) CreateEventTransferTransfers(ctx context.Context, arg []Create a.ToPkscript, a.ToSatpoint, a.ToOutputIndex, + a.SpentAsFee, a.Amount, } batch.Queue(createEventTransferTransfers, vals...) @@ -432,7 +434,7 @@ func (b *CreateInscriptionEntryStatesBatchResults) Close() error { } const createInscriptionTransfers = `-- name: CreateInscriptionTransfers :batchexec -INSERT INTO "brc20_inscription_transfers" ("inscription_id", "block_height", "tx_index", "old_satpoint_tx_hash", "old_satpoint_out_idx", "old_satpoint_offset", "new_satpoint_tx_hash", "new_satpoint_out_idx", "new_satpoint_offset", "new_pkscript", "new_output_value", "sent_as_fee") VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12) +INSERT INTO "brc20_inscription_transfers" ("inscription_id", "block_height", "tx_index", "tx_hash", "from_input_index", "old_satpoint_tx_hash", "old_satpoint_out_idx", "old_satpoint_offset", "new_satpoint_tx_hash", "new_satpoint_out_idx", "new_satpoint_offset", "new_pkscript", "new_output_value", "sent_as_fee", "transfer_count") VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15) ` type CreateInscriptionTransfersBatchResults struct { @@ -445,6 +447,8 @@ type CreateInscriptionTransfersParams struct { InscriptionID string BlockHeight int32 TxIndex int32 + TxHash string + FromInputIndex int32 OldSatpointTxHash pgtype.Text OldSatpointOutIdx pgtype.Int4 OldSatpointOffset pgtype.Int8 @@ -454,6 +458,7 @@ type CreateInscriptionTransfersParams struct { NewPkscript string NewOutputValue int64 SentAsFee bool + TransferCount int32 } func (q *Queries) CreateInscriptionTransfers(ctx context.Context, arg []CreateInscriptionTransfersParams) *CreateInscriptionTransfersBatchResults { @@ -463,6 +468,8 @@ func (q *Queries) CreateInscriptionTransfers(ctx context.Context, arg []CreateIn a.InscriptionID, a.BlockHeight, a.TxIndex, + a.TxHash, + a.FromInputIndex, a.OldSatpointTxHash, a.OldSatpointOutIdx, a.OldSatpointOffset, @@ -472,6 +479,7 @@ func (q *Queries) CreateInscriptionTransfers(ctx context.Context, arg []CreateIn a.NewPkscript, a.NewOutputValue, a.SentAsFee, + a.TransferCount, } batch.Queue(createInscriptionTransfers, vals...) } diff --git a/modules/brc20/internal/repository/postgres/gen/data.sql.go b/modules/brc20/internal/repository/postgres/gen/data.sql.go index b1bf33e..f71bf41 100644 --- a/modules/brc20/internal/repository/postgres/gen/data.sql.go +++ b/modules/brc20/internal/repository/postgres/gen/data.sql.go @@ -161,6 +161,87 @@ func (q *Queries) DeleteTickEntryStatesSinceHeight(ctx context.Context, blockHei return err } +const getBalancesBatchAtHeight = `-- name: GetBalancesBatchAtHeight :many +SELECT DISTINCT ON ("brc20_balances"."pkscript", "brc20_balances"."tick") brc20_balances.pkscript, brc20_balances.block_height, brc20_balances.tick, brc20_balances.overall_balance, brc20_balances.available_balance FROM "brc20_balances" + INNER JOIN ( + SELECT + unnest($1::text[]) AS "pkscript", + unnest($2::text[]) AS "tick" + ) "queries" ON "brc20_balances"."pkscript" = "queries"."pkscript" AND "brc20_balances"."tick" = "queries"."tick" AND "brc20_balances"."block_height" <= $3 + ORDER BY "brc20_balances"."pkscript", "brc20_balances"."tick", "block_height" DESC +` + +type GetBalancesBatchAtHeightParams struct { + PkscriptArr []string + TickArr []string + BlockHeight int32 +} + +func (q *Queries) GetBalancesBatchAtHeight(ctx context.Context, arg GetBalancesBatchAtHeightParams) ([]Brc20Balance, error) { + rows, err := q.db.Query(ctx, getBalancesBatchAtHeight, arg.PkscriptArr, arg.TickArr, arg.BlockHeight) + if err != nil { + return nil, err + } + defer rows.Close() + var items []Brc20Balance + for rows.Next() { + var i Brc20Balance + if err := rows.Scan( + &i.Pkscript, + &i.BlockHeight, + &i.Tick, + &i.OverallBalance, + &i.AvailableBalance, + ); err != nil { + return nil, err + } + items = append(items, i) + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} + +const getEventInscribeTransfersByInscriptionIds = `-- name: GetEventInscribeTransfersByInscriptionIds :many +SELECT id, inscription_id, inscription_number, tick, original_tick, tx_hash, block_height, tx_index, timestamp, pkscript, satpoint, output_index, sats_amount, amount FROM "brc20_event_inscribe_transfers" WHERE "inscription_id" = ANY($1::text[]) +` + +func (q *Queries) GetEventInscribeTransfersByInscriptionIds(ctx context.Context, inscriptionIds []string) ([]Brc20EventInscribeTransfer, error) { + rows, err := q.db.Query(ctx, getEventInscribeTransfersByInscriptionIds, inscriptionIds) + if err != nil { + return nil, err + } + defer rows.Close() + var items []Brc20EventInscribeTransfer + for rows.Next() { + var i Brc20EventInscribeTransfer + if err := rows.Scan( + &i.Id, + &i.InscriptionID, + &i.InscriptionNumber, + &i.Tick, + &i.OriginalTick, + &i.TxHash, + &i.BlockHeight, + &i.TxIndex, + &i.Timestamp, + &i.Pkscript, + &i.Satpoint, + &i.OutputIndex, + &i.SatsAmount, + &i.Amount, + ); err != nil { + return nil, err + } + items = append(items, i) + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} + const getIndexedBlockByHeight = `-- name: GetIndexedBlockByHeight :one SELECT height, hash, event_hash, cumulative_event_hash FROM "brc20_indexed_blocks" WHERE "height" = $1 ` @@ -247,8 +328,66 @@ func (q *Queries) GetInscriptionEntriesByIds(ctx context.Context, inscriptionIds return items, nil } +const getInscriptionNumbersByIds = `-- name: GetInscriptionNumbersByIds :many +SELECT id, number FROM "brc20_inscription_entries" WHERE "id" = ANY($1::text[]) +` + +type GetInscriptionNumbersByIdsRow struct { + Id string + Number int64 +} + +func (q *Queries) GetInscriptionNumbersByIds(ctx context.Context, inscriptionIds []string) ([]GetInscriptionNumbersByIdsRow, error) { + rows, err := q.db.Query(ctx, getInscriptionNumbersByIds, inscriptionIds) + if err != nil { + return nil, err + } + defer rows.Close() + var items []GetInscriptionNumbersByIdsRow + for rows.Next() { + var i GetInscriptionNumbersByIdsRow + if err := rows.Scan(&i.Id, &i.Number); err != nil { + return nil, err + } + items = append(items, i) + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} + +const getInscriptionParentsByIds = `-- name: GetInscriptionParentsByIds :many +SELECT id, parents FROM "brc20_inscription_entries" WHERE "id" = ANY($1::text[]) +` + +type GetInscriptionParentsByIdsRow struct { + Id string + Parents []string +} + +func (q *Queries) GetInscriptionParentsByIds(ctx context.Context, inscriptionIds []string) ([]GetInscriptionParentsByIdsRow, error) { + rows, err := q.db.Query(ctx, getInscriptionParentsByIds, inscriptionIds) + if err != nil { + return nil, err + } + defer rows.Close() + var items []GetInscriptionParentsByIdsRow + for rows.Next() { + var i GetInscriptionParentsByIdsRow + if err := rows.Scan(&i.Id, &i.Parents); err != nil { + return nil, err + } + items = append(items, i) + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} + const getInscriptionTransfersInOutPoints = `-- name: GetInscriptionTransfersInOutPoints :many -SELECT it.inscription_id, it.block_height, it.tx_index, it.old_satpoint_tx_hash, it.old_satpoint_out_idx, it.old_satpoint_offset, it.new_satpoint_tx_hash, it.new_satpoint_out_idx, it.new_satpoint_offset, it.new_pkscript, it.new_output_value, it.sent_as_fee, "ie"."content" FROM ( +SELECT it.inscription_id, it.block_height, it.tx_index, it.tx_hash, it.from_input_index, it.old_satpoint_tx_hash, it.old_satpoint_out_idx, it.old_satpoint_offset, it.new_satpoint_tx_hash, it.new_satpoint_out_idx, it.new_satpoint_offset, it.new_pkscript, it.new_output_value, it.sent_as_fee, it.transfer_count, "ie"."content" FROM ( SELECT unnest($1::text[]) AS "tx_hash", unnest($2::int[]) AS "tx_out_idx" @@ -266,6 +405,8 @@ type GetInscriptionTransfersInOutPointsRow struct { InscriptionID string BlockHeight int32 TxIndex int32 + TxHash string + FromInputIndex int32 OldSatpointTxHash pgtype.Text OldSatpointOutIdx pgtype.Int4 OldSatpointOffset pgtype.Int8 @@ -275,6 +416,7 @@ type GetInscriptionTransfersInOutPointsRow struct { NewPkscript string NewOutputValue int64 SentAsFee bool + TransferCount int32 Content []byte } @@ -291,6 +433,8 @@ func (q *Queries) GetInscriptionTransfersInOutPoints(ctx context.Context, arg Ge &i.InscriptionID, &i.BlockHeight, &i.TxIndex, + &i.TxHash, + &i.FromInputIndex, &i.OldSatpointTxHash, &i.OldSatpointOutIdx, &i.OldSatpointOffset, @@ -300,6 +444,7 @@ func (q *Queries) GetInscriptionTransfersInOutPoints(ctx context.Context, arg Ge &i.NewPkscript, &i.NewOutputValue, &i.SentAsFee, + &i.TransferCount, &i.Content, ); err != nil { return nil, err @@ -312,6 +457,45 @@ func (q *Queries) GetInscriptionTransfersInOutPoints(ctx context.Context, arg Ge return items, nil } +const getLatestEventIds = `-- name: GetLatestEventIds :one +WITH "latest_deploy_id" AS ( + SELECT "id" FROM "brc20_event_deploys" ORDER BY "id" DESC LIMIT 1 +), +"latest_mint_id" AS ( + SELECT "id" FROM "brc20_event_mints" ORDER BY "id" DESC LIMIT 1 +), +"latest_inscribe_transfer_id" AS ( + SELECT "id" FROM "brc20_event_inscribe_transfers" ORDER BY "id" DESC LIMIT 1 +), +"latest_transfer_transfer_id" AS ( + SELECT "id" FROM "brc20_event_transfer_transfers" ORDER BY "id" DESC LIMIT 1 +) +SELECT + (SELECT "id" FROM "latest_deploy_id") AS "event_deploy_id", + (SELECT "id" FROM "latest_mint_id") AS "event_mint_id", + (SELECT "id" FROM "latest_inscribe_transfer_id") AS "event_inscribe_transfer_id", + (SELECT "id" FROM "latest_transfer_transfer_id") AS "event_transfer_transfer_id" +` + +type GetLatestEventIdsRow struct { + EventDeployID int64 + EventMintID int64 + EventInscribeTransferID int64 + EventTransferTransferID int64 +} + +func (q *Queries) GetLatestEventIds(ctx context.Context) (GetLatestEventIdsRow, error) { + row := q.db.QueryRow(ctx, getLatestEventIds) + var i GetLatestEventIdsRow + err := row.Scan( + &i.EventDeployID, + &i.EventMintID, + &i.EventInscribeTransferID, + &i.EventTransferTransferID, + ) + return i, err +} + const getLatestIndexedBlock = `-- name: GetLatestIndexedBlock :one SELECT height, hash, event_hash, cumulative_event_hash FROM "brc20_indexed_blocks" ORDER BY "height" DESC LIMIT 1 ` diff --git a/modules/brc20/internal/repository/postgres/gen/models.go b/modules/brc20/internal/repository/postgres/gen/models.go index ea37717..87588dd 100644 --- a/modules/brc20/internal/repository/postgres/gen/models.go +++ b/modules/brc20/internal/repository/postgres/gen/models.go @@ -83,6 +83,7 @@ type Brc20EventTransferTransfer struct { ToPkscript string ToSatpoint string ToOutputIndex int32 + SpentAsFee bool Amount pgtype.Numeric } @@ -130,6 +131,8 @@ type Brc20InscriptionTransfer struct { InscriptionID string BlockHeight int32 TxIndex int32 + TxHash string + FromInputIndex int32 OldSatpointTxHash pgtype.Text OldSatpointOutIdx pgtype.Int4 OldSatpointOffset pgtype.Int8 @@ -139,6 +142,7 @@ type Brc20InscriptionTransfer struct { NewPkscript string NewOutputValue int64 SentAsFee bool + TransferCount int32 } type Brc20ProcessorStat struct { diff --git a/modules/brc20/internal/repository/postgres/mapper.go b/modules/brc20/internal/repository/postgres/mapper.go index 1a25f5d..d44844d 100644 --- a/modules/brc20/internal/repository/postgres/mapper.go +++ b/modules/brc20/internal/repository/postgres/mapper.go @@ -257,6 +257,10 @@ func mapInscriptionTransferModelToType(src gen.GetInscriptionTransfersInOutPoint if err != nil { return entity.InscriptionTransfer{}, errors.Wrap(err, "invalid inscription id") } + txHash, err := chainhash.NewHashFromStr(src.TxHash) + if err != nil { + return entity.InscriptionTransfer{}, errors.Wrap(err, "invalid tx hash") + } var oldSatPoint, newSatPoint ordinals.SatPoint if src.OldSatpointTxHash.Valid { if !src.OldSatpointOutIdx.Valid || !src.OldSatpointOffset.Valid { @@ -299,12 +303,15 @@ func mapInscriptionTransferModelToType(src gen.GetInscriptionTransfersInOutPoint InscriptionId: inscriptionId, BlockHeight: uint64(src.BlockHeight), TxIndex: uint32(src.TxIndex), + TxHash: *txHash, + FromInputIndex: uint32(src.FromInputIndex), Content: src.Content, OldSatPoint: oldSatPoint, NewSatPoint: newSatPoint, NewPkScript: newPkScript, NewOutputValue: uint64(src.NewOutputValue), SentAsFee: src.SentAsFee, + TransferCount: uint32(src.TransferCount), }, nil } @@ -313,6 +320,8 @@ func mapInscriptionTransferTypeToParams(src entity.InscriptionTransfer) gen.Crea InscriptionID: src.InscriptionId.String(), BlockHeight: int32(src.BlockHeight), TxIndex: int32(src.TxIndex), + TxHash: src.TxHash.String(), + FromInputIndex: int32(src.FromInputIndex), OldSatpointTxHash: lo.Ternary(src.OldSatPoint != ordinals.SatPoint{}, pgtype.Text{String: src.OldSatPoint.OutPoint.Hash.String(), Valid: true}, pgtype.Text{}), OldSatpointOutIdx: lo.Ternary(src.OldSatPoint != ordinals.SatPoint{}, pgtype.Int4{Int32: int32(src.OldSatPoint.OutPoint.Index), Valid: true}, pgtype.Int4{}), OldSatpointOffset: lo.Ternary(src.OldSatPoint != ordinals.SatPoint{}, pgtype.Int8{Int64: int64(src.OldSatPoint.Offset), Valid: true}, pgtype.Int8{}), @@ -322,6 +331,7 @@ func mapInscriptionTransferTypeToParams(src entity.InscriptionTransfer) gen.Crea NewPkscript: hex.EncodeToString(src.NewPkScript), NewOutputValue: int64(src.NewOutputValue), SentAsFee: src.SentAsFee, + TransferCount: int32(src.TransferCount), } } @@ -345,7 +355,7 @@ func mapEventDeployModelToType(src gen.Brc20EventDeploy) (entity.EventDeploy, er return entity.EventDeploy{ Id: uint64(src.Id), InscriptionId: inscriptionId, - InscriptionNumber: uint64(src.InscriptionNumber), + InscriptionNumber: src.InscriptionNumber, Tick: src.Tick, OriginalTick: src.OriginalTick, TxHash: *txHash, @@ -368,7 +378,7 @@ func mapEventDeployTypeToParams(src entity.EventDeploy) (gen.CreateEventDeploysP } return gen.CreateEventDeploysParams{ InscriptionID: src.InscriptionId.String(), - InscriptionNumber: int64(src.InscriptionNumber), + InscriptionNumber: src.InscriptionNumber, Tick: src.Tick, OriginalTick: src.OriginalTick, TxHash: src.TxHash.String(), @@ -412,7 +422,7 @@ func mapEventMintModelToType(src gen.Brc20EventMint) (entity.EventMint, error) { return entity.EventMint{ Id: uint64(src.Id), InscriptionId: inscriptionId, - InscriptionNumber: uint64(src.InscriptionNumber), + InscriptionNumber: src.InscriptionNumber, Tick: src.Tick, OriginalTick: src.OriginalTick, TxHash: *txHash, @@ -437,7 +447,7 @@ func mapEventMintTypeToParams(src entity.EventMint) (gen.CreateEventMintsParams, } return gen.CreateEventMintsParams{ InscriptionID: src.InscriptionId.String(), - InscriptionNumber: int64(src.InscriptionNumber), + InscriptionNumber: src.InscriptionNumber, Tick: src.Tick, OriginalTick: src.OriginalTick, TxHash: src.TxHash.String(), @@ -471,7 +481,7 @@ func mapEventInscribeTransferModelToType(src gen.Brc20EventInscribeTransfer) (en return entity.EventInscribeTransfer{ Id: uint64(src.Id), InscriptionId: inscriptionId, - InscriptionNumber: uint64(src.InscriptionNumber), + InscriptionNumber: src.InscriptionNumber, Tick: src.Tick, OriginalTick: src.OriginalTick, TxHash: *txHash, @@ -493,7 +503,7 @@ func mapEventInscribeTransferTypeToParams(src entity.EventInscribeTransfer) (gen } return gen.CreateEventInscribeTransfersParams{ InscriptionID: src.InscriptionId.String(), - InscriptionNumber: int64(src.InscriptionNumber), + InscriptionNumber: src.InscriptionNumber, Tick: src.Tick, OriginalTick: src.OriginalTick, TxHash: src.TxHash.String(), @@ -536,7 +546,7 @@ func mapEventTransferTransferModelToType(src gen.Brc20EventTransferTransfer) (en return entity.EventTransferTransfer{ Id: uint64(src.Id), InscriptionId: inscriptionId, - InscriptionNumber: uint64(src.InscriptionNumber), + InscriptionNumber: src.InscriptionNumber, Tick: src.Tick, OriginalTick: src.OriginalTick, TxHash: *txHash, @@ -549,6 +559,7 @@ func mapEventTransferTransferModelToType(src gen.Brc20EventTransferTransfer) (en ToPkScript: toPkScript, ToSatPoint: toSatPoint, ToOutputIndex: uint32(src.ToOutputIndex), + SpentAsFee: src.SpentAsFee, Amount: decimalFromNumeric(src.Amount).Decimal, }, nil } @@ -560,7 +571,7 @@ func mapEventTransferTransferTypeToParams(src entity.EventTransferTransfer) (gen } return gen.CreateEventTransferTransfersParams{ InscriptionID: src.InscriptionId.String(), - InscriptionNumber: int64(src.InscriptionNumber), + InscriptionNumber: src.InscriptionNumber, Tick: src.Tick, OriginalTick: src.OriginalTick, TxHash: src.TxHash.String(), @@ -573,6 +584,21 @@ func mapEventTransferTransferTypeToParams(src entity.EventTransferTransfer) (gen ToPkscript: hex.EncodeToString(src.ToPkScript), ToSatpoint: src.ToSatPoint.String(), ToOutputIndex: int32(src.ToOutputIndex), + SpentAsFee: src.SpentAsFee, Amount: numericFromDecimal(src.Amount), }, nil } + +func mapBalanceModelToType(src gen.Brc20Balance) (entity.Balance, error) { + pkScript, err := hex.DecodeString(src.Pkscript) + if err != nil { + return entity.Balance{}, errors.Wrap(err, "failed to parse pkscript") + } + return entity.Balance{ + PkScript: pkScript, + Tick: src.Tick, + BlockHeight: uint64(src.BlockHeight), + OverallBalance: decimalFromNumeric(src.OverallBalance).Decimal, + AvailableBalance: decimalFromNumeric(src.AvailableBalance).Decimal, + }, nil +} diff --git a/modules/brc20/processor.go b/modules/brc20/processor.go index d138d73..dd593d0 100644 --- a/modules/brc20/processor.go +++ b/modules/brc20/processor.go @@ -40,10 +40,18 @@ type Processor struct { // cache outPointValueCache *lru.Cache[wire.OutPoint, uint64] - // flush buffers + // flush buffers - inscription states newInscriptionTransfers []*entity.InscriptionTransfer newInscriptionEntries map[ordinals.InscriptionId]*ordinals.InscriptionEntry newInscriptionEntryStates map[ordinals.InscriptionId]*ordinals.InscriptionEntry + // flush buffers - brc20 states + newTickEntries map[string]*entity.TickEntry + newTickEntryStates map[string]*entity.TickEntry + newEventDeploys []*entity.EventDeploy + newEventMints []*entity.EventMint + newEventInscribeTransfers []*entity.EventInscribeTransfer + newEventTransferTransfers []*entity.EventTransferTransfer + newBalances map[string]map[string]*entity.Balance } // TODO: move this to config @@ -73,6 +81,14 @@ func NewProcessor(brc20Dg datagateway.BRC20DataGateway, indexerInfoDg datagatewa newInscriptionTransfers: make([]*entity.InscriptionTransfer, 0), newInscriptionEntries: make(map[ordinals.InscriptionId]*ordinals.InscriptionEntry), newInscriptionEntryStates: make(map[ordinals.InscriptionId]*ordinals.InscriptionEntry), + + newTickEntries: make(map[string]*entity.TickEntry), + newTickEntryStates: make(map[string]*entity.TickEntry), + newEventDeploys: make([]*entity.EventDeploy, 0), + newEventMints: make([]*entity.EventMint, 0), + newEventInscribeTransfers: make([]*entity.EventInscribeTransfer, 0), + newEventTransferTransfers: make([]*entity.EventTransferTransfer, 0), + newBalances: make(map[string]map[string]*entity.Balance), }, nil } diff --git a/modules/brc20/processor_brc20.go b/modules/brc20/processor_brc20.go index 78ec7a7..e76426f 100644 --- a/modules/brc20/processor_brc20.go +++ b/modules/brc20/processor_brc20.go @@ -1,13 +1,17 @@ package brc20 import ( + "bytes" "context" + "encoding/hex" "time" "github.com/cockroachdb/errors" "github.com/gaze-network/indexer-network/core/types" "github.com/gaze-network/indexer-network/modules/brc20/internal/brc20" + "github.com/gaze-network/indexer-network/modules/brc20/internal/datagateway" "github.com/gaze-network/indexer-network/modules/brc20/internal/entity" + "github.com/gaze-network/indexer-network/modules/brc20/internal/ordinals" "github.com/gaze-network/indexer-network/pkg/logger" "github.com/gaze-network/indexer-network/pkg/logger/slogx" "github.com/samber/lo" @@ -25,26 +29,93 @@ func (p *Processor) processBRC20States(ctx context.Context, transfers []*entity. payloads = append(payloads, payload) ticks[payload.Tick] = struct{}{} } - entries, err := p.getTickEntriesByTicks(ctx, lo.Keys(ticks)) + // TODO: concurrently fetch from db to optimize speed + tickEntries, err := p.brc20Dg.GetTickEntriesByTicks(ctx, lo.Keys(ticks)) if err != nil { return errors.Wrap(err, "failed to get inscription entries by ids") } + // preload required data to reduce individual data fetching during process + inscriptionIds := make([]ordinals.InscriptionId, 0) + inscriptionIdsToFetchParent := make([]ordinals.InscriptionId, 0) + inscriptionIdsToFetchEventInscribeTransfer := make([]ordinals.InscriptionId, 0) + balancesToFetch := make([]datagateway.GetBalancesBatchAtHeightQuery, 0) // pkscript -> tick -> struct{} + for _, payload := range payloads { + inscriptionIds = append(inscriptionIds, payload.Transfer.InscriptionId) + if payload.Op == brc20.OperationMint { + // preload parent id to validate mint events with self mint + if entry := tickEntries[payload.Tick]; entry.IsSelfMint { + inscriptionIdsToFetchParent = append(inscriptionIdsToFetchParent, payload.Transfer.InscriptionId) + } + } + if payload.Op == brc20.OperationTransfer { + if payload.Transfer.OldSatPoint == (ordinals.SatPoint{}) { + // preload balance to validate inscribe transfer event + balancesToFetch = append(balancesToFetch, datagateway.GetBalancesBatchAtHeightQuery{ + PkScriptHex: hex.EncodeToString(payload.Transfer.NewPkScript), + Tick: payload.Tick, + }) + } else { + // preload inscribe-transfer events to validate transfer-transfer event + inscriptionIdsToFetchEventInscribeTransfer = append(inscriptionIdsToFetchEventInscribeTransfer, payload.Transfer.InscriptionId) + } + } + } + inscriptionIdsToNumber, err := p.getInscriptionNumbersByIds(ctx, lo.Uniq(inscriptionIds)) + if err != nil { + return errors.Wrap(err, "failed to get inscription numbers by ids") + } + inscriptionIdsToParent, err := p.getInscriptionParentsByIds(ctx, lo.Uniq(inscriptionIdsToFetchParent)) + if err != nil { + return errors.Wrap(err, "failed to get inscription parents by ids") + } + latestEventId, err := p.brc20Dg.GetLatestEventId(ctx) + if err != nil { + return errors.Wrap(err, "failed to get latest event id") + } + // pkscript -> tick -> balance + balances, err := p.brc20Dg.GetBalancesBatchAtHeight(ctx, uint64(blockHeader.Height-1), balancesToFetch) + if err != nil { + return errors.Wrap(err, "failed to get balances batch at height") + } + eventInscribeTransfers, err := p.brc20Dg.GetEventInscribeTransfersByInscriptionIds(ctx, lo.Uniq(inscriptionIdsToFetchEventInscribeTransfer)) + if err != nil { + return errors.Wrap(err, "failed to get event inscribe transfers by inscription ids") + } + newTickEntries := make(map[string]*entity.TickEntry) newTickEntryStates := make(map[string]*entity.TickEntry) - // newDeployEvents := make([]*entity.EventDeploy, 0) - // newMintEvents := make([]*entity.EventMint, 0) - // newTransferEvents := make([]*entity.EventTransfer, 0) + newEventDeploys := make([]*entity.EventDeploy, 0) + newEventMints := make([]*entity.EventMint, 0) + newEventInscribeTransfers := make([]*entity.EventInscribeTransfer, 0) + newEventTransferTransfers := make([]*entity.EventTransferTransfer, 0) + newBalances := make(map[string]map[string]*entity.Balance) for _, payload := range payloads { - entry := entries[payload.Tick] + tickEntry := tickEntries[payload.Tick] + + if payload.Transfer.SentAsFee && payload.Transfer.OldSatPoint == (ordinals.SatPoint{}) { + logger.DebugContext(ctx, "found inscription inscribed as fee, skipping...", + slogx.String("tick", payload.Tick), + slogx.Stringer("inscriptionId", payload.Transfer.InscriptionId), + ) + continue + } switch payload.Op { case brc20.OperationDeploy: - if entry != nil { + if payload.Transfer.TransferCount > 1 { + logger.DebugContext(ctx, "found deploy inscription but it is already used, skipping...", + slogx.String("tick", payload.Tick), + slogx.Stringer("inscriptionId", payload.Transfer.InscriptionId), + slogx.Uint32("transferCount", payload.Transfer.TransferCount), + ) + continue + } + if tickEntry != nil { logger.DebugContext(ctx, "found deploy inscription but tick already exists, skipping...", slogx.String("tick", payload.Tick), - slogx.Stringer("entryInscriptionId", entry.DeployInscriptionId), + slogx.Stringer("entryInscriptionId", tickEntry.DeployInscriptionId), slogx.Stringer("currentInscriptionId", payload.Transfer.InscriptionId), ) continue @@ -58,7 +129,7 @@ func (p *Processor) processBRC20States(ctx context.Context, transfers []*entity. IsSelfMint: payload.SelfMint, DeployInscriptionId: payload.Transfer.InscriptionId, DeployedAt: blockHeader.Timestamp, - DeployedAtHeight: uint64(blockHeader.Height), + DeployedAtHeight: payload.Transfer.BlockHeight, MintedAmount: decimal.Zero, BurnedAmount: decimal.Zero, CompletedAt: time.Time{}, @@ -67,57 +138,335 @@ func (p *Processor) processBRC20States(ctx context.Context, transfers []*entity. newTickEntries[payload.Tick] = tickEntry newTickEntryStates[payload.Tick] = tickEntry // update entries for other operations in same block - entries[payload.Tick] = tickEntry + tickEntries[payload.Tick] = tickEntry - // TODO: handle deploy action + newEventDeploys = append(newEventDeploys, &entity.EventDeploy{ + Id: latestEventId + 1, + InscriptionId: payload.Transfer.InscriptionId, + InscriptionNumber: inscriptionIdsToNumber[payload.Transfer.InscriptionId], + Tick: payload.Tick, + OriginalTick: payload.OriginalTick, + TxHash: payload.Transfer.TxHash, + BlockHeight: payload.Transfer.BlockHeight, + TxIndex: payload.Transfer.TxIndex, + Timestamp: blockHeader.Timestamp, + PkScript: payload.Transfer.NewPkScript, + SatPoint: payload.Transfer.NewSatPoint, + TotalSupply: payload.Max, + Decimals: payload.Dec, + LimitPerMint: payload.Lim, + IsSelfMint: payload.SelfMint, + }) + latestEventId++ case brc20.OperationMint: - if entry == nil { + if payload.Transfer.TransferCount > 1 { + logger.DebugContext(ctx, "found mint inscription but it is already used, skipping...", + slogx.String("tick", payload.Tick), + slogx.Stringer("inscriptionId", payload.Transfer.InscriptionId), + slogx.Uint32("transferCount", payload.Transfer.TransferCount), + ) + continue + } + if tickEntry == nil { logger.DebugContext(ctx, "found mint inscription but tick does not exist, skipping...", slogx.String("tick", payload.Tick), slogx.Stringer("inscriptionId", payload.Transfer.InscriptionId), ) continue } - if -payload.Amt.Exponent() > int32(entry.Decimals) { + if -payload.Amt.Exponent() > int32(tickEntry.Decimals) { logger.DebugContext(ctx, "found mint inscription but amount has invalid decimals, skipping...", slogx.String("tick", payload.Tick), slogx.Stringer("inscriptionId", payload.Transfer.InscriptionId), slogx.Stringer("amount", payload.Amt), - slogx.Uint16("entryDecimals", entry.Decimals), + slogx.Uint16("entryDecimals", tickEntry.Decimals), slogx.Int32("payloadDecimals", -payload.Amt.Exponent()), ) continue } - // TODO: handle mint action + if tickEntry.MintedAmount.GreaterThanOrEqual(tickEntry.TotalSupply) { + logger.DebugContext(ctx, "found mint inscription but total supply is reached, skipping...", + slogx.String("tick", payload.Tick), + slogx.Stringer("inscriptionId", payload.Transfer.InscriptionId), + slogx.Stringer("mintedAmount", tickEntry.MintedAmount), + slogx.Stringer("totalSupply", tickEntry.TotalSupply), + ) + continue + } + if payload.Amt.GreaterThan(tickEntry.LimitPerMint) { + logger.DebugContext(ctx, "found mint inscription but amount exceeds limit per mint, skipping...", + slogx.String("tick", payload.Tick), + slogx.Stringer("inscriptionId", payload.Transfer.InscriptionId), + slogx.Stringer("amount", payload.Amt), + slogx.Stringer("limitPerMint", tickEntry.LimitPerMint), + ) + continue + } + mintableAmount := tickEntry.TotalSupply.Sub(tickEntry.MintedAmount) + if payload.Amt.GreaterThan(mintableAmount) { + payload.Amt = mintableAmount + } + var parentId *ordinals.InscriptionId + if tickEntry.IsSelfMint { + parentIdValue, ok := inscriptionIdsToParent[payload.Transfer.InscriptionId] + if !ok { + logger.DebugContext(ctx, "found mint inscription for self mint tick, but it does not have a parent, skipping...", + slogx.String("tick", payload.Tick), + slogx.Stringer("inscriptionId", payload.Transfer.InscriptionId), + ) + continue + } + if parentIdValue != tickEntry.DeployInscriptionId { + logger.DebugContext(ctx, "found mint inscription for self mint tick, but parent id does not match deploy inscription id, skipping...", + slogx.String("tick", payload.Tick), + slogx.Stringer("inscriptionId", payload.Transfer.InscriptionId), + slogx.Stringer("parentId", parentId), + slogx.Stringer("deployInscriptionId", tickEntry.DeployInscriptionId), + ) + } + parentId = &parentIdValue + } + + tickEntry.MintedAmount = tickEntry.MintedAmount.Add(payload.Amt) + if tickEntry.MintedAmount.GreaterThanOrEqual(tickEntry.TotalSupply) { + tickEntry.CompletedAt = blockHeader.Timestamp + tickEntry.CompletedAtHeight = payload.Transfer.BlockHeight + } + + newTickEntryStates[payload.Tick] = tickEntry + newEventMints = append(newEventMints, &entity.EventMint{ + Id: latestEventId + 1, + InscriptionId: payload.Transfer.InscriptionId, + InscriptionNumber: inscriptionIdsToNumber[payload.Transfer.InscriptionId], + Tick: payload.Tick, + OriginalTick: payload.OriginalTick, + TxHash: payload.Transfer.TxHash, + BlockHeight: payload.Transfer.BlockHeight, + TxIndex: payload.Transfer.TxIndex, + Timestamp: blockHeader.Timestamp, + PkScript: payload.Transfer.NewPkScript, + SatPoint: payload.Transfer.NewSatPoint, + Amount: payload.Amt, + ParentId: parentId, + }) + latestEventId++ case brc20.OperationTransfer: - if entry == nil { + if payload.Transfer.TransferCount > 2 { + logger.DebugContext(ctx, "found mint inscription but it is already used, skipping...", + slogx.String("tick", payload.Tick), + slogx.Stringer("inscriptionId", payload.Transfer.InscriptionId), + slogx.Uint32("transferCount", payload.Transfer.TransferCount), + ) + continue + } + if tickEntry == nil { logger.DebugContext(ctx, "found transfer inscription but tick does not exist, skipping...", slogx.String("tick", payload.Tick), slogx.Stringer("inscriptionId", payload.Transfer.InscriptionId), ) continue } - if -payload.Amt.Exponent() > int32(entry.Decimals) { + if -payload.Amt.Exponent() > int32(tickEntry.Decimals) { logger.DebugContext(ctx, "found transfer inscription but amount has invalid decimals, skipping...", slogx.String("tick", payload.Tick), slogx.Stringer("inscriptionId", payload.Transfer.InscriptionId), slogx.Stringer("amount", payload.Amt), - slogx.Uint16("entryDecimals", entry.Decimals), + slogx.Uint16("entryDecimals", tickEntry.Decimals), slogx.Int32("payloadDecimals", -payload.Amt.Exponent()), ) continue } - // TODO: handle transfer action + + if payload.Transfer.OldSatPoint == (ordinals.SatPoint{}) { + // inscribe transfer event + pkScriptHex := hex.EncodeToString(payload.Transfer.NewPkScript) + balance, ok := balances[pkScriptHex][payload.Tick] + if !ok { + balance = &entity.Balance{ + PkScript: payload.Transfer.NewPkScript, + Tick: payload.Tick, + BlockHeight: uint64(blockHeader.Height - 1), + OverallBalance: decimal.Zero, // defaults balance to zero if not found + AvailableBalance: decimal.Zero, + } + } + if payload.Amt.GreaterThan(balance.AvailableBalance) { + logger.DebugContext(ctx, "found transfer inscription but amount exceeds available balance, skipping...", + slogx.String("tick", payload.Tick), + slogx.Stringer("inscriptionId", payload.Transfer.InscriptionId), + slogx.Stringer("amount", payload.Amt), + slogx.Stringer("availableBalance", balance.AvailableBalance), + ) + continue + } + // update balance state + balance.BlockHeight = uint64(blockHeader.Height) + balance.AvailableBalance = balance.AvailableBalance.Sub(payload.Amt) + if _, ok := balances[pkScriptHex]; !ok { + balances[pkScriptHex] = make(map[string]*entity.Balance) + } + balances[pkScriptHex][payload.Tick] = balance + if _, ok := newBalances[pkScriptHex]; !ok { + newBalances[pkScriptHex] = make(map[string]*entity.Balance) + } + newBalances[pkScriptHex][payload.Tick] = &entity.Balance{} + + event := &entity.EventInscribeTransfer{ + Id: latestEventId + 1, + InscriptionId: payload.Transfer.InscriptionId, + InscriptionNumber: inscriptionIdsToNumber[payload.Transfer.InscriptionId], + Tick: payload.Tick, + OriginalTick: payload.OriginalTick, + TxHash: payload.Transfer.TxHash, + BlockHeight: payload.Transfer.BlockHeight, + TxIndex: payload.Transfer.TxIndex, + Timestamp: blockHeader.Timestamp, + PkScript: payload.Transfer.NewPkScript, + SatPoint: payload.Transfer.NewSatPoint, + OutputIndex: payload.Transfer.NewSatPoint.OutPoint.Index, + SatsAmount: payload.Transfer.NewOutputValue, + Amount: payload.Amt, + } + latestEventId++ + eventInscribeTransfers[payload.Transfer.InscriptionId] = event + newEventInscribeTransfers = append(newEventInscribeTransfers, event) + } else { + // transfer transfer event + inscribeTransfer, ok := eventInscribeTransfers[payload.Transfer.InscriptionId] + if !ok { + logger.DebugContext(ctx, "found transfer transfer event but inscribe transfer does not exist, skipping...", + slogx.String("tick", payload.Tick), + slogx.Stringer("inscriptionId", payload.Transfer.InscriptionId), + ) + continue + } + + if payload.Transfer.SentAsFee { + // return balance to sender + fromPkScriptHex := hex.EncodeToString(inscribeTransfer.PkScript) + fromBalance, ok := balances[fromPkScriptHex][payload.Tick] + if !ok { + logger.DebugContext(ctx, "found transfer transfer event but from balance does not exist, skipping...", + slogx.String("tick", payload.Tick), + slogx.Stringer("inscriptionId", payload.Transfer.InscriptionId), + slogx.String("pkScript", fromPkScriptHex), + ) + continue + } + fromBalance.BlockHeight = uint64(blockHeader.Height) + fromBalance.AvailableBalance = fromBalance.AvailableBalance.Sub(payload.Amt) + if _, ok := balances[fromPkScriptHex]; !ok { + balances[fromPkScriptHex] = make(map[string]*entity.Balance) + } + balances[fromPkScriptHex][payload.Tick] = fromBalance + if _, ok := newBalances[fromPkScriptHex]; !ok { + newBalances[fromPkScriptHex] = make(map[string]*entity.Balance) + } + newBalances[fromPkScriptHex][payload.Tick] = fromBalance + + newEventTransferTransfers = append(newEventTransferTransfers, &entity.EventTransferTransfer{ + Id: latestEventId + 1, + InscriptionId: payload.Transfer.InscriptionId, + InscriptionNumber: inscriptionIdsToNumber[payload.Transfer.InscriptionId], + Tick: payload.Tick, + OriginalTick: payload.OriginalTick, + TxHash: payload.Transfer.TxHash, + BlockHeight: payload.Transfer.BlockHeight, + TxIndex: payload.Transfer.TxIndex, + Timestamp: blockHeader.Timestamp, + FromPkScript: inscribeTransfer.PkScript, + FromSatPoint: inscribeTransfer.SatPoint, + FromInputIndex: payload.Transfer.FromInputIndex, + ToPkScript: payload.Transfer.NewPkScript, + ToSatPoint: payload.Transfer.NewSatPoint, + ToOutputIndex: payload.Transfer.NewSatPoint.OutPoint.Index, + SpentAsFee: true, + Amount: payload.Amt, + }) + } else { + // subtract balance from sender + fromPkScriptHex := hex.EncodeToString(inscribeTransfer.PkScript) + fromBalance, ok := balances[fromPkScriptHex][payload.Tick] + if !ok { + logger.DebugContext(ctx, "found transfer transfer event but from balance does not exist, skipping...", + slogx.String("tick", payload.Tick), + slogx.Stringer("inscriptionId", payload.Transfer.InscriptionId), + slogx.String("pkScript", fromPkScriptHex), + ) + continue + } + fromBalance.BlockHeight = uint64(blockHeader.Height) + fromBalance.OverallBalance = fromBalance.OverallBalance.Sub(payload.Amt) + if _, ok := balances[fromPkScriptHex]; !ok { + balances[fromPkScriptHex] = make(map[string]*entity.Balance) + } + balances[fromPkScriptHex][payload.Tick] = fromBalance + if _, ok := newBalances[fromPkScriptHex]; !ok { + newBalances[fromPkScriptHex] = make(map[string]*entity.Balance) + } + newBalances[fromPkScriptHex][payload.Tick] = fromBalance + + // add balance to receiver + if bytes.Equal(payload.Transfer.NewPkScript, []byte{0x6a}) { + // burn if sent to OP_RETURN + tickEntry.BurnedAmount = tickEntry.BurnedAmount.Add(payload.Amt) + tickEntries[payload.Tick] = tickEntry + newTickEntryStates[payload.Tick] = tickEntry + } else { + toPkScriptHex := hex.EncodeToString(payload.Transfer.NewPkScript) + toBalance, ok := balances[toPkScriptHex][payload.Tick] + if !ok { + toBalance = &entity.Balance{ + PkScript: payload.Transfer.NewPkScript, + Tick: payload.Tick, + BlockHeight: uint64(blockHeader.Height), + OverallBalance: decimal.Zero, // defaults balance to zero if not found + AvailableBalance: decimal.Zero, + } + } + toBalance.BlockHeight = uint64(blockHeader.Height) + toBalance.OverallBalance = toBalance.OverallBalance.Add(payload.Amt) + toBalance.AvailableBalance = toBalance.AvailableBalance.Add(payload.Amt) + if _, ok := balances[toPkScriptHex]; !ok { + balances[toPkScriptHex] = make(map[string]*entity.Balance) + } + balances[toPkScriptHex][payload.Tick] = toBalance + if _, ok := newBalances[toPkScriptHex]; !ok { + newBalances[toPkScriptHex] = make(map[string]*entity.Balance) + } + newBalances[toPkScriptHex][payload.Tick] = toBalance + } + + newEventTransferTransfers = append(newEventTransferTransfers, &entity.EventTransferTransfer{ + Id: latestEventId + 1, + InscriptionId: payload.Transfer.InscriptionId, + InscriptionNumber: inscriptionIdsToNumber[payload.Transfer.InscriptionId], + Tick: payload.Tick, + OriginalTick: payload.OriginalTick, + TxHash: payload.Transfer.TxHash, + BlockHeight: payload.Transfer.BlockHeight, + TxIndex: payload.Transfer.TxIndex, + Timestamp: blockHeader.Timestamp, + FromPkScript: inscribeTransfer.PkScript, + FromSatPoint: inscribeTransfer.SatPoint, + FromInputIndex: payload.Transfer.FromInputIndex, + ToPkScript: payload.Transfer.NewPkScript, + ToSatPoint: payload.Transfer.NewSatPoint, + ToOutputIndex: payload.Transfer.NewSatPoint.OutPoint.Index, + SpentAsFee: false, + Amount: payload.Amt, + }) + } + } } } + + p.newTickEntries = newTickEntries + p.newTickEntryStates = newTickEntryStates + p.newEventDeploys = newEventDeploys + p.newEventMints = newEventMints + p.newEventInscribeTransfers = newEventInscribeTransfers + p.newEventTransferTransfers = newEventTransferTransfers + p.newBalances = newBalances return nil } - -func (p *Processor) getTickEntriesByTicks(ctx context.Context, ticks []string) (map[string]*entity.TickEntry, error) { - // TODO: get from buffer if exists - result, err := p.brc20Dg.GetTickEntriesByTicks(ctx, ticks) - if err != nil { - return nil, errors.Wrap(err, "failed to get tick entries by ticks") - } - return result, nil -} diff --git a/modules/brc20/processor_inscription.go b/modules/brc20/processor_inscription.go index 53e0a24..0bc320c 100644 --- a/modules/brc20/processor_inscription.go +++ b/modules/brc20/processor_inscription.go @@ -80,6 +80,7 @@ func (p *Processor) processInscriptionTx(ctx context.Context, tx *types.Transact OriginOld: &entity.OriginOld{ OldSatPoint: satPoint, Content: transfer.Content, + InputIndex: uint32(i), }, }) if _, ok := inscribeOffsets[offset]; !ok { @@ -289,17 +290,6 @@ func (p *Processor) processInscriptionTx(ctx context.Context, tx *types.Transact func (p *Processor) updateInscriptionLocation(ctx context.Context, newSatPoint ordinals.SatPoint, flotsam *entity.Flotsam, sentAsFee bool, tx *types.Transaction, blockHeader types.BlockHeader) error { txOut := tx.TxOut[newSatPoint.OutPoint.Index] if flotsam.OriginOld != nil { - transfer := &entity.InscriptionTransfer{ - InscriptionId: flotsam.InscriptionId, - BlockHeight: uint64(flotsam.Tx.BlockHeight), // use flotsam's tx to track tx that initiated the transfer - TxIndex: flotsam.Tx.Index, // use flotsam's tx to track tx that initiated the transfer - Content: flotsam.OriginOld.Content, - OldSatPoint: flotsam.OriginOld.OldSatPoint, - NewSatPoint: newSatPoint, - NewPkScript: txOut.PkScript, - NewOutputValue: uint64(txOut.Value), - SentAsFee: sentAsFee, - } entry, err := p.getInscriptionEntryById(ctx, flotsam.InscriptionId) if err != nil { // skip inscriptions without entry (likely non-brc20 inscriptions) @@ -309,6 +299,20 @@ func (p *Processor) updateInscriptionLocation(ctx context.Context, newSatPoint o return errors.Wrap(err, "failed to get inscription entry") } entry.TransferCount++ + transfer := &entity.InscriptionTransfer{ + InscriptionId: flotsam.InscriptionId, + BlockHeight: uint64(flotsam.Tx.BlockHeight), // use flotsam's tx to track tx that initiated the transfer + TxIndex: flotsam.Tx.Index, // use flotsam's tx to track tx that initiated the transfer + TxHash: flotsam.Tx.TxHash, + Content: flotsam.OriginOld.Content, + FromInputIndex: flotsam.OriginOld.InputIndex, + OldSatPoint: flotsam.OriginOld.OldSatPoint, + NewSatPoint: newSatPoint, + NewPkScript: txOut.PkScript, + NewOutputValue: uint64(txOut.Value), + SentAsFee: sentAsFee, + TransferCount: entry.TransferCount, + } // track transfers even if transfer count exceeds 2 (because we need to check for reinscriptions) p.newInscriptionTransfers = append(p.newInscriptionTransfers, transfer) @@ -337,12 +341,15 @@ func (p *Processor) updateInscriptionLocation(ctx context.Context, newSatPoint o InscriptionId: flotsam.InscriptionId, BlockHeight: uint64(flotsam.Tx.BlockHeight), // use flotsam's tx to track tx that initiated the transfer TxIndex: flotsam.Tx.Index, // use flotsam's tx to track tx that initiated the transfer + TxHash: flotsam.Tx.TxHash, Content: origin.Inscription.Content, + FromInputIndex: 0, // unused OldSatPoint: ordinals.SatPoint{}, NewSatPoint: newSatPoint, NewPkScript: txOut.PkScript, NewOutputValue: uint64(txOut.Value), SentAsFee: sentAsFee, + TransferCount: 1, // count inscription as first transfer } entry := &ordinals.InscriptionEntry{ Id: flotsam.InscriptionId, @@ -510,6 +517,58 @@ func (p *Processor) getInscriptionEntriesByIds(ctx context.Context, ids []ordina return result, nil } +func (p *Processor) getInscriptionNumbersByIds(ctx context.Context, ids []ordinals.InscriptionId) (map[ordinals.InscriptionId]int64, error) { + // try to get from cache if exists + result := make(map[ordinals.InscriptionId]int64) + + idsToFetch := make([]ordinals.InscriptionId, 0) + for _, id := range ids { + if entry, ok := p.newInscriptionEntryStates[id]; ok { + result[id] = int64(entry.Number) + } else { + idsToFetch = append(idsToFetch, id) + } + } + + if len(idsToFetch) == 0 { + inscriptions, err := p.brc20Dg.GetInscriptionNumbersByIds(ctx, idsToFetch) + if err != nil { + return nil, errors.Wrap(err, "failed to get inscriptions by outpoint") + } + for id, number := range inscriptions { + result[id] = number + } + } + return result, nil +} + +func (p *Processor) getInscriptionParentsByIds(ctx context.Context, ids []ordinals.InscriptionId) (map[ordinals.InscriptionId]ordinals.InscriptionId, error) { + // try to get from cache if exists + result := make(map[ordinals.InscriptionId]ordinals.InscriptionId) + + idsToFetch := make([]ordinals.InscriptionId, 0) + for _, id := range ids { + if entry, ok := p.newInscriptionEntryStates[id]; ok { + if entry.Inscription.Parent != nil { + result[id] = *entry.Inscription.Parent + } + } else { + idsToFetch = append(idsToFetch, id) + } + } + + if len(idsToFetch) == 0 { + inscriptions, err := p.brc20Dg.GetInscriptionParentsByIds(ctx, idsToFetch) + if err != nil { + return nil, errors.Wrap(err, "failed to get inscriptions by outpoint") + } + for id, parent := range inscriptions { + result[id] = parent + } + } + return result, nil +} + func (p *Processor) getBlockSubsidy(blockHeight uint64) uint64 { return uint64(blockchain.CalcBlockSubsidy(int32(blockHeight), p.network.ChainParams())) } From f3ff5ecb7d35c248c56a05bd20f2f810c84f4561 Mon Sep 17 00:00:00 2001 From: Gaze Date: Sun, 9 Jun 2024 15:30:27 +0700 Subject: [PATCH 03/14] fix: bug transfer inscription in same block as inscribe --- modules/brc20/processor_inscription.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/brc20/processor_inscription.go b/modules/brc20/processor_inscription.go index 0bc320c..3b5697b 100644 --- a/modules/brc20/processor_inscription.go +++ b/modules/brc20/processor_inscription.go @@ -481,7 +481,7 @@ func (p *Processor) getInscriptionTransfersInOutPoints(ctx context.Context, outP } func (p *Processor) getInscriptionEntryById(ctx context.Context, id ordinals.InscriptionId) (*ordinals.InscriptionEntry, error) { - inscriptions, err := p.brc20Dg.GetInscriptionEntriesByIds(ctx, []ordinals.InscriptionId{id}) + inscriptions, err := p.getInscriptionEntriesByIds(ctx, []ordinals.InscriptionId{id}) if err != nil { return nil, errors.Wrap(err, "failed to get inscriptions by outpoint") } From 4228730a3483fcbbc35d644d34a4c76f41ea0493 Mon Sep 17 00:00:00 2001 From: Gaze Date: Sun, 9 Jun 2024 16:17:58 +0700 Subject: [PATCH 04/14] fix: inscription logic --- modules/brc20/processor_inscription.go | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/modules/brc20/processor_inscription.go b/modules/brc20/processor_inscription.go index 3b5697b..9bb88e2 100644 --- a/modules/brc20/processor_inscription.go +++ b/modules/brc20/processor_inscription.go @@ -123,7 +123,7 @@ func (p *Processor) processInscriptionTx(ctx context.Context, tx *types.Transact } else { initialInscriptionEntry, err := p.getInscriptionEntryById(ctx, initial.inscriptionId) if err != nil { - return errors.Wrap(err, "failed to get inscription entry") + return errors.Wrapf(err, "failed to get inscription entry id %s", initial.inscriptionId) } if !initialInscriptionEntry.Cursed { cursed = true // reinscription curse if initial inscription is not cursed @@ -292,11 +292,7 @@ func (p *Processor) updateInscriptionLocation(ctx context.Context, newSatPoint o if flotsam.OriginOld != nil { entry, err := p.getInscriptionEntryById(ctx, flotsam.InscriptionId) if err != nil { - // skip inscriptions without entry (likely non-brc20 inscriptions) - if errors.Is(err, errs.NotFound) { - return nil - } - return errors.Wrap(err, "failed to get inscription entry") + return errors.Wrapf(err, "failed to get inscription entry id %s", flotsam.InscriptionId) } entry.TransferCount++ transfer := &entity.InscriptionTransfer{ @@ -505,7 +501,7 @@ func (p *Processor) getInscriptionEntriesByIds(ctx context.Context, ids []ordina } } - if len(idsToFetch) == 0 { + if len(idsToFetch) > 0 { inscriptions, err := p.brc20Dg.GetInscriptionEntriesByIds(ctx, idsToFetch) if err != nil { return nil, errors.Wrap(err, "failed to get inscriptions by outpoint") @@ -530,7 +526,7 @@ func (p *Processor) getInscriptionNumbersByIds(ctx context.Context, ids []ordina } } - if len(idsToFetch) == 0 { + if len(idsToFetch) > 0 { inscriptions, err := p.brc20Dg.GetInscriptionNumbersByIds(ctx, idsToFetch) if err != nil { return nil, errors.Wrap(err, "failed to get inscriptions by outpoint") @@ -557,7 +553,7 @@ func (p *Processor) getInscriptionParentsByIds(ctx context.Context, ids []ordina } } - if len(idsToFetch) == 0 { + if len(idsToFetch) > 0 { inscriptions, err := p.brc20Dg.GetInscriptionParentsByIds(ctx, idsToFetch) if err != nil { return nil, errors.Wrap(err, "failed to get inscriptions by outpoint") From 132dcde715326cf1cec9ec7c85c3a8fccb4873d1 Mon Sep 17 00:00:00 2001 From: Gaze Date: Sun, 9 Jun 2024 16:28:42 +0700 Subject: [PATCH 05/14] fix: transfer order --- modules/brc20/processor_process.go | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/modules/brc20/processor_process.go b/modules/brc20/processor_process.go index d7c4862..f8d7598 100644 --- a/modules/brc20/processor_process.go +++ b/modules/brc20/processor_process.go @@ -36,6 +36,13 @@ func (p *Processor) Process(ctx context.Context, blocks []*types.Block) error { if t1.TxIndex != t2.TxIndex { return int(t1.TxIndex) - int(t2.TxIndex) } + if t1.SentAsFee != t2.SentAsFee { + // transfers sent as fee should be ordered after non-fees + if t1.SentAsFee { + return 1 + } + return -1 + } if t1.NewSatPoint.OutPoint.Index != t2.NewSatPoint.OutPoint.Index { return int(t1.NewSatPoint.OutPoint.Index) - int(t2.NewSatPoint.OutPoint.Index) } From 2ae5b0835d180b0aae139d543e033fab24ea8b16 Mon Sep 17 00:00:00 2001 From: Gaze Date: Sun, 9 Jun 2024 16:29:00 +0700 Subject: [PATCH 06/14] feat: process brc20 states --- modules/brc20/processor_inscription.go | 2 +- modules/brc20/processor_process.go | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/modules/brc20/processor_inscription.go b/modules/brc20/processor_inscription.go index 9bb88e2..9f71a32 100644 --- a/modules/brc20/processor_inscription.go +++ b/modules/brc20/processor_inscription.go @@ -354,7 +354,7 @@ func (p *Processor) updateInscriptionLocation(ctx context.Context, newSatPoint o Cursed: origin.Cursed, CursedForBRC20: origin.CursedForBRC20, CreatedAt: blockHeader.Timestamp, - CreatedAtHeight: uint64(tx.BlockHeight), + CreatedAtHeight: uint64(blockHeader.Height), Inscription: origin.Inscription, TransferCount: 1, // count inscription as first transfer } diff --git a/modules/brc20/processor_process.go b/modules/brc20/processor_process.go index f8d7598..0afdf12 100644 --- a/modules/brc20/processor_process.go +++ b/modules/brc20/processor_process.go @@ -49,7 +49,9 @@ func (p *Processor) Process(ctx context.Context, blocks []*types.Block) error { return int(t1.NewSatPoint.Offset) - int(t2.NewSatPoint.Offset) }) - // TODO: process brc20 states + if err := p.processBRC20States(ctx, p.newInscriptionTransfers, block.Header); err != nil { + return errors.Wrap(err, "failed to process brc20 states") + } if err := p.flushBlock(ctx, block.Header); err != nil { return errors.Wrap(err, "failed to flush block") From b45dfd066a9eeddef9c1871e3936c5eb5481cdb8 Mon Sep 17 00:00:00 2001 From: Gaze Date: Sun, 9 Jun 2024 16:41:58 +0700 Subject: [PATCH 07/14] fix: remove debug logs --- modules/brc20/processor_brc20.go | 117 +++++++------------------------ 1 file changed, 25 insertions(+), 92 deletions(-) diff --git a/modules/brc20/processor_brc20.go b/modules/brc20/processor_brc20.go index e76426f..ddd0323 100644 --- a/modules/brc20/processor_brc20.go +++ b/modules/brc20/processor_brc20.go @@ -12,8 +12,6 @@ import ( "github.com/gaze-network/indexer-network/modules/brc20/internal/datagateway" "github.com/gaze-network/indexer-network/modules/brc20/internal/entity" "github.com/gaze-network/indexer-network/modules/brc20/internal/ordinals" - "github.com/gaze-network/indexer-network/pkg/logger" - "github.com/gaze-network/indexer-network/pkg/logger/slogx" "github.com/samber/lo" "github.com/shopspring/decimal" ) @@ -95,29 +93,18 @@ func (p *Processor) processBRC20States(ctx context.Context, transfers []*entity. tickEntry := tickEntries[payload.Tick] if payload.Transfer.SentAsFee && payload.Transfer.OldSatPoint == (ordinals.SatPoint{}) { - logger.DebugContext(ctx, "found inscription inscribed as fee, skipping...", - slogx.String("tick", payload.Tick), - slogx.Stringer("inscriptionId", payload.Transfer.InscriptionId), - ) + // skip inscriptions inscribed as fee continue } switch payload.Op { case brc20.OperationDeploy: if payload.Transfer.TransferCount > 1 { - logger.DebugContext(ctx, "found deploy inscription but it is already used, skipping...", - slogx.String("tick", payload.Tick), - slogx.Stringer("inscriptionId", payload.Transfer.InscriptionId), - slogx.Uint32("transferCount", payload.Transfer.TransferCount), - ) + // skip used deploy inscriptions continue } if tickEntry != nil { - logger.DebugContext(ctx, "found deploy inscription but tick already exists, skipping...", - slogx.String("tick", payload.Tick), - slogx.Stringer("entryInscriptionId", tickEntry.DeployInscriptionId), - slogx.Stringer("currentInscriptionId", payload.Transfer.InscriptionId), - ) + // skip deploy inscriptions for duplicate ticks continue } tickEntry := &entity.TickEntry{ @@ -160,46 +147,23 @@ func (p *Processor) processBRC20States(ctx context.Context, transfers []*entity. latestEventId++ case brc20.OperationMint: if payload.Transfer.TransferCount > 1 { - logger.DebugContext(ctx, "found mint inscription but it is already used, skipping...", - slogx.String("tick", payload.Tick), - slogx.Stringer("inscriptionId", payload.Transfer.InscriptionId), - slogx.Uint32("transferCount", payload.Transfer.TransferCount), - ) + // skip used mint inscriptions that are already used continue } if tickEntry == nil { - logger.DebugContext(ctx, "found mint inscription but tick does not exist, skipping...", - slogx.String("tick", payload.Tick), - slogx.Stringer("inscriptionId", payload.Transfer.InscriptionId), - ) + // skip mint inscriptions for non-existent ticks continue } if -payload.Amt.Exponent() > int32(tickEntry.Decimals) { - logger.DebugContext(ctx, "found mint inscription but amount has invalid decimals, skipping...", - slogx.String("tick", payload.Tick), - slogx.Stringer("inscriptionId", payload.Transfer.InscriptionId), - slogx.Stringer("amount", payload.Amt), - slogx.Uint16("entryDecimals", tickEntry.Decimals), - slogx.Int32("payloadDecimals", -payload.Amt.Exponent()), - ) + // skip mint inscriptions with decimals greater than allowed continue } if tickEntry.MintedAmount.GreaterThanOrEqual(tickEntry.TotalSupply) { - logger.DebugContext(ctx, "found mint inscription but total supply is reached, skipping...", - slogx.String("tick", payload.Tick), - slogx.Stringer("inscriptionId", payload.Transfer.InscriptionId), - slogx.Stringer("mintedAmount", tickEntry.MintedAmount), - slogx.Stringer("totalSupply", tickEntry.TotalSupply), - ) + // skip mint inscriptions for ticks with completed mints continue } if payload.Amt.GreaterThan(tickEntry.LimitPerMint) { - logger.DebugContext(ctx, "found mint inscription but amount exceeds limit per mint, skipping...", - slogx.String("tick", payload.Tick), - slogx.Stringer("inscriptionId", payload.Transfer.InscriptionId), - slogx.Stringer("amount", payload.Amt), - slogx.Stringer("limitPerMint", tickEntry.LimitPerMint), - ) + // skip mint inscriptions with amount greater than limit per mint continue } mintableAmount := tickEntry.TotalSupply.Sub(tickEntry.MintedAmount) @@ -210,19 +174,12 @@ func (p *Processor) processBRC20States(ctx context.Context, transfers []*entity. if tickEntry.IsSelfMint { parentIdValue, ok := inscriptionIdsToParent[payload.Transfer.InscriptionId] if !ok { - logger.DebugContext(ctx, "found mint inscription for self mint tick, but it does not have a parent, skipping...", - slogx.String("tick", payload.Tick), - slogx.Stringer("inscriptionId", payload.Transfer.InscriptionId), - ) + // skip mint inscriptions for self mint ticks without parent inscription continue } if parentIdValue != tickEntry.DeployInscriptionId { - logger.DebugContext(ctx, "found mint inscription for self mint tick, but parent id does not match deploy inscription id, skipping...", - slogx.String("tick", payload.Tick), - slogx.Stringer("inscriptionId", payload.Transfer.InscriptionId), - slogx.Stringer("parentId", parentId), - slogx.Stringer("deployInscriptionId", tickEntry.DeployInscriptionId), - ) + // skip mint inscriptions for self mint ticks with invalid parent inscription + continue } parentId = &parentIdValue } @@ -252,28 +209,15 @@ func (p *Processor) processBRC20States(ctx context.Context, transfers []*entity. latestEventId++ case brc20.OperationTransfer: if payload.Transfer.TransferCount > 2 { - logger.DebugContext(ctx, "found mint inscription but it is already used, skipping...", - slogx.String("tick", payload.Tick), - slogx.Stringer("inscriptionId", payload.Transfer.InscriptionId), - slogx.Uint32("transferCount", payload.Transfer.TransferCount), - ) + // skip used transfer inscriptions continue } if tickEntry == nil { - logger.DebugContext(ctx, "found transfer inscription but tick does not exist, skipping...", - slogx.String("tick", payload.Tick), - slogx.Stringer("inscriptionId", payload.Transfer.InscriptionId), - ) + // skip transfer inscriptions for non-existent ticks continue } if -payload.Amt.Exponent() > int32(tickEntry.Decimals) { - logger.DebugContext(ctx, "found transfer inscription but amount has invalid decimals, skipping...", - slogx.String("tick", payload.Tick), - slogx.Stringer("inscriptionId", payload.Transfer.InscriptionId), - slogx.Stringer("amount", payload.Amt), - slogx.Uint16("entryDecimals", tickEntry.Decimals), - slogx.Int32("payloadDecimals", -payload.Amt.Exponent()), - ) + // skip transfer inscriptions with decimals greater than allowed continue } @@ -291,12 +235,7 @@ func (p *Processor) processBRC20States(ctx context.Context, transfers []*entity. } } if payload.Amt.GreaterThan(balance.AvailableBalance) { - logger.DebugContext(ctx, "found transfer inscription but amount exceeds available balance, skipping...", - slogx.String("tick", payload.Tick), - slogx.Stringer("inscriptionId", payload.Transfer.InscriptionId), - slogx.Stringer("amount", payload.Amt), - slogx.Stringer("availableBalance", balance.AvailableBalance), - ) + // skip inscribe transfer event if amount exceeds available balance continue } // update balance state @@ -334,10 +273,7 @@ func (p *Processor) processBRC20States(ctx context.Context, transfers []*entity. // transfer transfer event inscribeTransfer, ok := eventInscribeTransfers[payload.Transfer.InscriptionId] if !ok { - logger.DebugContext(ctx, "found transfer transfer event but inscribe transfer does not exist, skipping...", - slogx.String("tick", payload.Tick), - slogx.Stringer("inscriptionId", payload.Transfer.InscriptionId), - ) + // skip transfer transfer event if prior inscribe transfer event does not exist continue } @@ -346,15 +282,16 @@ func (p *Processor) processBRC20States(ctx context.Context, transfers []*entity. fromPkScriptHex := hex.EncodeToString(inscribeTransfer.PkScript) fromBalance, ok := balances[fromPkScriptHex][payload.Tick] if !ok { - logger.DebugContext(ctx, "found transfer transfer event but from balance does not exist, skipping...", - slogx.String("tick", payload.Tick), - slogx.Stringer("inscriptionId", payload.Transfer.InscriptionId), - slogx.String("pkScript", fromPkScriptHex), - ) - continue + fromBalance = &entity.Balance{ + PkScript: inscribeTransfer.PkScript, + Tick: payload.Tick, + BlockHeight: uint64(blockHeader.Height), + OverallBalance: decimal.Zero, // defaults balance to zero if not found + AvailableBalance: decimal.Zero, + } } fromBalance.BlockHeight = uint64(blockHeader.Height) - fromBalance.AvailableBalance = fromBalance.AvailableBalance.Sub(payload.Amt) + fromBalance.AvailableBalance = fromBalance.AvailableBalance.Add(payload.Amt) if _, ok := balances[fromPkScriptHex]; !ok { balances[fromPkScriptHex] = make(map[string]*entity.Balance) } @@ -388,11 +325,7 @@ func (p *Processor) processBRC20States(ctx context.Context, transfers []*entity. fromPkScriptHex := hex.EncodeToString(inscribeTransfer.PkScript) fromBalance, ok := balances[fromPkScriptHex][payload.Tick] if !ok { - logger.DebugContext(ctx, "found transfer transfer event but from balance does not exist, skipping...", - slogx.String("tick", payload.Tick), - slogx.Stringer("inscriptionId", payload.Transfer.InscriptionId), - slogx.String("pkScript", fromPkScriptHex), - ) + // skip transfer transfer event if from balance does not exist continue } fromBalance.BlockHeight = uint64(blockHeader.Height) From e4d41cc7a44b937be555528f2ed0d9c3baddac38 Mon Sep 17 00:00:00 2001 From: Gaze Date: Sun, 9 Jun 2024 17:00:01 +0700 Subject: [PATCH 08/14] feat: skip non-brc20 transfers --- .../brc20/database/postgresql/queries/data.sql | 8 ++++---- modules/brc20/internal/datagateway/brc20.go | 2 +- modules/brc20/internal/entity/event_deploy.go | 2 +- .../internal/entity/event_inscribe_transfer.go | 2 +- modules/brc20/internal/entity/event_mint.go | 2 +- .../internal/entity/event_transfer_transfer.go | 2 +- .../brc20/internal/repository/postgres/brc20.go | 4 ++-- .../internal/repository/postgres/gen/data.sql.go | 16 ++++++++-------- .../brc20/internal/repository/postgres/mapper.go | 8 ++++---- modules/brc20/processor_brc20.go | 8 ++++++++ 10 files changed, 31 insertions(+), 23 deletions(-) diff --git a/modules/brc20/database/postgresql/queries/data.sql b/modules/brc20/database/postgresql/queries/data.sql index 14ca824..fa2664b 100644 --- a/modules/brc20/database/postgresql/queries/data.sql +++ b/modules/brc20/database/postgresql/queries/data.sql @@ -55,10 +55,10 @@ WITH "latest_deploy_id" AS ( SELECT "id" FROM "brc20_event_transfer_transfers" ORDER BY "id" DESC LIMIT 1 ) SELECT - (SELECT "id" FROM "latest_deploy_id") AS "event_deploy_id", - (SELECT "id" FROM "latest_mint_id") AS "event_mint_id", - (SELECT "id" FROM "latest_inscribe_transfer_id") AS "event_inscribe_transfer_id", - (SELECT "id" FROM "latest_transfer_transfer_id") AS "event_transfer_transfer_id"; + COALESCE((SELECT "id" FROM "latest_deploy_id"), -1) AS "event_deploy_id", + COALESCE((SELECT "id" FROM "latest_mint_id"), -1) AS "event_mint_id", + COALESCE((SELECT "id" FROM "latest_inscribe_transfer_id"), -1) AS "event_inscribe_transfer_id", + COALESCE((SELECT "id" FROM "latest_transfer_transfer_id"), -1) AS "event_transfer_transfer_id"; -- name: GetBalancesBatchAtHeight :many SELECT DISTINCT ON ("brc20_balances"."pkscript", "brc20_balances"."tick") "brc20_balances".* FROM "brc20_balances" diff --git a/modules/brc20/internal/datagateway/brc20.go b/modules/brc20/internal/datagateway/brc20.go index 9d0d429..ef1fb98 100644 --- a/modules/brc20/internal/datagateway/brc20.go +++ b/modules/brc20/internal/datagateway/brc20.go @@ -33,7 +33,7 @@ type BRC20ReaderDataGateway interface { GetBalancesBatchAtHeight(ctx context.Context, blockHeight uint64, queries []GetBalancesBatchAtHeightQuery) (map[string]map[string]*entity.Balance, error) GetTickEntriesByTicks(ctx context.Context, ticks []string) (map[string]*entity.TickEntry, error) GetEventInscribeTransfersByInscriptionIds(ctx context.Context, ids []ordinals.InscriptionId) (map[ordinals.InscriptionId]*entity.EventInscribeTransfer, error) - GetLatestEventId(ctx context.Context) (uint64, error) + GetLatestEventId(ctx context.Context) (int64, error) } type BRC20WriterDataGateway interface { diff --git a/modules/brc20/internal/entity/event_deploy.go b/modules/brc20/internal/entity/event_deploy.go index 55277d5..02a59fa 100644 --- a/modules/brc20/internal/entity/event_deploy.go +++ b/modules/brc20/internal/entity/event_deploy.go @@ -9,7 +9,7 @@ import ( ) type EventDeploy struct { - Id uint64 + Id int64 InscriptionId ordinals.InscriptionId InscriptionNumber int64 Tick string diff --git a/modules/brc20/internal/entity/event_inscribe_transfer.go b/modules/brc20/internal/entity/event_inscribe_transfer.go index 34293cd..37ed8f3 100644 --- a/modules/brc20/internal/entity/event_inscribe_transfer.go +++ b/modules/brc20/internal/entity/event_inscribe_transfer.go @@ -9,7 +9,7 @@ import ( ) type EventInscribeTransfer struct { - Id uint64 + Id int64 InscriptionId ordinals.InscriptionId InscriptionNumber int64 Tick string diff --git a/modules/brc20/internal/entity/event_mint.go b/modules/brc20/internal/entity/event_mint.go index d358ed1..2978f28 100644 --- a/modules/brc20/internal/entity/event_mint.go +++ b/modules/brc20/internal/entity/event_mint.go @@ -9,7 +9,7 @@ import ( ) type EventMint struct { - Id uint64 + Id int64 InscriptionId ordinals.InscriptionId InscriptionNumber int64 Tick string diff --git a/modules/brc20/internal/entity/event_transfer_transfer.go b/modules/brc20/internal/entity/event_transfer_transfer.go index 2058184..bfb62e6 100644 --- a/modules/brc20/internal/entity/event_transfer_transfer.go +++ b/modules/brc20/internal/entity/event_transfer_transfer.go @@ -9,7 +9,7 @@ import ( ) type EventTransferTransfer struct { - Id uint64 + Id int64 InscriptionId ordinals.InscriptionId InscriptionNumber int64 Tick string diff --git a/modules/brc20/internal/repository/postgres/brc20.go b/modules/brc20/internal/repository/postgres/brc20.go index 21e3d53..bb9914f 100644 --- a/modules/brc20/internal/repository/postgres/brc20.go +++ b/modules/brc20/internal/repository/postgres/brc20.go @@ -158,12 +158,12 @@ func (r *Repository) GetInscriptionParentsByIds(ctx context.Context, ids []ordin return result, nil } -func (r *Repository) GetLatestEventId(ctx context.Context) (uint64, error) { +func (r *Repository) GetLatestEventId(ctx context.Context) (int64, error) { row, err := r.queries.GetLatestEventIds(ctx) if err != nil { return 0, errors.WithStack(err) } - return uint64(max(row.EventDeployID, row.EventMintID, row.EventInscribeTransferID, row.EventTransferTransferID)), nil + return max(row.EventDeployID.(int64), row.EventMintID.(int64), row.EventInscribeTransferID.(int64), row.EventTransferTransferID.(int64)), nil } func (r *Repository) GetBalancesBatchAtHeight(ctx context.Context, blockHeight uint64, queries []datagateway.GetBalancesBatchAtHeightQuery) (map[string]map[string]*entity.Balance, error) { diff --git a/modules/brc20/internal/repository/postgres/gen/data.sql.go b/modules/brc20/internal/repository/postgres/gen/data.sql.go index f71bf41..ade188c 100644 --- a/modules/brc20/internal/repository/postgres/gen/data.sql.go +++ b/modules/brc20/internal/repository/postgres/gen/data.sql.go @@ -471,17 +471,17 @@ WITH "latest_deploy_id" AS ( SELECT "id" FROM "brc20_event_transfer_transfers" ORDER BY "id" DESC LIMIT 1 ) SELECT - (SELECT "id" FROM "latest_deploy_id") AS "event_deploy_id", - (SELECT "id" FROM "latest_mint_id") AS "event_mint_id", - (SELECT "id" FROM "latest_inscribe_transfer_id") AS "event_inscribe_transfer_id", - (SELECT "id" FROM "latest_transfer_transfer_id") AS "event_transfer_transfer_id" + COALESCE((SELECT "id" FROM "latest_deploy_id"), -1) AS "event_deploy_id", + COALESCE((SELECT "id" FROM "latest_mint_id"), -1) AS "event_mint_id", + COALESCE((SELECT "id" FROM "latest_inscribe_transfer_id"), -1) AS "event_inscribe_transfer_id", + COALESCE((SELECT "id" FROM "latest_transfer_transfer_id"), -1) AS "event_transfer_transfer_id" ` type GetLatestEventIdsRow struct { - EventDeployID int64 - EventMintID int64 - EventInscribeTransferID int64 - EventTransferTransferID int64 + EventDeployID interface{} + EventMintID interface{} + EventInscribeTransferID interface{} + EventTransferTransferID interface{} } func (q *Queries) GetLatestEventIds(ctx context.Context) (GetLatestEventIdsRow, error) { diff --git a/modules/brc20/internal/repository/postgres/mapper.go b/modules/brc20/internal/repository/postgres/mapper.go index d44844d..1feb2a7 100644 --- a/modules/brc20/internal/repository/postgres/mapper.go +++ b/modules/brc20/internal/repository/postgres/mapper.go @@ -353,7 +353,7 @@ func mapEventDeployModelToType(src gen.Brc20EventDeploy) (entity.EventDeploy, er return entity.EventDeploy{}, errors.Wrap(err, "cannot parse satpoint") } return entity.EventDeploy{ - Id: uint64(src.Id), + Id: src.Id, InscriptionId: inscriptionId, InscriptionNumber: src.InscriptionNumber, Tick: src.Tick, @@ -420,7 +420,7 @@ func mapEventMintModelToType(src gen.Brc20EventMint) (entity.EventMint, error) { parentId = &parentIdValue } return entity.EventMint{ - Id: uint64(src.Id), + Id: src.Id, InscriptionId: inscriptionId, InscriptionNumber: src.InscriptionNumber, Tick: src.Tick, @@ -479,7 +479,7 @@ func mapEventInscribeTransferModelToType(src gen.Brc20EventInscribeTransfer) (en return entity.EventInscribeTransfer{}, errors.Wrap(err, "cannot parse satPoint") } return entity.EventInscribeTransfer{ - Id: uint64(src.Id), + Id: src.Id, InscriptionId: inscriptionId, InscriptionNumber: src.InscriptionNumber, Tick: src.Tick, @@ -544,7 +544,7 @@ func mapEventTransferTransferModelToType(src gen.Brc20EventTransferTransfer) (en return entity.EventTransferTransfer{}, errors.Wrap(err, "cannot parse toSatPoint") } return entity.EventTransferTransfer{ - Id: uint64(src.Id), + Id: src.Id, InscriptionId: inscriptionId, InscriptionNumber: src.InscriptionNumber, Tick: src.Tick, diff --git a/modules/brc20/processor_brc20.go b/modules/brc20/processor_brc20.go index ddd0323..a5a9941 100644 --- a/modules/brc20/processor_brc20.go +++ b/modules/brc20/processor_brc20.go @@ -20,6 +20,10 @@ func (p *Processor) processBRC20States(ctx context.Context, transfers []*entity. payloads := make([]*brc20.Payload, 0) ticks := make(map[string]struct{}) for _, transfer := range transfers { + if transfer.Content == nil { + // skip empty content + continue + } payload, err := brc20.ParsePayload(transfer) if err != nil { return errors.Wrap(err, "failed to parse payload") @@ -27,6 +31,10 @@ func (p *Processor) processBRC20States(ctx context.Context, transfers []*entity. payloads = append(payloads, payload) ticks[payload.Tick] = struct{}{} } + if len(payloads) == 0 { + // skip if no valid payloads + return nil + } // TODO: concurrently fetch from db to optimize speed tickEntries, err := p.brc20Dg.GetTickEntriesByTicks(ctx, lo.Keys(ticks)) if err != nil { From 49eff4f3ba5bc69901817a09af50792f5ae571bc Mon Sep 17 00:00:00 2001 From: Gaze Date: Sun, 9 Jun 2024 23:40:36 +0700 Subject: [PATCH 09/14] refactor: separate brc20 processing logic to smaller funcs --- modules/brc20/processor_brc20.go | 545 ++++++++++++++++--------------- 1 file changed, 281 insertions(+), 264 deletions(-) diff --git a/modules/brc20/processor_brc20.go b/modules/brc20/processor_brc20.go index a5a9941..4e52da0 100644 --- a/modules/brc20/processor_brc20.go +++ b/modules/brc20/processor_brc20.go @@ -97,6 +97,282 @@ func (p *Processor) processBRC20States(ctx context.Context, transfers []*entity. newEventTransferTransfers := make([]*entity.EventTransferTransfer, 0) newBalances := make(map[string]map[string]*entity.Balance) + handleEventDeploy := func(payload *brc20.Payload, tickEntry *entity.TickEntry) { + if payload.Transfer.TransferCount > 1 { + // skip used deploy inscriptions + return + } + if tickEntry != nil { + // skip deploy inscriptions for duplicate ticks + return + } + newEntry := &entity.TickEntry{ + Tick: payload.Tick, + OriginalTick: payload.OriginalTick, + TotalSupply: payload.Max, + Decimals: payload.Dec, + LimitPerMint: payload.Lim, + IsSelfMint: payload.SelfMint, + DeployInscriptionId: payload.Transfer.InscriptionId, + DeployedAt: blockHeader.Timestamp, + DeployedAtHeight: payload.Transfer.BlockHeight, + MintedAmount: decimal.Zero, + BurnedAmount: decimal.Zero, + CompletedAt: time.Time{}, + CompletedAtHeight: 0, + } + newTickEntries[payload.Tick] = newEntry + newTickEntryStates[payload.Tick] = newEntry + // update entries for other operations in same block + tickEntries[payload.Tick] = newEntry + + newEventDeploys = append(newEventDeploys, &entity.EventDeploy{ + Id: latestEventId + 1, + InscriptionId: payload.Transfer.InscriptionId, + InscriptionNumber: inscriptionIdsToNumber[payload.Transfer.InscriptionId], + Tick: payload.Tick, + OriginalTick: payload.OriginalTick, + TxHash: payload.Transfer.TxHash, + BlockHeight: payload.Transfer.BlockHeight, + TxIndex: payload.Transfer.TxIndex, + Timestamp: blockHeader.Timestamp, + PkScript: payload.Transfer.NewPkScript, + SatPoint: payload.Transfer.NewSatPoint, + TotalSupply: payload.Max, + Decimals: payload.Dec, + LimitPerMint: payload.Lim, + IsSelfMint: payload.SelfMint, + }) + latestEventId++ + } + handleEventMint := func(payload *brc20.Payload, tickEntry *entity.TickEntry) { + if payload.Transfer.TransferCount > 1 { + // skip used mint inscriptions that are already used + return + } + if tickEntry == nil { + // skip mint inscriptions for non-existent ticks + return + } + if -payload.Amt.Exponent() > int32(tickEntry.Decimals) { + // skip mint inscriptions with decimals greater than allowed + return + } + if tickEntry.MintedAmount.GreaterThanOrEqual(tickEntry.TotalSupply) { + // skip mint inscriptions for ticks with completed mints + return + } + if payload.Amt.GreaterThan(tickEntry.LimitPerMint) { + // skip mint inscriptions with amount greater than limit per mint + return + } + mintableAmount := tickEntry.TotalSupply.Sub(tickEntry.MintedAmount) + if payload.Amt.GreaterThan(mintableAmount) { + payload.Amt = mintableAmount + } + var parentId *ordinals.InscriptionId + if tickEntry.IsSelfMint { + parentIdValue, ok := inscriptionIdsToParent[payload.Transfer.InscriptionId] + if !ok { + // skip mint inscriptions for self mint ticks without parent inscription + return + } + if parentIdValue != tickEntry.DeployInscriptionId { + // skip mint inscriptions for self mint ticks with invalid parent inscription + return + } + parentId = &parentIdValue + } + + tickEntry.MintedAmount = tickEntry.MintedAmount.Add(payload.Amt) + // mark as completed if this mint completes the total supply + if tickEntry.MintedAmount.GreaterThanOrEqual(tickEntry.TotalSupply) { + tickEntry.CompletedAt = blockHeader.Timestamp + tickEntry.CompletedAtHeight = payload.Transfer.BlockHeight + } + + newTickEntryStates[payload.Tick] = tickEntry + newEventMints = append(newEventMints, &entity.EventMint{ + Id: latestEventId + 1, + InscriptionId: payload.Transfer.InscriptionId, + InscriptionNumber: inscriptionIdsToNumber[payload.Transfer.InscriptionId], + Tick: payload.Tick, + OriginalTick: payload.OriginalTick, + TxHash: payload.Transfer.TxHash, + BlockHeight: payload.Transfer.BlockHeight, + TxIndex: payload.Transfer.TxIndex, + Timestamp: blockHeader.Timestamp, + PkScript: payload.Transfer.NewPkScript, + SatPoint: payload.Transfer.NewSatPoint, + Amount: payload.Amt, + ParentId: parentId, + }) + latestEventId++ + } + handleEventInscribeTransfer := func(payload *brc20.Payload) { + // inscribe transfer event + pkScriptHex := hex.EncodeToString(payload.Transfer.NewPkScript) + balance, ok := balances[pkScriptHex][payload.Tick] + if !ok { + balance = &entity.Balance{ + PkScript: payload.Transfer.NewPkScript, + Tick: payload.Tick, + BlockHeight: uint64(blockHeader.Height - 1), + OverallBalance: decimal.Zero, // defaults balance to zero if not found + AvailableBalance: decimal.Zero, + } + } + if payload.Amt.GreaterThan(balance.AvailableBalance) { + // skip inscribe transfer event if amount exceeds available balance + return + } + // update balance state + balance.BlockHeight = uint64(blockHeader.Height) + balance.AvailableBalance = balance.AvailableBalance.Sub(payload.Amt) + if _, ok := balances[pkScriptHex]; !ok { + balances[pkScriptHex] = make(map[string]*entity.Balance) + } + balances[pkScriptHex][payload.Tick] = balance + if _, ok := newBalances[pkScriptHex]; !ok { + newBalances[pkScriptHex] = make(map[string]*entity.Balance) + } + newBalances[pkScriptHex][payload.Tick] = &entity.Balance{} + + event := &entity.EventInscribeTransfer{ + Id: latestEventId + 1, + InscriptionId: payload.Transfer.InscriptionId, + InscriptionNumber: inscriptionIdsToNumber[payload.Transfer.InscriptionId], + Tick: payload.Tick, + OriginalTick: payload.OriginalTick, + TxHash: payload.Transfer.TxHash, + BlockHeight: payload.Transfer.BlockHeight, + TxIndex: payload.Transfer.TxIndex, + Timestamp: blockHeader.Timestamp, + PkScript: payload.Transfer.NewPkScript, + SatPoint: payload.Transfer.NewSatPoint, + OutputIndex: payload.Transfer.NewSatPoint.OutPoint.Index, + SatsAmount: payload.Transfer.NewOutputValue, + Amount: payload.Amt, + } + latestEventId++ + eventInscribeTransfers[payload.Transfer.InscriptionId] = event + newEventInscribeTransfers = append(newEventInscribeTransfers, event) + } + handleEventTransferTransferAsFee := func(payload *brc20.Payload, inscribeTransfer *entity.EventInscribeTransfer) { + // return balance to sender + fromPkScriptHex := hex.EncodeToString(inscribeTransfer.PkScript) + fromBalance, ok := balances[fromPkScriptHex][payload.Tick] + if !ok { + fromBalance = &entity.Balance{ + PkScript: inscribeTransfer.PkScript, + Tick: payload.Tick, + BlockHeight: uint64(blockHeader.Height), + OverallBalance: decimal.Zero, // defaults balance to zero if not found + AvailableBalance: decimal.Zero, + } + } + fromBalance.BlockHeight = uint64(blockHeader.Height) + fromBalance.AvailableBalance = fromBalance.AvailableBalance.Add(payload.Amt) + if _, ok := balances[fromPkScriptHex]; !ok { + balances[fromPkScriptHex] = make(map[string]*entity.Balance) + } + balances[fromPkScriptHex][payload.Tick] = fromBalance + if _, ok := newBalances[fromPkScriptHex]; !ok { + newBalances[fromPkScriptHex] = make(map[string]*entity.Balance) + } + newBalances[fromPkScriptHex][payload.Tick] = fromBalance + + newEventTransferTransfers = append(newEventTransferTransfers, &entity.EventTransferTransfer{ + Id: latestEventId + 1, + InscriptionId: payload.Transfer.InscriptionId, + InscriptionNumber: inscriptionIdsToNumber[payload.Transfer.InscriptionId], + Tick: payload.Tick, + OriginalTick: payload.OriginalTick, + TxHash: payload.Transfer.TxHash, + BlockHeight: payload.Transfer.BlockHeight, + TxIndex: payload.Transfer.TxIndex, + Timestamp: blockHeader.Timestamp, + FromPkScript: inscribeTransfer.PkScript, + FromSatPoint: inscribeTransfer.SatPoint, + FromInputIndex: payload.Transfer.FromInputIndex, + ToPkScript: payload.Transfer.NewPkScript, + ToSatPoint: payload.Transfer.NewSatPoint, + ToOutputIndex: payload.Transfer.NewSatPoint.OutPoint.Index, + SpentAsFee: true, + Amount: payload.Amt, + }) + } + handleEventTransferTransferNormal := func(payload *brc20.Payload, tickEntry *entity.TickEntry, inscribeTransfer *entity.EventInscribeTransfer) { + // subtract balance from sender + fromPkScriptHex := hex.EncodeToString(inscribeTransfer.PkScript) + fromBalance, ok := balances[fromPkScriptHex][payload.Tick] + if !ok { + // skip transfer transfer event if from balance does not exist + return + } + fromBalance.BlockHeight = uint64(blockHeader.Height) + fromBalance.OverallBalance = fromBalance.OverallBalance.Sub(payload.Amt) + if _, ok := balances[fromPkScriptHex]; !ok { + balances[fromPkScriptHex] = make(map[string]*entity.Balance) + } + balances[fromPkScriptHex][payload.Tick] = fromBalance + if _, ok := newBalances[fromPkScriptHex]; !ok { + newBalances[fromPkScriptHex] = make(map[string]*entity.Balance) + } + newBalances[fromPkScriptHex][payload.Tick] = fromBalance + + // add balance to receiver + if bytes.Equal(payload.Transfer.NewPkScript, []byte{0x6a}) { + // burn if sent to OP_RETURN + tickEntry.BurnedAmount = tickEntry.BurnedAmount.Add(payload.Amt) + tickEntries[payload.Tick] = tickEntry + newTickEntryStates[payload.Tick] = tickEntry + } else { + toPkScriptHex := hex.EncodeToString(payload.Transfer.NewPkScript) + toBalance, ok := balances[toPkScriptHex][payload.Tick] + if !ok { + toBalance = &entity.Balance{ + PkScript: payload.Transfer.NewPkScript, + Tick: payload.Tick, + BlockHeight: uint64(blockHeader.Height), + OverallBalance: decimal.Zero, // defaults balance to zero if not found + AvailableBalance: decimal.Zero, + } + } + toBalance.BlockHeight = uint64(blockHeader.Height) + toBalance.OverallBalance = toBalance.OverallBalance.Add(payload.Amt) + toBalance.AvailableBalance = toBalance.AvailableBalance.Add(payload.Amt) + if _, ok := balances[toPkScriptHex]; !ok { + balances[toPkScriptHex] = make(map[string]*entity.Balance) + } + balances[toPkScriptHex][payload.Tick] = toBalance + if _, ok := newBalances[toPkScriptHex]; !ok { + newBalances[toPkScriptHex] = make(map[string]*entity.Balance) + } + newBalances[toPkScriptHex][payload.Tick] = toBalance + } + + newEventTransferTransfers = append(newEventTransferTransfers, &entity.EventTransferTransfer{ + Id: latestEventId + 1, + InscriptionId: payload.Transfer.InscriptionId, + InscriptionNumber: inscriptionIdsToNumber[payload.Transfer.InscriptionId], + Tick: payload.Tick, + OriginalTick: payload.OriginalTick, + TxHash: payload.Transfer.TxHash, + BlockHeight: payload.Transfer.BlockHeight, + TxIndex: payload.Transfer.TxIndex, + Timestamp: blockHeader.Timestamp, + FromPkScript: inscribeTransfer.PkScript, + FromSatPoint: inscribeTransfer.SatPoint, + FromInputIndex: payload.Transfer.FromInputIndex, + ToPkScript: payload.Transfer.NewPkScript, + ToSatPoint: payload.Transfer.NewSatPoint, + ToOutputIndex: payload.Transfer.NewSatPoint.OutPoint.Index, + SpentAsFee: false, + Amount: payload.Amt, + }) + } + for _, payload := range payloads { tickEntry := tickEntries[payload.Tick] @@ -107,114 +383,9 @@ func (p *Processor) processBRC20States(ctx context.Context, transfers []*entity. switch payload.Op { case brc20.OperationDeploy: - if payload.Transfer.TransferCount > 1 { - // skip used deploy inscriptions - continue - } - if tickEntry != nil { - // skip deploy inscriptions for duplicate ticks - continue - } - tickEntry := &entity.TickEntry{ - Tick: payload.Tick, - OriginalTick: payload.OriginalTick, - TotalSupply: payload.Max, - Decimals: payload.Dec, - LimitPerMint: payload.Lim, - IsSelfMint: payload.SelfMint, - DeployInscriptionId: payload.Transfer.InscriptionId, - DeployedAt: blockHeader.Timestamp, - DeployedAtHeight: payload.Transfer.BlockHeight, - MintedAmount: decimal.Zero, - BurnedAmount: decimal.Zero, - CompletedAt: time.Time{}, - CompletedAtHeight: 0, - } - newTickEntries[payload.Tick] = tickEntry - newTickEntryStates[payload.Tick] = tickEntry - // update entries for other operations in same block - tickEntries[payload.Tick] = tickEntry - - newEventDeploys = append(newEventDeploys, &entity.EventDeploy{ - Id: latestEventId + 1, - InscriptionId: payload.Transfer.InscriptionId, - InscriptionNumber: inscriptionIdsToNumber[payload.Transfer.InscriptionId], - Tick: payload.Tick, - OriginalTick: payload.OriginalTick, - TxHash: payload.Transfer.TxHash, - BlockHeight: payload.Transfer.BlockHeight, - TxIndex: payload.Transfer.TxIndex, - Timestamp: blockHeader.Timestamp, - PkScript: payload.Transfer.NewPkScript, - SatPoint: payload.Transfer.NewSatPoint, - TotalSupply: payload.Max, - Decimals: payload.Dec, - LimitPerMint: payload.Lim, - IsSelfMint: payload.SelfMint, - }) - latestEventId++ + handleEventDeploy(payload, tickEntry) case brc20.OperationMint: - if payload.Transfer.TransferCount > 1 { - // skip used mint inscriptions that are already used - continue - } - if tickEntry == nil { - // skip mint inscriptions for non-existent ticks - continue - } - if -payload.Amt.Exponent() > int32(tickEntry.Decimals) { - // skip mint inscriptions with decimals greater than allowed - continue - } - if tickEntry.MintedAmount.GreaterThanOrEqual(tickEntry.TotalSupply) { - // skip mint inscriptions for ticks with completed mints - continue - } - if payload.Amt.GreaterThan(tickEntry.LimitPerMint) { - // skip mint inscriptions with amount greater than limit per mint - continue - } - mintableAmount := tickEntry.TotalSupply.Sub(tickEntry.MintedAmount) - if payload.Amt.GreaterThan(mintableAmount) { - payload.Amt = mintableAmount - } - var parentId *ordinals.InscriptionId - if tickEntry.IsSelfMint { - parentIdValue, ok := inscriptionIdsToParent[payload.Transfer.InscriptionId] - if !ok { - // skip mint inscriptions for self mint ticks without parent inscription - continue - } - if parentIdValue != tickEntry.DeployInscriptionId { - // skip mint inscriptions for self mint ticks with invalid parent inscription - continue - } - parentId = &parentIdValue - } - - tickEntry.MintedAmount = tickEntry.MintedAmount.Add(payload.Amt) - if tickEntry.MintedAmount.GreaterThanOrEqual(tickEntry.TotalSupply) { - tickEntry.CompletedAt = blockHeader.Timestamp - tickEntry.CompletedAtHeight = payload.Transfer.BlockHeight - } - - newTickEntryStates[payload.Tick] = tickEntry - newEventMints = append(newEventMints, &entity.EventMint{ - Id: latestEventId + 1, - InscriptionId: payload.Transfer.InscriptionId, - InscriptionNumber: inscriptionIdsToNumber[payload.Transfer.InscriptionId], - Tick: payload.Tick, - OriginalTick: payload.OriginalTick, - TxHash: payload.Transfer.TxHash, - BlockHeight: payload.Transfer.BlockHeight, - TxIndex: payload.Transfer.TxIndex, - Timestamp: blockHeader.Timestamp, - PkScript: payload.Transfer.NewPkScript, - SatPoint: payload.Transfer.NewSatPoint, - Amount: payload.Amt, - ParentId: parentId, - }) - latestEventId++ + handleEventMint(payload, tickEntry) case brc20.OperationTransfer: if payload.Transfer.TransferCount > 2 { // skip used transfer inscriptions @@ -230,53 +401,7 @@ func (p *Processor) processBRC20States(ctx context.Context, transfers []*entity. } if payload.Transfer.OldSatPoint == (ordinals.SatPoint{}) { - // inscribe transfer event - pkScriptHex := hex.EncodeToString(payload.Transfer.NewPkScript) - balance, ok := balances[pkScriptHex][payload.Tick] - if !ok { - balance = &entity.Balance{ - PkScript: payload.Transfer.NewPkScript, - Tick: payload.Tick, - BlockHeight: uint64(blockHeader.Height - 1), - OverallBalance: decimal.Zero, // defaults balance to zero if not found - AvailableBalance: decimal.Zero, - } - } - if payload.Amt.GreaterThan(balance.AvailableBalance) { - // skip inscribe transfer event if amount exceeds available balance - continue - } - // update balance state - balance.BlockHeight = uint64(blockHeader.Height) - balance.AvailableBalance = balance.AvailableBalance.Sub(payload.Amt) - if _, ok := balances[pkScriptHex]; !ok { - balances[pkScriptHex] = make(map[string]*entity.Balance) - } - balances[pkScriptHex][payload.Tick] = balance - if _, ok := newBalances[pkScriptHex]; !ok { - newBalances[pkScriptHex] = make(map[string]*entity.Balance) - } - newBalances[pkScriptHex][payload.Tick] = &entity.Balance{} - - event := &entity.EventInscribeTransfer{ - Id: latestEventId + 1, - InscriptionId: payload.Transfer.InscriptionId, - InscriptionNumber: inscriptionIdsToNumber[payload.Transfer.InscriptionId], - Tick: payload.Tick, - OriginalTick: payload.OriginalTick, - TxHash: payload.Transfer.TxHash, - BlockHeight: payload.Transfer.BlockHeight, - TxIndex: payload.Transfer.TxIndex, - Timestamp: blockHeader.Timestamp, - PkScript: payload.Transfer.NewPkScript, - SatPoint: payload.Transfer.NewSatPoint, - OutputIndex: payload.Transfer.NewSatPoint.OutPoint.Index, - SatsAmount: payload.Transfer.NewOutputValue, - Amount: payload.Amt, - } - latestEventId++ - eventInscribeTransfers[payload.Transfer.InscriptionId] = event - newEventInscribeTransfers = append(newEventInscribeTransfers, event) + handleEventInscribeTransfer(payload) } else { // transfer transfer event inscribeTransfer, ok := eventInscribeTransfers[payload.Transfer.InscriptionId] @@ -286,117 +411,9 @@ func (p *Processor) processBRC20States(ctx context.Context, transfers []*entity. } if payload.Transfer.SentAsFee { - // return balance to sender - fromPkScriptHex := hex.EncodeToString(inscribeTransfer.PkScript) - fromBalance, ok := balances[fromPkScriptHex][payload.Tick] - if !ok { - fromBalance = &entity.Balance{ - PkScript: inscribeTransfer.PkScript, - Tick: payload.Tick, - BlockHeight: uint64(blockHeader.Height), - OverallBalance: decimal.Zero, // defaults balance to zero if not found - AvailableBalance: decimal.Zero, - } - } - fromBalance.BlockHeight = uint64(blockHeader.Height) - fromBalance.AvailableBalance = fromBalance.AvailableBalance.Add(payload.Amt) - if _, ok := balances[fromPkScriptHex]; !ok { - balances[fromPkScriptHex] = make(map[string]*entity.Balance) - } - balances[fromPkScriptHex][payload.Tick] = fromBalance - if _, ok := newBalances[fromPkScriptHex]; !ok { - newBalances[fromPkScriptHex] = make(map[string]*entity.Balance) - } - newBalances[fromPkScriptHex][payload.Tick] = fromBalance - - newEventTransferTransfers = append(newEventTransferTransfers, &entity.EventTransferTransfer{ - Id: latestEventId + 1, - InscriptionId: payload.Transfer.InscriptionId, - InscriptionNumber: inscriptionIdsToNumber[payload.Transfer.InscriptionId], - Tick: payload.Tick, - OriginalTick: payload.OriginalTick, - TxHash: payload.Transfer.TxHash, - BlockHeight: payload.Transfer.BlockHeight, - TxIndex: payload.Transfer.TxIndex, - Timestamp: blockHeader.Timestamp, - FromPkScript: inscribeTransfer.PkScript, - FromSatPoint: inscribeTransfer.SatPoint, - FromInputIndex: payload.Transfer.FromInputIndex, - ToPkScript: payload.Transfer.NewPkScript, - ToSatPoint: payload.Transfer.NewSatPoint, - ToOutputIndex: payload.Transfer.NewSatPoint.OutPoint.Index, - SpentAsFee: true, - Amount: payload.Amt, - }) + handleEventTransferTransferAsFee(payload, inscribeTransfer) } else { - // subtract balance from sender - fromPkScriptHex := hex.EncodeToString(inscribeTransfer.PkScript) - fromBalance, ok := balances[fromPkScriptHex][payload.Tick] - if !ok { - // skip transfer transfer event if from balance does not exist - continue - } - fromBalance.BlockHeight = uint64(blockHeader.Height) - fromBalance.OverallBalance = fromBalance.OverallBalance.Sub(payload.Amt) - if _, ok := balances[fromPkScriptHex]; !ok { - balances[fromPkScriptHex] = make(map[string]*entity.Balance) - } - balances[fromPkScriptHex][payload.Tick] = fromBalance - if _, ok := newBalances[fromPkScriptHex]; !ok { - newBalances[fromPkScriptHex] = make(map[string]*entity.Balance) - } - newBalances[fromPkScriptHex][payload.Tick] = fromBalance - - // add balance to receiver - if bytes.Equal(payload.Transfer.NewPkScript, []byte{0x6a}) { - // burn if sent to OP_RETURN - tickEntry.BurnedAmount = tickEntry.BurnedAmount.Add(payload.Amt) - tickEntries[payload.Tick] = tickEntry - newTickEntryStates[payload.Tick] = tickEntry - } else { - toPkScriptHex := hex.EncodeToString(payload.Transfer.NewPkScript) - toBalance, ok := balances[toPkScriptHex][payload.Tick] - if !ok { - toBalance = &entity.Balance{ - PkScript: payload.Transfer.NewPkScript, - Tick: payload.Tick, - BlockHeight: uint64(blockHeader.Height), - OverallBalance: decimal.Zero, // defaults balance to zero if not found - AvailableBalance: decimal.Zero, - } - } - toBalance.BlockHeight = uint64(blockHeader.Height) - toBalance.OverallBalance = toBalance.OverallBalance.Add(payload.Amt) - toBalance.AvailableBalance = toBalance.AvailableBalance.Add(payload.Amt) - if _, ok := balances[toPkScriptHex]; !ok { - balances[toPkScriptHex] = make(map[string]*entity.Balance) - } - balances[toPkScriptHex][payload.Tick] = toBalance - if _, ok := newBalances[toPkScriptHex]; !ok { - newBalances[toPkScriptHex] = make(map[string]*entity.Balance) - } - newBalances[toPkScriptHex][payload.Tick] = toBalance - } - - newEventTransferTransfers = append(newEventTransferTransfers, &entity.EventTransferTransfer{ - Id: latestEventId + 1, - InscriptionId: payload.Transfer.InscriptionId, - InscriptionNumber: inscriptionIdsToNumber[payload.Transfer.InscriptionId], - Tick: payload.Tick, - OriginalTick: payload.OriginalTick, - TxHash: payload.Transfer.TxHash, - BlockHeight: payload.Transfer.BlockHeight, - TxIndex: payload.Transfer.TxIndex, - Timestamp: blockHeader.Timestamp, - FromPkScript: inscribeTransfer.PkScript, - FromSatPoint: inscribeTransfer.SatPoint, - FromInputIndex: payload.Transfer.FromInputIndex, - ToPkScript: payload.Transfer.NewPkScript, - ToSatPoint: payload.Transfer.NewSatPoint, - ToOutputIndex: payload.Transfer.NewSatPoint.OutPoint.Index, - SpentAsFee: false, - Amount: payload.Amt, - }) + handleEventTransferTransferNormal(payload, tickEntry, inscribeTransfer) } } } From 05d7fecf69ce07cfb8c50ef85b784f41cbb32f38 Mon Sep 17 00:00:00 2001 From: Gaze Date: Mon, 10 Jun 2024 00:09:18 +0700 Subject: [PATCH 10/14] feat: add event hash logic --- modules/brc20/event_hash.go | 69 +++++++++++++++++++ .../brc20/internal/entity/indexed_block.go | 4 +- .../internal/repository/postgres/mapper.go | 12 ++-- modules/brc20/processor.go | 2 + modules/brc20/processor_brc20.go | 41 +++++++---- modules/brc20/processor_process.go | 41 ++++++++--- 6 files changed, 139 insertions(+), 30 deletions(-) create mode 100644 modules/brc20/event_hash.go diff --git a/modules/brc20/event_hash.go b/modules/brc20/event_hash.go new file mode 100644 index 0000000..ea9106b --- /dev/null +++ b/modules/brc20/event_hash.go @@ -0,0 +1,69 @@ +package brc20 + +import ( + "encoding/hex" + "strconv" + "strings" + + "github.com/gaze-network/indexer-network/modules/brc20/internal/entity" + "github.com/samber/lo" +) + +const eventHashSeparator = "|" + +func getEventDeployString(event *entity.EventDeploy) string { + var sb strings.Builder + sb.WriteString("deploy-inscribe;") + sb.WriteString(event.InscriptionId.String() + ";") + sb.WriteString(hex.EncodeToString(event.PkScript) + ";") + sb.WriteString(event.Tick + ";") + sb.WriteString(event.OriginalTick + ";") + sb.WriteString(event.TotalSupply.StringFixed(int32(event.Decimals)) + ";") + sb.WriteString(strconv.Itoa(int(event.Decimals)) + ";") + sb.WriteString(event.LimitPerMint.StringFixed(int32(event.Decimals)) + ";") + sb.WriteString(lo.Ternary(event.IsSelfMint, "True", "False")) + return sb.String() +} + +func getEventMintString(event *entity.EventMint, decimals uint16) string { + var sb strings.Builder + var parentId string + if event.ParentId != nil { + parentId = event.ParentId.String() + } + sb.WriteString("mint-inscribe;") + sb.WriteString(event.InscriptionId.String() + ";") + sb.WriteString(hex.EncodeToString(event.PkScript) + ";") + sb.WriteString(event.Tick + ";") + sb.WriteString(event.OriginalTick + ";") + sb.WriteString(event.Amount.StringFixed(int32(decimals)) + ";") + sb.WriteString(parentId) + return sb.String() +} + +func getEventInscribeTransferString(event *entity.EventInscribeTransfer, decimals uint16) string { + var sb strings.Builder + sb.WriteString("inscribe-transfer;") + sb.WriteString(event.InscriptionId.String() + ";") + sb.WriteString(hex.EncodeToString(event.PkScript) + ";") + sb.WriteString(event.Tick + ";") + sb.WriteString(event.OriginalTick + ";") + sb.WriteString(event.Amount.StringFixed(int32(decimals))) + return sb.String() +} + +func getEventTransferTransferString(event *entity.EventTransferTransfer, decimals uint16) string { + var sb strings.Builder + sb.WriteString("transfer-transfer;") + sb.WriteString(event.InscriptionId.String() + ";") + sb.WriteString(hex.EncodeToString(event.FromPkScript) + ";") + if event.SpentAsFee { + sb.WriteString(";") + } else { + sb.WriteString(hex.EncodeToString(event.ToPkScript) + ";") + } + sb.WriteString(event.Tick + ";") + sb.WriteString(event.OriginalTick + ";") + sb.WriteString(event.Amount.StringFixed(int32(decimals))) + return sb.String() +} diff --git a/modules/brc20/internal/entity/indexed_block.go b/modules/brc20/internal/entity/indexed_block.go index a20451d..b899457 100644 --- a/modules/brc20/internal/entity/indexed_block.go +++ b/modules/brc20/internal/entity/indexed_block.go @@ -5,6 +5,6 @@ import "github.com/btcsuite/btcd/chaincfg/chainhash" type IndexedBlock struct { Height uint64 Hash chainhash.Hash - EventHash chainhash.Hash - CumulativeEventHash chainhash.Hash + EventHash []byte + CumulativeEventHash []byte } diff --git a/modules/brc20/internal/repository/postgres/mapper.go b/modules/brc20/internal/repository/postgres/mapper.go index 1feb2a7..178574d 100644 --- a/modules/brc20/internal/repository/postgres/mapper.go +++ b/modules/brc20/internal/repository/postgres/mapper.go @@ -70,19 +70,19 @@ func mapIndexedBlockModelToType(src gen.Brc20IndexedBlock) (entity.IndexedBlock, if err != nil { return entity.IndexedBlock{}, errors.Wrap(err, "invalid block hash") } - eventHash, err := chainhash.NewHashFromStr(src.EventHash) + eventHash, err := hex.DecodeString(src.EventHash) if err != nil { return entity.IndexedBlock{}, errors.Wrap(err, "invalid event hash") } - cumulativeEventHash, err := chainhash.NewHashFromStr(src.CumulativeEventHash) + cumulativeEventHash, err := hex.DecodeString(src.CumulativeEventHash) if err != nil { return entity.IndexedBlock{}, errors.Wrap(err, "invalid cumulative event hash") } return entity.IndexedBlock{ Height: uint64(src.Height), Hash: *hash, - EventHash: *eventHash, - CumulativeEventHash: *cumulativeEventHash, + EventHash: eventHash, + CumulativeEventHash: cumulativeEventHash, }, nil } @@ -90,8 +90,8 @@ func mapIndexedBlockTypeToParams(src entity.IndexedBlock) gen.CreateIndexedBlock return gen.CreateIndexedBlockParams{ Height: int32(src.Height), Hash: src.Hash.String(), - EventHash: src.EventHash.String(), - CumulativeEventHash: src.CumulativeEventHash.String(), + EventHash: hex.EncodeToString(src.EventHash), + CumulativeEventHash: hex.EncodeToString(src.CumulativeEventHash), } } diff --git a/modules/brc20/processor.go b/modules/brc20/processor.go index dd593d0..f211abe 100644 --- a/modules/brc20/processor.go +++ b/modules/brc20/processor.go @@ -52,6 +52,8 @@ type Processor struct { newEventInscribeTransfers []*entity.EventInscribeTransfer newEventTransferTransfers []*entity.EventTransferTransfer newBalances map[string]map[string]*entity.Balance + + eventHashString string } // TODO: move this to config diff --git a/modules/brc20/processor_brc20.go b/modules/brc20/processor_brc20.go index 4e52da0..c887b7e 100644 --- a/modules/brc20/processor_brc20.go +++ b/modules/brc20/processor_brc20.go @@ -4,6 +4,7 @@ import ( "bytes" "context" "encoding/hex" + "strings" "time" "github.com/cockroachdb/errors" @@ -96,6 +97,7 @@ func (p *Processor) processBRC20States(ctx context.Context, transfers []*entity. newEventInscribeTransfers := make([]*entity.EventInscribeTransfer, 0) newEventTransferTransfers := make([]*entity.EventTransferTransfer, 0) newBalances := make(map[string]map[string]*entity.Balance) + var eventHashBuilder strings.Builder handleEventDeploy := func(payload *brc20.Payload, tickEntry *entity.TickEntry) { if payload.Transfer.TransferCount > 1 { @@ -126,7 +128,7 @@ func (p *Processor) processBRC20States(ctx context.Context, transfers []*entity. // update entries for other operations in same block tickEntries[payload.Tick] = newEntry - newEventDeploys = append(newEventDeploys, &entity.EventDeploy{ + event := &entity.EventDeploy{ Id: latestEventId + 1, InscriptionId: payload.Transfer.InscriptionId, InscriptionNumber: inscriptionIdsToNumber[payload.Transfer.InscriptionId], @@ -142,8 +144,11 @@ func (p *Processor) processBRC20States(ctx context.Context, transfers []*entity. Decimals: payload.Dec, LimitPerMint: payload.Lim, IsSelfMint: payload.SelfMint, - }) + } + newEventDeploys = append(newEventDeploys, event) latestEventId++ + + eventHashBuilder.WriteString(getEventDeployString(event) + eventHashSeparator) } handleEventMint := func(payload *brc20.Payload, tickEntry *entity.TickEntry) { if payload.Transfer.TransferCount > 1 { @@ -192,7 +197,7 @@ func (p *Processor) processBRC20States(ctx context.Context, transfers []*entity. } newTickEntryStates[payload.Tick] = tickEntry - newEventMints = append(newEventMints, &entity.EventMint{ + event := &entity.EventMint{ Id: latestEventId + 1, InscriptionId: payload.Transfer.InscriptionId, InscriptionNumber: inscriptionIdsToNumber[payload.Transfer.InscriptionId], @@ -206,10 +211,13 @@ func (p *Processor) processBRC20States(ctx context.Context, transfers []*entity. SatPoint: payload.Transfer.NewSatPoint, Amount: payload.Amt, ParentId: parentId, - }) + } + newEventMints = append(newEventMints, event) latestEventId++ + + eventHashBuilder.WriteString(getEventMintString(event, tickEntry.Decimals) + eventHashSeparator) } - handleEventInscribeTransfer := func(payload *brc20.Payload) { + handleEventInscribeTransfer := func(payload *brc20.Payload, tickEntry *entity.TickEntry) { // inscribe transfer event pkScriptHex := hex.EncodeToString(payload.Transfer.NewPkScript) balance, ok := balances[pkScriptHex][payload.Tick] @@ -257,8 +265,10 @@ func (p *Processor) processBRC20States(ctx context.Context, transfers []*entity. latestEventId++ eventInscribeTransfers[payload.Transfer.InscriptionId] = event newEventInscribeTransfers = append(newEventInscribeTransfers, event) + + eventHashBuilder.WriteString(getEventInscribeTransferString(event, tickEntry.Decimals) + eventHashSeparator) } - handleEventTransferTransferAsFee := func(payload *brc20.Payload, inscribeTransfer *entity.EventInscribeTransfer) { + handleEventTransferTransferAsFee := func(payload *brc20.Payload, tickEntry *entity.TickEntry, inscribeTransfer *entity.EventInscribeTransfer) { // return balance to sender fromPkScriptHex := hex.EncodeToString(inscribeTransfer.PkScript) fromBalance, ok := balances[fromPkScriptHex][payload.Tick] @@ -282,7 +292,7 @@ func (p *Processor) processBRC20States(ctx context.Context, transfers []*entity. } newBalances[fromPkScriptHex][payload.Tick] = fromBalance - newEventTransferTransfers = append(newEventTransferTransfers, &entity.EventTransferTransfer{ + event := &entity.EventTransferTransfer{ Id: latestEventId + 1, InscriptionId: payload.Transfer.InscriptionId, InscriptionNumber: inscriptionIdsToNumber[payload.Transfer.InscriptionId], @@ -300,7 +310,10 @@ func (p *Processor) processBRC20States(ctx context.Context, transfers []*entity. ToOutputIndex: payload.Transfer.NewSatPoint.OutPoint.Index, SpentAsFee: true, Amount: payload.Amt, - }) + } + newEventTransferTransfers = append(newEventTransferTransfers, event) + + eventHashBuilder.WriteString(getEventTransferTransferString(event, tickEntry.Decimals) + eventHashSeparator) } handleEventTransferTransferNormal := func(payload *brc20.Payload, tickEntry *entity.TickEntry, inscribeTransfer *entity.EventInscribeTransfer) { // subtract balance from sender @@ -352,7 +365,7 @@ func (p *Processor) processBRC20States(ctx context.Context, transfers []*entity. newBalances[toPkScriptHex][payload.Tick] = toBalance } - newEventTransferTransfers = append(newEventTransferTransfers, &entity.EventTransferTransfer{ + event := &entity.EventTransferTransfer{ Id: latestEventId + 1, InscriptionId: payload.Transfer.InscriptionId, InscriptionNumber: inscriptionIdsToNumber[payload.Transfer.InscriptionId], @@ -370,7 +383,10 @@ func (p *Processor) processBRC20States(ctx context.Context, transfers []*entity. ToOutputIndex: payload.Transfer.NewSatPoint.OutPoint.Index, SpentAsFee: false, Amount: payload.Amt, - }) + } + newEventTransferTransfers = append(newEventTransferTransfers, event) + + eventHashBuilder.WriteString(getEventTransferTransferString(event, tickEntry.Decimals) + eventHashSeparator) } for _, payload := range payloads { @@ -401,7 +417,7 @@ func (p *Processor) processBRC20States(ctx context.Context, transfers []*entity. } if payload.Transfer.OldSatPoint == (ordinals.SatPoint{}) { - handleEventInscribeTransfer(payload) + handleEventInscribeTransfer(payload, tickEntry) } else { // transfer transfer event inscribeTransfer, ok := eventInscribeTransfers[payload.Transfer.InscriptionId] @@ -411,7 +427,7 @@ func (p *Processor) processBRC20States(ctx context.Context, transfers []*entity. } if payload.Transfer.SentAsFee { - handleEventTransferTransferAsFee(payload, inscribeTransfer) + handleEventTransferTransferAsFee(payload, tickEntry, inscribeTransfer) } else { handleEventTransferTransferNormal(payload, tickEntry, inscribeTransfer) } @@ -426,5 +442,6 @@ func (p *Processor) processBRC20States(ctx context.Context, transfers []*entity. p.newEventInscribeTransfers = newEventInscribeTransfers p.newEventTransferTransfers = newEventTransferTransfers p.newBalances = newBalances + p.eventHashString = eventHashBuilder.String() return nil } diff --git a/modules/brc20/processor_process.go b/modules/brc20/processor_process.go index 0afdf12..d4892d5 100644 --- a/modules/brc20/processor_process.go +++ b/modules/brc20/processor_process.go @@ -2,10 +2,11 @@ package brc20 import ( "context" + "crypto/sha256" "slices" - "github.com/btcsuite/btcd/chaincfg/chainhash" "github.com/cockroachdb/errors" + "github.com/gaze-network/indexer-network/common/errs" "github.com/gaze-network/indexer-network/core/types" "github.com/gaze-network/indexer-network/modules/brc20/internal/entity" "github.com/gaze-network/indexer-network/modules/brc20/internal/ordinals" @@ -78,15 +79,35 @@ func (p *Processor) flushBlock(ctx context.Context, blockHeader types.BlockHeade blockHeight := uint64(blockHeader.Height) - // CreateIndexedBlock must be performed before other flush methods to correctly calculate event hash - // TODO: calculate event hash - if err := brc20DgTx.CreateIndexedBlock(ctx, &entity.IndexedBlock{ - Height: blockHeight, - Hash: blockHeader.Hash, - EventHash: chainhash.Hash{}, - CumulativeEventHash: chainhash.Hash{}, - }); err != nil { - return errors.Wrap(err, "failed to create indexed block") + // calculate event hash + { + eventHashString := p.eventHashString + if len(eventHashString) > 0 && eventHashString[len(eventHashString)-1:] == eventHashSeparator { + eventHashString = eventHashString[:len(eventHashString)-1] + } + eventHash := sha256.Sum256([]byte(eventHashString)) + prevIndexedBlock, err := brc20DgTx.GetIndexedBlockByHeight(ctx, blockHeader.Height-1) + if err != nil && errors.Is(err, errs.NotFound) && blockHeader.Height-1 == startingBlockHeader[p.network].Height { + prevIndexedBlock = &entity.IndexedBlock{ + Height: uint64(startingBlockHeader[p.network].Height), + Hash: startingBlockHeader[p.network].Hash, + EventHash: []byte{}, + CumulativeEventHash: []byte{}, + } + err = nil + } + if err != nil { + return errors.Wrap(err, "failed to get previous indexed block") + } + cumulativeEventHash := sha256.Sum256(append(prevIndexedBlock.CumulativeEventHash[:], eventHash[:]...)) + if err := brc20DgTx.CreateIndexedBlock(ctx, &entity.IndexedBlock{ + Height: blockHeight, + Hash: blockHeader.Hash, + EventHash: eventHash[:], + CumulativeEventHash: cumulativeEventHash[:], + }); err != nil { + return errors.Wrap(err, "failed to create indexed block") + } } // flush new inscription entries From 2c5a6076ff4f3c07dc064dbb8b4abf1bb75e583b Mon Sep 17 00:00:00 2001 From: Gaze Date: Mon, 10 Jun 2024 00:14:28 +0700 Subject: [PATCH 11/14] feat: add flush brc20 states --- .../database/postgresql/queries/data.sql | 3 + modules/brc20/internal/datagateway/brc20.go | 1 + .../internal/repository/postgres/brc20.go | 17 ++++++ .../internal/repository/postgres/gen/batch.go | 55 +++++++++++++++++ .../internal/repository/postgres/mapper.go | 10 ++++ modules/brc20/processor.go | 2 +- modules/brc20/processor_process.go | 60 +++++++++++++++++++ 7 files changed, 147 insertions(+), 1 deletion(-) diff --git a/modules/brc20/database/postgresql/queries/data.sql b/modules/brc20/database/postgresql/queries/data.sql index fa2664b..b990846 100644 --- a/modules/brc20/database/postgresql/queries/data.sql +++ b/modules/brc20/database/postgresql/queries/data.sql @@ -105,6 +105,9 @@ INSERT INTO "brc20_event_inscribe_transfers" ("inscription_id", "inscription_num -- name: CreateEventTransferTransfers :batchexec INSERT INTO "brc20_event_transfer_transfers" ("inscription_id", "inscription_number", "tick", "original_tick", "tx_hash", "block_height", "tx_index", "timestamp", "from_pkscript", "from_satpoint", "from_input_index", "to_pkscript", "to_satpoint", "to_output_index", "spent_as_fee", "amount") VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16); +-- name: CreateBalances :batchexec +INSERT INTO "brc20_balances" ("pkscript", "block_height", "tick", "overall_balance", "available_balance") VALUES ($1, $2, $3, $4, $5); + -- name: DeleteIndexedBlocksSinceHeight :exec DELETE FROM "brc20_indexed_blocks" WHERE "height" >= $1; diff --git a/modules/brc20/internal/datagateway/brc20.go b/modules/brc20/internal/datagateway/brc20.go index ef1fb98..010965a 100644 --- a/modules/brc20/internal/datagateway/brc20.go +++ b/modules/brc20/internal/datagateway/brc20.go @@ -48,6 +48,7 @@ type BRC20WriterDataGateway interface { CreateEventMints(ctx context.Context, events []*entity.EventMint) error CreateEventInscribeTransfers(ctx context.Context, events []*entity.EventInscribeTransfer) error CreateEventTransferTransfers(ctx context.Context, events []*entity.EventTransferTransfer) error + CreateBalances(ctx context.Context, balances []*entity.Balance) error // used for revert data DeleteIndexedBlocksSinceHeight(ctx context.Context, height uint64) error diff --git a/modules/brc20/internal/repository/postgres/brc20.go b/modules/brc20/internal/repository/postgres/brc20.go index bb9914f..6533db1 100644 --- a/modules/brc20/internal/repository/postgres/brc20.go +++ b/modules/brc20/internal/repository/postgres/brc20.go @@ -440,6 +440,23 @@ func (r *Repository) CreateEventTransferTransfers(ctx context.Context, events [] return nil } +func (r *Repository) CreateBalances(ctx context.Context, balances []*entity.Balance) error { + params := lo.Map(balances, func(balance *entity.Balance, _ int) gen.CreateBalancesParams { + return mapBalanceTypeToParams(*balance) + }) + results := r.queries.CreateBalances(ctx, params) + var execErrors []error + results.Exec(func(i int, err error) { + if err != nil { + execErrors = append(execErrors, err) + } + }) + if len(execErrors) > 0 { + return errors.Wrap(errors.Join(execErrors...), "error during exec") + } + return nil +} + func (r *Repository) DeleteIndexedBlocksSinceHeight(ctx context.Context, height uint64) error { if err := r.queries.DeleteIndexedBlocksSinceHeight(ctx, int32(height)); err != nil { return errors.Wrap(err, "error during exec") diff --git a/modules/brc20/internal/repository/postgres/gen/batch.go b/modules/brc20/internal/repository/postgres/gen/batch.go index 10fe598..85a2900 100644 --- a/modules/brc20/internal/repository/postgres/gen/batch.go +++ b/modules/brc20/internal/repository/postgres/gen/batch.go @@ -17,6 +17,61 @@ var ( ErrBatchAlreadyClosed = errors.New("batch already closed") ) +const createBalances = `-- name: CreateBalances :batchexec +INSERT INTO "brc20_balances" ("pkscript", "block_height", "tick", "overall_balance", "available_balance") VALUES ($1, $2, $3, $4, $5) +` + +type CreateBalancesBatchResults struct { + br pgx.BatchResults + tot int + closed bool +} + +type CreateBalancesParams struct { + Pkscript string + BlockHeight int32 + Tick string + OverallBalance pgtype.Numeric + AvailableBalance pgtype.Numeric +} + +func (q *Queries) CreateBalances(ctx context.Context, arg []CreateBalancesParams) *CreateBalancesBatchResults { + batch := &pgx.Batch{} + for _, a := range arg { + vals := []interface{}{ + a.Pkscript, + a.BlockHeight, + a.Tick, + a.OverallBalance, + a.AvailableBalance, + } + batch.Queue(createBalances, vals...) + } + br := q.db.SendBatch(ctx, batch) + return &CreateBalancesBatchResults{br, len(arg), false} +} + +func (b *CreateBalancesBatchResults) Exec(f func(int, error)) { + defer b.br.Close() + for t := 0; t < b.tot; t++ { + if b.closed { + if f != nil { + f(t, ErrBatchAlreadyClosed) + } + continue + } + _, err := b.br.Exec() + if f != nil { + f(t, err) + } + } +} + +func (b *CreateBalancesBatchResults) Close() error { + b.closed = true + return b.br.Close() +} + const createEventDeploys = `-- name: CreateEventDeploys :batchexec INSERT INTO "brc20_event_deploys" ("inscription_id", "inscription_number", "tick", "original_tick", "tx_hash", "block_height", "tx_index", "timestamp", "pkscript", "satpoint", "total_supply", "decimals", "limit_per_mint", "is_self_mint") VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14) ` diff --git a/modules/brc20/internal/repository/postgres/mapper.go b/modules/brc20/internal/repository/postgres/mapper.go index 178574d..e7c649e 100644 --- a/modules/brc20/internal/repository/postgres/mapper.go +++ b/modules/brc20/internal/repository/postgres/mapper.go @@ -602,3 +602,13 @@ func mapBalanceModelToType(src gen.Brc20Balance) (entity.Balance, error) { AvailableBalance: decimalFromNumeric(src.AvailableBalance).Decimal, }, nil } + +func mapBalanceTypeToParams(src entity.Balance) gen.CreateBalancesParams { + return gen.CreateBalancesParams{ + Pkscript: hex.EncodeToString(src.PkScript), + Tick: src.Tick, + BlockHeight: int32(src.BlockHeight), + OverallBalance: numericFromDecimal(src.OverallBalance), + AvailableBalance: numericFromDecimal(src.AvailableBalance), + } +} diff --git a/modules/brc20/processor.go b/modules/brc20/processor.go index f211abe..055751b 100644 --- a/modules/brc20/processor.go +++ b/modules/brc20/processor.go @@ -51,7 +51,7 @@ type Processor struct { newEventMints []*entity.EventMint newEventInscribeTransfers []*entity.EventInscribeTransfer newEventTransferTransfers []*entity.EventTransferTransfer - newBalances map[string]map[string]*entity.Balance + newBalances map[string]map[string]*entity.Balance // pkscript -> tick -> balance eventHashString string } diff --git a/modules/brc20/processor_process.go b/modules/brc20/processor_process.go index d4892d5..0d2f7f1 100644 --- a/modules/brc20/processor_process.go +++ b/modules/brc20/processor_process.go @@ -108,6 +108,7 @@ func (p *Processor) flushBlock(ctx context.Context, blockHeader types.BlockHeade }); err != nil { return errors.Wrap(err, "failed to create indexed block") } + p.eventHashString = "" } // flush new inscription entries @@ -148,6 +149,65 @@ func (p *Processor) flushBlock(ctx context.Context, blockHeader types.BlockHeade return errors.Wrap(err, "failed to create processor stats") } } + // newTickEntries map[string]*entity.TickEntry + // newTickEntryStates map[string]*entity.TickEntry + // newEventDeploys []*entity.EventDeploy + // newEventMints []*entity.EventMint + // newEventInscribeTransfers []*entity.EventInscribeTransfer + // newEventTransferTransfers []*entity.EventTransferTransfer + // newBalances map[string]map[string]*entity.Balance + + // flush new tick entries + { + newTickEntries := lo.Values(p.newTickEntries) + if err := brc20DgTx.CreateTickEntries(ctx, blockHeight, newTickEntries); err != nil { + return errors.Wrap(err, "failed to create tick entries") + } + p.newTickEntries = make(map[string]*entity.TickEntry) + } + + // flush new tick entry states + { + newTickEntryStates := lo.Values(p.newTickEntryStates) + if err := brc20DgTx.CreateTickEntryStates(ctx, blockHeight, newTickEntryStates); err != nil { + return errors.Wrap(err, "failed to create tick entry states") + } + p.newTickEntryStates = make(map[string]*entity.TickEntry) + } + + // flush new events + { + if err := brc20DgTx.CreateEventDeploys(ctx, p.newEventDeploys); err != nil { + return errors.Wrap(err, "failed to create event deploys") + } + if err := brc20DgTx.CreateEventMints(ctx, p.newEventMints); err != nil { + return errors.Wrap(err, "failed to create event mints") + } + if err := brc20DgTx.CreateEventInscribeTransfers(ctx, p.newEventInscribeTransfers); err != nil { + return errors.Wrap(err, "failed to create event inscribe transfers") + } + if err := brc20DgTx.CreateEventTransferTransfers(ctx, p.newEventTransferTransfers); err != nil { + return errors.Wrap(err, "failed to create event transfer transfers") + } + p.newEventDeploys = make([]*entity.EventDeploy, 0) + p.newEventMints = make([]*entity.EventMint, 0) + p.newEventInscribeTransfers = make([]*entity.EventInscribeTransfer, 0) + p.newEventTransferTransfers = make([]*entity.EventTransferTransfer, 0) + } + + // flush new balances + { + newBalances := make([]*entity.Balance, 0) + for _, tickBalances := range p.newBalances { + for _, balance := range tickBalances { + newBalances = append(newBalances, balance) + } + } + if err := brc20DgTx.CreateBalances(ctx, newBalances); err != nil { + return errors.Wrap(err, "failed to create balances") + } + p.newBalances = make(map[string]map[string]*entity.Balance) + } if err := brc20DgTx.Commit(ctx); err != nil { return errors.Wrap(err, "failed to commit transaction") From 3603248485a05cbe92abee38a32af9e8c7089428 Mon Sep 17 00:00:00 2001 From: Gaze Date: Mon, 10 Jun 2024 01:45:17 +0700 Subject: [PATCH 12/14] fix: start cumulative hash with first event hash --- modules/brc20/processor_process.go | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/modules/brc20/processor_process.go b/modules/brc20/processor_process.go index 0d2f7f1..1a306a1 100644 --- a/modules/brc20/processor_process.go +++ b/modules/brc20/processor_process.go @@ -99,7 +99,12 @@ func (p *Processor) flushBlock(ctx context.Context, blockHeader types.BlockHeade if err != nil { return errors.Wrap(err, "failed to get previous indexed block") } - cumulativeEventHash := sha256.Sum256(append(prevIndexedBlock.CumulativeEventHash[:], eventHash[:]...)) + var cumulativeEventHash [32]byte + if len(prevIndexedBlock.CumulativeEventHash) == 0 { + cumulativeEventHash = eventHash + } else { + cumulativeEventHash = sha256.Sum256(append(prevIndexedBlock.CumulativeEventHash[:], eventHash[:]...)) + } if err := brc20DgTx.CreateIndexedBlock(ctx, &entity.IndexedBlock{ Height: blockHeight, Hash: blockHeader.Hash, From 8110434e18f24ae364aebce33e5fbc518422b509 Mon Sep 17 00:00:00 2001 From: Gaze Date: Mon, 10 Jun 2024 01:52:24 +0700 Subject: [PATCH 13/14] fix: concat hex-encoded hashes instead --- modules/brc20/processor_process.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/brc20/processor_process.go b/modules/brc20/processor_process.go index 1a306a1..7af28aa 100644 --- a/modules/brc20/processor_process.go +++ b/modules/brc20/processor_process.go @@ -103,7 +103,7 @@ func (p *Processor) flushBlock(ctx context.Context, blockHeader types.BlockHeade if len(prevIndexedBlock.CumulativeEventHash) == 0 { cumulativeEventHash = eventHash } else { - cumulativeEventHash = sha256.Sum256(append(prevIndexedBlock.CumulativeEventHash[:], eventHash[:]...)) + cumulativeEventHash = sha256.Sum256([]byte(hex.EncodeToString(prevIndexedBlock.CumulativeEventHash[:]) + hex.EncodeToString(eventHash[:]))) } if err := brc20DgTx.CreateIndexedBlock(ctx, &entity.IndexedBlock{ Height: blockHeight, From 980163900c2b0840c943a7a960aeb23f6b9c3525 Mon Sep 17 00:00:00 2001 From: Gaze Date: Mon, 10 Jun 2024 01:53:13 +0700 Subject: [PATCH 14/14] fix: imports --- modules/brc20/processor_process.go | 1 + 1 file changed, 1 insertion(+) diff --git a/modules/brc20/processor_process.go b/modules/brc20/processor_process.go index 7af28aa..4fbcaba 100644 --- a/modules/brc20/processor_process.go +++ b/modules/brc20/processor_process.go @@ -3,6 +3,7 @@ package brc20 import ( "context" "crypto/sha256" + "encoding/hex" "slices" "github.com/cockroachdb/errors"