fix(rosetta): incorrect nonce in rosetta /account/balance endpoint #955 (#959)

docs/openapi.yaml

@@ -1358,6 +1358,19 @@ paths:
           required: true
           schema:
             type: string
+        - name: block_height
+          in: query
+          description: Optionally get the nonce at a given block height
+          required: false
+          schema:
+            type: number
+        - name: block_hash
+          in: query
+          description: Optionally get the nonce at a given block hash
+          required: false
+          schema:
+            type: string
+
       responses:
         200:
           description: Success

src/api/routes/address.ts

@@ -1,7 +1,7 @@
 import * as express from 'express';
 import { addAsync, RouterWithAsync } from '@awaitjs/express';
 import * as Bluebird from 'bluebird';
-import { DataStore } from '../../datastore/common';
+import { BlockIdentifier, DataStore } from '../../datastore/common';
 import { parseLimitQuery, parsePagingQueryInput } from '../pagination';
 import { isUnanchoredRequest, getBlockParams, parseUntilBlockQuery } from '../query-helpers';
 import {
@@ -510,16 +510,59 @@ export function createAddressRouter(db: DataStore, chainId: ChainID): RouterWith
     if (!isValidPrincipal(stxAddress)) {
       return res.status(400).json({ error: `invalid STX address "${stxAddress}"` });
     }
-    const nonces = await db.getAddressNonces({
-      stxAddress,
-    });
-    const results: AddressNonces = {
-      last_executed_tx_nonce: nonces.lastExecutedTxNonce as number,
-      last_mempool_tx_nonce: nonces.lastMempoolTxNonce as number,
-      possible_next_nonce: nonces.possibleNextNonce,
-      detected_missing_nonces: nonces.detectedMissingNonces,
-    };
-    res.json(results);
+    let blockIdentifier: BlockIdentifier | undefined;
+    const blockHeightQuery = req.query['block_height'];
+    const blockHashQuery = req.query['block_hash'];
+    if (blockHeightQuery && blockHashQuery) {
+      res.status(400).json({ error: `Multiple block query parameters specified` });
+      return;
+    }
+    if (blockHeightQuery) {
+      const blockHeight = Number(blockHeightQuery);
+      if (!Number.isInteger(blockHeight) || blockHeight < 1) {
+        res.status(400).json({
+          error: `Query parameter 'block_height' is not a valid integer: ${blockHeightQuery}`,
+        });
+        return;
+      }
+      blockIdentifier = { height: blockHeight };
+    } else if (blockHashQuery) {
+      if (typeof blockHashQuery !== 'string' || !has0xPrefix(blockHashQuery)) {
+        res.status(400).json({
+          error: `Query parameter 'block_hash' is not a valid block hash hex string: ${blockHashQuery}`,
+        });
+        return;
+      }
+      blockIdentifier = { hash: blockHashQuery };
+    }
+    if (blockIdentifier) {
+      const nonceQuery = await db.getAddressNonceAtBlock({ stxAddress, blockIdentifier });
+      if (!nonceQuery.found) {
+        res.status(404).json({
+          error: `No block found for ${JSON.stringify(blockIdentifier)}`,
+        });
+        return;
+      }
+      const results: AddressNonces = {
+        last_executed_tx_nonce: nonceQuery.result.nonce,
+        possible_next_nonce: nonceQuery.result.nonce + 1,
+        // Note: OpenAPI type generator doesn't support `nullable: true` so force cast it here
+        last_mempool_tx_nonce: (null as unknown) as number,
+        detected_missing_nonces: [],
+      };
+      res.json(results);
+    } else {
+      const nonces = await db.getAddressNonces({
+        stxAddress,
+      });
+      const results: AddressNonces = {
+        last_executed_tx_nonce: nonces.lastExecutedTxNonce as number,
+        last_mempool_tx_nonce: nonces.lastMempoolTxNonce as number,
+        possible_next_nonce: nonces.possibleNextNonce,
+        detected_missing_nonces: nonces.detectedMissingNonces,
+      };
+      res.json(results);
+    }
   });
 
   return router;

src/api/routes/rosetta/account.ts

@@ -67,7 +67,10 @@ export function createRosettaAccountRouter(db: DataStore, chainId: ChainID): Rou
     // return spendable balance (liquid) if no sub-account is specified
     let balance = (stxBalance.balance - stxBalance.locked).toString();
 
-    const accountInfo = await new StacksCoreRpcClient().getAccount(accountIdentifier.address);
+    const accountNonceQuery = await db.getAddressNonceAtBlock({
+      stxAddress: accountIdentifier.address,
+      blockIdentifier: { height: block.block_height },
+    });
 
     const extra_metadata: any = {};
 
@@ -117,7 +120,7 @@ export function createRosettaAccountRouter(db: DataStore, chainId: ChainID): Rou
         },
       ],
       metadata: {
-        sequence_number: accountInfo.nonce ? accountInfo.nonce : 0,
+        sequence_number: accountNonceQuery.found ? accountNonceQuery.result.nonce ?? 0 : 0,
       },
     };
 

src/datastore/common.ts

@@ -777,6 +777,11 @@ export interface DataStore extends DataStoreEventEmitter {
     offset: number;
   }): Promise<{ results: DbEvent[]; total: number }>;
 
+  getAddressNonceAtBlock(args: {
+    stxAddress: string;
+    blockIdentifier: BlockIdentifier;
+  }): Promise<FoundOrNot<{ nonce: number }>>;
+
   getAddressNonces(args: {
     stxAddress: string;
   }): Promise<{

src/datastore/memory-store.ts

@@ -570,6 +570,13 @@ export class MemoryDataStore
     throw new Error('not yet implemented');
   }
 
+  getAddressNonceAtBlock(args: {
+    stxAddress: string;
+    blockIdentifier: BlockIdentifier;
+  }): Promise<FoundOrNot<{ nonce: number }>> {
+    throw new Error('not yet implemented');
+  }
+
   getAddressNonces(args: {
     stxAddress: string;
   }): Promise<{

src/datastore/postgres-store.ts

@@ -1722,6 +1722,30 @@ export class PgDataStore
     });
   }
 
+  async getAddressNonceAtBlock(args: {
+    stxAddress: string;
+    blockIdentifier: BlockIdentifier;
+  }): Promise<FoundOrNot<{ nonce: number }>> {
+    return await this.queryTx(async client => {
+      const dbBlock = await this.getBlockInternal(client, args.blockIdentifier);
+      if (!dbBlock.found) {
+        return { found: false };
+      }
+      const executedTxNonce = await client.query<{ nonce: number | null }>(
+        `
+        SELECT MAX(nonce) nonce
+        FROM txs
+        WHERE ((sender_address = $1 AND sponsored = false) OR (sponsor_address = $1 AND sponsored = true))
+        AND canonical = true AND microblock_canonical = true
+        AND block_height <= $2
+        `,
+        [args.stxAddress, dbBlock.result.block_height]
+      );
+      const nonce = executedTxNonce.rows[0]?.nonce ?? 0;
+      return { found: true, result: { nonce } };
+    });
+  }
+
   async getAddressNonces(args: {
     stxAddress: string;
   }): Promise<{

src/tests-rosetta/api.ts

@@ -624,6 +624,147 @@ describe('Rosetta API', () => {
     expect(JSON.parse(result1.text)).toEqual(expectedResponse);
   });
 
+  test('account/balance - nonce calculated properly', async () => {
+    const testAddr1 = 'STNN931GWC0XMRBWXYJQXTEKT4YFB1Z7YTCV3RZN';
+    const testAddr1Key = '532d5ff9f0d4980225a031f65a2dff75b351d675b086766917d43372cedf762901';
+    const testAddr2 = 'ST2WFY0H48AS2VYPA7N69V2VJ8VKS8FSPQSPFE1Z8';
+    let expectedTxId: string = '';
+    const broadcastTx = new Promise<DbTx>(resolve => {
+      const listener: (txId: string) => void = async txId => {
+        const dbTxQuery = await api.datastore.getTx({ txId: txId, includeUnanchored: false });
+        if (!dbTxQuery.found) {
+          return;
+        }
+        const dbTx = dbTxQuery.result as DbTx;
+        if (dbTx.tx_id === expectedTxId && dbTx.status === DbTxStatus.Success) {
+          api.datastore.removeListener('txUpdate', listener);
+          resolve(dbTx);
+        }
+      };
+      api.datastore.addListener('txUpdate', listener);
+    });
+    const transferTx = await makeSTXTokenTransfer({
+      recipient: testAddr1,
+      amount: new BN(10000000),
+      senderKey: 'c71700b07d520a8c9731e4d0f095aa6efb91e16e25fb27ce2b72e7b698f8127a01',
+      network: getStacksTestnetNetwork(),
+      memo: 'test1234',
+      anchorMode: AnchorMode.Any
+    });
+    expectedTxId = '0x' + transferTx.txid();
+    const submitResult = await new StacksCoreRpcClient().sendTransaction(transferTx.serialize());
+    expect(submitResult.txId).toBe(expectedTxId);
+    let tx1 = await broadcastTx;
+    const txDb = await api.datastore.getTx({ txId: expectedTxId, includeUnanchored: false });
+    assert(txDb.found);
+
+    // Send three transactions from `testAddr1` so its nonce at chaintip should be 2 (the third nonce in a zero-based index)
+    let tx2: DbTx;
+    let tx3: DbTx;
+    let tx4: DbTx;
+    for (let i = 0; i < 3; i++) {
+      const broadcastTx2 = new Promise<DbTx>(resolve => {
+        const listener: (txId: string) => void = async txId => {
+          const dbTxQuery = await api.datastore.getTx({ txId: txId, includeUnanchored: false });
+          if (!dbTxQuery.found) {
+            return;
+          }
+          const dbTx = dbTxQuery.result as DbTx;
+          if (dbTx.tx_id === expectedTxId && dbTx.status === DbTxStatus.Success) {
+            api.datastore.removeListener('txUpdate', listener);
+            resolve(dbTx);
+          }
+        };
+        api.datastore.addListener('txUpdate', listener);
+      });
+      const transferTx2 = await makeSTXTokenTransfer({
+        recipient: testAddr2,
+        amount: new BN(10),
+        senderKey: testAddr1Key,
+        network: getStacksTestnetNetwork(),
+        memo: 'test1234',
+        anchorMode: AnchorMode.Any
+      });
+      expectedTxId = '0x' + transferTx2.txid();
+      const submitResult2 = await new StacksCoreRpcClient().sendTransaction(transferTx2.serialize());
+      expect(submitResult2.txId).toBe(expectedTxId);
+      const tx = await broadcastTx2;
+      if (i === 0) {
+        tx2 = tx;
+      } else if (i === 1) {
+        tx3 = tx;
+      } else {
+        tx4 = tx;
+      }
+    }
+
+    const request1: RosettaAccountBalanceRequest = {
+      network_identifier: {
+        blockchain: 'stacks',
+        network: 'testnet',
+      },
+      block_identifier: {
+        index: tx3!.block_height,
+      },
+      account_identifier: {
+        address: testAddr1,
+      },
+    };
+    const nonceResult1 = await supertest(api.server).post(`/rosetta/v1/account/balance/`).send(request1);
+    expect(nonceResult1.status).toBe(200);
+    expect(nonceResult1.type).toBe('application/json');
+    const expectedResponse1: RosettaAccountBalanceResponse = {
+      block_identifier: {
+        hash: tx3!.block_hash,
+        index: tx3!.block_height,
+      },
+      balances: [{
+        value: '9999620',
+        currency: {
+          symbol: 'STX',
+          decimals: 6,
+        },
+      }],
+      metadata: {
+        sequence_number: 1,
+      },
+    };
+    expect(JSON.parse(nonceResult1.text)).toEqual(expectedResponse1);
+
+    const request2: RosettaAccountBalanceRequest = {
+      network_identifier: {
+        blockchain: 'stacks',
+        network: 'testnet',
+      },
+      block_identifier: {
+        index: tx2!.block_height,
+      },
+      account_identifier: {
+        address: testAddr1,
+      },
+    };
+    const nonceResult2 = await supertest(api.server).post(`/rosetta/v1/account/balance/`).send(request2);
+    expect(nonceResult2.status).toBe(200);
+    expect(nonceResult2.type).toBe('application/json');
+    const expectedResponse2: RosettaAccountBalanceResponse = {
+      block_identifier: {
+        hash: tx2!.block_hash,
+        index: tx2!.block_height,
+      },
+      balances: [{
+        value: '9999810',
+        currency: {
+          symbol: 'STX',
+          decimals: 6,
+        },
+      }],
+      metadata: {
+        sequence_number: 0,
+      },
+    };
+    expect(JSON.parse(nonceResult2.text)).toEqual(expectedResponse2);
+  });
+
   test('account/balance - fees calculated properly', async () => {
     // this account has made one transaction
     // ensure that the fees for it are calculated after it makes
@@ -665,7 +806,7 @@ describe('Rosetta API', () => {
       balances: [amount],
 
       metadata: {
-        sequence_number: 1,
+        sequence_number: 0,
       },
     };
 

src/tests/api-tests.ts

@@ -4014,6 +4014,131 @@ describe('api tests', () => {
     expect(JSON.parse(fetch2.text)).toEqual(expected2);
   });
 
+  test('address nonce', async () => {
+    const testAddr1 = 'ST3DWSXBPYDB484QXFTR81K4AWG4ZB5XZNFF3H70C';
+
+    const block1 = new TestBlockBuilder({
+      block_height: 1,
+      block_hash: '0x0001',
+      index_block_hash: '0x9001',
+    })
+      .addTx({ tx_id: '0x0101', nonce: 1, sender_address: testAddr1 })
+      .build();
+    await db.update(block1);
+
+    const block2 = new TestBlockBuilder({
+      block_height: 2,
+      block_hash: '0x0002',
+      index_block_hash: '0x9002',
+      parent_index_block_hash: block1.block.index_block_hash,
+    })
+      .addTx({ tx_id: '0x0201', nonce: 2, sender_address: testAddr1 })
+      .build();
+    await db.update(block2);
+
+    const block3 = new TestBlockBuilder({
+      block_height: 3,
+      block_hash: '0x0003',
+      index_block_hash: '0x9003',
+      parent_index_block_hash: block2.block.index_block_hash,
+    })
+      .addTx({ tx_id: '0x0301', nonce: 3, sender_address: testAddr1 })
+      .build();
+    await db.update(block3);
+
+    const mempoolTx1 = new TestMempoolTxBuilder({
+      tx_id: '0x1401',
+      nonce: 4,
+      type_id: DbTxTypeId.TokenTransfer,
+      sender_address: testAddr1,
+    }).build();
+    await db.updateMempoolTxs({ mempoolTxs: [mempoolTx1] });
+
+    // Chain-tip nonce
+    const expectedNonceResults1 = {
+      detected_missing_nonces: [],
+      last_executed_tx_nonce: 3,
+      last_mempool_tx_nonce: 4,
+      possible_next_nonce: 5,
+    };
+    const nonceResults1 = await supertest(api.server).get(
+      `/extended/v1/address/${testAddr1}/nonces`
+    );
+    expect(nonceResults1.status).toBe(200);
+    expect(nonceResults1.type).toBe('application/json');
+    expect(nonceResults1.body).toEqual(expectedNonceResults1);
+
+    // Detect missing nonce
+    const mempoolTx2 = new TestMempoolTxBuilder({
+      tx_id: '0x1402',
+      nonce: 7,
+      type_id: DbTxTypeId.TokenTransfer,
+      sender_address: testAddr1,
+    }).build();
+    await db.updateMempoolTxs({ mempoolTxs: [mempoolTx2] });
+    const expectedNonceResults2 = {
+      detected_missing_nonces: [6, 5],
+      last_executed_tx_nonce: 3,
+      last_mempool_tx_nonce: 7,
+      possible_next_nonce: 8,
+    };
+    const nonceResults2 = await supertest(api.server).get(
+      `/extended/v1/address/${testAddr1}/nonces`
+    );
+    expect(nonceResults2.status).toBe(200);
+    expect(nonceResults2.type).toBe('application/json');
+    expect(nonceResults2.body).toEqual(expectedNonceResults2);
+
+    // Get nonce at block height
+    const expectedNonceResults3 = {
+      detected_missing_nonces: [],
+      last_executed_tx_nonce: 2,
+      last_mempool_tx_nonce: null,
+      possible_next_nonce: 3,
+    };
+    const nonceResults3 = await supertest(api.server).get(
+      `/extended/v1/address/${testAddr1}/nonces?block_height=${block2.block.block_height}`
+    );
+    expect(nonceResults3.status).toBe(200);
+    expect(nonceResults3.type).toBe('application/json');
+    expect(nonceResults3.body).toEqual(expectedNonceResults3);
+
+    // Get nonce at block hash
+    const expectedNonceResults4 = {
+      detected_missing_nonces: [],
+      last_executed_tx_nonce: 2,
+      last_mempool_tx_nonce: null,
+      possible_next_nonce: 3,
+    };
+    const nonceResults4 = await supertest(api.server).get(
+      `/extended/v1/address/${testAddr1}/nonces?block_hash=${block2.block.block_hash}`
+    );
+    expect(nonceResults4.status).toBe(200);
+    expect(nonceResults4.type).toBe('application/json');
+    expect(nonceResults4.body).toEqual(expectedNonceResults4);
+
+    // Bad requests
+    const nonceResults5 = await supertest(api.server).get(
+      `/extended/v1/address/${testAddr1}/nonces?block_hash=xcvbnmn`
+    );
+    expect(nonceResults5.status).toBe(400);
+
+    const nonceResults6 = await supertest(api.server).get(
+      `/extended/v1/address/${testAddr1}/nonces?block_height=xcvbnmn`
+    );
+    expect(nonceResults6.status).toBe(400);
+
+    const nonceResults7 = await supertest(api.server).get(
+      `/extended/v1/address/${testAddr1}/nonces?block_height=xcvbnmn&block_hash=xcvbnmn`
+    );
+    expect(nonceResults7.status).toBe(400);
+
+    const nonceResults8 = await supertest(api.server).get(
+      `/extended/v1/address/${testAddr1}/nonces?block_height=999999999`
+    );
+    expect(nonceResults8.status).toBe(404);
+  });
+
   test('address info', async () => {
     const testAddr1 = 'ST3J8EVYHVKH6XXPD61EE8XEHW4Y2K83861225AB1';
     const testAddr2 = 'ST1HB64MAJ1MBV4CQ80GF01DZS4T1DSMX20ADCRA4';

src/tests/test-helpers.ts

@@ -115,12 +115,17 @@ export class TestBlockBuilder {
   private data: DataStoreBlockUpdateData;
   private txIndex = 0;
 
-  constructor(args?: { block_height?: number; block_hash?: string }) {
+  constructor(args?: {
+    block_height?: number;
+    block_hash?: string;
+    index_block_hash?: string;
+    parent_index_block_hash?: string;
+  }) {
     this.data = {
       block: {
         block_hash: args?.block_hash ?? '0x1234',
-        index_block_hash: '0xdeadbeef',
-        parent_index_block_hash: '0x00',
+        index_block_hash: args?.index_block_hash ?? '0xdeadbeef',
+        parent_index_block_hash: args?.parent_index_block_hash ?? '0x00',
         parent_block_hash: '0xff0011',
         parent_microblock_hash: '',
         block_height: args?.block_height ?? 1,
@@ -146,13 +151,14 @@ export class TestBlockBuilder {
     sender_address?: string;
     type_id?: DbTxTypeId;
     tx_id?: string;
+    nonce?: number;
   }): TestBlockBuilder {
     this.data.txs.push({
       tx: {
         tx_id: args?.tx_id ?? '0x01',
         tx_index: 0,
         anchor_mode: 3,
-        nonce: 0,
+        nonce: args?.nonce ?? 0,
         raw_tx: Buffer.alloc(0),
         index_block_hash: this.data.block.index_block_hash,
         block_hash: this.data.block.block_hash,
@@ -240,6 +246,7 @@ export class TestMempoolTxBuilder {
     type_id?: DbTxTypeId;
     sender_address?: string;
     tx_id?: string;
+    nonce?: number;
     smart_contract_contract_id?: string;
     contract_call_contract_id?: string;
     contract_call_function_name?: string;
@@ -250,7 +257,7 @@ export class TestMempoolTxBuilder {
       pruned: false,
       tx_id: args?.tx_id ?? `0x1234`,
       anchor_mode: 3,
-      nonce: 0,
+      nonce: args?.nonce ?? 0,
       raw_tx: Buffer.from('test-raw-tx'),
       type_id: args?.type_id ?? DbTxTypeId.TokenTransfer,
       receipt_time: (new Date().getTime() / 1000) | 0,