Dex custom backfill (#3485)

* Add missing fetchs to balancer

* Add balancer customBackfill function

* dexVolumes/helper/chains.js -> dexVolumes/helper/chains.ts

* Fix types

* Refactor function custom backfill

dexVolumes/balancer/index.ts

@@ -1,6 +1,7 @@
 import { DexVolumeAdapter } from "../dexVolume.type";
 import { getChainVolume } from "../helper/getUniSubgraphVolume";
 import { ARBITRUM, ETHEREUM, POLYGON } from "../helper/chains";
+import customBackfill from "../helper/customBackfill";
 
 const endpoints = {
   [ETHEREUM]: "https://api.thegraph.com/subgraphs/name/balancer-labs/balancer",
@@ -28,11 +29,20 @@ const adapter: DexVolumeAdapter = {
     [ETHEREUM]: {
       fetch: graphs(ETHEREUM),
       start: 0,
-      customBackfill: () => {},
+      customBackfill: customBackfill(ETHEREUM, graphs),
     },
     // POLYGON
-
+    [POLYGON]: {
+      fetch: graphs(POLYGON),
+      start: 0,
+      customBackfill: customBackfill(POLYGON, graphs),
+    },
     // ARBITRUM
+    [ARBITRUM]: {
+      fetch: graphs(ARBITRUM),
+      start: 0,
+      customBackfill: customBackfill(ARBITRUM, graphs),
+    },
   },
 };
 

dexVolumes/bancor/index.ts

@@ -42,7 +42,7 @@ const adapter: DexVolumeAdapter = {
     ethereum: {
       fetch: graphs("ethereum"),
       runAtCurrTime: true,
-      customBackfill: () => {},
+      customBackfill: undefined,
       start: 0,
     },
     // CUSTOM BACKFILL

dexVolumes/cli/testAdapter.ts

@@ -3,7 +3,8 @@ import type { ChainBlocks, DexAdapter, VolumeAdapter } from '../dexVolume.type';
 import { chainsForBlocks } from "@defillama/sdk/build/computeTVL/blocks";
 import { Chain } from '@defillama/sdk/build/general';
 import handleError from '../../utils/handleError';
-import { checkArguments, getLatestBlockRetry, printVolumes } from './utils';
+import { checkArguments, printVolumes } from './utils';
+import { getBlock } from '../../projects/helper/getBlock';
 
 // Add handler to rejections/exceptions
 process.on('unhandledRejection', handleError)
@@ -19,15 +20,16 @@ const passedFile = path.resolve(process.cwd(), `dexVolumes/${process.argv[2]}`);
   // Import module to test
   let module: DexAdapter = (await import(passedFile)).default
 
+  const unixTimestamp = +process.argv[3] || Math.round(Date.now() / 1000) - 60;
   if ("volume" in module) {
     // Get adapter
     const volumeAdapter = module.volume
-    const volumes = await runAdapter(volumeAdapter)
+    const volumes = await runAdapter(volumeAdapter, unixTimestamp)
     printVolumes(volumes)
   } else if ("breakdown" in module) {
     const breakdownAdapter = module.breakdown
     const allVolumes = await Promise.all(Object.entries(breakdownAdapter).map(async ([version, adapter]) =>
-      await runAdapter(adapter).then(res => ({ version, res }))
+      await runAdapter(adapter, unixTimestamp).then(res => ({ version, res }))
     ))
     allVolumes.forEach((promise) => {
       console.info(promise.version)
@@ -36,7 +38,7 @@ const passedFile = path.resolve(process.cwd(), `dexVolumes/${process.argv[2]}`);
   } else console.info("No compatible adapter found")
 })()
 
-async function runAdapter(volumeAdapter: VolumeAdapter) {
+async function runAdapter(volumeAdapter: VolumeAdapter, timestamp: number) {
   // Get chains to check
   const chains = Object.keys(volumeAdapter).filter(item => typeof volumeAdapter[item] === 'object');
   // Get lastest block 
@@ -45,19 +47,19 @@ async function runAdapter(volumeAdapter: VolumeAdapter) {
     chains.map(async (chainRaw) => {
       const chain: Chain = chainRaw === "ava" ? "avax" : chainRaw as Chain
       if (chainsForBlocks.includes(chain as Chain) || chain === "ethereum") {
-        const latestBlock = await getLatestBlockRetry(chain)
+        const latestBlock = await getBlock(timestamp, chain, chainBlocks)
         if (!latestBlock) throw new Error("latestBlock")
-        chainBlocks[chain] = latestBlock.number - 10
+        chainBlocks[chain] = latestBlock - 10
       }
     })
   );
   // Get volumes
-  const unixTimestamp = Math.round(Date.now() / 1000) - 60;
   const volumes = await Promise.all(Object.keys(chainBlocks).map(
-    async chain => volumeAdapter[chain].fetch(unixTimestamp, chainBlocks)
-      .then(res => {
-        return { timestamp: unixTimestamp, totalVolume: res.totalVolume, dailyVolume: res.dailyVolume }
-      })
+    async chain => {
+      const fetchVolumeFunc = volumeAdapter[chain].customBackfill ?? volumeAdapter[chain].fetch
+      return fetchVolumeFunc(timestamp, chainBlocks)
+        .then(res => ({ timestamp: res.timestamp, totalVolume: res.totalVolume, dailyVolume: res.dailyVolume }))
+    }
   ))
   return volumes
 }

dexVolumes/curve/index.ts

@@ -29,7 +29,7 @@ const adapter: DexVolumeAdapter = {
     ethereum: {
       fetch: graphs("ethereum"),
       runAtCurrTime: true,
-      customBackfill: () => {},
+      customBackfill: undefined,
       start: 0,
     },
     // TODO custom backfill

dexVolumes/dexVolume.type.ts

@@ -19,7 +19,7 @@ export type VolumeAdapter = {
     start: number | any;
     fetch: Fetch;
     runAtCurrTime?: boolean;
-    customBackfill?: any;
+    customBackfill?: Fetch;
   };
 };
 

dexVolumes/helper/chains.ts

@@ -14,7 +14,7 @@ const RONIN = "ronin";
 const XDAI = "xdai";
 const AURORA = "aurora";
 
-module.exports = {
+export {
   ARBITRUM,
   AVAX,
   BOBA,

dexVolumes/helper/customBackfill.ts

@@ -0,0 +1,18 @@
+import { Chain } from "@defillama/sdk/build/general"
+import { Fetch } from "../dexVolume.type"
+import { getChainVolume } from "./getUniSubgraphVolume"
+import { getBlock } from "../../projects/helper/getBlock"
+
+export default (chain: Chain, graphs: ReturnType<typeof getChainVolume>): Fetch => async (timestamp: number, chainBlocks: ChainBlocks) => {
+    const fetchGetVolume = graphs(chain)
+    const resultDayN = await fetchGetVolume(timestamp, chainBlocks)
+    const timestampPreviousDay = timestamp - 60 * 60 * 24
+    const chainBlocksPreviousDay = (await getBlock(timestampPreviousDay, chain, {})) - 20
+    const resultPreviousDayN = await fetchGetVolume(timestampPreviousDay, { [chain]: chainBlocksPreviousDay })
+    return {
+        block: resultDayN.block,
+        timestamp: resultDayN.timestamp,
+        totalVolume: resultDayN.totalVolume,
+        dailyVolume: `${Number(resultDayN.totalVolume) - Number(resultPreviousDayN.totalVolume)}`,
+    }
+}

dexVolumes/klayswap/index.ts

@@ -32,7 +32,7 @@ const adapter: DexVolumeAdapter = {
     klatyn: {
       fetch,
       runAtCurrTime: true,
-      customBackfill: () => {},
+      customBackfill: undefined,
       start: 0,
     },
     // TODO custom backfill

dexVolumes/koyo/index.ts

@@ -23,12 +23,12 @@ const adapter: DexVolumeAdapter = {
     [BOBA]: {
       fetch: graphs(BOBA),
       start: 1655104044,
-      customBackfill: () => {},
+      customBackfill: undefined,
     },
     [AURORA]: {
       fetch: graphs(AURORA),
       start: 1657617165,
-      customBackfill: () => {},
+      customBackfill: undefined,
     },
   },
 };

dexVolumes/osmosis/index.ts

@@ -34,7 +34,7 @@ const adapter: DexVolumeAdapter = {
     cosmos: {
       fetch: graphs,
       runAtCurrTime: true,
-      customBackfill: () => {},
+      customBackfill: undefined,
       start: 0,
     },
     // TODO custom backfill

dexVolumes/raydium/index.ts

@@ -26,7 +26,7 @@ const adapter: DexVolumeAdapter = {
     solana: {
       fetch: graphs("solana"),
       runAtCurrTime: true,
-      customBackfill: () => {},
+      customBackfill: undefined,
       start: 0,
     },
     // TODO custom backfill

dexVolumes/saros/index.ts

@@ -28,7 +28,7 @@ const adapter: DexVolumeAdapter = {
     solana: {
       fetch: graphs("solana"),
       runAtCurrTime: true,
-      customBackfill: () => {},
+      customBackfill: undefined,
       start: 0,
     },
   },

dexVolumes/serum/index.ts

@@ -41,7 +41,7 @@ const adapter: DexVolumeAdapter = {
       fetch,
       start: 0,
       runAtCurrTime: true,
-      customBackfill: () => {},
+      customBackfill: undefined,
     },
   },
 };

dexVolumes/terraswap/index.ts

@@ -66,7 +66,7 @@ const adapter: DexVolumeAdapter = {
     terra: {
       fetch,
       runAtCurrTime: true,
-      customBackfill: () => {},
+      customBackfill: undefined,
       start: 0,
     },
     // TODO custom backfill

typings/index.d.ts

@@ -1 +0,0 @@
-declare module '*/helper/chains';