Everything is a file (#356)

2026-01-12 16:53:14 +08:00 · 2024-07-10 09:41:46 +03:00
parent 1502a69005
commit a4fd231c44
7 changed files with 269 additions and 161 deletions
--- a/api2/cache.ts
+++ b/api2/cache.ts
@@ -26,14 +26,16 @@ export const cache: {
 const MINUTES = 60 * 1000
 const HOUR = 60 * MINUTES

+const cacheFile = 'stablecoin-cache'
+
 export async function initCache() {
  console.time('Cache initialized')
-  const _cache = await readFromPGCache('cron-cache') ?? {}
+  const _cache = await readFromPGCache(cacheFile) ?? {}
  Object.keys(_cache).forEach(key => cache[key] = _cache[key])
  cache.rates = await getLastRecord(historicalRates);
  console.timeEnd('Cache initialized')
 }

 export async function saveCache() {
-  await writeToPGCache('cron-cache', cache)
+  await writeToPGCache(cacheFile, cache)
 }
--- a/api2/cron-task/getStableCoins.ts
+++ b/api2/cron-task/getStableCoins.ts
@@ -19,7 +19,7 @@ function getTVLOfRecordClosestToTimestamp(
  for (let i = 0; i < items.length; i++) {
    if (items[i].SK <= endSK && items[i].SK >= endSK - searchWidth) {
      if (!record || items[i].SK > record.SK)
-      record = items[i]
+        record = items[i]
    }
  }

@@ -32,89 +32,89 @@ function craftProtocolsResponse(
  let prices = cache.peggedPrices!

  const response = (
-      peggedAssets.map((pegged) => {
-        const pegType = pegged.pegType;
-        const { balances, lastBalance } = cache.peggedAssetsData?.[pegged.id] ?? {}
-        const lastHourlyRecord = lastBalance
-        if (lastHourlyRecord === undefined) {
-          return null;
+    peggedAssets.map((pegged) => {
+      const pegType = pegged.pegType;
+      const { balances, lastBalance } = cache.peggedAssetsData?.[pegged.id] ?? {}
+      const lastHourlyRecord = lastBalance
+      if (lastHourlyRecord === undefined) {
+        return null;
+      }
+      const lastSK = lastHourlyRecord.SK;
+      if (lastSK === undefined) {
+        return null;
+      }
+      const lastDailyPeggedRecord = getTVLOfRecordClosestToTimestamp(
+        balances,
+        lastSK - secondsInDay,
+        secondsInDay // temporary, update later
+      );
+      const lastWeeklyPeggedRecord = getTVLOfRecordClosestToTimestamp(
+        balances,
+        lastSK - secondsInWeek,
+        secondsInDay // temporary, update later
+      );
+      const lastMonthlyPeggedRecord = getTVLOfRecordClosestToTimestamp(
+        balances,
+        lastSK - secondsInDay * 30,
+        secondsInDay // temporary, update later
+      );
+      const chainCirculating = {} as {
+        [chain: string]: any;
+      };
+      const chains: string[] = [];
+      Object.entries(lastHourlyRecord).forEach(([chain, issuances]: any) => {
+        if (nonChains.includes(chain)) {
+          return;
        }
-        const lastSK = lastHourlyRecord.SK;
-        if (lastSK === undefined) {
-          return null;
-        }
-        const lastDailyPeggedRecord = getTVLOfRecordClosestToTimestamp(
-          balances,
-          lastSK - secondsInDay,
-          secondsInDay // temporary, update later
-        );
-        const lastWeeklyPeggedRecord = getTVLOfRecordClosestToTimestamp(
-          balances,
-          lastSK - secondsInWeek,
-          secondsInDay // temporary, update later
-        );
-        const lastMonthlyPeggedRecord = getTVLOfRecordClosestToTimestamp(
-          balances,
-          lastSK - secondsInDay * 30,
-          secondsInDay // temporary, update later
-        );
-        const chainCirculating = {} as {
-          [chain: string]: any;
-        };
-        const chains: string[] = [];
-        Object.entries(lastHourlyRecord).forEach(([chain, issuances]: any) => {
-          if (nonChains.includes(chain)) {
-            return;
-          }
-          const chainDisplayName = getChainDisplayName(chain, useNewChainNames);
-          chainCirculating[chainDisplayName] =
-            chainCirculating[chainDisplayName] || {};
-          chainCirculating[chainDisplayName].current = issuances.circulating;
-          chainCirculating[chainDisplayName].circulatingPrevDay =
-            lastDailyPeggedRecord && lastDailyPeggedRecord[chain]
-              ? lastDailyPeggedRecord[chain].circulating ?? 0
-              : 0;
-          chainCirculating[chainDisplayName].circulatingPrevWeek =
-            lastWeeklyPeggedRecord && lastWeeklyPeggedRecord[chain]
-              ? lastWeeklyPeggedRecord[chain].circulating ?? 0
-              : 0;
-          chainCirculating[chainDisplayName].circulatingPrevMonth =
-            lastMonthlyPeggedRecord && lastMonthlyPeggedRecord[chain]
-              ? lastMonthlyPeggedRecord[chain].circulating ?? 0
-              : 0;
-          addToChains(chains, chainDisplayName);
-        });
+        const chainDisplayName = getChainDisplayName(chain, useNewChainNames);
+        chainCirculating[chainDisplayName] =
+          chainCirculating[chainDisplayName] || {};
+        chainCirculating[chainDisplayName].current = issuances.circulating;
+        chainCirculating[chainDisplayName].circulatingPrevDay =
+          lastDailyPeggedRecord && lastDailyPeggedRecord[chain]
+            ? lastDailyPeggedRecord[chain].circulating ?? 0
+            : 0;
+        chainCirculating[chainDisplayName].circulatingPrevWeek =
+          lastWeeklyPeggedRecord && lastWeeklyPeggedRecord[chain]
+            ? lastWeeklyPeggedRecord[chain].circulating ?? 0
+            : 0;
+        chainCirculating[chainDisplayName].circulatingPrevMonth =
+          lastMonthlyPeggedRecord && lastMonthlyPeggedRecord[chain]
+            ? lastMonthlyPeggedRecord[chain].circulating ?? 0
+            : 0;
+        addToChains(chains, chainDisplayName);
+      });

-        const dataToReturn = {
-          id: pegged.id,
-          name: pegged.name,
-          symbol: pegged.symbol,
-          gecko_id: pegged.gecko_id,
-          pegType: pegged.pegType,
-          priceSource: pegged.priceSource,
-          pegMechanism: pegged.pegMechanism,
-          circulating: lastHourlyRecord.totalCirculating.circulating,
-          circulatingPrevDay: lastDailyPeggedRecord
-            ? lastDailyPeggedRecord.totalCirculating.circulating
-            : 0,
-          circulatingPrevWeek: lastWeeklyPeggedRecord
-            ? lastWeeklyPeggedRecord.totalCirculating.circulating
-            : 0,
-          circulatingPrevMonth: lastMonthlyPeggedRecord
-            ? lastMonthlyPeggedRecord.totalCirculating.circulating
-            : 0,
-          chainCirculating,
-          chains: chains.sort(
-            (a, b) =>
-              chainCirculating[b].current[pegType] -
-              chainCirculating[a].current[pegType]
-          ),
-        } as any;
-        dataToReturn.price = prices[pegged.gecko_id] ?? null;
-        if (pegged.delisted) dataToReturn.delisted = true;
-        return dataToReturn;
-      })
-    )
+      const dataToReturn = {
+        id: pegged.id,
+        name: pegged.name,
+        symbol: pegged.symbol,
+        gecko_id: pegged.gecko_id,
+        pegType: pegged.pegType,
+        priceSource: pegged.priceSource,
+        pegMechanism: pegged.pegMechanism,
+        circulating: lastHourlyRecord.totalCirculating.circulating,
+        circulatingPrevDay: lastDailyPeggedRecord
+          ? lastDailyPeggedRecord.totalCirculating.circulating
+          : 0,
+        circulatingPrevWeek: lastWeeklyPeggedRecord
+          ? lastWeeklyPeggedRecord.totalCirculating.circulating
+          : 0,
+        circulatingPrevMonth: lastMonthlyPeggedRecord
+          ? lastMonthlyPeggedRecord.totalCirculating.circulating
+          : 0,
+        chainCirculating,
+        chains: chains.sort(
+          (a, b) =>
+            chainCirculating[b].current[pegType] -
+            chainCirculating[a].current[pegType]
+        ),
+      } as any;
+      dataToReturn.price = prices[pegged.gecko_id] ?? null;
+      if (pegged.delisted) dataToReturn.delisted = true;
+      return dataToReturn;
+    })
+  )
    .filter((pegged) => pegged !== null)
    .sort((a, b) => b.circulating - a.circulating);
  return response;
@@ -125,8 +125,9 @@ export default async function handler() {
  let response: any = {
    peggedAssets: pegged,
  };
-    const chainData = await craftStablecoinChainsResponse();
-    response.chains = chainData;
-    await storeRouteData('stablecoins', response)
+  const chainData = await craftStablecoinChainsResponse();
+  response.chains = chainData;
+  await storeRouteData('stablecoins', response)
+  return response;
 };

--- a/api2/cron-task/index.ts
+++ b/api2/cron-task/index.ts
@@ -3,13 +3,16 @@ import * as rates from "../../src/getRates";
 import sluggifyPegged from "../../src/peggedAssets/utils/sluggifyPegged";
 import { storeRouteData } from "../file-cache";
 import storePeggedPrices from "./storePeggedPrices";
-import storeCharts from "./storeCharts";
+import storeCharts, { craftChartsResponse } from "./storeCharts";
 import storeStablecoins from "./getStableCoins";
 import { craftStablecoinPricesResponse } from "./getStablecoinPrices";
 import { craftStablecoinChainsResponse } from "./getStablecoinChains";
 import { cache, initCache, saveCache } from "../cache";
 import { getCurrentUnixTimestamp } from "../../src/utils/date";
 import { sendMessage } from "../../src/utils/discord";
+import { getStablecoinData } from "../routes/getStableCoin";
+import { craftChainDominanceResponse } from "../routes/getChainDominance";
+import { normalizeChain } from "../../src/utils/normalizeChain";

 run().catch(console.error).then(() => process.exit(0))

@@ -24,39 +27,120 @@ async function run() {

  // this also pulls data from ddb and sets to cache
  await storeCharts()
-  await storeStablecoins()
+  const allStablecoinsData = await storeStablecoins()
  await storePrices()
  await storeStablecoinChains()
+  const allChainsSet: Set<string> = new Set()
+  const assetChainMap: {
+    [asset: string]: Set<string>
+  } = {}
+  allStablecoinsData.peggedAssets.forEach((asset: any) => {
+    const _chains = asset.chains.map(normalizeChain)
+    assetChainMap[asset.id] = new Set(_chains)
+    _chains.forEach((chain) => allChainsSet.add(chain))
+  })
+
+  await storePeggedAssets()
+  await storeStablecoinDominance()
+  await storeChainChartData()

  await saveCache()

  await alertOutdated()

-  async function storeConfig() {
-    let configJSON: any = Object.fromEntries(
-      peggedAssets.map((pegged) => [sluggifyPegged(pegged), pegged.id])
-    )
-    await storeRouteData('config', configJSON)
+
+  async function storePeggedAssets() {
+    for (const { id } of allStablecoinsData.peggedAssets) {
+      try {
+        const data = await getStablecoinData(id)
+        data.chainBalances = data.chainBalances ?? {}
+        for (const [chain, chainData] of Object.entries(data.chainBalances)) {
+          const nChain = normalizeChain(chain)
+          allChainsSet.add(nChain)
+          assetChainMap[id].add(nChain);
+          (chainData as any).tokens = removeEmptyItems((chainData as any).tokens)
+        }
+        data.tokens = removeEmptyItems(data.tokens)
+        await storeRouteData('stablecoin/' + id, data)
+      } catch (e) {
+        console.error('Error fetching asset data', e)
+      }
+    }
  }

-  async function storeRates() {
-    await storeRouteData('rates', await rates.craftRatesResponse())
+  async function storeStablecoinDominance() {
+    for (const chain of [...allChainsSet]) {
+      try {
+        const data = await craftChainDominanceResponse(chain)
+        await storeRouteData('stablecoindominance/' + chain, data)
+      } catch (e) {
+        console.error('Error fetching chain data', e)
+      }
+    }
  }

-  async function storePrices() {
-    await storeRouteData('stablecoinprices', craftStablecoinPricesResponse())
-  }

-  async function storeStablecoinChains() {
-    await storeRouteData('stablecoinchains', craftStablecoinChainsResponse())
+
+  async function storeChainChartData() {
+    const frontendKey = 'all-llama-app'
+    const allChartsStartTimestamp = 1617148800 // for /stablecoins page, charts begin on April 1, 2021, to reduce size of page
+    const allData = await getChainData('all')
+    await storeRouteData('stablecoincharts2/all', allData)
+    const allDataShortened = await getChainData(frontendKey)
+    await storeRouteData('stablecoincharts2/' + frontendKey, allDataShortened)
+
+    for (const chain of [...allChainsSet]) {
+      try {
+        const data = await getChainData(chain)
+        await storeRouteData('stablecoincharts2/' + chain, data)
+      } catch (e) {
+        console.error('Error fetching chain data', e)
+      }
+    }
+
+    async function getChainData(chain: string) {
+      let startTimestamp = chain === frontendKey ? allChartsStartTimestamp : undefined
+      chain = chain === frontendKey ? 'all' : chain
+      const aggregated = removeEmptyItems(await craftChartsResponse({ chain, startTimestamp, }))
+      const breakdown: any = {}
+
+      for (const [peggedAsset, chainMap] of Object.entries(assetChainMap)) {
+        if (chain !== 'all' && !(chainMap as any).has(chain))
+          continue
+        breakdown[peggedAsset] = removeEmptyItems(await craftChartsResponse({ chain, peggedID: peggedAsset, startTimestamp }))
+      }
+
+
+      return { aggregated, breakdown }
+    }
  }
 }

+async function storeConfig() {
+  let configJSON: any = Object.fromEntries(
+    peggedAssets.map((pegged) => [sluggifyPegged(pegged), pegged.id])
+  )
+  await storeRouteData('config', configJSON)
+}
+
+async function storeRates() {
+  await storeRouteData('rates', await rates.craftRatesResponse())
+}
+
+async function storePrices() {
+  await storeRouteData('stablecoinprices', craftStablecoinPricesResponse())
+}
+
+async function storeStablecoinChains() {
+  await storeRouteData('stablecoinchains', craftStablecoinChainsResponse())
+}
+
+
 async function alertOutdated() {
  const now = getCurrentUnixTimestamp();
  const outdated = (
    peggedAssets.map((asset) => {
-      if (asset.delisted) return null;
+      if (asset.delisted || asset.name === 'TerraClassicUSD') return null;
      const last = cache.peggedAssetsData?.[asset.id]?.lastBalance
      if (last?.SK < now - 5 * 3600) {
        return {
@@ -75,4 +159,27 @@ async function alertOutdated() {
    await sendMessage(message, process.env.OUTDATED_WEBHOOK!);
  }

+}
+
+
+function removeEmptyItems(array: any[] = []) {
+  return array.map(removeEmpty).filter((item: any) => item)
+}
+
+function removeEmpty(item: any) {
+  if (!item) return item
+  if (typeof item === 'object') {
+    const { date, ...rest } = item
+    for (const key in rest) {
+      rest[key] = removeEmpty(rest[key])
+      if (!rest[key])
+        delete rest[key]
+    }
+    if (Object.keys(rest).length === 0)
+      return null
+    return { date, ...rest }
+  } else if (typeof item === 'number') {
+    return Math.round(item)
+  }
+  return item
 }
--- a/api2/cron-task/storeCharts.ts
+++ b/api2/cron-task/storeCharts.ts
@@ -45,45 +45,45 @@ export default async function handler() {

  console.time('storeCharts')

-/*   const commonOptions = {
-    lastPrices,
-    historicalPrices,
-    historicalRates,
-    priceTimestamps,
-    rateTimestamps,
-    peggedAssetsData: cache.peggedAssetsData,
-  }
-  // store overall chart
-  const allData = await craftChartsResponse({ ...commonOptions, chain: "all" });  
-  await storeRouteData('charts/all/all', allData)
-
-  // store chain charts
-  const chains = [Object.keys(chainCoingeckoIds), Object.values(normalizedChainReplacements)].flat()
-  for (let chain of chains) {
-    const normalizedChain = normalizeChain(chain);
-    const chainData = await craftChartsResponse({ ...commonOptions, chain, });
-    if (chainData.length) {
-      await storeRouteData(`charts/${normalizedChain}`, chainData)
-    } else {
-      console.log(`No data for ${chain}`)
+  /*   const commonOptions = {
+      lastPrices,
+      historicalPrices,
+      historicalRates,
+      priceTimestamps,
+      rateTimestamps,
+      peggedAssetsData: cache.peggedAssetsData,
    }
-  }
-
-  // store pegged asset charts
-  for (const pegged of peggedAssets) {
-    const id = pegged.id;
-    const chart = await craftChartsResponse({ ...commonOptions, peggedID: id });
-    await storeRouteData(`charts/all/${id}`, chart)
-  }
-  console.timeEnd('storeCharts') */
+    // store overall chart
+    const allData = await craftChartsResponse({ ...commonOptions, chain: "all" });  
+    await storeRouteData('charts/all/all', allData)
+  
+    // store chain charts
+    const chains = [Object.keys(chainCoingeckoIds), Object.values(normalizedChainReplacements)].flat()
+    for (let chain of chains) {
+      const normalizedChain = normalizeChain(chain);
+      const chainData = await craftChartsResponse({ ...commonOptions, chain, });
+      if (chainData.length) {
+        await storeRouteData(`charts/${normalizedChain}`, chainData)
+      } else {
+        console.log(`No data for ${chain}`)
+      }
+    }
+  
+    // store pegged asset charts
+    for (const pegged of peggedAssets) {
+      const id = pegged.id;
+      const chart = await craftChartsResponse({ ...commonOptions, peggedID: id });
+      await storeRouteData(`charts/all/${id}`, chart)
+    }
+    console.timeEnd('storeCharts') */

 }


 async function getPeggedAssetsData() {
  if (!cache.peggedAssetsData)
-      cache.peggedAssetsData = {}
-  
+    cache.peggedAssetsData = {}
+
  await Promise.all(peggedAssets.map(async (pegged) => {

    const lastBalance = await getLastRecord(hourlyPeggedBalances(pegged.id));
@@ -159,20 +159,21 @@ export function craftChartsResponse(
  { chain = 'all', peggedID, startTimestamp }: {
    chain?: string,
    peggedID?: string,
-    startTimestamp?: string,
+    startTimestamp?: string | number,
  }
 ) {
+  if (startTimestamp && typeof startTimestamp === 'string') startTimestamp = parseInt(startTimestamp)

  const filterChart = (chart: any) => {
    return chart.map((entry: any) => {
      if (!startTimestamp) return entry;
-      if (entry.date < parseInt(startTimestamp)) {
+      if (entry.date < startTimestamp) {
        return null;
      }
      return entry;
    }).filter((entry: any) => entry);
  }
-  
+
  const { historicalPrices, historicalRates, lastPrices, priceTimestamps, rateTimestamps, peggedAssetsData, } = cache as any
  const sumDailyBalances = {} as {
    [timestamp: number]: {
--- a/api2/routes/getChainDominance.ts
+++ b/api2/routes/getChainDominance.ts
@@ -19,16 +19,6 @@ export function craftChainDominanceResponse(chain: string | undefined) {
      };
    };
  };
-  // quick fix; need to update later
-  if (chain === "gnosis") {
-    chain = "xdai";
-  }
-  if (chain === "terra%20classic") {
-    chain = "terra";
-  }
-  if (chain === "ethereumpow") {
-    chain = "ethpow";
-  }

  if (chain === undefined) 
    throw new Error("Must include chain as path parameter.")
--- a/api2/routes/index.ts
+++ b/api2/routes/index.ts
@@ -5,6 +5,7 @@ import { readRouteData } from "../file-cache";
 import { craftChartsResponse } from "../cron-task/storeCharts";
 import { getStablecoinData } from "./getStableCoin";
 import { craftChainDominanceResponse } from "./getChainDominance";
+import { normalizeChain } from "../../src/utils/normalizeChain";

 export default function setRoutes(router: HyperExpress.Router) {

@@ -13,7 +14,18 @@ export default function setRoutes(router: HyperExpress.Router) {
  router.get("/stablecoin", defaultFileHandler);
  router.get("/stablecoinprices", defaultFileHandler);
  router.get("/stablecoinchains", defaultFileHandler);
+  router.get("/stablecoins", defaultFileHandler);
+  router.get("/stablecoin/:stablecoin", defaultFileHandler);

+  router.get("/stablecoindominance/:chain", ew(async (req: any, res: any) => {
+    let { chain } = req.path_parameters;
+    chain = normalizeChain(chain)
+    return fileResponse('/stablecoindominance/'+chain, res);
+  }))
+  router.get("/stablecoincharts2/:chain", defaultFileHandler);
+  router.get("/stablecoincharts2/all-llama-app", defaultFileHandler);
+
+  /* Ignore optional query parameters for now
  router.get("/stablecoins", ew(async (req: any, res: any) => {
    const { includePrices, includeChains } = req.query;
    const data = await readRouteData('stablecoins');
@@ -26,7 +38,7 @@ export default function setRoutes(router: HyperExpress.Router) {
    }

    return successResponse(res, data);
-  }));
+  })); 

  router.get("/stablecoin/:stablecoin", ew(async (req: any, res: any) => {
    const { stablecoin } = req.path_parameters;
@@ -40,12 +52,17 @@ export default function setRoutes(router: HyperExpress.Router) {
    return successResponse(res, craftChainDominanceResponse(chain.toLowerCase()));
  }));

+
+  */
+
+
+  // TOO: nuke this route to reduce load on the server
  router.get("/stablecoincharts/:chain", ew(async (req: any, res: any) => {
    const { chain } = req.path_parameters;
-    let { stablecoin, starts } = req.query;
+    let { stablecoin, starts, startts } = req.query;
    const peggedID = stablecoin?.toLowerCase()

-    return successResponse(res, await craftChartsResponse({ chain, peggedID, startTimestamp: starts }));
+    return successResponse(res, await craftChartsResponse({ chain, peggedID, startTimestamp: starts ?? startts }));
  }));

  function defaultFileHandler(req: HyperExpress.Request, res: HyperExpress.Response) {