Everything is a file (#356)

api2/cache.ts

@@ -26,14 +26,16 @@ export const cache: {
 const MINUTES = 60 * 1000
 const HOUR = 60 * MINUTES
 
+const cacheFile = 'stablecoin-cache'
+
 export async function initCache() {
   console.time('Cache initialized')
-  const _cache = await readFromPGCache('cron-cache') ?? {}
+  const _cache = await readFromPGCache(cacheFile) ?? {}
   Object.keys(_cache).forEach(key => cache[key] = _cache[key])
   cache.rates = await getLastRecord(historicalRates);
   console.timeEnd('Cache initialized')
 }
 
 export async function saveCache() {
-  await writeToPGCache('cron-cache', cache)
+  await writeToPGCache(cacheFile, cache)
 }

api2/cron-task/getStableCoins.ts

@@ -128,5 +128,6 @@ export default async function handler() {
   const chainData = await craftStablecoinChainsResponse();
   response.chains = chainData;
   await storeRouteData('stablecoins', response)
+  return response;
 };
 

api2/cron-task/index.ts

@@ -3,13 +3,16 @@ import * as rates from "../../src/getRates";
 import sluggifyPegged from "../../src/peggedAssets/utils/sluggifyPegged";
 import { storeRouteData } from "../file-cache";
 import storePeggedPrices from "./storePeggedPrices";
-import storeCharts from "./storeCharts";
+import storeCharts, { craftChartsResponse } from "./storeCharts";
 import storeStablecoins from "./getStableCoins";
 import { craftStablecoinPricesResponse } from "./getStablecoinPrices";
 import { craftStablecoinChainsResponse } from "./getStablecoinChains";
 import { cache, initCache, saveCache } from "../cache";
 import { getCurrentUnixTimestamp } from "../../src/utils/date";
 import { sendMessage } from "../../src/utils/discord";
+import { getStablecoinData } from "../routes/getStableCoin";
+import { craftChainDominanceResponse } from "../routes/getChainDominance";
+import { normalizeChain } from "../../src/utils/normalizeChain";
 
 run().catch(console.error).then(() => process.exit(0))
 
@@ -24,39 +27,120 @@ async function run() {
 
   // this also pulls data from ddb and sets to cache
   await storeCharts()
-  await storeStablecoins()
+  const allStablecoinsData = await storeStablecoins()
   await storePrices()
   await storeStablecoinChains()
+  const allChainsSet: Set<string> = new Set()
+  const assetChainMap: {
+    [asset: string]: Set<string>
+  } = {}
+  allStablecoinsData.peggedAssets.forEach((asset: any) => {
+    const _chains = asset.chains.map(normalizeChain)
+    assetChainMap[asset.id] = new Set(_chains)
+    _chains.forEach((chain) => allChainsSet.add(chain))
+  })
+
+  await storePeggedAssets()
+  await storeStablecoinDominance()
+  await storeChainChartData()
 
   await saveCache()
 
   await alertOutdated()
 
-  async function storeConfig() {
+
+  async function storePeggedAssets() {
+    for (const { id } of allStablecoinsData.peggedAssets) {
+      try {
+        const data = await getStablecoinData(id)
+        data.chainBalances = data.chainBalances ?? {}
+        for (const [chain, chainData] of Object.entries(data.chainBalances)) {
+          const nChain = normalizeChain(chain)
+          allChainsSet.add(nChain)
+          assetChainMap[id].add(nChain);
+          (chainData as any).tokens = removeEmptyItems((chainData as any).tokens)
+        }
+        data.tokens = removeEmptyItems(data.tokens)
+        await storeRouteData('stablecoin/' + id, data)
+      } catch (e) {
+        console.error('Error fetching asset data', e)
+      }
+    }
+  }
+
+  async function storeStablecoinDominance() {
+    for (const chain of [...allChainsSet]) {
+      try {
+        const data = await craftChainDominanceResponse(chain)
+        await storeRouteData('stablecoindominance/' + chain, data)
+      } catch (e) {
+        console.error('Error fetching chain data', e)
+      }
+    }
+  }
+
+
+
+  async function storeChainChartData() {
+    const frontendKey = 'all-llama-app'
+    const allChartsStartTimestamp = 1617148800 // for /stablecoins page, charts begin on April 1, 2021, to reduce size of page
+    const allData = await getChainData('all')
+    await storeRouteData('stablecoincharts2/all', allData)
+    const allDataShortened = await getChainData(frontendKey)
+    await storeRouteData('stablecoincharts2/' + frontendKey, allDataShortened)
+
+    for (const chain of [...allChainsSet]) {
+      try {
+        const data = await getChainData(chain)
+        await storeRouteData('stablecoincharts2/' + chain, data)
+      } catch (e) {
+        console.error('Error fetching chain data', e)
+      }
+    }
+
+    async function getChainData(chain: string) {
+      let startTimestamp = chain === frontendKey ? allChartsStartTimestamp : undefined
+      chain = chain === frontendKey ? 'all' : chain
+      const aggregated = removeEmptyItems(await craftChartsResponse({ chain, startTimestamp, }))
+      const breakdown: any = {}
+
+      for (const [peggedAsset, chainMap] of Object.entries(assetChainMap)) {
+        if (chain !== 'all' && !(chainMap as any).has(chain))
+          continue
+        breakdown[peggedAsset] = removeEmptyItems(await craftChartsResponse({ chain, peggedID: peggedAsset, startTimestamp }))
+      }
+
+
+      return { aggregated, breakdown }
+    }
+  }
+}
+
+async function storeConfig() {
   let configJSON: any = Object.fromEntries(
     peggedAssets.map((pegged) => [sluggifyPegged(pegged), pegged.id])
   )
   await storeRouteData('config', configJSON)
-  }
-
-  async function storeRates() {
-    await storeRouteData('rates', await rates.craftRatesResponse())
-  }
-
-  async function storePrices() {
-    await storeRouteData('stablecoinprices', craftStablecoinPricesResponse())
-  }
-
-  async function storeStablecoinChains() {
-    await storeRouteData('stablecoinchains', craftStablecoinChainsResponse())
-  }
 }
 
+async function storeRates() {
+  await storeRouteData('rates', await rates.craftRatesResponse())
+}
+
+async function storePrices() {
+  await storeRouteData('stablecoinprices', craftStablecoinPricesResponse())
+}
+
+async function storeStablecoinChains() {
+  await storeRouteData('stablecoinchains', craftStablecoinChainsResponse())
+}
+
+
 async function alertOutdated() {
   const now = getCurrentUnixTimestamp();
   const outdated = (
     peggedAssets.map((asset) => {
-      if (asset.delisted) return null;
+      if (asset.delisted || asset.name === 'TerraClassicUSD') return null;
       const last = cache.peggedAssetsData?.[asset.id]?.lastBalance
       if (last?.SK < now - 5 * 3600) {
         return {
@@ -76,3 +160,26 @@ async function alertOutdated() {
   }
 
 }
+
+
+function removeEmptyItems(array: any[] = []) {
+  return array.map(removeEmpty).filter((item: any) => item)
+}
+
+function removeEmpty(item: any) {
+  if (!item) return item
+  if (typeof item === 'object') {
+    const { date, ...rest } = item
+    for (const key in rest) {
+      rest[key] = removeEmpty(rest[key])
+      if (!rest[key])
+        delete rest[key]
+    }
+    if (Object.keys(rest).length === 0)
+      return null
+    return { date, ...rest }
+  } else if (typeof item === 'number') {
+    return Math.round(item)
+  }
+  return item
+}

api2/cron-task/storeCharts.ts

@@ -45,7 +45,7 @@ export default async function handler() {
 
   console.time('storeCharts')
 
-/*   const commonOptions = {
+  /*   const commonOptions = {
       lastPrices,
       historicalPrices,
       historicalRates,
@@ -159,14 +159,15 @@ export function craftChartsResponse(
   { chain = 'all', peggedID, startTimestamp }: {
     chain?: string,
     peggedID?: string,
-    startTimestamp?: string,
+    startTimestamp?: string | number,
   }
 ) {
+  if (startTimestamp && typeof startTimestamp === 'string') startTimestamp = parseInt(startTimestamp)
 
   const filterChart = (chart: any) => {
     return chart.map((entry: any) => {
       if (!startTimestamp) return entry;
-      if (entry.date < parseInt(startTimestamp)) {
+      if (entry.date < startTimestamp) {
         return null;
       }
       return entry;

api2/routes/getChainDominance.ts

@@ -19,16 +19,6 @@ export function craftChainDominanceResponse(chain: string | undefined) {
       };
     };
   };
-  // quick fix; need to update later
-  if (chain === "gnosis") {
-    chain = "xdai";
-  }
-  if (chain === "terra%20classic") {
-    chain = "terra";
-  }
-  if (chain === "ethereumpow") {
-    chain = "ethpow";
-  }
 
   if (chain === undefined) 
     throw new Error("Must include chain as path parameter.")

api2/routes/index.ts

@@ -5,6 +5,7 @@ import { readRouteData } from "../file-cache";
 import { craftChartsResponse } from "../cron-task/storeCharts";
 import { getStablecoinData } from "./getStableCoin";
 import { craftChainDominanceResponse } from "./getChainDominance";
+import { normalizeChain } from "../../src/utils/normalizeChain";
 
 export default function setRoutes(router: HyperExpress.Router) {
 
@@ -13,7 +14,18 @@ export default function setRoutes(router: HyperExpress.Router) {
   router.get("/stablecoin", defaultFileHandler);
   router.get("/stablecoinprices", defaultFileHandler);
   router.get("/stablecoinchains", defaultFileHandler);
+  router.get("/stablecoins", defaultFileHandler);
+  router.get("/stablecoin/:stablecoin", defaultFileHandler);
 
+  router.get("/stablecoindominance/:chain", ew(async (req: any, res: any) => {
+    let { chain } = req.path_parameters;
+    chain = normalizeChain(chain)
+    return fileResponse('/stablecoindominance/'+chain, res);
+  }))
+  router.get("/stablecoincharts2/:chain", defaultFileHandler);
+  router.get("/stablecoincharts2/all-llama-app", defaultFileHandler);
+
+  /* Ignore optional query parameters for now
   router.get("/stablecoins", ew(async (req: any, res: any) => {
     const { includePrices, includeChains } = req.query;
     const data = await readRouteData('stablecoins');
@@ -40,12 +52,17 @@ export default function setRoutes(router: HyperExpress.Router) {
     return successResponse(res, craftChainDominanceResponse(chain.toLowerCase()));
   }));
 
+
+  */
+
+
+  // TOO: nuke this route to reduce load on the server
   router.get("/stablecoincharts/:chain", ew(async (req: any, res: any) => {
     const { chain } = req.path_parameters;
-    let { stablecoin, starts } = req.query;
+    let { stablecoin, starts, startts } = req.query;
     const peggedID = stablecoin?.toLowerCase()
 
-    return successResponse(res, await craftChartsResponse({ chain, peggedID, startTimestamp: starts }));
+    return successResponse(res, await craftChartsResponse({ chain, peggedID, startTimestamp: starts ?? startts }));
   }));
 
   function defaultFileHandler(req: HyperExpress.Request, res: HyperExpress.Response) {

src/getChainDominance.ts

@@ -30,16 +30,6 @@ export async function craftChainDominanceResponse(chain: string | undefined) {
       };
     };
   };
-  // quick fix; need to update later
-  if (chain === "gnosis") {
-    chain = "xdai";
-  }
-  if (chain === "terra%20classic") {
-    chain = "terra";
-  }
-  if (chain === "ethereumpow") {
-    chain = "ethpow";
-  }
   
   if (chain === undefined) {
     return errorResponse({