diff --git a/api2/cache.ts b/api2/cache.ts index 8d6a2c8..7cab4e4 100644 --- a/api2/cache.ts +++ b/api2/cache.ts @@ -26,14 +26,16 @@ export const cache: { const MINUTES = 60 * 1000 const HOUR = 60 * MINUTES +const cacheFile = 'stablecoin-cache' + export async function initCache() { console.time('Cache initialized') - const _cache = await readFromPGCache('cron-cache') ?? {} + const _cache = await readFromPGCache(cacheFile) ?? {} Object.keys(_cache).forEach(key => cache[key] = _cache[key]) cache.rates = await getLastRecord(historicalRates); console.timeEnd('Cache initialized') } export async function saveCache() { - await writeToPGCache('cron-cache', cache) + await writeToPGCache(cacheFile, cache) } diff --git a/api2/cron-task/getStableCoins.ts b/api2/cron-task/getStableCoins.ts index 1183092..c983fa0 100644 --- a/api2/cron-task/getStableCoins.ts +++ b/api2/cron-task/getStableCoins.ts @@ -19,7 +19,7 @@ function getTVLOfRecordClosestToTimestamp( for (let i = 0; i < items.length; i++) { if (items[i].SK <= endSK && items[i].SK >= endSK - searchWidth) { if (!record || items[i].SK > record.SK) - record = items[i] + record = items[i] } } @@ -32,89 +32,89 @@ function craftProtocolsResponse( let prices = cache.peggedPrices! const response = ( - peggedAssets.map((pegged) => { - const pegType = pegged.pegType; - const { balances, lastBalance } = cache.peggedAssetsData?.[pegged.id] ?? {} - const lastHourlyRecord = lastBalance - if (lastHourlyRecord === undefined) { - return null; + peggedAssets.map((pegged) => { + const pegType = pegged.pegType; + const { balances, lastBalance } = cache.peggedAssetsData?.[pegged.id] ?? {} + const lastHourlyRecord = lastBalance + if (lastHourlyRecord === undefined) { + return null; + } + const lastSK = lastHourlyRecord.SK; + if (lastSK === undefined) { + return null; + } + const lastDailyPeggedRecord = getTVLOfRecordClosestToTimestamp( + balances, + lastSK - secondsInDay, + secondsInDay // temporary, update later + ); + const lastWeeklyPeggedRecord = getTVLOfRecordClosestToTimestamp( + balances, + lastSK - secondsInWeek, + secondsInDay // temporary, update later + ); + const lastMonthlyPeggedRecord = getTVLOfRecordClosestToTimestamp( + balances, + lastSK - secondsInDay * 30, + secondsInDay // temporary, update later + ); + const chainCirculating = {} as { + [chain: string]: any; + }; + const chains: string[] = []; + Object.entries(lastHourlyRecord).forEach(([chain, issuances]: any) => { + if (nonChains.includes(chain)) { + return; } - const lastSK = lastHourlyRecord.SK; - if (lastSK === undefined) { - return null; - } - const lastDailyPeggedRecord = getTVLOfRecordClosestToTimestamp( - balances, - lastSK - secondsInDay, - secondsInDay // temporary, update later - ); - const lastWeeklyPeggedRecord = getTVLOfRecordClosestToTimestamp( - balances, - lastSK - secondsInWeek, - secondsInDay // temporary, update later - ); - const lastMonthlyPeggedRecord = getTVLOfRecordClosestToTimestamp( - balances, - lastSK - secondsInDay * 30, - secondsInDay // temporary, update later - ); - const chainCirculating = {} as { - [chain: string]: any; - }; - const chains: string[] = []; - Object.entries(lastHourlyRecord).forEach(([chain, issuances]: any) => { - if (nonChains.includes(chain)) { - return; - } - const chainDisplayName = getChainDisplayName(chain, useNewChainNames); - chainCirculating[chainDisplayName] = - chainCirculating[chainDisplayName] || {}; - chainCirculating[chainDisplayName].current = issuances.circulating; - chainCirculating[chainDisplayName].circulatingPrevDay = - lastDailyPeggedRecord && lastDailyPeggedRecord[chain] - ? lastDailyPeggedRecord[chain].circulating ?? 0 - : 0; - chainCirculating[chainDisplayName].circulatingPrevWeek = - lastWeeklyPeggedRecord && lastWeeklyPeggedRecord[chain] - ? lastWeeklyPeggedRecord[chain].circulating ?? 0 - : 0; - chainCirculating[chainDisplayName].circulatingPrevMonth = - lastMonthlyPeggedRecord && lastMonthlyPeggedRecord[chain] - ? lastMonthlyPeggedRecord[chain].circulating ?? 0 - : 0; - addToChains(chains, chainDisplayName); - }); + const chainDisplayName = getChainDisplayName(chain, useNewChainNames); + chainCirculating[chainDisplayName] = + chainCirculating[chainDisplayName] || {}; + chainCirculating[chainDisplayName].current = issuances.circulating; + chainCirculating[chainDisplayName].circulatingPrevDay = + lastDailyPeggedRecord && lastDailyPeggedRecord[chain] + ? lastDailyPeggedRecord[chain].circulating ?? 0 + : 0; + chainCirculating[chainDisplayName].circulatingPrevWeek = + lastWeeklyPeggedRecord && lastWeeklyPeggedRecord[chain] + ? lastWeeklyPeggedRecord[chain].circulating ?? 0 + : 0; + chainCirculating[chainDisplayName].circulatingPrevMonth = + lastMonthlyPeggedRecord && lastMonthlyPeggedRecord[chain] + ? lastMonthlyPeggedRecord[chain].circulating ?? 0 + : 0; + addToChains(chains, chainDisplayName); + }); - const dataToReturn = { - id: pegged.id, - name: pegged.name, - symbol: pegged.symbol, - gecko_id: pegged.gecko_id, - pegType: pegged.pegType, - priceSource: pegged.priceSource, - pegMechanism: pegged.pegMechanism, - circulating: lastHourlyRecord.totalCirculating.circulating, - circulatingPrevDay: lastDailyPeggedRecord - ? lastDailyPeggedRecord.totalCirculating.circulating - : 0, - circulatingPrevWeek: lastWeeklyPeggedRecord - ? lastWeeklyPeggedRecord.totalCirculating.circulating - : 0, - circulatingPrevMonth: lastMonthlyPeggedRecord - ? lastMonthlyPeggedRecord.totalCirculating.circulating - : 0, - chainCirculating, - chains: chains.sort( - (a, b) => - chainCirculating[b].current[pegType] - - chainCirculating[a].current[pegType] - ), - } as any; - dataToReturn.price = prices[pegged.gecko_id] ?? null; - if (pegged.delisted) dataToReturn.delisted = true; - return dataToReturn; - }) - ) + const dataToReturn = { + id: pegged.id, + name: pegged.name, + symbol: pegged.symbol, + gecko_id: pegged.gecko_id, + pegType: pegged.pegType, + priceSource: pegged.priceSource, + pegMechanism: pegged.pegMechanism, + circulating: lastHourlyRecord.totalCirculating.circulating, + circulatingPrevDay: lastDailyPeggedRecord + ? lastDailyPeggedRecord.totalCirculating.circulating + : 0, + circulatingPrevWeek: lastWeeklyPeggedRecord + ? lastWeeklyPeggedRecord.totalCirculating.circulating + : 0, + circulatingPrevMonth: lastMonthlyPeggedRecord + ? lastMonthlyPeggedRecord.totalCirculating.circulating + : 0, + chainCirculating, + chains: chains.sort( + (a, b) => + chainCirculating[b].current[pegType] - + chainCirculating[a].current[pegType] + ), + } as any; + dataToReturn.price = prices[pegged.gecko_id] ?? null; + if (pegged.delisted) dataToReturn.delisted = true; + return dataToReturn; + }) + ) .filter((pegged) => pegged !== null) .sort((a, b) => b.circulating - a.circulating); return response; @@ -125,8 +125,9 @@ export default async function handler() { let response: any = { peggedAssets: pegged, }; - const chainData = await craftStablecoinChainsResponse(); - response.chains = chainData; - await storeRouteData('stablecoins', response) + const chainData = await craftStablecoinChainsResponse(); + response.chains = chainData; + await storeRouteData('stablecoins', response) + return response; }; diff --git a/api2/cron-task/index.ts b/api2/cron-task/index.ts index 5f55809..a047a5d 100644 --- a/api2/cron-task/index.ts +++ b/api2/cron-task/index.ts @@ -3,13 +3,16 @@ import * as rates from "../../src/getRates"; import sluggifyPegged from "../../src/peggedAssets/utils/sluggifyPegged"; import { storeRouteData } from "../file-cache"; import storePeggedPrices from "./storePeggedPrices"; -import storeCharts from "./storeCharts"; +import storeCharts, { craftChartsResponse } from "./storeCharts"; import storeStablecoins from "./getStableCoins"; import { craftStablecoinPricesResponse } from "./getStablecoinPrices"; import { craftStablecoinChainsResponse } from "./getStablecoinChains"; import { cache, initCache, saveCache } from "../cache"; import { getCurrentUnixTimestamp } from "../../src/utils/date"; import { sendMessage } from "../../src/utils/discord"; +import { getStablecoinData } from "../routes/getStableCoin"; +import { craftChainDominanceResponse } from "../routes/getChainDominance"; +import { normalizeChain } from "../../src/utils/normalizeChain"; run().catch(console.error).then(() => process.exit(0)) @@ -24,39 +27,120 @@ async function run() { // this also pulls data from ddb and sets to cache await storeCharts() - await storeStablecoins() + const allStablecoinsData = await storeStablecoins() await storePrices() await storeStablecoinChains() + const allChainsSet: Set = new Set() + const assetChainMap: { + [asset: string]: Set + } = {} + allStablecoinsData.peggedAssets.forEach((asset: any) => { + const _chains = asset.chains.map(normalizeChain) + assetChainMap[asset.id] = new Set(_chains) + _chains.forEach((chain) => allChainsSet.add(chain)) + }) + + await storePeggedAssets() + await storeStablecoinDominance() + await storeChainChartData() await saveCache() await alertOutdated() - async function storeConfig() { - let configJSON: any = Object.fromEntries( - peggedAssets.map((pegged) => [sluggifyPegged(pegged), pegged.id]) - ) - await storeRouteData('config', configJSON) + + async function storePeggedAssets() { + for (const { id } of allStablecoinsData.peggedAssets) { + try { + const data = await getStablecoinData(id) + data.chainBalances = data.chainBalances ?? {} + for (const [chain, chainData] of Object.entries(data.chainBalances)) { + const nChain = normalizeChain(chain) + allChainsSet.add(nChain) + assetChainMap[id].add(nChain); + (chainData as any).tokens = removeEmptyItems((chainData as any).tokens) + } + data.tokens = removeEmptyItems(data.tokens) + await storeRouteData('stablecoin/' + id, data) + } catch (e) { + console.error('Error fetching asset data', e) + } + } } - async function storeRates() { - await storeRouteData('rates', await rates.craftRatesResponse()) + async function storeStablecoinDominance() { + for (const chain of [...allChainsSet]) { + try { + const data = await craftChainDominanceResponse(chain) + await storeRouteData('stablecoindominance/' + chain, data) + } catch (e) { + console.error('Error fetching chain data', e) + } + } } - async function storePrices() { - await storeRouteData('stablecoinprices', craftStablecoinPricesResponse()) - } - async function storeStablecoinChains() { - await storeRouteData('stablecoinchains', craftStablecoinChainsResponse()) + + async function storeChainChartData() { + const frontendKey = 'all-llama-app' + const allChartsStartTimestamp = 1617148800 // for /stablecoins page, charts begin on April 1, 2021, to reduce size of page + const allData = await getChainData('all') + await storeRouteData('stablecoincharts2/all', allData) + const allDataShortened = await getChainData(frontendKey) + await storeRouteData('stablecoincharts2/' + frontendKey, allDataShortened) + + for (const chain of [...allChainsSet]) { + try { + const data = await getChainData(chain) + await storeRouteData('stablecoincharts2/' + chain, data) + } catch (e) { + console.error('Error fetching chain data', e) + } + } + + async function getChainData(chain: string) { + let startTimestamp = chain === frontendKey ? allChartsStartTimestamp : undefined + chain = chain === frontendKey ? 'all' : chain + const aggregated = removeEmptyItems(await craftChartsResponse({ chain, startTimestamp, })) + const breakdown: any = {} + + for (const [peggedAsset, chainMap] of Object.entries(assetChainMap)) { + if (chain !== 'all' && !(chainMap as any).has(chain)) + continue + breakdown[peggedAsset] = removeEmptyItems(await craftChartsResponse({ chain, peggedID: peggedAsset, startTimestamp })) + } + + + return { aggregated, breakdown } + } } } +async function storeConfig() { + let configJSON: any = Object.fromEntries( + peggedAssets.map((pegged) => [sluggifyPegged(pegged), pegged.id]) + ) + await storeRouteData('config', configJSON) +} + +async function storeRates() { + await storeRouteData('rates', await rates.craftRatesResponse()) +} + +async function storePrices() { + await storeRouteData('stablecoinprices', craftStablecoinPricesResponse()) +} + +async function storeStablecoinChains() { + await storeRouteData('stablecoinchains', craftStablecoinChainsResponse()) +} + + async function alertOutdated() { const now = getCurrentUnixTimestamp(); const outdated = ( peggedAssets.map((asset) => { - if (asset.delisted) return null; + if (asset.delisted || asset.name === 'TerraClassicUSD') return null; const last = cache.peggedAssetsData?.[asset.id]?.lastBalance if (last?.SK < now - 5 * 3600) { return { @@ -75,4 +159,27 @@ async function alertOutdated() { await sendMessage(message, process.env.OUTDATED_WEBHOOK!); } +} + + +function removeEmptyItems(array: any[] = []) { + return array.map(removeEmpty).filter((item: any) => item) +} + +function removeEmpty(item: any) { + if (!item) return item + if (typeof item === 'object') { + const { date, ...rest } = item + for (const key in rest) { + rest[key] = removeEmpty(rest[key]) + if (!rest[key]) + delete rest[key] + } + if (Object.keys(rest).length === 0) + return null + return { date, ...rest } + } else if (typeof item === 'number') { + return Math.round(item) + } + return item } \ No newline at end of file diff --git a/api2/cron-task/storeCharts.ts b/api2/cron-task/storeCharts.ts index 349716f..710603f 100644 --- a/api2/cron-task/storeCharts.ts +++ b/api2/cron-task/storeCharts.ts @@ -45,45 +45,45 @@ export default async function handler() { console.time('storeCharts') -/* const commonOptions = { - lastPrices, - historicalPrices, - historicalRates, - priceTimestamps, - rateTimestamps, - peggedAssetsData: cache.peggedAssetsData, - } - // store overall chart - const allData = await craftChartsResponse({ ...commonOptions, chain: "all" }); - await storeRouteData('charts/all/all', allData) - - // store chain charts - const chains = [Object.keys(chainCoingeckoIds), Object.values(normalizedChainReplacements)].flat() - for (let chain of chains) { - const normalizedChain = normalizeChain(chain); - const chainData = await craftChartsResponse({ ...commonOptions, chain, }); - if (chainData.length) { - await storeRouteData(`charts/${normalizedChain}`, chainData) - } else { - console.log(`No data for ${chain}`) + /* const commonOptions = { + lastPrices, + historicalPrices, + historicalRates, + priceTimestamps, + rateTimestamps, + peggedAssetsData: cache.peggedAssetsData, } - } - - // store pegged asset charts - for (const pegged of peggedAssets) { - const id = pegged.id; - const chart = await craftChartsResponse({ ...commonOptions, peggedID: id }); - await storeRouteData(`charts/all/${id}`, chart) - } - console.timeEnd('storeCharts') */ + // store overall chart + const allData = await craftChartsResponse({ ...commonOptions, chain: "all" }); + await storeRouteData('charts/all/all', allData) + + // store chain charts + const chains = [Object.keys(chainCoingeckoIds), Object.values(normalizedChainReplacements)].flat() + for (let chain of chains) { + const normalizedChain = normalizeChain(chain); + const chainData = await craftChartsResponse({ ...commonOptions, chain, }); + if (chainData.length) { + await storeRouteData(`charts/${normalizedChain}`, chainData) + } else { + console.log(`No data for ${chain}`) + } + } + + // store pegged asset charts + for (const pegged of peggedAssets) { + const id = pegged.id; + const chart = await craftChartsResponse({ ...commonOptions, peggedID: id }); + await storeRouteData(`charts/all/${id}`, chart) + } + console.timeEnd('storeCharts') */ } async function getPeggedAssetsData() { if (!cache.peggedAssetsData) - cache.peggedAssetsData = {} - + cache.peggedAssetsData = {} + await Promise.all(peggedAssets.map(async (pegged) => { const lastBalance = await getLastRecord(hourlyPeggedBalances(pegged.id)); @@ -159,20 +159,21 @@ export function craftChartsResponse( { chain = 'all', peggedID, startTimestamp }: { chain?: string, peggedID?: string, - startTimestamp?: string, + startTimestamp?: string | number, } ) { + if (startTimestamp && typeof startTimestamp === 'string') startTimestamp = parseInt(startTimestamp) const filterChart = (chart: any) => { return chart.map((entry: any) => { if (!startTimestamp) return entry; - if (entry.date < parseInt(startTimestamp)) { + if (entry.date < startTimestamp) { return null; } return entry; }).filter((entry: any) => entry); } - + const { historicalPrices, historicalRates, lastPrices, priceTimestamps, rateTimestamps, peggedAssetsData, } = cache as any const sumDailyBalances = {} as { [timestamp: number]: { diff --git a/api2/routes/getChainDominance.ts b/api2/routes/getChainDominance.ts index 09ea46f..6175812 100644 --- a/api2/routes/getChainDominance.ts +++ b/api2/routes/getChainDominance.ts @@ -19,16 +19,6 @@ export function craftChainDominanceResponse(chain: string | undefined) { }; }; }; - // quick fix; need to update later - if (chain === "gnosis") { - chain = "xdai"; - } - if (chain === "terra%20classic") { - chain = "terra"; - } - if (chain === "ethereumpow") { - chain = "ethpow"; - } if (chain === undefined) throw new Error("Must include chain as path parameter.") diff --git a/api2/routes/index.ts b/api2/routes/index.ts index 864e2d5..771a225 100644 --- a/api2/routes/index.ts +++ b/api2/routes/index.ts @@ -5,6 +5,7 @@ import { readRouteData } from "../file-cache"; import { craftChartsResponse } from "../cron-task/storeCharts"; import { getStablecoinData } from "./getStableCoin"; import { craftChainDominanceResponse } from "./getChainDominance"; +import { normalizeChain } from "../../src/utils/normalizeChain"; export default function setRoutes(router: HyperExpress.Router) { @@ -13,7 +14,18 @@ export default function setRoutes(router: HyperExpress.Router) { router.get("/stablecoin", defaultFileHandler); router.get("/stablecoinprices", defaultFileHandler); router.get("/stablecoinchains", defaultFileHandler); + router.get("/stablecoins", defaultFileHandler); + router.get("/stablecoin/:stablecoin", defaultFileHandler); + router.get("/stablecoindominance/:chain", ew(async (req: any, res: any) => { + let { chain } = req.path_parameters; + chain = normalizeChain(chain) + return fileResponse('/stablecoindominance/'+chain, res); + })) + router.get("/stablecoincharts2/:chain", defaultFileHandler); + router.get("/stablecoincharts2/all-llama-app", defaultFileHandler); + + /* Ignore optional query parameters for now router.get("/stablecoins", ew(async (req: any, res: any) => { const { includePrices, includeChains } = req.query; const data = await readRouteData('stablecoins'); @@ -26,7 +38,7 @@ export default function setRoutes(router: HyperExpress.Router) { } return successResponse(res, data); - })); + })); router.get("/stablecoin/:stablecoin", ew(async (req: any, res: any) => { const { stablecoin } = req.path_parameters; @@ -40,12 +52,17 @@ export default function setRoutes(router: HyperExpress.Router) { return successResponse(res, craftChainDominanceResponse(chain.toLowerCase())); })); + + */ + + + // TOO: nuke this route to reduce load on the server router.get("/stablecoincharts/:chain", ew(async (req: any, res: any) => { const { chain } = req.path_parameters; - let { stablecoin, starts } = req.query; + let { stablecoin, starts, startts } = req.query; const peggedID = stablecoin?.toLowerCase() - return successResponse(res, await craftChartsResponse({ chain, peggedID, startTimestamp: starts })); + return successResponse(res, await craftChartsResponse({ chain, peggedID, startTimestamp: starts ?? startts })); })); function defaultFileHandler(req: HyperExpress.Request, res: HyperExpress.Response) { diff --git a/src/getChainDominance.ts b/src/getChainDominance.ts index b972104..5653161 100644 --- a/src/getChainDominance.ts +++ b/src/getChainDominance.ts @@ -30,17 +30,7 @@ export async function craftChainDominanceResponse(chain: string | undefined) { }; }; }; - // quick fix; need to update later - if (chain === "gnosis") { - chain = "xdai"; - } - if (chain === "terra%20classic") { - chain = "terra"; - } - if (chain === "ethereumpow") { - chain = "ethpow"; - } - + if (chain === undefined) { return errorResponse({ message: "Must include chain as path parameter.",