diff --git a/.github/workflows/build-modules.yml b/.github/workflows/build-modules.yml new file mode 100644 index 000000000..40ae80d3b --- /dev/null +++ b/.github/workflows/build-modules.yml @@ -0,0 +1,33 @@ +name: Build tvlModules and upload artifact + +on: + push: + branches: [ "main" ] + +jobs: + build-and-upload: + runs-on: ubuntu-latest + steps: + - name: Checkout repository + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Setup Node.js + uses: actions/setup-node@v4 + with: + node-version: "20" + + - name: Install dependencies + run: npm ci + + - name: Run buildImports script + run: node scripts/buildImports.js + + - name: Publish tvlModules.json as "latest" release + uses: ncipollo/release-action@v1 + with: + tag: latest + name: Latest tvlModules + artifacts: scripts/tvlModules.json + allowUpdates: true \ No newline at end of file diff --git a/.gitignore b/.gitignore index ce7c779ab..a1fbddb5f 100644 --- a/.gitignore +++ b/.gitignore @@ -10,4 +10,6 @@ yarn.lock .vscode projects/binance/data.csv -*.log \ No newline at end of file +*.log + +scripts/tvlModules.json \ No newline at end of file diff --git a/package.json b/package.json index 596d7475b..8190b33ae 100644 --- a/package.json +++ b/package.json @@ -7,6 +7,7 @@ "test": "echo \"Error: no test specified\" && exit 1", "weeklyChanges": "git pull && git diff $(git log -1 --before=@{7.days.ago} --format=%H) --stat | grep -E \"projects/\" | cut -d / -f 2 | cut -d \" \" -f 1 | uniq | wc -l", "dev": "babel-watch curve.js", + "build": "node scripts/buildImports.js", "lint": "eslint -c .eslintrc.js .", "eslint:github-action": "eslint -c .eslintrc.js .", "test-interactive": "node utils/testInteractive", diff --git a/projects/helper/cache.js b/projects/helper/cache.js index 8966c6906..5462c0177 100644 --- a/projects/helper/cache.js +++ b/projects/helper/cache.js @@ -15,12 +15,12 @@ function getLink(project, chain) { return `https://${Bucket}.s3.eu-central-1.amazonaws.com/${getKey(project, chain)}` } -async function getCache(project, chain, { _ } = {}) { +async function getCache(project, chain, { skipCompression } = {}) { const Key = getKey(project, chain) const fileKey = getFileKey(project, chain) try { - const json = await sdk.cache.readCache(fileKey) + const json = await sdk.cache.readCache(fileKey, { skipCompression}) if (!json || Object.keys(json).length === 0) throw new Error('Invalid data') return json } catch (e) { @@ -36,11 +36,13 @@ async function getCache(project, chain, { _ } = {}) { } } -async function setCache(project, chain, cache, { _ } = {}) { +async function setCache(project, chain, cache, { skipCompression } = {}) { const Key = getKey(project, chain) try { - await sdk.cache.writeCache(getFileKey(project, chain), cache) + await sdk.cache.writeCache(getFileKey(project, chain), cache, { + skipCompression, + }) } catch (e) { sdk.log('failed to write data to s3 bucket: ', Key) sdk.log(e) diff --git a/scripts/buildImports.js b/scripts/buildImports.js new file mode 100644 index 000000000..fb91ca3cc --- /dev/null +++ b/scripts/buildImports.js @@ -0,0 +1,85 @@ +const fs = require("fs") +const { get } = require("../projects/helper/http") +// const { setCache, getCache } = require("../projects/helper/cache") + +async function run() { + // await getCache('defi-configs', 'tvlModules') + const configs = await get('https://api.llama.fi/_fe/static/configs') + + const moduleMap = {} + const protocols = configs.protocols.concat(configs.treasuries).concat(configs.entities) + + console.log('# of protocols/treasuries/entities:', protocols.length) + + for (const protocol of protocols) { + try { + + if (moduleMap[protocol.module]) continue; // already imported + + const modulePath = `../projects/${protocol.module}` + const importedModule = mockFunctions(require(modulePath)) + + if (importedModule.hallmarks) + importedModule.hallmarks = convertHallmarkStrings(importedModule.hallmarks) + + moduleMap[protocol.module] = importedModule + } catch (e) { + console.error(`Error importing ${protocol.module}:`, e) + } + } + + fs.writeFileSync('scripts/tvlModules.json', JSON.stringify(moduleMap)) + // await setCache('defi-configs', 'tvlModules', moduleMap, { + // skipCompression: true, + // }) + + process.exit(0) +} + +run().catch((e) => { + console.error(e) + process.exit(1) +}) + + +function convertHallmarkStrings(hallmarks) { + if (!Array.isArray(hallmarks)) return hallmarks + return hallmarks.map((item) => { + if (typeof item?.[0] === 'string') { + item[0] = dateStringToTimestamp(item[0]) + } + if (Array.isArray(item?.[0])) { + item[0].forEach((subItem, index) => { + if (typeof subItem === 'string') { + item[0][index] = dateStringToTimestamp(subItem) + } + }) + } + return item + }).filter((item) => { + if (typeof item?.[0] === 'number') return true + // if it is a range hallmark + if (Array.isArray(item?.[0] && typeof item[0][0] === 'number' && typeof item[0][1] === 'number')) { + return true + } + return false + }) +} + +//Replace all fuctions with mock functions in an object all the way down +function mockFunctions(obj) { + if (typeof obj === "function") { + return '_f' // llamaMockedTVLFunction + } else if (typeof obj === "object") { + Object.keys(obj).forEach((key) => obj[key] = mockFunctions(obj[key])) + } + return obj +} + +function dateStringToTimestamp(dateString) { + + let timestamp = Math.floor(+new Date(dateString) / 1e3) + if (!isNaN(timestamp)) + return timestamp + return dateString +} \ No newline at end of file