kylefang/redstone-cache-layer
1
0
Fork 0
mirror of https://github.com/alexgo-io/redstone-cache-layer.git synced 2026-01-12 08:34:36 +08:00
Code Issues Packages Projects Releases Wiki Activity

feat: migrate to Typescript (#5)

Browse Source
This commit is contained in:
Cezary Haliniarz
2022-06-06 09:13:33 +02:00
committed by GitHub
parent 0e584f2126
commit 857f3c84f4
46 changed files with 1612 additions and 1501 deletions
Download Patch File Download Diff File Expand all files Collapse all files

23
app.js
View File

@@ -1,23 +0,0 @@
const express = require("express");
const cors = require("cors");
const errorhandler = require("errorhandler");
const { getRouter } = require("./routes");
const logger = require("./helpers/logger");
const app = express();
// This allows to get request ip address from req.ip
app.set("trust proxy", true);
app.use(cors());
app.use(express.json({ limit: "20mb", extended: true }));
app.use(express.urlencoded({ extended: true }));
app.use("/", getRouter(express));
app.use(errorhandler({ log: errorNotification }));
function errorNotification(err, str, req) {
const title = `Error in ${req.method} ${req.url}`;
logger.error(title, str, err.stack);
}
module.exports = app;

22
app.ts Normal file
View File

@@ -0,0 +1,22 @@
import express, { Request } from "express";
import cors from "cors";
import errorhandler from "errorhandler";
import { getRouter } from "./routes";
import { logger } from "./helpers/logger";
export const app = express();
// This allows to get request ip address from req.ip
app.set("trust proxy", true);
app.use(cors());
app.use(express.json({ limit: "20mb" }));
app.use(express.urlencoded({ extended: true }));
app.use("/", getRouter());
const errorNotification = (error: Error, string: string, request: Request) => {
const title = `Error in ${request.method} ${request.url}`;
logger.error(title, string, error.stack);
}
app.use(errorhandler({ log: errorNotification }));

44
config.js
View File

@@ -1,44 +0,0 @@
const cacheTTLMilliseconds = 3 * 60 * 1000; // 3 minutes
const enableLiteMode = !!getEnv("LIGHT_MODE", false);
const dbUrls = {
local: "mongodb://localhost:27017/redstone",
};
if (!enableLiteMode) {
const secrets = require("./.secrets.json");
dbUrls["prod"] = secrets.dbUrl;
}
function getDbUrl() {
if (getMode() === "LOCAL") {
return dbUrls.local;
} else {
return dbUrls.prod;
}
}
function getEnv(name, defaultValue) {
return process.env[name] || defaultValue;
}
function getMode() {
return getEnv("MODE", "LOCAL");
}
function isProd() {
return getMode() === "PROD";
}
module.exports = {
enableLiteMode,
dbUrl: getDbUrl(),
bigLimitWithMargin: 1200,
defaultLimit: 1,
defaultLocalPort: 9000,
awsSesRegion: "eu-north-1",
enableJsonLogs: isProd(),
maxLimitForPrices: 3000,
enableAmplitudeLogging: getEnv("ENABLE_AMPLITUDE_LOGGING", false),
isProd,
cacheTTLMilliseconds,
};

55
config.ts Normal file
View File

@@ -0,0 +1,55 @@
const getEnv = (name: string, defaultValue: any) => {
return process.env[name] || defaultValue;
}
const getMode = () => {
return getEnv("MODE", "LOCAL");
}
const isProd = () => {
return getMode() === "PROD";
}
const isTest = process.env.NODE_ENV === "test";
const enableLiteMode = getEnv("LIGHT_MODE", false) === "true";
const dbUrls = {
local: "mongodb://localhost:27017/redstone",
prod: ""
};
if (!enableLiteMode && !isTest) {
const secrets = require("./.secrets.json");
dbUrls["prod"] = secrets.dbUrl;
}
function getDbUrl() {
if (getMode() === "LOCAL") {
return dbUrls.local;
} else {
return dbUrls.prod;
}
}
const cacheTTLMilliseconds = !isTest ? 3 * 60 * 1000 : 0; // 3 minutes
const dbUrl = getDbUrl();
const bigLimitWithMargin = 1200;
const defaultLimit = 1;
const defaultLocalPort = 9000;
const awsSesRegion = "eu-north-1";
const isProduction = isProd();
const maxLimitForPrices = 3000;
const enableAmplitudeLogging = getEnv("ENABLE_AMPLITUDE_LOGGING", false);
export {
enableLiteMode,
dbUrl,
bigLimitWithMargin,
defaultLimit,
defaultLocalPort,
awsSesRegion,
isProduction,
maxLimitForPrices,
enableAmplitudeLogging,
cacheTTLMilliseconds
};

19
helpers/amplitude-event-logger.js → helpers/amplitude-event-logger.ts
View File

@@ -1,6 +1,6 @@
const Amplitude = require("@amplitude/node");
const logger = require("./logger");
const config = require("../config");
import Amplitude from "@amplitude/node";
import { logger } from "./logger";
import { enableAmplitudeLogging } from "../config";
// Amplitude is a web analytics system. Learn more at https://amplitude.com
// We use it to simply measure get and post requests to the RedStone Http Api
@@ -9,14 +9,15 @@ const config = require("../config");
// Check the analytics dashboard using the link below
// https://analytics.amplitude.com/limestone/dashboard/ttoropr
const client = Amplitude.init("4990f7285c58e8a009f7818b54fc01eb");
function logEvent({
export const logEvent = ({
eventName,
eventProps,
ip,
}) {
if (config.enableAmplitudeLogging) {
}) => {
if (enableAmplitudeLogging) {
const client = Amplitude.init("4990f7285c58e8a009f7818b54fc01eb");
logger.info(
`Logging event "${eventName}" in amplitude for ip: "${ip}". With props: `
+ JSON.stringify(eventProps));
@@ -30,8 +31,4 @@ function logEvent({
event_properties: eventProps,
});
}
}
module.exports = {
logEvent,
};

10
helpers/cloudwatch.js → helpers/cloudwatch.ts
View File

@@ -1,9 +1,9 @@
const AWS = require("aws-sdk");
const logger = require("../helpers/logger");
const { isProd } = require("../config");
import AWS from "aws-sdk";
import { logger } from "../helpers/logger";
import { isProduction } from "../config";
module.exports.saveMetric = async ({ label, value }) => {
if (isProd()) {
export const saveMetric = async ({ label, value }) => {
if (isProduction) {
const cloudwatch = new AWS.CloudWatch({apiVersion: "2010-08-01"});
const metricParams = {

8
helpers/logger.js
View File

@@ -1,8 +0,0 @@
const consola = require("consola");
const config = require("../config");
if (config.enableJsonLogs) {
consola.setReporters([new consola.JSONReporter()]);
}
module.exports = consola;

8
helpers/logger.ts Normal file
View File

@@ -0,0 +1,8 @@
import consola, { JSONReporter } from "consola";
import { isProduction } from "../config";
if (isProduction) {
consola.setReporters([new JSONReporter()]);
}
export const logger = consola;

6
helpers/mail-sender.js → helpers/mail-sender.ts
View File

@@ -1,7 +1,7 @@
const aws = require("aws-sdk");
const ses = new aws.SES({ region: config.awsSesRegion });
import aws from "aws-sdk";
const ses = new aws.SES();
module.exports.sendEmail = async ({ to, subject, text }) => {
export const sendEmail = async ({ to, subject, text }) => {
var params = {
Destination: {
ToAddresses: [to],

43
helpers/mongo.js
View File

@@ -1,43 +0,0 @@
const mongoose = require("mongoose");
const { MongoMemoryServer } = require('mongodb-memory-server');
const logger = require("./logger");
const MAX_COLLECTION_SIZE_TO_CLEAN = 5000;
async function tryCleanCollection(model, query) {
const collectionSize = await model.countDocuments(query).exec();
if (collectionSize > MAX_COLLECTION_SIZE_TO_CLEAN) {
logger.warn('Unsafe collection cleaning skipped: '
+ JSON.stringify({collectionSize, MAX_COLLECTION_SIZE_TO_CLEAN}));
} else {
logger.info(
`Cleaning collection: ${model.collection.collectionName}. `
+ `Query: ${JSON.stringify(query)}. Items to be removed: ${collectionSize}`);
await model.deleteMany(query);
}
}
async function connectToRemoteMongo(url) {
await mongoose.connect(url, {
useNewUrlParser: true,
useUnifiedTopology: true,
useFindAndModify: false,
useCreateIndex: true,
}).then("Connected to mongoDB");
}
async function connectToMongoMemoryServer() {
mongo = await MongoMemoryServer.create();
const uri = mongo.getUri();
await mongoose.connect(uri, {
useNewUrlParser: true,
useUnifiedTopology: true,
useFindAndModify: false,
});
}
module.exports = {
tryCleanCollection,
connectToRemoteMongo,
connectToMongoMemoryServer,
};

29
helpers/mongo.ts Normal file
View File

@@ -0,0 +1,29 @@
import mongoose, { FilterQuery, Model } from "mongoose";
import { MongoMemoryServer } from 'mongodb-memory-server';
import { logger } from "./logger";
const MAX_COLLECTION_SIZE_TO_CLEAN = 5000;
export const tryCleanCollection = async (model: Model<any>, query: FilterQuery<any>) => {
const collectionSize = await model.countDocuments(query).exec();
if (collectionSize > MAX_COLLECTION_SIZE_TO_CLEAN) {
logger.warn('Unsafe collection cleaning skipped: '
+ JSON.stringify({collectionSize, MAX_COLLECTION_SIZE_TO_CLEAN}));
} else {
logger.info(
`Cleaning collection: ${model.collection.collectionName}. `
+ `Query: ${JSON.stringify(query)}. Items to be removed: ${collectionSize}`);
await model.deleteMany(query);
}
}
export const connectToRemoteMongo = async (url: string) => {
await mongoose.connect(url)
logger.info("Connected to mongoDB");
}
export const connectToMongoMemoryServer = async () => {
const mongo = await MongoMemoryServer.create();
const uri = mongo.getUri();
await mongoose.connect(uri);
}

33
helpers/signature-verifier.js → helpers/signature-verifier.ts
View File

@@ -1,11 +1,12 @@
const _ = require("lodash");
const Arweave = require("arweave/node");
const deepSortObject = require("deep-sort-object");
const Price = require("../models/price");
const { getPublicKeyForProviderAddress } = require("../providers");
const logger = require("./logger");
import _ from "lodash";
import Arweave from "arweave/node";
import deepSortObject from "deep-sort-object";
import { Price } from "../models/price";
import { getPublicKeyForProviderAddress } from "../providers";
import { logger } from "./logger";
import { PriceWithParams } from "../routes/prices";
async function assertValidSignature(price, skipNewerPricesCheck = true) {
export const assertValidSignature = async (price: PriceWithParams, skipNewerPricesCheck = true) => {
if (!skipNewerPricesCheck) {
// Checking if price with a greater timestamp is already in DB
// If so, then we raise an Error, because we allow to publish
@@ -33,7 +34,7 @@ async function assertValidSignature(price, skipNewerPricesCheck = true) {
}
}
async function verifySignature(price) {
export const verifySignature = async (price: PriceWithParams) => {
// Time measurement: start
const startTime = Date.now();
@@ -66,9 +67,9 @@ async function verifySignature(price) {
`Signature verification time elapsed: ${signVerificationTime} ms`);
return validSignature;
}
};
function getPriceSignedData(price) {
const getPriceSignedData = (price: PriceWithParams) => {
const priceWithPickedProps = _.pick(price, [
"id",
"source",
@@ -86,12 +87,8 @@ function getPriceSignedData(price) {
} else {
return JSON.stringify(priceWithPickedProps);
}
}
function shouldApplyDeepSort(price) {
return price.version && (price.version === "3" || price.version.includes("."));
}
module.exports = {
assertValidSignature,
};
const shouldApplyDeepSort = (price: PriceWithParams) => {
return price.version && (price.version === "3" || price.version.includes("."));
};

32
index.js
View File

@@ -1,32 +0,0 @@
const awsServerlessExpress = require("aws-serverless-express");
const yargs = require("yargs/yargs");
const { hideBin } = require("yargs/helpers");
const config = require("./config");
const app = require("./app");
const logger = require("./helpers/logger");
const {
connectToMongoMemoryServer,
connectToRemoteMongo,
} = require("./helpers/mongo");
const argv = yargs(hideBin(process.argv)).argv;
// Connecting to mongoDB
if (config.enableLiteMode) {
connectToMongoMemoryServer();
} else {
connectToRemoteMongo(argv.db || config.dbUrl);
}
// Exporting method for docker container for AWS lambda
const server = awsServerlessExpress.createServer(app);
exports.handler = (event, context) =>
awsServerlessExpress.proxy(server, event, context);
// Method for locals server execution
exports.runLocalServer = () => {
const port = argv.port || config.defaultLocalPort;
app.listen(port, () => {
logger.info(`Express api listening at http://localhost:${port}`);
});
};

30
index.ts Normal file
View File

@@ -0,0 +1,30 @@
import awsServerlessExpress from "aws-serverless-express";
import { APIGatewayProxyEvent, Context } from 'aws-lambda';
import yargs from "yargs/yargs";
import { hideBin } from "yargs/helpers";
import { enableLiteMode, dbUrl, defaultLocalPort } from "./config";
import { app } from "./app";
import { logger } from "./helpers/logger";
import { connectToMongoMemoryServer, connectToRemoteMongo } from "./helpers/mongo";
const argv = yargs(hideBin(process.argv)).argv;
// Connecting to mongoDB
if (enableLiteMode) {
connectToMongoMemoryServer();
} else {
connectToRemoteMongo(argv["db"] || dbUrl);
}
// Exporting method for docker container for AWS lambda
const server = awsServerlessExpress.createServer(app);
exports.handler = (event: APIGatewayProxyEvent, context: Context) =>
awsServerlessExpress.proxy(server, event, context);
// Method for locals server execution
exports.runLocalServer = () => {
const port = argv["port"] || defaultLocalPort;
app.listen(port, () => {
logger.info(`Express api listening at http://localhost:${port}`);
});
};

4
jest.config.js
View File

@@ -1,5 +1,9 @@
module.exports = {
preset: 'ts-jest',
testEnvironment: 'node',
verbose: true,
testTimeout: 20000,
testEnvironment: "node",
modulePathIgnorePatterns: ['./dist'],
setupFiles: ['./jest.setup.ts']
};

1
jest.setup.ts Normal file
View File

@@ -0,0 +1 @@
process.env.LIGHT_MODE="true"

18
models/package.js → models/package.ts
View File

@@ -1,7 +1,17 @@
const mongoose = require("mongoose");
import mongoose from "mongoose";
import { Price, PriceSchema } from "./price";
const Schema = mongoose.Schema;
const PackageSchema = new Schema({
export interface Package {
timestamp: number;
signature?: string;
liteSignature?: string;
provider: string;
signer: string;
prices: Price[];
}
const PackageSchema = new Schema<Package>({
timestamp: {
type: Number,
required: true,
@@ -23,9 +33,9 @@ const PackageSchema = new Schema({
required: false,
},
prices: {
type: Array,
type: [PriceSchema],
required: true,
},
});
module.exports = mongoose.model("Package", PackageSchema);
export const Package = mongoose.model("Package", PackageSchema);

36
models/price.js → models/price.ts
View File

@@ -1,9 +1,23 @@
const _ = require("lodash");
const mongoose = require("mongoose");
const Schema = mongoose.Schema;
const { getPublicKeyForProviderAddress } = require("../providers");
import _ from "lodash";
import mongoose, { Document, Schema } from "mongoose";
import { getPublicKeyForProviderAddress } from "../providers";
const PriceSchema = new Schema({
export interface Price {
id: string;
symbol: string;
provider: string;
value: number;
signature?: Buffer;
evmSignature?: Buffer;
liteEvmSignature?: Buffer;
permawebTx: string;
version: string;
source: object;
timestamp: number;
minutes?: number;
}
export const PriceSchema = new Schema<Price>({
id: {
type: String,
required: true,
@@ -54,16 +68,16 @@ const PriceSchema = new Schema({
},
});
PriceSchema.statics.toObj = function(price) {
export const priceToObject = (price: Document<unknown, any, Price> & Price) => {
let result = price;
if (result.toObject !== undefined) {
result = result.toObject();
}
result.providerPublicKey = getPublicKeyForProviderAddress(result.provider);
const providerPublicKey = { providerPublicKey: getPublicKeyForProviderAddress(result.provider) }
const resultWithPublicKey = Object.assign(result, providerPublicKey);
return _.omit(result, ["__v", "_id"]);
};
return _.omit(resultWithPublicKey, ["__v", "_id"]);
}
module.exports = mongoose.model("Price", PriceSchema);
export const Price = mongoose.model("Price", PriceSchema);

16
package.json
View File

@@ -7,7 +7,8 @@
"start": "node -e 'require(\"./index\").runLocalServer()'",
"dev": "MODE=LOCAL node -e 'require(\"./index\").runLocalServer()'",
"test": "jest --coverage",
"test:ci": "NODE_ENV=test jest --ci --reporters='default' --reporters='./helpers/GithubActionsReporter'"
"test:ci": "NODE_ENV=test jest --ci --reporters='default' --reporters='./helpers/GithubActionsReporter'",
"build": "tsc"
},
"dependencies": {
"@amplitude/node": "^1.9.1",
@@ -23,16 +24,25 @@
"express-async-handler": "^1.1.4",
"lodash": "^4.17.21",
"mongodb-memory-server": "^7.5.1",
"mongoose": "^5.12.3",
"mongoose": "^6.3.4",
"redstone-node": "^0.4.20",
"yargs": "^17.0.1"
},
"devDependencies": {
"@types/aws-serverless-express": "^3.3.5",
"@types/cors": "^2.8.12",
"@types/errorhandler": "^1.5.0",
"@types/express": "^4.17.13",
"@types/jest": "^27.5.1",
"@types/lodash": "^4.14.182",
"@types/supertest": "^2.0.12",
"express": "^4.17.1",
"jest": "^27.0.4",
"jest": "^28.0.1",
"mongodb": "^3.6.6",
"sleep-promise": "^9.1.0",
"supertest": "^6.1.3",
"ts-jest": "^28.0.3",
"typescript": "^4.7.2",
"uuid-random": "^1.3.2"
}
}

17
providers/index.js → providers/index.ts
View File

@@ -1,20 +1,20 @@
const providers = require("redstone-node/dist/src/config/providers.json");
import providers from "redstone-node/dist/src/config/providers.json";
function getProviders() {
export const getProviders = () => {
return providers;
}
function getPublicKeyForProviderAddress(address) {
export const getPublicKeyForProviderAddress = (address: string) => {
const details = findProviderDetailsByAddress(address);
return details.publicKey;
}
function getEvmAddressForProviderAddress(address) {
export const getEvmAddressForProviderAddress = (address: string) => {
const details = findProviderDetailsByAddress(address);
return details.evmAddress;
}
function findProviderDetailsByAddress(address) {
export const findProviderDetailsByAddress = (address: string) => {
for (const providerName in providers) {
const details = providers[providerName];
if (details.address === address) {
@@ -25,10 +25,3 @@ function findProviderDetailsByAddress(address) {
// throw new Error(`Public key not found for provider address: ${address}`);
return {};
}
module.exports = {
getProviders,
getPublicKeyForProviderAddress,
getEvmAddressForProviderAddress,
};

5
routes/configs.js → routes/configs.ts
View File

@@ -1,6 +1,7 @@
const tokens = require("redstone-node/dist/src/config/tokens.json");
import { Router } from "express";
import tokens from "redstone-node/dist/src/config/tokens.json";
module.exports = (router) => {
export const configs = (router: Router) => {
/**
* This endpoint is used for returning tokens config file

7
routes/errors.js → routes/errors.ts
View File

@@ -1,8 +1,9 @@
const asyncHandler = require("express-async-handler");
const logger = require("../helpers/logger");
import { Router } from "express";
import asyncHandler from "express-async-handler";
import { logger } from "../helpers/logger";
// const { sendEmail } = require("../helpers/mail-sender");
module.exports = (router) => {
export const errors = (router: Router) => {
/**
* This endpoint is used for error saving

23
routes/index.js
View File

@@ -1,23 +0,0 @@
const prices = require("./prices");
const packages = require("./packages");
const metrics = require("./metrics");
const errors = require("./errors");
const configs = require("./configs");
const providers = require("./providers");
const config = require("../config");
module.exports.getRouter = (express) => {
const router = express.Router();
prices(router);
packages(router);
configs(router);
providers(router);
if (!config.enableLiteMode) {
metrics(router);
errors(router);
}
return router;
};

24
routes/index.ts Normal file
View File

@@ -0,0 +1,24 @@
import express from "express";
import { prices } from "./prices";
import { packages } from "./packages";
import { metrics } from "./metrics";
import { errors } from "./errors";
import { configs } from "./configs";
import { providers } from "./providers";
import { enableLiteMode } from "../config";
export const getRouter = () => {
const router = express.Router();
prices(router);
packages(router);
configs(router);
providers(router);
if (!enableLiteMode) {
metrics(router);
errors(router);
}
return router;
};

9
routes/metrics.js → routes/metrics.ts
View File

@@ -1,7 +1,8 @@
const asyncHandler = require("express-async-handler");
const cloudwatch = require("../helpers/cloudwatch");
import { Router } from "express";
import asyncHandler from "express-async-handler";
import { saveMetric } from "../helpers/cloudwatch";
module.exports = (router) => {
export const metrics = (router: Router) => {
/**
* This endpoint is used for saving metric values in AWS Cloudwatch.
@@ -10,7 +11,7 @@ module.exports = (router) => {
*/
router.post("/metrics", asyncHandler(async (req, res) => {
const { label, value } = req.body;
await cloudwatch.saveMetric({ label, value });
await saveMetric({ label, value });
return res.json({
msg: "Metric saved",

42
routes/packages.js → routes/packages.ts
View File

@@ -1,16 +1,18 @@
const asyncHandler = require("express-async-handler");
const _ = require("lodash");
const Package = require("../models/package");
const Price = require("../models/price");
const { getProviderFromParams } = require("../utils");
const { tryCleanCollection } = require("../helpers/mongo");
const config = require("../config");
import asyncHandler from "express-async-handler";
import _ from "lodash";
import { Package } from "../models/package";
import { Price } from "../models/price";
import { getProviderFromParams } from "../utils";
import { tryCleanCollection } from "../helpers/mongo";
import { enableLiteMode, cacheTTLMilliseconds } from "../config";
import { Router } from "express";
import { Document } from "mongoose";
function dbItemToObj(item) {
const dbItemToObj = (item: Document<unknown, any, Package> & Package) => {
return _.omit(item.toObject(), ["_id", "__v"]);
}
module.exports = (router) => {
export const packages = (router: Router) => {
/**
* This endpoint is used for publishing a new price package
*/
@@ -20,10 +22,10 @@ module.exports = (router) => {
await newPackage.save();
// Cleaning older packages of the same provider before in the lite mode
if (config.enableLiteMode) {
if (enableLiteMode) {
await tryCleanCollection(Package, {
signer: req.body.signer,
timestamp: { $lt: newPackage.timestamp - config.cacheTTLMilliseconds },
timestamp: { $lt: newPackage.timestamp - cacheTTLMilliseconds },
});
}
@@ -39,14 +41,13 @@ module.exports = (router) => {
* packages for the specified provider
*/
router.get("/packages/latest", asyncHandler(async (req, res) => {
const provider = await getProviderFromParams(req.query);
const provider = await getProviderFromParams(req.query as { provider: string; });
if (!provider.address) {
throw new Error("Provider address is required");
}
let responseObj;
const symbol = req.query.symbol;
const symbol = req.query.symbol as string;
if (symbol) {
// Fetching latest price for symbol from DB
@@ -60,13 +61,15 @@ module.exports = (router) => {
throw new Error(`Value not found for symbol: ${symbol}`);
}
responseObj = {
const responseObj = {
..._.pick(price, ["timestamp", "provider"]),
signature: price.evmSignature?.toString("base64"),
liteSignature: price.liteEvmSignature.toString("base64"),
prices: [{ symbol: req.query.symbol, value: price.value }],
prices: [{ symbol, value: price.value }],
signer: provider.evmAddress, // TODO: we don't really need signer, as it must be fetched from a trusted source or hardcoded in the redstone-evm-connector
};
return res.json(responseObj);
} else {
// Fetching latest package from DB
const packageFromDB = await Package.findOne({
@@ -78,11 +81,8 @@ module.exports = (router) => {
throw new Error(`Latest package not found`);
}
responseObj = dbItemToObj(packageFromDB);
const responseObj = dbItemToObj(packageFromDB);
return res.json(responseObj);
}
return res.json(responseObj);
}));
};

112
routes/prices.js → routes/prices.ts
View File

@@ -1,20 +1,34 @@
const asyncHandler = require("express-async-handler");
const _ = require("lodash");
import { Request, Router } from "express";
import asyncHandler from "express-async-handler";
import { FilterQuery, PipelineStage, Document } from "mongoose";
import _ from "lodash";
import { bigLimitWithMargin, defaultLimit, cacheTTLMilliseconds, maxLimitForPrices, enableLiteMode } from "../config";
import { Price, priceToObject } from "../models/price";
import { logEvent } from "../helpers/amplitude-event-logger";
import { assertValidSignature } from "../helpers/signature-verifier";
import { priceParamsToPriceObj, getProviderFromParams } from "../utils";
import { logger } from "../helpers/logger";
import { tryCleanCollection } from "../helpers/mongo";
const config = require("../config");
const Price = require("../models/price");
const { logEvent } = require("../helpers/amplitude-event-logger");
const { assertValidSignature } = require("../helpers/signature-verifier");
const { priceParamsToPriceObj, getProviderFromParams } = require("../utils");
const logger = require("../helpers/logger");
const { tryCleanCollection } = require("../helpers/mongo");
export interface PriceWithParams extends Omit<Price, "signature" | "evmSignature" | "liteEvmSignature"> {
limit?: number;
offset?: number;
fromTimestamp?: number;
toTimestamp?: number;
interval?: number;
providerPublicKey?: string;
symbols?: string;
signature?: string;
evmSignature?: string;
liteEvmSignature?: string;
}
async function addSinglePrice(params) {
const addSinglePrice = async (params: PriceWithParams) => {
const price = new Price(priceParamsToPriceObj(params));
await price.save();
}
async function getLatestPricesForSingleToken(params) {
const getLatestPricesForSingleToken = async (params: PriceWithParams) => {
validateParams(params, ["symbol"]);
const prices = await getPrices({
filters: {
@@ -24,10 +38,10 @@ async function getLatestPricesForSingleToken(params) {
limit: params.limit,
offset: params.offset,
});
return prices.map(Price.toObj);
return prices.map(priceToObject);
}
async function addSeveralPrices(params) {
const addSeveralPrices = async (params: PriceWithParams[]) => {
const ops = [];
for (const price of params) {
ops.push({
@@ -39,7 +53,7 @@ async function addSeveralPrices(params) {
await Price.bulkWrite(ops);
}
async function getPriceForManyTokens(params) {
const getPriceForManyTokens = async (params: PriceWithParams) => {
// Parsing symbols params
let tokens = [];
if (params.symbols !== undefined) {
@@ -55,7 +69,8 @@ async function getPriceForManyTokens(params) {
// Fetching prices from DB
const prices = await getPrices({
filters,
limit: config.bigLimitWithMargin,
limit: bigLimitWithMargin,
offset: 0,
});
// Building tokens object
@@ -65,10 +80,10 @@ async function getPriceForManyTokens(params) {
// We currently filter here
if (tokens.length === 0 || tokens.includes(price.symbol)) {
if (tokensResponse[price.symbol] === undefined) {
tokensResponse[price.symbol] = Price.toObj(price);
tokensResponse[price.symbol] = priceToObject(price);
} else {
if (tokensResponse[price.symbol].timestamp < price.timestamp) {
tokensResponse[price.symbol] = Price.toObj(price);
tokensResponse[price.symbol] = priceToObject(price);
}
}
}
@@ -77,12 +92,12 @@ async function getPriceForManyTokens(params) {
return tokensResponse;
}
async function getHistoricalPricesForSingleToken(params) {
const getHistoricalPricesForSingleToken = async (params: PriceWithParams) => {
validateParams(params, ["symbol"]);
const filters = {
symbol: params.symbol,
provider: params.provider,
timestamp: { $lte: params.toTimestamp },
timestamp: { $lte: params.toTimestamp } as { $lte: number, $gte?: number },
};
if (params.fromTimestamp) {
@@ -92,13 +107,13 @@ async function getHistoricalPricesForSingleToken(params) {
const prices = await getPrices({
filters,
offset: Number(params.offset || 0),
limit: params.limit || config.defaultLimit,
limit: params.limit || defaultLimit,
});
return prices.map(Price.toObj);
return prices.map(priceToObject);
}
// This function is used to return data for charts
async function getPricesInTimeRangeForSingleToken(params) {
const getPricesInTimeRangeForSingleToken = async (params: PriceWithParams) => {
validateParams(params,
["symbol", "fromTimestamp", "toTimestamp", "interval", "provider"]);
const { symbol, provider, fromTimestamp, toTimestamp, interval, offset, limit } = params;
@@ -113,7 +128,7 @@ async function getPricesInTimeRangeForSingleToken(params) {
},
},
},
];
] as PipelineStage[];
if (interval >= 3600 * 1000) {
pipeline.push({
@@ -138,7 +153,7 @@ async function getPricesInTimeRangeForSingleToken(params) {
}
const fetchedPrices = await Price.aggregate(pipeline);
let prices = fetchedPrices.map(Price.toObj);
let prices = fetchedPrices.map(priceToObject);
// TODO: sorting may be moved to aggregation pipeline later
// it caused performance problems, that's why now we do it here
@@ -164,16 +179,20 @@ async function getPricesInTimeRangeForSingleToken(params) {
return prices;
}
async function getPrices({
filters = {},
limit = config.defaultLimit,
const getPrices = async ({
filters = {},
limit = defaultLimit,
offset,
}) {
}: {
filters: FilterQuery<Price>;
limit: number;
offset: number;
}) => {
// Query building
let pricesQuery = Price
.find(filters)
.sort({ timestamp: -1 })
.limit(Math.min(Number(limit), config.maxLimitForPrices));
.limit(Math.min(Number(limit), maxLimitForPrices));
if (offset) {
pricesQuery = pricesQuery.skip(Number(offset));
}
@@ -184,7 +203,7 @@ async function getPrices({
return prices;
}
function validateParams(params, requiredParams) {
const validateParams = (params: Record<string, any>, requiredParams: string[]) => {
const errors = [];
for (const requiredParam of requiredParams) {
if (params[requiredParam] === undefined) {
@@ -197,7 +216,7 @@ function validateParams(params, requiredParams) {
}
}
function getPricesCount(reqBody) {
const getPricesCount = (reqBody: PriceWithParams) => {
if (Array.isArray(reqBody)) {
return reqBody.length;
} else {
@@ -205,20 +224,27 @@ function getPricesCount(reqBody) {
}
}
function getIp(req) {
const getIp = (req: Request) => {
const ip = req.ip;
logger.info("Request IP address: " + ip);
return ip;
}
module.exports = (router) => {
interface QueryParams extends PriceWithParams{
provider: string;
symbols?: string;
tokens?: string[];
providerPublicKey?: string;
}
export const prices = (router: Router) => {
/**
* This endpoint is used for fetching prices data.
* It is used in redstone-api
*/
router.get("/prices", asyncHandler(async (req, res) => {
// Request validation
const params = req.query;
const params = req.query as unknown as QueryParams;
// Saving API read event in amplitude
logEvent({
@@ -234,6 +260,7 @@ module.exports = (router) => {
// If query params contain "symbol" we fetch price for this symbol
if (params.symbol !== undefined) {
let body: _.Omit<Document<unknown, any, Price> & Price & { providerPublicKey: any; }, "_id" | "__v">[];
if (params.interval !== undefined) {
body = await getPricesInTimeRangeForSingleToken(params);
} else if (params.toTimestamp !== undefined) {
@@ -241,6 +268,7 @@ module.exports = (router) => {
} else {
body = await getLatestPricesForSingleToken(params);
}
return res.json(body);
}
// Otherwise we fetch prices for many symbols
else {
@@ -250,18 +278,18 @@ module.exports = (router) => {
}
params.tokens = tokens;
body = await getPriceForManyTokens(params);
const body = await getPriceForManyTokens(params);
return res.json(body);
}
return res.json(body);
}));
/**
* This endpoint is used for posting a new price data.
* It supports posting a single price and several prices
*/
router.post("/prices", asyncHandler(async (req, res) => {
const reqBody = req.body;
const reqBody = req.body as PriceWithParams;
let pricesSavedCount = 0;
// Saving API post event in amplitude
@@ -290,10 +318,10 @@ module.exports = (router) => {
// Cleaning older prices for the same provider after posting
// new ones in the lite mode
if (config.enableLiteMode) {
if (enableLiteMode) {
await tryCleanCollection(Price, {
provider: reqBody[0].provider,
timestamp: { $lt: Number(reqBody[0].timestamp) - config.cacheTTLMilliseconds },
timestamp: { $lt: Number(reqBody[0].timestamp) - cacheTTLMilliseconds },
});
}
@@ -308,11 +336,11 @@ module.exports = (router) => {
// Cleaning prices for the same provider and symbol before posting
// a new one in the lite mode
if (config.enableLiteMode) {
if (enableLiteMode) {
await tryCleanCollection(Price, {
provider: reqBody.provider,
symbol: reqBody.symbol,
timestamp: { $lt: Number(reqBody.timestamp) - config.cacheTTLMilliseconds },
timestamp: { $lt: Number(reqBody.timestamp) - cacheTTLMilliseconds },
});
}
}

12
routes/providers.js
View File

@@ -1,12 +0,0 @@
const providers = require("redstone-node/dist/src/config/providers.json");
module.exports = (router) => {
/**
* This endpoint is used for returning providers details
*/
router.get("/providers", (req, res) => {
res.json(providers);
});
};

12
routes/providers.ts Normal file
View File

@@ -0,0 +1,12 @@
import { Router } from "express";
import providersJson from "redstone-node/dist/src/config/providers.json";
export const providers = (router: Router) => {
/**
* This endpoint is used for returning providers details
*/
router.get("/providers", (req, res) => {
res.json(providersJson);
});
};

9
test/helpers/lite-mode-config.js
View File

@@ -1,9 +0,0 @@
module.exports = {
bigLimitWithMargin: 1200,
defaultLimit: 1,
defaultLocalPort: 9000,
enableJsonLogs: false,
maxLimitForPrices: 3000,
enableLiteMode: true,
cacheTTLMilliseconds: 0, // for tests
};

10
test/helpers/signature-verifier.test.js → test/helpers/signature-verifier.test.ts
View File

@@ -1,7 +1,7 @@
const { assertValidSignature } = require("../../helpers/signature-verifier");
const testJwk = require("./test-jwk.json");
const Arweave = require("arweave");
const deepSortObject = require("deep-sort-object");
import Arweave from "arweave";
import deepSortObject from "deep-sort-object";
import { assertValidSignature } from "../../helpers/signature-verifier";
import testJwk from "./test-jwk.json";
const mockProviders = {
"redstone": {
@@ -52,7 +52,7 @@ describe("Testing signature verifier", () => {
signature: await getSignature(initialPriceData),
};
const skipLatestPriceCheck = true;
await expect(assertValidSignature(price, skipLatestPriceCheck)).resolves;
expect(assertValidSignature(price, skipLatestPriceCheck)).resolves;
});
test("Should not verify invalid signature", async () => {

16
test/helpers/test-db.ts Normal file
View File

@@ -0,0 +1,16 @@
import mongoose from 'mongoose';
import { MongoMemoryServer } from 'mongodb-memory-server';
let mongo: MongoMemoryServer;
export const connect = async () => {
mongo = await MongoMemoryServer.create();
const uri = mongo.getUri();
await mongoose.connect(uri);
};
export const closeDatabase = async () => {
await mongoose.connection.dropDatabase();
await mongoose.connection.close();
await mongo.stop();
};

14
test/lite-mode-routes/configs.test.js → test/lite-mode-routes/configs.test.ts
View File

@@ -1,12 +1,10 @@
const request = require("supertest");
const app = require("../../app");
const testDB = require("../test-db");
import request from "supertest";
import { app } from "../../app";
import { connect, closeDatabase } from "../helpers/test-db";
jest.mock("../../config", () => require("../helpers/lite-mode-config"));
describe("Testing configs route", () => {
beforeAll(async () => await testDB.connect());
afterAll(async () => await testDB.closeDatabase());
describe.only("Testing configs route", () => {
beforeAll(async () => await connect());
afterAll(async () => await closeDatabase());
test("Should return tokens config", async () => {

20
test/lite-mode-routes/packages.test.js → test/lite-mode-routes/packages.test.ts
View File

@@ -1,18 +1,16 @@
const request = require("supertest");
const _ = require("lodash");
const app = require("../../app");
const testDB = require("../test-db");
const Price = require("../../models/price");
const Package = require("../../models/package");
const { getProviders } = require("../../providers");
jest.mock("../../config", () => require("../helpers/lite-mode-config"));
import request from "supertest";
import _ from "lodash";
import { app } from "../../app";
import { connect, closeDatabase } from "../helpers/test-db";
import { Price } from "../../models/price";
import { Package } from "../../models/package";
import { getProviders } from "../../providers";
const provider = getProviderForTests();
describe("Testing packages route", () => {
beforeEach(async () => await testDB.connect());
afterEach(async () => await testDB.closeDatabase());
beforeEach(async () => await connect());
afterEach(async () => await closeDatabase());
const testTimestamp = Date.now();
const arweaveSignature = "dGVzdC1zaWduYXR1cmU="; // test-signature in Base64

21
test/lite-mode-routes/prices.test.js → test/lite-mode-routes/prices.test.ts
View File

@@ -1,21 +1,18 @@
const request = require("supertest");
const _ = require("lodash");
const sleep = require("sleep-promise");
const app = require("../../app");
const uuid = require("uuid-random");
const testDB = require("../test-db");
const { getProviders } = require("../../providers");
const Price = require("../../models/price");
jest.mock("../../config", () => require("../helpers/lite-mode-config"));
import request from "supertest";
import _ from "lodash";
import { app } from "../../app";
import uuid from "uuid-random";
import { connect, closeDatabase } from "../helpers/test-db";
import { getProviders } from "../../providers";
import { Price } from "../../models/price";
const provider = getProviderForTests();
jest.mock("../../helpers/signature-verifier");
describe("Testing prices route", () => {
beforeAll(async () => await testDB.connect());
afterAll(async () => await testDB.closeDatabase());
beforeAll(async () => await connect());
afterAll(async () => await closeDatabase());
test("Should post and get a single price", async () => {
// Posting a price

10
test/routes/providers.test.js → test/lite-mode-routes/providers.test.ts
View File

@@ -1,6 +1,6 @@
const request = require("supertest");
const app = require("../../app");
const testDB = require("../test-db");
import request from "supertest";
import { app } from "../../app";
import { connect, closeDatabase } from "../helpers/test-db";
const providers = [
"redstone",
@@ -9,8 +9,8 @@ const providers = [
];
describe("Testing providers route", () => {
beforeAll(async () => await testDB.connect());
afterAll(async () => await testDB.closeDatabase());
beforeAll(async () => await connect());
afterAll(async () => await closeDatabase());
test("Should return providers config", async () => {
const response = await request(app)

10
test/routes/configs.test.js → test/routes/configs.test.ts
View File

@@ -1,10 +1,10 @@
const request = require("supertest");
const app = require("../../app");
const testDB = require("../test-db");
import request from "supertest";
import { app } from "../../app";
import { connect, closeDatabase } from "../helpers/test-db";
describe("Testing configs route", () => {
beforeAll(async () => await testDB.connect());
afterAll(async () => await testDB.closeDatabase());
beforeAll(async () => await connect());
afterAll(async () => await closeDatabase());
test("Should return tokens config", async () => {
const response = await request(app)

11
test/routes/errors.test.js → test/routes/errors.test.ts
View File

@@ -1,10 +1,11 @@
const request = require("supertest");
const app = require("../../app");
const testDB = require("../test-db");
process.env.LIGHT_MODE = "false";
import request from "supertest";
import { app } from "../../app";
import { connect, closeDatabase } from "../helpers/test-db";
describe("Testing errors route", () => {
beforeAll(async () => await testDB.connect());
afterAll(async () => await testDB.closeDatabase());
beforeAll(async () => await connect());
afterAll(async () => await closeDatabase());
const testError = {
error: "test-error",

13
test/routes/metrics.test.js → test/routes/metrics.test.ts
View File

@@ -1,11 +1,12 @@
const request = require("supertest");
const app = require("../../app");
const testDB = require("../test-db");
const cloudwatch = require("../../helpers/cloudwatch");
process.env.LIGHT_MODE = "false";
import request from "supertest";
import { app } from "../../app";
import { connect, closeDatabase } from "../helpers/test-db";
import * as cloudwatch from "../../helpers/cloudwatch";
describe("Testing metrics route", () => {
beforeAll(async () => await testDB.connect());
afterAll(async () => await testDB.closeDatabase());
beforeAll(async () => await connect());
afterAll(async () => await closeDatabase());
const testMetricValue = {
label: "test-metric",

17
test/routes/packages.test.js → test/routes/packages.test.ts
View File

@@ -1,16 +1,15 @@
const request = require("supertest");
const _ = require("lodash");
const app = require("../../app");
const testDB = require("../test-db");
const Price = require("../../models/price");
const Package = require("../../models/package");
const { getProviders } = require("../../providers");
import request from "supertest";
import _ from "lodash";
import { app } from "../../app";
import { connect, closeDatabase } from "../helpers/test-db";
import { Price } from "../../models/price";
import { getProviders } from "../../providers";
const provider = getProviderForTests();
describe("Testing packages route", () => {
beforeEach(async () => await testDB.connect());
afterEach(async () => await testDB.closeDatabase());
beforeEach(async () => await connect());
afterEach(async () => await closeDatabase());
const testTimestamp = Date.now();
const arweaveSignature = "dGVzdC1zaWduYXR1cmU="; // test-signature in Base64

36
test/routes/prices.test.js → test/routes/prices.test.ts
View File

@@ -1,18 +1,18 @@
const request = require("supertest");
const _ = require("lodash");
const sleep = require("sleep-promise");
const app = require("../../app");
const uuid = require("uuid-random");
const testDB = require("../test-db");
const { getProviders } = require("../../providers");
import request from "supertest";
import _ from "lodash";
import sleep from "sleep-promise";
import { app } from "../../app";
import uuid from "uuid-random";
import { connect, closeDatabase } from "../helpers/test-db";
import { getProviders } from "../../providers";
const provider = getProviderForTests();
jest.mock("../../helpers/signature-verifier");
describe("Testing prices route", () => {
beforeAll(async () => await testDB.connect());
afterAll(async () => await testDB.closeDatabase());
beforeAll(async () => await connect());
afterAll(async () => await closeDatabase());
test("Should post and get a single price", async () => {
// Posting a price
@@ -183,10 +183,10 @@ describe("Testing prices route", () => {
describe("Testing EVM signature", () => {
beforeEach(async () => {
await testDB.connect();
await connect();
// TODO: put intial data to DB
});
afterEach(async () => await testDB.closeDatabase());
afterEach(async () => await closeDatabase());
test("Should post and get a price without EVM signature", async () => {
const price = getMockPriceData();
@@ -208,17 +208,19 @@ describe("Testing EVM signature", () => {
expect(getResponse).toHaveProperty("body");
expect(getResponse.body.length).toBe(1);
expect(getResponse.body[0]).toHaveProperty("signature");
expect(getResponse.body[0]).toHaveProperty("signature", "dGVzdC1zaWduYXR1cmU=");
expect(getResponse.body[0]).not.toHaveProperty("evmSignature");
});
test("Should post and get a price with EVM signature", async () => {
const price = getMockPriceData();
price.evmSignature = "dGVzdC1ldm0tc2lnbmF0dXJl"; // "test-evm-signature" in base64
const priceWithEvmSignature = Object.assign(price, {
evmSignature: "dGVzdC1ldm0tc2lnbmF0dXJl"
}); // "test-evm-signature" in base64
const postResponse = await request(app)
.post("/prices")
.send([price])
.send([priceWithEvmSignature])
.expect(200);
expect(postResponse.body).toHaveProperty("msg", "Prices saved. count: 1");
@@ -226,15 +228,15 @@ describe("Testing EVM signature", () => {
const getResponse = await request(app)
.get("/prices")
.query({
symbol: price.symbol,
provider: provider.name,
symbol: priceWithEvmSignature.symbol,
provider: priceWithEvmSignature.provider,
limit: 1,
});
expect(getResponse).toHaveProperty("body");
expect(getResponse.body.length).toBe(1);
expect(getResponse.body[0]).toHaveProperty("signature");
expect(getResponse.body[0]).toHaveProperty("evmSignature", price.evmSignature);
expect(getResponse.body[0]).toHaveProperty("evmSignature", priceWithEvmSignature.evmSignature);
});
});

12
test/lite-mode-routes/providers.test.js → test/routes/providers.test.ts
View File

@@ -1,8 +1,6 @@
const request = require("supertest");
const app = require("../../app");
const testDB = require("../test-db");
jest.mock("../../config", () => require("../helpers/lite-mode-config"));
import request from "supertest";
import { app } from "../../app";
import { connect, closeDatabase } from "../helpers/test-db";
const providers = [
"redstone",
@@ -11,8 +9,8 @@ const providers = [
];
describe("Testing providers route", () => {
beforeAll(async () => await testDB.connect());
afterAll(async () => await testDB.closeDatabase());
beforeAll(async () => await connect());
afterAll(async () => await closeDatabase());
test("Should return providers config", async () => {
const response = await request(app)

20
test/test-db.js
View File

@@ -1,20 +0,0 @@
const mongoose = require('mongoose');
const { MongoMemoryServer } = require('mongodb-memory-server');
let mongo;
module.exports.connect = async () => {
mongo = await MongoMemoryServer.create();
const uri = mongo.getUri();
await mongoose.connect(uri, {
useNewUrlParser: true,
useUnifiedTopology: true,
useFindAndModify: false,
});
};
module.exports.closeDatabase = async () => {
await mongoose.connection.dropDatabase();
await mongoose.connection.close();
await mongo.stop();
};

12
tsconfig.json Normal file
View File

@@ -0,0 +1,12 @@
{
"compilerOptions": {
"module": "commonjs",
"esModuleInterop": true,
"target": "es6",
"moduleResolution": "node",
"sourceMap": true,
"outDir": "dist",
"allowJs": true,
"resolveJsonModule": true
}
}

17
utils.js → utils.ts
View File

@@ -1,6 +1,7 @@
const { getProviders } = require("./providers");
import { getProviders } from "./providers";
import { PriceWithParams } from "./routes/prices";
function priceParamsToPriceObj(price) {
export const priceParamsToPriceObj = (price: PriceWithParams) => {
const priceObj = {
...price,
minutes: getMinutesFromTimestamp(price.timestamp),
@@ -15,11 +16,11 @@ function priceParamsToPriceObj(price) {
return priceObj;
}
function getMinutesFromTimestamp(timestamp) {
export const getMinutesFromTimestamp = (timestamp: number) => {
return new Date(timestamp).getMinutes();
}
async function getProviderFromParams(params) {
export const getProviderFromParams = async (params: { provider: string; }) => {
// TODO: load this mapping directly from arweave
// TODO: we also can implement caching
const providers = getProviders();
@@ -42,7 +43,7 @@ async function getProviderFromParams(params) {
}
function formatDate(date) {
export const formatDate = (date: number) => {
const d = new Date(Number(date));
const year = String(d.getFullYear());
let month = String((d.getMonth() + 1));
@@ -57,9 +58,3 @@ function formatDate(date) {
return `${year}-${month}-${day}`;
}
module.exports = {
formatDate,
getProviderFromParams,
priceParamsToPriceObj,
};

2165
yarn.lock
View File

File diff suppressed because it is too large Load Diff