kylefang/redstone-cache-layer
1
0
Fork 0
mirror of https://github.com/alexgo-io/redstone-cache-layer.git synced 2026-01-12 22:43:30 +08:00
Code Issues Packages Projects Releases Wiki Activity

chore: removed arweave signatures verification

Browse Source
This commit is contained in:
hatskier
2023-03-06 12:07:53 +00:00
parent 6c2de9fbad
commit 750fd637c3
3 changed files with 206 additions and 178 deletions
Download Patch File Download Diff File Expand all files Collapse all files

82
helpers/signature-verifier.ts
View File

@@ -6,7 +6,10 @@ import { getPublicKeyForProviderAddress } from "../providers";
import { logger } from "./logger"; import { logger } from "./logger";
import { PriceWithParams } from "../routes/prices"; import { PriceWithParams } from "../routes/prices";
export const assertValidSignature = async (price: PriceWithParams, skipNewerPricesCheck = true) => { export const assertValidSignature = async (
price: PriceWithParams,
skipNewerPricesCheck = true
) => {
if (!skipNewerPricesCheck) { if (!skipNewerPricesCheck) {
// Checking if price with a greater timestamp is already in DB // Checking if price with a greater timestamp is already in DB
// If so, then we raise an Error, because we allow to publish // If so, then we raise an Error, because we allow to publish
@@ -18,9 +21,10 @@ export const assertValidSignature = async (price: PriceWithParams, skipNewerPric
}); });
if (newerPriceFound) { if (newerPriceFound) {
throw new Error( throw new Error(
`A newer price found in DB. ` `A newer price found in DB. ` +
+ `Newer price: ${JSON.stringify(newerPriceFound)}. ` `Newer price: ${JSON.stringify(newerPriceFound)}. ` +
+ `Failed price: ${JSON.stringify(price)}.`); `Failed price: ${JSON.stringify(price)}.`
);
} }
} else { } else {
logger.info("Newer prices check skipped"); logger.info("Newer prices check skipped");
@@ -29,44 +33,41 @@ export const assertValidSignature = async (price: PriceWithParams, skipNewerPric
// Signature verification // Signature verification
const isValid = await verifySignature(price); const isValid = await verifySignature(price);
if (!isValid) { if (!isValid) {
throw new Error( throw new Error("Price signature is invalid: " + JSON.stringify(price));
"Price signature is invalid: " + JSON.stringify(price));
}
} }
};
// We should implement ECDSA signature verification
export const verifySignature = async (price: PriceWithParams) => { export const verifySignature = async (price: PriceWithParams) => {
// Time measurement: start
const startTime = Date.now();
// Data preparation
const publicKey = getPublicKeyForProviderAddress(price.provider);
const signedData = getPriceSignedData(price);
const signedBytes = new TextEncoder().encode(signedData);
const signatureBytes = Uint8Array.from(Buffer.from(price.signature, "base64"));
// It allows other providers (not registered in our api) post their data here
// In future, we should implement some kind of authorization for this kind of providers
if (!publicKey) {
return true; return true;
} // // Time measurement: start
// const startTime = Date.now();
const dataPrepTime = Date.now() - startTime; // // Data preparation
logger.info( // const publicKey = getPublicKeyForProviderAddress(price.provider);
`Data prep time elapsed: ${dataPrepTime} ms`); // const signedData = getPriceSignedData(price);
// const signedBytes = new TextEncoder().encode(signedData);
// Signature verification // const signatureBytes = Uint8Array.from(
const validSignature = await Arweave.crypto.verify( // Buffer.from(price.signature, "base64")
publicKey, // );
signedBytes, // // It allows other providers (not registered in our api) post their data here
signatureBytes, // // In future, we should implement some kind of authorization for this kind of providers
); // if (!publicKey) {
// return true;
// Time measurement: end // }
const signVerificationTime = Date.now() - startTime; // const dataPrepTime = Date.now() - startTime;
logger.info( // logger.info(`Data prep time elapsed: ${dataPrepTime} ms`);
`Signature verification time elapsed: ${signVerificationTime} ms`); // // Signature verification
// const validSignature = await Arweave.crypto.verify(
return validSignature; // publicKey,
// signedBytes,
// signatureBytes
// );
// // Time measurement: end
// const signVerificationTime = Date.now() - startTime;
// logger.info(
// `Signature verification time elapsed: ${signVerificationTime} ms`
// );
// return validSignature;
}; };
const getPriceSignedData = (price: PriceWithParams) => { const getPriceSignedData = (price: PriceWithParams) => {
@@ -81,7 +82,6 @@ const getPriceSignedData = (price: PriceWithParams) => {
"provider", "provider",
]); ]);
if (shouldApplyDeepSort(price)) { if (shouldApplyDeepSort(price)) {
return JSON.stringify(deepSortObject(priceWithPickedProps)); return JSON.stringify(deepSortObject(priceWithPickedProps));
} else { } else {
@@ -90,5 +90,7 @@ const getPriceSignedData = (price: PriceWithParams) => {
}; };
const shouldApplyDeepSort = (price: PriceWithParams) => { const shouldApplyDeepSort = (price: PriceWithParams) => {
return price.version && (price.version === "3" || price.version.includes(".")); return (
price.version && (price.version === "3" || price.version.includes("."))
);
}; };

97
routes/prices.ts
View File

@@ -2,7 +2,13 @@ import { Request, Router } from "express";
import asyncHandler from "express-async-handler"; import asyncHandler from "express-async-handler";
import { FilterQuery, PipelineStage, Document } from "mongoose"; import { FilterQuery, PipelineStage, Document } from "mongoose";
import _ from "lodash"; import _ from "lodash";
import { bigLimitWithMargin, defaultLimit, cacheTTLMilliseconds, maxLimitForPrices, enableLiteMode } from "../config"; import {
bigLimitWithMargin,
defaultLimit,
cacheTTLMilliseconds,
maxLimitForPrices,
enableLiteMode,
} from "../config";
import { Price, priceToObject } from "../models/price"; import { Price, priceToObject } from "../models/price";
import { logEvent } from "../helpers/amplitude-event-logger"; import { logEvent } from "../helpers/amplitude-event-logger";
import { assertValidSignature } from "../helpers/signature-verifier"; import { assertValidSignature } from "../helpers/signature-verifier";
@@ -10,7 +16,8 @@ import { priceParamsToPriceObj, getProviderFromParams } from "../utils";
import { logger } from "../helpers/logger"; import { logger } from "../helpers/logger";
import { tryCleanCollection } from "../helpers/mongo"; import { tryCleanCollection } from "../helpers/mongo";
export interface PriceWithParams extends Omit<Price, "signature" | "evmSignature" | "liteEvmSignature"> { export interface PriceWithParams
extends Omit<Price, "signature" | "evmSignature" | "liteEvmSignature"> {
limit?: number; limit?: number;
offset?: number; offset?: number;
fromTimestamp?: number; fromTimestamp?: number;
@@ -26,7 +33,7 @@ export interface PriceWithParams extends Omit<Price, "signature" | "evmSignature
const addSinglePrice = async (params: PriceWithParams) => { const addSinglePrice = async (params: PriceWithParams) => {
const price = new Price(priceParamsToPriceObj(params)); const price = new Price(priceParamsToPriceObj(params));
await price.save(); await price.save();
} };
const getLatestPricesForSingleToken = async (params: PriceWithParams) => { const getLatestPricesForSingleToken = async (params: PriceWithParams) => {
validateParams(params, ["symbol"]); validateParams(params, ["symbol"]);
@@ -39,7 +46,7 @@ const getLatestPricesForSingleToken = async (params: PriceWithParams) => {
offset: params.offset, offset: params.offset,
}); });
return prices.map(priceToObject); return prices.map(priceToObject);
} };
const addSeveralPrices = async (params: PriceWithParams[]) => { const addSeveralPrices = async (params: PriceWithParams[]) => {
const ops = []; const ops = [];
@@ -51,7 +58,7 @@ const addSeveralPrices = async (params: PriceWithParams[]) => {
}); });
} }
await Price.bulkWrite(ops); await Price.bulkWrite(ops);
} };
const getPriceForManyTokens = async (params: PriceWithParams) => { const getPriceForManyTokens = async (params: PriceWithParams) => {
// Parsing symbols params // Parsing symbols params
@@ -76,7 +83,6 @@ const getPriceForManyTokens = async (params: PriceWithParams) => {
// Building tokens object // Building tokens object
const tokensResponse = {}; const tokensResponse = {};
for (const price of prices) { for (const price of prices) {
// We currently filter here // We currently filter here
if (tokens.length === 0 || tokens.includes(price.symbol)) { if (tokens.length === 0 || tokens.includes(price.symbol)) {
if (tokensResponse[price.symbol] === undefined) { if (tokensResponse[price.symbol] === undefined) {
@@ -90,14 +96,14 @@ const getPriceForManyTokens = async (params: PriceWithParams) => {
} }
return tokensResponse; return tokensResponse;
} };
const getHistoricalPricesForSingleToken = async (params: PriceWithParams) => { const getHistoricalPricesForSingleToken = async (params: PriceWithParams) => {
validateParams(params, ["symbol"]); validateParams(params, ["symbol"]);
const filters = { const filters = {
symbol: params.symbol, symbol: params.symbol,
provider: params.provider, provider: params.provider,
timestamp: { $lte: params.toTimestamp } as { $lte: number, $gte?: number }, timestamp: { $lte: params.toTimestamp } as { $lte: number; $gte?: number },
}; };
if (params.fromTimestamp) { if (params.fromTimestamp) {
@@ -110,13 +116,26 @@ const getHistoricalPricesForSingleToken = async (params: PriceWithParams) => {
limit: params.limit || defaultLimit, limit: params.limit || defaultLimit,
}); });
return prices.map(priceToObject); return prices.map(priceToObject);
} };
// This function is used to return data for charts // This function is used to return data for charts
const getPricesInTimeRangeForSingleToken = async (params: PriceWithParams) => { const getPricesInTimeRangeForSingleToken = async (params: PriceWithParams) => {
validateParams(params, validateParams(params, [
["symbol", "fromTimestamp", "toTimestamp", "interval", "provider"]); "symbol",
const { symbol, provider, fromTimestamp, toTimestamp, interval, offset, limit } = params; "fromTimestamp",
"toTimestamp",
"interval",
"provider",
]);
const {
symbol,
provider,
fromTimestamp,
toTimestamp,
interval,
offset,
limit,
} = params;
const pipeline = [ const pipeline = [
{ {
$match: { $match: {
@@ -165,7 +184,8 @@ const getPricesInTimeRangeForSingleToken = async (params: PriceWithParams) => {
// so we can not filter them out in DB query level // so we can not filter them out in DB query level
const millisecondsInMinute = 60 * 1000; const millisecondsInMinute = 60 * 1000;
if (interval > millisecondsInMinute && prices.length > 0) { if (interval > millisecondsInMinute && prices.length > 0) {
let filteredPrices = [], prevTimestamp = prices[0].timestamp; let filteredPrices = [],
prevTimestamp = prices[0].timestamp;
for (const price of prices) { for (const price of prices) {
const diff = price.timestamp - prevTimestamp; const diff = price.timestamp - prevTimestamp;
if (diff === 0 || diff > millisecondsInMinute) { if (diff === 0 || diff > millisecondsInMinute) {
@@ -177,7 +197,7 @@ const getPricesInTimeRangeForSingleToken = async (params: PriceWithParams) => {
} }
return prices; return prices;
} };
const getPrices = async ({ const getPrices = async ({
filters = {}, filters = {},
@@ -189,8 +209,7 @@ const getPrices = async ({
offset: number; offset: number;
}) => { }) => {
// Query building // Query building
let pricesQuery = Price let pricesQuery = Price.find(filters)
.find(filters)
.sort({ timestamp: -1 }) .sort({ timestamp: -1 })
.limit(Math.min(Number(limit), maxLimitForPrices)); .limit(Math.min(Number(limit), maxLimitForPrices));
if (offset) { if (offset) {
@@ -201,9 +220,12 @@ const getPrices = async ({
const prices = await pricesQuery.exec(); const prices = await pricesQuery.exec();
return prices; return prices;
} };
const validateParams = (params: Record<string, any>, requiredParams: string[]) => { const validateParams = (
params: Record<string, any>,
requiredParams: string[]
) => {
const errors = []; const errors = [];
for (const requiredParam of requiredParams) { for (const requiredParam of requiredParams) {
if (params[requiredParam] === undefined) { if (params[requiredParam] === undefined) {
@@ -214,7 +236,7 @@ const validateParams = (params: Record<string, any>, requiredParams: string[]) =
if (errors.length > 0) { if (errors.length > 0) {
throw new Error(JSON.stringify(errors)); throw new Error(JSON.stringify(errors));
} }
} };
const getPricesCount = (reqBody: PriceWithParams) => { const getPricesCount = (reqBody: PriceWithParams) => {
if (Array.isArray(reqBody)) { if (Array.isArray(reqBody)) {
@@ -222,13 +244,13 @@ const getPricesCount = (reqBody: PriceWithParams) => {
} else { } else {
return 1; return 1;
} }
} };
const getIp = (req: Request) => { const getIp = (req: Request) => {
const ip = req.ip; const ip = req.ip;
logger.info("Request IP address: " + ip); logger.info("Request IP address: " + ip);
return ip; return ip;
} };
interface QueryParams extends PriceWithParams { interface QueryParams extends PriceWithParams {
provider: string; provider: string;
@@ -242,7 +264,9 @@ export const prices = (router: Router) => {
* This endpoint is used for fetching prices data. * This endpoint is used for fetching prices data.
* It is used in redstone-api * It is used in redstone-api
*/ */
router.get("/prices", asyncHandler(async (req, res) => { router.get(
"/prices",
asyncHandler(async (req, res) => {
// Request validation // Request validation
const params = req.query as unknown as QueryParams; const params = req.query as unknown as QueryParams;
@@ -260,7 +284,10 @@ export const prices = (router: Router) => {
// If query params contain "symbol" we fetch price for this symbol // If query params contain "symbol" we fetch price for this symbol
if (params.symbol !== undefined) { if (params.symbol !== undefined) {
let body: _.Omit<Document<unknown, any, Price> & Price & { providerPublicKey: any; }, "_id" | "__v">[]; let body: _.Omit<
Document<unknown, any, Price> & Price & { providerPublicKey: any },
"_id" | "__v"
>[];
if (params.interval !== undefined) { if (params.interval !== undefined) {
body = await getPricesInTimeRangeForSingleToken(params); body = await getPricesInTimeRangeForSingleToken(params);
} else if (params.toTimestamp !== undefined) { } else if (params.toTimestamp !== undefined) {
@@ -281,14 +308,16 @@ export const prices = (router: Router) => {
const body = await getPriceForManyTokens(params); const body = await getPriceForManyTokens(params);
return res.json(body); return res.json(body);
} }
})); })
);
/** /**
* This endpoint is used for posting a new price data. * This endpoint is used for posting a new price data.
* It supports posting a single price and several prices * It supports posting a single price and several prices
*/ */
router.post("/prices", asyncHandler(async (req, res) => { router.post(
"/prices",
asyncHandler(async (req, res) => {
const reqBody = req.body as PriceWithParams; const reqBody = req.body as PriceWithParams;
let pricesSavedCount = 0; let pricesSavedCount = 0;
@@ -302,10 +331,11 @@ export const prices = (router: Router) => {
}); });
if (Array.isArray(reqBody)) { if (Array.isArray(reqBody)) {
const invalidPrices = reqBody.filter(p => !p.value); const invalidPrices = reqBody.filter((p) => !p.value);
if (invalidPrices.length > 0) { if (invalidPrices.length > 0) {
logger.error( logger.error(
"Invalid prices with empty value: " + JSON.stringify(invalidPrices)); "Invalid prices with empty value: " + JSON.stringify(invalidPrices)
);
} }
// Validating a signature of a randomly selected price // Validating a signature of a randomly selected price
@@ -321,7 +351,9 @@ export const prices = (router: Router) => {
if (enableLiteMode) { if (enableLiteMode) {
await tryCleanCollection(Price, { await tryCleanCollection(Price, {
provider: reqBody[0].provider, provider: reqBody[0].provider,
timestamp: { $lt: Number(reqBody[0].timestamp) - cacheTTLMilliseconds }, timestamp: {
$lt: Number(reqBody[0].timestamp) - cacheTTLMilliseconds,
},
}); });
} }
@@ -340,11 +372,14 @@ export const prices = (router: Router) => {
await tryCleanCollection(Price, { await tryCleanCollection(Price, {
provider: reqBody.provider, provider: reqBody.provider,
symbol: reqBody.symbol, symbol: reqBody.symbol,
timestamp: { $lt: Number(reqBody.timestamp) - cacheTTLMilliseconds }, timestamp: {
$lt: Number(reqBody.timestamp) - cacheTTLMilliseconds,
},
}); });
} }
} }
return res.json({ msg: `Prices saved. count: ${pricesSavedCount}` }); return res.json({ msg: `Prices saved. count: ${pricesSavedCount}` });
})); })
);
}; };

29
test/helpers/signature-verifier.test.ts
View File

@@ -4,18 +4,21 @@ import { assertValidSignature } from "../../helpers/signature-verifier";
import testJwk from "./test-jwk.json"; import testJwk from "./test-jwk.json";
const mockProviders = { const mockProviders = {
"redstone": { redstone: {
"address": "MlV6DeOtRmakDOf6vgOBlif795tcWimgyPsYYNQ8q1Y", address: "MlV6DeOtRmakDOf6vgOBlif795tcWimgyPsYYNQ8q1Y",
"publicKey": "zhTx5Kr9VNQrXGarf0EXySfbSePBbIQuSOpb07s3pM3q8HKCx-bbd_py8t-JxgwnKAmpGKt6UhOP0FeobGITCwr_O7ATFPrFgTbM-xLYG0JOzxUlPScyqdJ8rFRcSSpevfUyJ6UVTpA3LDQHEzf7kebjfMPeYwpsWuT3c9LP3j0kyPDOBini-LRUpKX3n4ljhJIHzl-Jdv6Z31U65kZRBR1LPwnjcBUg4hoc50i8JZsSLsrUYFfpYVuxM0L4ch0l2-FvPtmZs831mOQgT8e1s7GPB7kJBhrQBagGF3eVnAiImJjslXNQhy4eQr6Nffb5Wa61Tec52LX5-gmoNSuA0PW5yuYGuDO2faULW74u8ZfmMUxd2x3E3M6E0deP_rj27FUQCECdbO6ATVanA16wnW7MrySu2m-Kt83XyATdVoNDls-coxA4UxuX7Rmlr2eGM7ZRKtypt12GziKnZgNglK5c_4mmMP2xeeLU1fneBLkvuHSEnoFjqZnAaI0ei6pW8Jy3k8txI5MucaRkXdPOhCm3Nwj8B9rBAh0hU64NVVb7C28Gz8LCwZkRhtGRY_v2vzcS0DaomK2G63vyQMKx3VUc9_RnkxcI6bwy6xG2GBEjpV8tHxXgw8zGc53_8EMo-9EM1PpjOHHYyaYoubDbxHaSJPwCPqi_OlGbl2h8gIM", publicKey:
} "zhTx5Kr9VNQrXGarf0EXySfbSePBbIQuSOpb07s3pM3q8HKCx-bbd_py8t-JxgwnKAmpGKt6UhOP0FeobGITCwr_O7ATFPrFgTbM-xLYG0JOzxUlPScyqdJ8rFRcSSpevfUyJ6UVTpA3LDQHEzf7kebjfMPeYwpsWuT3c9LP3j0kyPDOBini-LRUpKX3n4ljhJIHzl-Jdv6Z31U65kZRBR1LPwnjcBUg4hoc50i8JZsSLsrUYFfpYVuxM0L4ch0l2-FvPtmZs831mOQgT8e1s7GPB7kJBhrQBagGF3eVnAiImJjslXNQhy4eQr6Nffb5Wa61Tec52LX5-gmoNSuA0PW5yuYGuDO2faULW74u8ZfmMUxd2x3E3M6E0deP_rj27FUQCECdbO6ATVanA16wnW7MrySu2m-Kt83XyATdVoNDls-coxA4UxuX7Rmlr2eGM7ZRKtypt12GziKnZgNglK5c_4mmMP2xeeLU1fneBLkvuHSEnoFjqZnAaI0ei6pW8Jy3k8txI5MucaRkXdPOhCm3Nwj8B9rBAh0hU64NVVb7C28Gz8LCwZkRhtGRY_v2vzcS0DaomK2G63vyQMKx3VUc9_RnkxcI6bwy6xG2GBEjpV8tHxXgw8zGc53_8EMo-9EM1PpjOHHYyaYoubDbxHaSJPwCPqi_OlGbl2h8gIM",
},
}; };
jest.mock("../../providers/index", () => ({ jest.mock("../../providers/index", () => ({
getProviders: () => mockProviders, getProviders: () => mockProviders,
getPublicKeyForProviderAddress: (address) => { getPublicKeyForProviderAddress: (address) => {
if (address !== mockProviders.redstone.address) { if (address !== mockProviders.redstone.address) {
throw new Error( throw new Error(
"Mock getPublicKeyForProviderAddress should not be " "Mock getPublicKeyForProviderAddress should not be " +
+ "called with this address: " + address); "called with this address: " +
address
);
} else { } else {
return mockProviders.redstone.publicKey; return mockProviders.redstone.publicKey;
} }
@@ -33,7 +36,6 @@ async function getSignature(price) {
return buffer.toString("base64"); return buffer.toString("base64");
} }
describe("Testing signature verifier", () => { describe("Testing signature verifier", () => {
const initialPriceData = { const initialPriceData = {
id: "test-id", id: "test-id",
@@ -41,7 +43,7 @@ describe("Testing signature verifier", () => {
provider: mockProviders.redstone.address, provider: mockProviders.redstone.address,
value: Number((Math.random() * 100).toFixed(3)), value: Number((Math.random() * 100).toFixed(3)),
permawebTx: "test-permaweb-tx", permawebTx: "test-permaweb-tx",
source: {"test": 123}, source: { test: 123 },
timestamp: Date.now(), timestamp: Date.now(),
version: "0.4", version: "0.4",
}; };
@@ -54,15 +56,4 @@ describe("Testing signature verifier", () => {
const skipLatestPriceCheck = true; const skipLatestPriceCheck = true;
expect(assertValidSignature(price, skipLatestPriceCheck)).resolves; expect(assertValidSignature(price, skipLatestPriceCheck)).resolves;
}); });
test("Should not verify invalid signature", async () => {
const price = {
...initialPriceData,
signature: "bad-signature",
};
const skipLatestPriceCheck = true;
await expect(assertValidSignature(price, skipLatestPriceCheck))
.rejects
.toThrow();
});
}); });