kylefang/redstone-cache-layer
1
0
Fork 0
mirror of https://github.com/alexgo-io/redstone-cache-layer.git synced 2026-01-12 08:34:36 +08:00
Code Issues Packages Projects Releases Wiki Activity

chore: removed arweave signatures verification

Browse Source
This commit is contained in:
hatskier
2023-03-06 12:07:53 +00:00
parent 6c2de9fbad
commit 750fd637c3
3 changed files with 206 additions and 178 deletions
Download Patch File Download Diff File Expand all files Collapse all files

84
helpers/signature-verifier.ts
View File

@@ -6,7 +6,10 @@ import { getPublicKeyForProviderAddress } from "../providers";
import { logger } from "./logger";
import { PriceWithParams } from "../routes/prices";
export const assertValidSignature = async (price: PriceWithParams, skipNewerPricesCheck = true) => {
export const assertValidSignature = async (
price: PriceWithParams,
skipNewerPricesCheck = true
) => {
if (!skipNewerPricesCheck) {
// Checking if price with a greater timestamp is already in DB
// If so, then we raise an Error, because we allow to publish
@@ -18,9 +21,10 @@ export const assertValidSignature = async (price: PriceWithParams, skipNewerPric
});
if (newerPriceFound) {
throw new Error(
`A newer price found in DB. `
+ `Newer price: ${JSON.stringify(newerPriceFound)}. `
+ `Failed price: ${JSON.stringify(price)}.`);
`A newer price found in DB. ` +
`Newer price: ${JSON.stringify(newerPriceFound)}. ` +
`Failed price: ${JSON.stringify(price)}.`
);
}
} else {
logger.info("Newer prices check skipped");
@@ -29,44 +33,41 @@ export const assertValidSignature = async (price: PriceWithParams, skipNewerPric
// Signature verification
const isValid = await verifySignature(price);
if (!isValid) {
throw new Error(
"Price signature is invalid: " + JSON.stringify(price));
throw new Error("Price signature is invalid: " + JSON.stringify(price));
}
}
};
// We should implement ECDSA signature verification
export const verifySignature = async (price: PriceWithParams) => {
// Time measurement: start
const startTime = Date.now();
// Data preparation
const publicKey = getPublicKeyForProviderAddress(price.provider);
const signedData = getPriceSignedData(price);
const signedBytes = new TextEncoder().encode(signedData);
const signatureBytes = Uint8Array.from(Buffer.from(price.signature, "base64"));
// It allows other providers (not registered in our api) post their data here
// In future, we should implement some kind of authorization for this kind of providers
if (!publicKey) {
return true;
}
const dataPrepTime = Date.now() - startTime;
logger.info(
`Data prep time elapsed: ${dataPrepTime} ms`);
// Signature verification
const validSignature = await Arweave.crypto.verify(
publicKey,
signedBytes,
signatureBytes,
);
// Time measurement: end
const signVerificationTime = Date.now() - startTime;
logger.info(
`Signature verification time elapsed: ${signVerificationTime} ms`);
return validSignature;
return true;
// // Time measurement: start
// const startTime = Date.now();
// // Data preparation
// const publicKey = getPublicKeyForProviderAddress(price.provider);
// const signedData = getPriceSignedData(price);
// const signedBytes = new TextEncoder().encode(signedData);
// const signatureBytes = Uint8Array.from(
// Buffer.from(price.signature, "base64")
// );
// // It allows other providers (not registered in our api) post their data here
// // In future, we should implement some kind of authorization for this kind of providers
// if (!publicKey) {
// return true;
// }
// const dataPrepTime = Date.now() - startTime;
// logger.info(`Data prep time elapsed: ${dataPrepTime} ms`);
// // Signature verification
// const validSignature = await Arweave.crypto.verify(
// publicKey,
// signedBytes,
// signatureBytes
// );
// // Time measurement: end
// const signVerificationTime = Date.now() - startTime;
// logger.info(
// `Signature verification time elapsed: ${signVerificationTime} ms`
// );
// return validSignature;
};
const getPriceSignedData = (price: PriceWithParams) => {
@@ -81,7 +82,6 @@ const getPriceSignedData = (price: PriceWithParams) => {
"provider",
]);
if (shouldApplyDeepSort(price)) {
return JSON.stringify(deepSortObject(priceWithPickedProps));
} else {
@@ -90,5 +90,7 @@ const getPriceSignedData = (price: PriceWithParams) => {
};
const shouldApplyDeepSort = (price: PriceWithParams) => {
return price.version && (price.version === "3" || price.version.includes("."));
return (
price.version && (price.version === "3" || price.version.includes("."))
);
};

269
routes/prices.ts
View File

@@ -2,7 +2,13 @@ import { Request, Router } from "express";
import asyncHandler from "express-async-handler";
import { FilterQuery, PipelineStage, Document } from "mongoose";
import _ from "lodash";
import { bigLimitWithMargin, defaultLimit, cacheTTLMilliseconds, maxLimitForPrices, enableLiteMode } from "../config";
import {
bigLimitWithMargin,
defaultLimit,
cacheTTLMilliseconds,
maxLimitForPrices,
enableLiteMode,
} from "../config";
import { Price, priceToObject } from "../models/price";
import { logEvent } from "../helpers/amplitude-event-logger";
import { assertValidSignature } from "../helpers/signature-verifier";
@@ -10,7 +16,8 @@ import { priceParamsToPriceObj, getProviderFromParams } from "../utils";
import { logger } from "../helpers/logger";
import { tryCleanCollection } from "../helpers/mongo";
export interface PriceWithParams extends Omit<Price, "signature" | "evmSignature" | "liteEvmSignature"> {
export interface PriceWithParams
extends Omit<Price, "signature" | "evmSignature" | "liteEvmSignature"> {
limit?: number;
offset?: number;
fromTimestamp?: number;
@@ -26,7 +33,7 @@ export interface PriceWithParams extends Omit<Price, "signature" | "evmSignature
const addSinglePrice = async (params: PriceWithParams) => {
const price = new Price(priceParamsToPriceObj(params));
await price.save();
}
};
const getLatestPricesForSingleToken = async (params: PriceWithParams) => {
validateParams(params, ["symbol"]);
@@ -39,7 +46,7 @@ const getLatestPricesForSingleToken = async (params: PriceWithParams) => {
offset: params.offset,
});
return prices.map(priceToObject);
}
};
const addSeveralPrices = async (params: PriceWithParams[]) => {
const ops = [];
@@ -51,7 +58,7 @@ const addSeveralPrices = async (params: PriceWithParams[]) => {
});
}
await Price.bulkWrite(ops);
}
};
const getPriceForManyTokens = async (params: PriceWithParams) => {
// Parsing symbols params
@@ -76,7 +83,6 @@ const getPriceForManyTokens = async (params: PriceWithParams) => {
// Building tokens object
const tokensResponse = {};
for (const price of prices) {
// We currently filter here
if (tokens.length === 0 || tokens.includes(price.symbol)) {
if (tokensResponse[price.symbol] === undefined) {
@@ -90,14 +96,14 @@ const getPriceForManyTokens = async (params: PriceWithParams) => {
}
return tokensResponse;
}
};
const getHistoricalPricesForSingleToken = async (params: PriceWithParams) => {
validateParams(params, ["symbol"]);
const filters = {
symbol: params.symbol,
provider: params.provider,
timestamp: { $lte: params.toTimestamp } as { $lte: number, $gte?: number },
timestamp: { $lte: params.toTimestamp } as { $lte: number; $gte?: number },
};
if (params.fromTimestamp) {
@@ -110,13 +116,26 @@ const getHistoricalPricesForSingleToken = async (params: PriceWithParams) => {
limit: params.limit || defaultLimit,
});
return prices.map(priceToObject);
}
};
// This function is used to return data for charts
const getPricesInTimeRangeForSingleToken = async (params: PriceWithParams) => {
validateParams(params,
["symbol", "fromTimestamp", "toTimestamp", "interval", "provider"]);
const { symbol, provider, fromTimestamp, toTimestamp, interval, offset, limit } = params;
validateParams(params, [
"symbol",
"fromTimestamp",
"toTimestamp",
"interval",
"provider",
]);
const {
symbol,
provider,
fromTimestamp,
toTimestamp,
interval,
offset,
limit,
} = params;
const pipeline = [
{
$match: {
@@ -165,7 +184,8 @@ const getPricesInTimeRangeForSingleToken = async (params: PriceWithParams) => {
// so we can not filter them out in DB query level
const millisecondsInMinute = 60 * 1000;
if (interval > millisecondsInMinute && prices.length > 0) {
let filteredPrices = [], prevTimestamp = prices[0].timestamp;
let filteredPrices = [],
prevTimestamp = prices[0].timestamp;
for (const price of prices) {
const diff = price.timestamp - prevTimestamp;
if (diff === 0 || diff > millisecondsInMinute) {
@@ -177,10 +197,10 @@ const getPricesInTimeRangeForSingleToken = async (params: PriceWithParams) => {
}
return prices;
}
};
const getPrices = async ({
filters = {},
filters = {},
limit = defaultLimit,
offset,
}: {
@@ -189,21 +209,23 @@ const getPrices = async ({
offset: number;
}) => {
// Query building
let pricesQuery = Price
.find(filters)
let pricesQuery = Price.find(filters)
.sort({ timestamp: -1 })
.limit(Math.min(Number(limit), maxLimitForPrices));
if (offset) {
pricesQuery = pricesQuery.skip(Number(offset));
}
// Query executing
const prices = await pricesQuery.exec();
return prices;
}
};
const validateParams = (params: Record<string, any>, requiredParams: string[]) => {
const validateParams = (
params: Record<string, any>,
requiredParams: string[]
) => {
const errors = [];
for (const requiredParam of requiredParams) {
if (params[requiredParam] === undefined) {
@@ -214,7 +236,7 @@ const validateParams = (params: Record<string, any>, requiredParams: string[]) =
if (errors.length > 0) {
throw new Error(JSON.stringify(errors));
}
}
};
const getPricesCount = (reqBody: PriceWithParams) => {
if (Array.isArray(reqBody)) {
@@ -222,15 +244,15 @@ const getPricesCount = (reqBody: PriceWithParams) => {
} else {
return 1;
}
}
};
const getIp = (req: Request) => {
const ip = req.ip;
logger.info("Request IP address: " + ip);
return ip;
}
};
interface QueryParams extends PriceWithParams{
interface QueryParams extends PriceWithParams {
provider: string;
symbols?: string;
tokens?: string[];
@@ -241,110 +263,123 @@ export const prices = (router: Router) => {
/**
* This endpoint is used for fetching prices data.
* It is used in redstone-api
*/
router.get("/prices", asyncHandler(async (req, res) => {
// Request validation
const params = req.query as unknown as QueryParams;
*/
router.get(
"/prices",
asyncHandler(async (req, res) => {
// Request validation
const params = req.query as unknown as QueryParams;
// Saving API read event in amplitude
logEvent({
eventName: "api-get-request",
eventProps: params,
ip: getIp(req),
});
// Saving API read event in amplitude
logEvent({
eventName: "api-get-request",
eventProps: params,
ip: getIp(req),
});
// Getting provider details
const providerDetails = await getProviderFromParams(params);
params.provider = providerDetails.address;
params.providerPublicKey = providerDetails.publicKey;
// Getting provider details
const providerDetails = await getProviderFromParams(params);
params.provider = providerDetails.address;
params.providerPublicKey = providerDetails.publicKey;
// If query params contain "symbol" we fetch price for this symbol
if (params.symbol !== undefined) {
let body: _.Omit<Document<unknown, any, Price> & Price & { providerPublicKey: any; }, "_id" | "__v">[];
if (params.interval !== undefined) {
body = await getPricesInTimeRangeForSingleToken(params);
} else if (params.toTimestamp !== undefined) {
body = await getHistoricalPricesForSingleToken(params);
} else {
body = await getLatestPricesForSingleToken(params);
// If query params contain "symbol" we fetch price for this symbol
if (params.symbol !== undefined) {
let body: _.Omit<
Document<unknown, any, Price> & Price & { providerPublicKey: any },
"_id" | "__v"
>[];
if (params.interval !== undefined) {
body = await getPricesInTimeRangeForSingleToken(params);
} else if (params.toTimestamp !== undefined) {
body = await getHistoricalPricesForSingleToken(params);
} else {
body = await getLatestPricesForSingleToken(params);
}
return res.json(body);
}
return res.json(body);
}
// Otherwise we fetch prices for many symbols
else {
let tokens = [];
if (params.symbols !== undefined) {
tokens = params.symbols.split(",");
// Otherwise we fetch prices for many symbols
else {
let tokens = [];
if (params.symbols !== undefined) {
tokens = params.symbols.split(",");
}
params.tokens = tokens;
const body = await getPriceForManyTokens(params);
return res.json(body);
}
params.tokens = tokens;
const body = await getPriceForManyTokens(params);
return res.json(body);
}
}));
})
);
/**
* This endpoint is used for posting a new price data.
* It supports posting a single price and several prices
*/
router.post("/prices", asyncHandler(async (req, res) => {
const reqBody = req.body as PriceWithParams;
let pricesSavedCount = 0;
*/
router.post(
"/prices",
asyncHandler(async (req, res) => {
const reqBody = req.body as PriceWithParams;
let pricesSavedCount = 0;
// Saving API post event in amplitude
logEvent({
eventName: "api-post-request",
eventProps: {
pricesCount: getPricesCount(reqBody),
},
ip: getIp(req),
});
// Saving API post event in amplitude
logEvent({
eventName: "api-post-request",
eventProps: {
pricesCount: getPricesCount(reqBody),
},
ip: getIp(req),
});
if (Array.isArray(reqBody)) {
const invalidPrices = reqBody.filter(p => !p.value);
if (invalidPrices.length > 0) {
logger.error(
"Invalid prices with empty value: " + JSON.stringify(invalidPrices));
if (Array.isArray(reqBody)) {
const invalidPrices = reqBody.filter((p) => !p.value);
if (invalidPrices.length > 0) {
logger.error(
"Invalid prices with empty value: " + JSON.stringify(invalidPrices)
);
}
// Validating a signature of a randomly selected price
// We got rid of arweave signatures
// const priceToVerify = _.sample(reqBody);
// await assertValidSignature(priceToVerify);
// Adding several prices
await addSeveralPrices(reqBody);
// Cleaning older prices for the same provider after posting
// new ones in the lite mode
if (enableLiteMode) {
await tryCleanCollection(Price, {
provider: reqBody[0].provider,
timestamp: {
$lt: Number(reqBody[0].timestamp) - cacheTTLMilliseconds,
},
});
}
pricesSavedCount = reqBody.length;
} else {
// Validating the price signature
await assertValidSignature(reqBody);
// Adding a single price
await addSinglePrice(reqBody);
pricesSavedCount = 1;
// Cleaning prices for the same provider and symbol before posting
// a new one in the lite mode
if (enableLiteMode) {
await tryCleanCollection(Price, {
provider: reqBody.provider,
symbol: reqBody.symbol,
timestamp: {
$lt: Number(reqBody.timestamp) - cacheTTLMilliseconds,
},
});
}
}
// Validating a signature of a randomly selected price
// We got rid of arweave signatures
// const priceToVerify = _.sample(reqBody);
// await assertValidSignature(priceToVerify);
// Adding several prices
await addSeveralPrices(reqBody);
// Cleaning older prices for the same provider after posting
// new ones in the lite mode
if (enableLiteMode) {
await tryCleanCollection(Price, {
provider: reqBody[0].provider,
timestamp: { $lt: Number(reqBody[0].timestamp) - cacheTTLMilliseconds },
});
}
pricesSavedCount = reqBody.length;
} else {
// Validating the price signature
await assertValidSignature(reqBody);
// Adding a single price
await addSinglePrice(reqBody);
pricesSavedCount = 1;
// Cleaning prices for the same provider and symbol before posting
// a new one in the lite mode
if (enableLiteMode) {
await tryCleanCollection(Price, {
provider: reqBody.provider,
symbol: reqBody.symbol,
timestamp: { $lt: Number(reqBody.timestamp) - cacheTTLMilliseconds },
});
}
}
return res.json({ msg: `Prices saved. count: ${pricesSavedCount}` });
}));
return res.json({ msg: `Prices saved. count: ${pricesSavedCount}` });
})
);
};

31
test/helpers/signature-verifier.test.ts
View File

@@ -4,18 +4,21 @@ import { assertValidSignature } from "../../helpers/signature-verifier";
import testJwk from "./test-jwk.json";
const mockProviders = {
"redstone": {
"address": "MlV6DeOtRmakDOf6vgOBlif795tcWimgyPsYYNQ8q1Y",
"publicKey": "zhTx5Kr9VNQrXGarf0EXySfbSePBbIQuSOpb07s3pM3q8HKCx-bbd_py8t-JxgwnKAmpGKt6UhOP0FeobGITCwr_O7ATFPrFgTbM-xLYG0JOzxUlPScyqdJ8rFRcSSpevfUyJ6UVTpA3LDQHEzf7kebjfMPeYwpsWuT3c9LP3j0kyPDOBini-LRUpKX3n4ljhJIHzl-Jdv6Z31U65kZRBR1LPwnjcBUg4hoc50i8JZsSLsrUYFfpYVuxM0L4ch0l2-FvPtmZs831mOQgT8e1s7GPB7kJBhrQBagGF3eVnAiImJjslXNQhy4eQr6Nffb5Wa61Tec52LX5-gmoNSuA0PW5yuYGuDO2faULW74u8ZfmMUxd2x3E3M6E0deP_rj27FUQCECdbO6ATVanA16wnW7MrySu2m-Kt83XyATdVoNDls-coxA4UxuX7Rmlr2eGM7ZRKtypt12GziKnZgNglK5c_4mmMP2xeeLU1fneBLkvuHSEnoFjqZnAaI0ei6pW8Jy3k8txI5MucaRkXdPOhCm3Nwj8B9rBAh0hU64NVVb7C28Gz8LCwZkRhtGRY_v2vzcS0DaomK2G63vyQMKx3VUc9_RnkxcI6bwy6xG2GBEjpV8tHxXgw8zGc53_8EMo-9EM1PpjOHHYyaYoubDbxHaSJPwCPqi_OlGbl2h8gIM",
}
redstone: {
address: "MlV6DeOtRmakDOf6vgOBlif795tcWimgyPsYYNQ8q1Y",
publicKey:
"zhTx5Kr9VNQrXGarf0EXySfbSePBbIQuSOpb07s3pM3q8HKCx-bbd_py8t-JxgwnKAmpGKt6UhOP0FeobGITCwr_O7ATFPrFgTbM-xLYG0JOzxUlPScyqdJ8rFRcSSpevfUyJ6UVTpA3LDQHEzf7kebjfMPeYwpsWuT3c9LP3j0kyPDOBini-LRUpKX3n4ljhJIHzl-Jdv6Z31U65kZRBR1LPwnjcBUg4hoc50i8JZsSLsrUYFfpYVuxM0L4ch0l2-FvPtmZs831mOQgT8e1s7GPB7kJBhrQBagGF3eVnAiImJjslXNQhy4eQr6Nffb5Wa61Tec52LX5-gmoNSuA0PW5yuYGuDO2faULW74u8ZfmMUxd2x3E3M6E0deP_rj27FUQCECdbO6ATVanA16wnW7MrySu2m-Kt83XyATdVoNDls-coxA4UxuX7Rmlr2eGM7ZRKtypt12GziKnZgNglK5c_4mmMP2xeeLU1fneBLkvuHSEnoFjqZnAaI0ei6pW8Jy3k8txI5MucaRkXdPOhCm3Nwj8B9rBAh0hU64NVVb7C28Gz8LCwZkRhtGRY_v2vzcS0DaomK2G63vyQMKx3VUc9_RnkxcI6bwy6xG2GBEjpV8tHxXgw8zGc53_8EMo-9EM1PpjOHHYyaYoubDbxHaSJPwCPqi_OlGbl2h8gIM",
},
};
jest.mock("../../providers/index", () => ({
getProviders: () => mockProviders,
getPublicKeyForProviderAddress: (address) => {
if (address !== mockProviders.redstone.address) {
throw new Error(
"Mock getPublicKeyForProviderAddress should not be "
+ "called with this address: " + address);
"Mock getPublicKeyForProviderAddress should not be " +
"called with this address: " +
address
);
} else {
return mockProviders.redstone.publicKey;
}
@@ -33,7 +36,6 @@ async function getSignature(price) {
return buffer.toString("base64");
}
describe("Testing signature verifier", () => {
const initialPriceData = {
id: "test-id",
@@ -41,12 +43,12 @@ describe("Testing signature verifier", () => {
provider: mockProviders.redstone.address,
value: Number((Math.random() * 100).toFixed(3)),
permawebTx: "test-permaweb-tx",
source: {"test": 123},
source: { test: 123 },
timestamp: Date.now(),
version: "0.4",
};
test("Should verify valid signature", async () => {
test("Should verify valid signature", async () => {
const price = {
...initialPriceData,
signature: await getSignature(initialPriceData),
@@ -54,15 +56,4 @@ describe("Testing signature verifier", () => {
const skipLatestPriceCheck = true;
expect(assertValidSignature(price, skipLatestPriceCheck)).resolves;
});
test("Should not verify invalid signature", async () => {
const price = {
...initialPriceData,
signature: "bad-signature",
};
const skipLatestPriceCheck = true;
await expect(assertValidSignature(price, skipLatestPriceCheck))
.rejects
.toThrow();
});
});