From bd1d8c61ae5eaae09d032aeb92227b16e16aed69 Mon Sep 17 00:00:00 2001 From: Pelotfr Date: Wed, 1 Nov 2023 13:18:50 +0100 Subject: [PATCH] Support new /sales/aggregate endpoint --- src/fetch/GET.ts | 2 ++ src/fetch/openapi.ts | 72 +++++++++++++++++++++++++++++++++++++++++++- src/queries.spec.ts | 14 +++++---- src/queries.ts | 47 +++++++++++++++++++++++++++-- 4 files changed, 126 insertions(+), 9 deletions(-) diff --git a/src/fetch/GET.ts b/src/fetch/GET.ts index 0c27548..c90aefc 100644 --- a/src/fetch/GET.ts +++ b/src/fetch/GET.ts @@ -2,6 +2,7 @@ import { registry } from "../prometheus.js"; import openapi from "./openapi.js"; import health from "./health.js"; import sales from "./sales.js"; +import aggregate from "./aggregate.js"; import * as prometheus from "../prometheus.js"; import { logger } from "../logger.js"; import swaggerHtml from "../../swagger/index.html" @@ -16,6 +17,7 @@ export default async function (req: Request) { if ( pathname === "/metrics" ) return new Response(await registry.metrics(), {headers: {"Content-Type": registry.contentType}}); if ( pathname === "/openapi" ) return new Response(openapi, {headers: {"Content-Type": "application/json"}}); if ( pathname === "/sales" ) return sales(req); + if ( pathname === "/sales/aggregate" ) return aggregate(req); logger.warn(`Not found: ${pathname}`); prometheus.request_error.inc({pathname, status: 404}); return new Response("Not found", { status: 404 }); diff --git a/src/fetch/openapi.ts b/src/fetch/openapi.ts index f7114f8..0730013 100644 --- a/src/fetch/openapi.ts +++ b/src/fetch/openapi.ts @@ -2,7 +2,7 @@ import pkg from "../../package.json" assert { type: "json" }; import { OpenApiBuilder, SchemaObject, ExampleObject, ParameterObject } from "openapi3-ts/oas31"; import { config } from "../config.js"; -import { getSale } from "../queries.js"; +import { getSale, getAggregate } from "../queries.js"; import { registry } from "../prometheus.js"; import { makeQuery } from "../clickhouse/makeQuery.js"; @@ -14,6 +14,7 @@ const TAGS = { } as const; const sale_example = (await makeQuery(await getSale( new URLSearchParams({limit: "2"})))).data; +const aggregate_example = (await makeQuery(await getAggregate( new URLSearchParams({aggregate_function: "count"})))).data; const timestampSchema: SchemaObject = { anyOf: [ {type: "number"}, @@ -154,6 +155,75 @@ export default new OpenApiBuilder() }, }, }) + .addPath("/sales/aggregate", { + get: { + tags: [TAGS.USAGE], + summary: "Get aggregate of sales", + description: "Get aggregate of sales by `collection_name`, `timestamp` or `block_number`", + parameters: [ + { + name: "aggregate_function", + in: "query", + description: "Aggregate function", + required: true, + schema: {enum: ['count', 'min', 'max', 'sum', 'avg', 'median'] }, + }, + { + name: "aggregate_column", + in: "query", + description: "Aggregate column", + required: false, + schema: {enum: ['listing_price_amount'] }, + }, + { + name: "collection_name", + in: "query", + description: "Filter by collection name (ex: 'pomelo')", + required: false, + schema: {type: "string"}, + }, + { + name: 'timestamp', + in: 'query', + description: 'Filter by exact timestamp', + required: false, + schema: timestampSchema, + examples: timestampExamples, + }, + { + name: "block_number", + description: "Filter by Block number (ex: 18399498)", + in: "query", + required: false, + schema: { type: "number" }, + }, + ...["greater_or_equals_by_timestamp", "greater_by_timestamp", "less_or_equals_by_timestamp", "less_by_timestamp"].map(name => { + return { + name, + in: "query", + description: "Filter " + name.replace(/_/g, " "), + required: false, + schema: timestampSchema, + examples: timestampExamples, + } as ParameterObject + }), + ...["greater_or_equals_by_block_number", "greater_by_block_number", "less_or_equals_by_block_number", "less_by_block_number"].map(name => { + return { + name, + in: "query", + description: "Filter " + name.replace(/_/g, " "), + required: false, + schema: { type: "number" }, + } as ParameterObject + }), + ], + responses: { + 200: { description: "Aggregate of sales", content: { "text/plain": { example: aggregate_example} } }, + 400: { description: "Bad request" }, + }, + }, + + }) .addPath("/health", { get: { tags: [TAGS.HEALTH], diff --git a/src/queries.spec.ts b/src/queries.spec.ts index 802b010..2eac5e0 100644 --- a/src/queries.spec.ts +++ b/src/queries.spec.ts @@ -1,14 +1,11 @@ // from: https://github.com/pinax-network/substreams-clock-api/blob/main/src/queries.spec.ts import { expect, test } from "bun:test"; -import { getSalesCount, getSale } from "./queries.js"; +import { getSale, getAggregate } from "./queries.js"; const collection_name = "pomelo"; - -test("getSalesCount", () => { - expect(getSalesCount(new URLSearchParams({collection_name}))) - .toBe(`SELECT count(sale_id) FROM Sales WHERE collection_name = '${collection_name}'`); -}); +const aggregate_function = "min"; +const aggregate_column = "listing_price_amount"; test("getSale", () => { expect(getSale(new URLSearchParams({collection_name}))) @@ -22,4 +19,9 @@ JOIN blocks ON blocks.block_id = s.block_id WHERE (collection_name == '${collect expect(getSale(new URLSearchParams({asset_id_in_asset_ids: '2199024044581'}))) .toBe(`SELECT * FROM Sales JOIN blocks ON blocks.block_id = Sales.block_id WHERE (has(asset_ids, 2199024044581)) ORDER BY sale_id DESC LIMIT 1`);*/ +}); + +test("getAggregate", () => { + expect(getAggregate(new URLSearchParams({aggregate_function, aggregate_column}))) + .toBe(`SELECT ${aggregate_function}(${aggregate_column}) FROM Sales`); }); \ No newline at end of file diff --git a/src/queries.ts b/src/queries.ts index ba7f0d9..fa4ec35 100644 --- a/src/queries.ts +++ b/src/queries.ts @@ -48,8 +48,8 @@ s.collection_name as collection_name, template_id, block_number, timestamp FROM` if (asset_id_in_asset_ids) where.push(`has(asset_ids, ${asset_id_in_asset_ids})`) // equals - const collection_name = searchParams.get("collection_name"); - const sale_id = searchParams.get("sale_id"); + const collection_name = searchParams.get('collection_name'); + const sale_id = searchParams.get('sale_id'); const block_number = searchParams.get('block_number'); const timestamp = parseTimestamp(searchParams.get('timestamp')); const listing_price_amount = searchParams.get('listing_price_amount'); @@ -73,5 +73,48 @@ s.collection_name as collection_name, template_id, block_number, timestamp FROM` const sort_by = searchParams.get("sort_by"); query += ` ORDER BY sale_id ${sort_by ?? DEFAULT_SORT_BY}` query += ` LIMIT ${limit}` + return query; +} + +export function getAggregate(searchParams: URLSearchParams) { + // SQL Query + let query = `SELECT`; + + // Aggregate Function + const aggregate_function = searchParams.get("aggregate_function"); + const aggregate_column = searchParams.get("aggregate_column"); + if (aggregate_function == "count" && !aggregate_column) query += ` count()`; + else if (aggregate_function && aggregate_column) query += ` ${aggregate_function}(${aggregate_column})`; + else throw new Error("Invalid aggregate function or column"); + + + query += ` FROM Sales`; + const where = []; + // Clickhouse Operators + // https://clickhouse.com/docs/en/sql-reference/operators + const operators = [ + ["greater_or_equals", ">="], + ["greater", ">"], + ["less_or_equals", "<="], + ["less", "<"], + ] + for ( const [key, operator] of operators ) { + const block_number = searchParams.get(`${key}_by_block_number`); + const timestamp = parseTimestamp(searchParams.get(`${key}_by_timestamp`)); + if (block_number) where.push(`block_number ${operator} ${block_number}`); + if (timestamp) where.push(`toUnixTimestamp(timestamp) ${operator} ${timestamp}`); + } + + // equals + const collection_name = searchParams.get('collection_name'); + const block_number = searchParams.get('block_number'); + const timestamp = parseTimestamp(searchParams.get('timestamp')); + if (collection_name) where.push(`collection_name == '${collection_name}'`); + if (block_number) where.push(`block_number == '${block_number}'`); + if (timestamp) where.push(`toUnixTimestamp(timestamp) == ${timestamp}`); + + // Join WHERE statements with AND + if ( where.length ) query += ` WHERE (${where.join(' AND ')})`; + return query; } \ No newline at end of file