diff --git a/apps/aggregator/README.md b/apps/aggregator/README.md index bd6bc45..9c054d7 100644 --- a/apps/aggregator/README.md +++ b/apps/aggregator/README.md @@ -426,6 +426,100 @@ export class MyCustomAggregator implements IAggregator { } } ``` +apps/aggregator/ +├── src/ +│ ├── main.ts # Application entry point +│ ├── app.module.ts # Root module +│ ├── interfaces/ # Type definitions +│ │ ├── normalized-price.interface.ts +│ │ └── normalizer.interface.ts +│ ├── normalizers/ # Source-specific normalizers +│ │ ├── base.normalizer.ts +│ │ ├── alpha-vantage.normalizer.ts +│ │ ├── finnhub.normalizer.ts +│ │ ├── yahoo-finance.normalizer.ts +│ │ └── mock.normalizer.ts +│ ├── services/ # Business logic +│ │ └── normalization.service.ts +│ ├── modules/ # Feature modules +│ │ └── normalization.module.ts +│ └── exceptions/ # Custom exceptions +│ └── normalization.exception.ts +├── .env.example # Example environment variables +├── nest-cli.json # NestJS CLI configuration +├── package.json # Dependencies and scripts +├── tsconfig.json # TypeScript configuration +└── README.md # This file +``` + +## Data Normalization + +### NormalizedPrice Interface + +The standard internal format for normalized price data: + +```typescript +interface NormalizedPrice { + symbol: string; // Normalized ticker (e.g., 'AAPL') + price: number; // Price rounded to 4 decimal places + timestamp: string; // ISO 8601 UTC (e.g., '2024-01-15T14:30:00.000Z') + originalTimestamp: number; // Original Unix timestamp in milliseconds + source: NormalizedSource; // Enum: 'alpha_vantage' | 'finnhub' | 'yahoo_finance' | 'mock' + metadata: { + originalSource: string; // Original source string + originalSymbol: string; // Original symbol before normalization + normalizedAt: string; // When normalization occurred + normalizerVersion: string; // Version of normalizer used + wasTransformed: boolean; // Whether transformations were applied + transformations: string[]; // List of transformations applied + }; +} +``` + +### Supported Sources and Transformations + +| Source | Detected By | Symbol Transformations | +|--------|-------------|------------------------| +| **Alpha Vantage** | `alphavantage`, `alpha_vantage`, `alpha-vantage` | Removes `.US`, `.NYSE`, `.NASDAQ`, `.LSE`, `.TSX`, `.ASX`, `.HK` suffixes | +| **Finnhub** | `finnhub` | Removes `US-`, `CRYPTO-`, `FX-`, `INDICES-` prefixes | +| **Yahoo Finance** | `yahoo`, `yahoofinance`, `yahoo_finance`, `yahoo-finance` | Removes `.L`, `.T`, `.AX`, `.HK`, `.SI`, `.KS`, `.TW`, `.NS`, `.BO`, `.TO`, `.DE`, `.PA` suffixes; removes `^` index prefix | +| **Mock** | `mock` | Basic cleanup (trim, uppercase) | + +### Common Transformations + +All normalizers apply these transformations: +- **Symbol**: Trimmed and uppercased +- **Price**: Rounded to 4 decimal places +- **Timestamp**: Converted to ISO 8601 UTC format + +### Usage Example + +```typescript +import { NormalizationService } from './services/normalization.service'; + +// Inject via NestJS DI +constructor(private readonly normalizationService: NormalizationService) {} + +// Normalize a single price +const rawPrice = { + symbol: 'AAPL.US', + price: 150.123456, + timestamp: Date.now(), + source: 'AlphaVantage', +}; +const normalized = this.normalizationService.normalize(rawPrice); +// Result: { symbol: 'AAPL', price: 150.1235, timestamp: '2024-01-15T14:30:00.000Z', ... } + +// Normalize multiple prices (skips failures) +const results = this.normalizationService.normalizeMany(rawPrices); + +// Normalize with error tracking +const { successful, failed } = this.normalizationService.normalizeManyWithErrors(rawPrices); +``` + +## Status + +🚧 Under construction - Aggregation and filtering logic will be implemented in subsequent issues. 2. Register in `AggregationService` constructor: diff --git a/apps/aggregator/package.json b/apps/aggregator/package.json index cffe802..463083e 100644 --- a/apps/aggregator/package.json +++ b/apps/aggregator/package.json @@ -21,6 +21,7 @@ "@nestjs/core": "^10.0.0", "@nestjs/event-emitter": "^3.0.1", "@nestjs/platform-express": "^10.0.0", + "@oracle-stocks/shared": "*", "@nestjs/terminus": "^10.0.0", "axios": "^1.6.0", "ioredis": "^5.3.2", @@ -69,6 +70,9 @@ "**/*.(t|j)s" ], "coverageDirectory": "../coverage", - "testEnvironment": "node" + "testEnvironment": "node", + "moduleNameMapper": { + "^@oracle-stocks/shared$": "/../../packages/shared/src" + } } } diff --git a/apps/aggregator/src/__mocks__/raw-price.fixtures.ts b/apps/aggregator/src/__mocks__/raw-price.fixtures.ts new file mode 100644 index 0000000..ee296f1 --- /dev/null +++ b/apps/aggregator/src/__mocks__/raw-price.fixtures.ts @@ -0,0 +1,101 @@ +import { RawPrice } from '@oracle-stocks/shared'; + +/** + * Test fixtures for raw price data from different sources + */ +export const mockRawPrices: Record = { + alphaVantage: { + symbol: 'AAPL.US', + price: 150.1234567, + timestamp: 1705329000000, // 2024-01-15T14:30:00.000Z + source: 'AlphaVantage', + }, + alphaVantageNYSE: { + symbol: 'MSFT.NYSE', + price: 380.5, + timestamp: 1705330200000, + source: 'alpha_vantage', + }, + finnhub: { + symbol: 'US-GOOGL', + price: 140.999, + timestamp: 1705330200000, + source: 'Finnhub', + }, + finnhubCrypto: { + symbol: 'CRYPTO-BTC', + price: 42000.0, + timestamp: 1705330200000, + source: 'finnhub', + }, + yahooFinance: { + symbol: 'MSFT.L', + price: 380.12345, + timestamp: 1705330200000, + source: 'Yahoo Finance', + }, + yahooFinanceIndex: { + symbol: '^DJI', + price: 37500.0, + timestamp: 1705330200000, + source: 'yahoo_finance', + }, + yahooFinanceAustralia: { + symbol: 'BHP.AX', + price: 45.67, + timestamp: 1705330200000, + source: 'YahooFinance', + }, + mock: { + symbol: 'TSLA', + price: 250.5, + timestamp: 1705330200000, + source: 'MockProvider', + }, + mockLowercase: { + symbol: ' aapl ', + price: 150.0, + timestamp: 1705330200000, + source: 'mock', + }, + unknown: { + symbol: 'BTC', + price: 42000.0, + timestamp: 1705330200000, + source: 'UnknownSource', + }, +}; + +/** + * Malformed price data for testing validation + */ +export const malformedPrices: Array | null | undefined> = [ + { symbol: '', price: 100, timestamp: Date.now(), source: 'Test' }, + { symbol: 'TEST', price: NaN, timestamp: Date.now(), source: 'Test' }, + { symbol: 'TEST', price: -100, timestamp: Date.now(), source: 'Test' }, + { symbol: 'TEST', price: 100, timestamp: null as unknown as number, source: 'Test' }, + { symbol: 'TEST', price: 100, timestamp: Date.now(), source: '' }, + { price: 100, timestamp: Date.now(), source: 'Test' } as Partial, + { symbol: 'TEST', timestamp: Date.now(), source: 'Test' } as Partial, + null, + undefined, +]; + +/** + * Valid raw prices for batch testing + */ +export const validRawPrices: RawPrice[] = [ + mockRawPrices.alphaVantage, + mockRawPrices.finnhub, + mockRawPrices.yahooFinance, + mockRawPrices.mock, +]; + +/** + * Mixed valid and invalid prices for error handling tests + */ +export const mixedRawPrices: RawPrice[] = [ + mockRawPrices.alphaVantage, + mockRawPrices.unknown, + mockRawPrices.finnhub, +]; diff --git a/apps/aggregator/src/app.module.ts b/apps/aggregator/src/app.module.ts index c54cbf1..071749a 100644 --- a/apps/aggregator/src/app.module.ts +++ b/apps/aggregator/src/app.module.ts @@ -1,4 +1,5 @@ import { Module } from '@nestjs/common'; +import { NormalizationModule } from './modules/normalization.module'; import { ConfigModule } from '@nestjs/config'; import { AggregationService } from './services/aggregation.service'; import { WeightedAverageAggregator } from './strategies/aggregators/weighted-average.aggregator'; @@ -10,6 +11,7 @@ import { DebugModule } from './debug/debug.module'; @Module({ imports: [ + NormalizationModule, ConfigModule.forRoot({ isGlobal: true, envFilePath: '.env' }), HealthModule, MetricsModule, diff --git a/apps/aggregator/src/exceptions/index.ts b/apps/aggregator/src/exceptions/index.ts new file mode 100644 index 0000000..7e5e647 --- /dev/null +++ b/apps/aggregator/src/exceptions/index.ts @@ -0,0 +1 @@ +export * from './normalization.exception'; diff --git a/apps/aggregator/src/exceptions/normalization.exception.ts b/apps/aggregator/src/exceptions/normalization.exception.ts new file mode 100644 index 0000000..c74e686 --- /dev/null +++ b/apps/aggregator/src/exceptions/normalization.exception.ts @@ -0,0 +1,36 @@ +import { RawPrice } from '@oracle-stocks/shared'; + +/** + * Base exception for normalization errors + */ +export class NormalizationException extends Error { + constructor( + message: string, + public readonly rawPrice?: RawPrice, + public readonly cause?: Error, + ) { + super(message); + this.name = 'NormalizationException'; + Error.captureStackTrace(this, this.constructor); + } +} + +/** + * Exception for validation failures + */ +export class ValidationException extends NormalizationException { + constructor(message: string, rawPrice?: RawPrice) { + super(message, rawPrice); + this.name = 'ValidationException'; + } +} + +/** + * Exception when no normalizer is found for a source + */ +export class NoNormalizerFoundException extends NormalizationException { + constructor(source: string, rawPrice?: RawPrice) { + super(`No normalizer found for source: ${source}`, rawPrice); + this.name = 'NoNormalizerFoundException'; + } +} diff --git a/apps/aggregator/src/interfaces/index.ts b/apps/aggregator/src/interfaces/index.ts new file mode 100644 index 0000000..f90f132 --- /dev/null +++ b/apps/aggregator/src/interfaces/index.ts @@ -0,0 +1,2 @@ +export * from './normalized-price.interface'; +export * from './normalizer.interface'; diff --git a/apps/aggregator/src/interfaces/normalized-price.interface.ts b/apps/aggregator/src/interfaces/normalized-price.interface.ts index 9cc57ef..4e45e80 100644 --- a/apps/aggregator/src/interfaces/normalized-price.interface.ts +++ b/apps/aggregator/src/interfaces/normalized-price.interface.ts @@ -1,3 +1,62 @@ +/** + * Enum for standardized source identifiers + */ +export enum NormalizedSource { + ALPHA_VANTAGE = 'alpha_vantage', + FINNHUB = 'finnhub', + YAHOO_FINANCE = 'yahoo_finance', + MOCK = 'mock', + UNKNOWN = 'unknown', +} + +/** + * Metadata tracking normalization processing + */ +export interface NormalizationMetadata { + /** Original source string before normalization */ + originalSource: string; + + /** Original symbol before normalization (e.g., 'AAPL.US') */ + originalSymbol: string; + + /** ISO 8601 timestamp when normalization was performed */ + normalizedAt: string; + + /** Version of the normalization logic used */ + normalizerVersion: string; + + /** Whether any transformations were applied */ + wasTransformed: boolean; + + /** List of transformations applied (for debugging/audit) */ + transformations: string[]; +} + +/** + * Represents a fully normalized price record with standard formatting + * and metadata for audit/tracking purposes. + * Used as output from the normalization service. + */ +export interface NormalizedPriceRecord { + /** Normalized ticker symbol (e.g., 'AAPL' - stripped of exchange suffixes) */ + symbol: string; + + /** Price value normalized to minimum 4 decimal places */ + price: number; + + /** ISO 8601 UTC timestamp string (e.g., '2024-01-15T14:30:00.000Z') */ + timestamp: string; + + /** Original Unix timestamp in milliseconds (preserved for precision) */ + originalTimestamp: number; + + /** Normalized source identifier */ + source: NormalizedSource; + + /** Metadata for tracking and audit purposes */ + metadata: NormalizationMetadata; +} + /** * Normalized price data structure from various sources * This is the input to the aggregation engine diff --git a/apps/aggregator/src/interfaces/normalizer.interface.ts b/apps/aggregator/src/interfaces/normalizer.interface.ts new file mode 100644 index 0000000..90cdff6 --- /dev/null +++ b/apps/aggregator/src/interfaces/normalizer.interface.ts @@ -0,0 +1,55 @@ +import { RawPrice } from '@oracle-stocks/shared'; +import { NormalizedPriceRecord, NormalizedSource } from './normalized-price.interface'; + +/** + * Interface for source-specific normalization strategies. + * Each data source should implement this interface. + */ +export interface Normalizer { + /** Unique identifier for this normalizer */ + readonly name: string; + + /** The source this normalizer handles */ + readonly source: NormalizedSource; + + /** Version string for tracking normalization logic changes */ + readonly version: string; + + /** + * Check if this normalizer can handle the given raw price + * @param rawPrice - The raw price to check + * @returns true if this normalizer supports the source + */ + canNormalize(rawPrice: RawPrice): boolean; + + /** + * Normalize a single raw price record + * @param rawPrice - The raw price to normalize + * @returns Normalized price or throws NormalizationException + */ + normalize(rawPrice: RawPrice): NormalizedPriceRecord; + + /** + * Normalize multiple raw price records + * @param rawPrices - Array of raw prices to normalize + * @returns Array of normalized prices (invalid entries filtered out) + */ + normalizeMany(rawPrices: RawPrice[]): NormalizedPriceRecord[]; +} + +/** + * Result type for batch normalization with error tracking + */ +export interface NormalizationResult { + successful: NormalizedPriceRecord[]; + failed: NormalizationFailure[]; +} + +/** + * Represents a failed normalization attempt + */ +export interface NormalizationFailure { + rawPrice: RawPrice; + error: string; + timestamp: string; +} diff --git a/apps/aggregator/src/modules/normalization.module.ts b/apps/aggregator/src/modules/normalization.module.ts new file mode 100644 index 0000000..7f6a8d7 --- /dev/null +++ b/apps/aggregator/src/modules/normalization.module.ts @@ -0,0 +1,8 @@ +import { Module } from '@nestjs/common'; +import { NormalizationService } from '../services/normalization.service'; + +@Module({ + providers: [NormalizationService], + exports: [NormalizationService], +}) +export class NormalizationModule {} diff --git a/apps/aggregator/src/normalizers/alpha-vantage.normalizer.spec.ts b/apps/aggregator/src/normalizers/alpha-vantage.normalizer.spec.ts new file mode 100644 index 0000000..770ca47 --- /dev/null +++ b/apps/aggregator/src/normalizers/alpha-vantage.normalizer.spec.ts @@ -0,0 +1,140 @@ +import { AlphaVantageNormalizer } from './alpha-vantage.normalizer'; +import { NormalizedSource } from '../interfaces/normalized-price.interface'; +import { RawPrice } from '@oracle-stocks/shared'; + +describe('AlphaVantageNormalizer', () => { + let normalizer: AlphaVantageNormalizer; + + const createMockPrice = (overrides: Partial = {}): RawPrice => ({ + symbol: 'AAPL', + price: 150.0, + timestamp: Date.now(), + source: 'AlphaVantage', + ...overrides, + }); + + beforeEach(() => { + normalizer = new AlphaVantageNormalizer(); + }); + + describe('properties', () => { + it('should have correct name', () => { + expect(normalizer.name).toBe('AlphaVantageNormalizer'); + }); + + it('should have correct source', () => { + expect(normalizer.source).toBe(NormalizedSource.ALPHA_VANTAGE); + }); + + it('should have version', () => { + expect(normalizer.version).toBe('1.0.0'); + }); + }); + + describe('canNormalize', () => { + it('should return true for AlphaVantage source', () => { + expect(normalizer.canNormalize(createMockPrice({ source: 'AlphaVantage' }))).toBe(true); + }); + + it('should return true for alpha_vantage source', () => { + expect(normalizer.canNormalize(createMockPrice({ source: 'alpha_vantage' }))).toBe(true); + }); + + it('should return true for alpha-vantage source', () => { + expect(normalizer.canNormalize(createMockPrice({ source: 'alpha-vantage' }))).toBe(true); + }); + + it('should return true for ALPHAVANTAGE source', () => { + expect(normalizer.canNormalize(createMockPrice({ source: 'ALPHAVANTAGE' }))).toBe(true); + }); + + it('should return false for Finnhub source', () => { + expect(normalizer.canNormalize(createMockPrice({ source: 'Finnhub' }))).toBe(false); + }); + + it('should return false for Yahoo source', () => { + expect(normalizer.canNormalize(createMockPrice({ source: 'Yahoo Finance' }))).toBe(false); + }); + }); + + describe('normalizeSymbol', () => { + it('should remove .US suffix', () => { + expect(normalizer.normalizeSymbol('AAPL.US')).toBe('AAPL'); + }); + + it('should remove .NYSE suffix', () => { + expect(normalizer.normalizeSymbol('MSFT.NYSE')).toBe('MSFT'); + }); + + it('should remove .NASDAQ suffix', () => { + expect(normalizer.normalizeSymbol('GOOGL.NASDAQ')).toBe('GOOGL'); + }); + + it('should remove .LSE suffix', () => { + expect(normalizer.normalizeSymbol('BP.LSE')).toBe('BP'); + }); + + it('should remove .TSX suffix', () => { + expect(normalizer.normalizeSymbol('RY.TSX')).toBe('RY'); + }); + + it('should remove .ASX suffix', () => { + expect(normalizer.normalizeSymbol('BHP.ASX')).toBe('BHP'); + }); + + it('should remove .HK suffix', () => { + expect(normalizer.normalizeSymbol('0005.HK')).toBe('0005'); + }); + + it('should handle already clean symbols', () => { + expect(normalizer.normalizeSymbol('GOOGL')).toBe('GOOGL'); + }); + + it('should uppercase symbols', () => { + expect(normalizer.normalizeSymbol('aapl.us')).toBe('AAPL'); + }); + + it('should trim whitespace', () => { + expect(normalizer.normalizeSymbol(' AAPL.US ')).toBe('AAPL'); + }); + }); + + describe('normalize', () => { + it('should produce correct normalized price', () => { + const rawPrice = createMockPrice({ symbol: 'AAPL.US', price: 150.123456 }); + const result = normalizer.normalize(rawPrice); + + expect(result.symbol).toBe('AAPL'); + expect(result.price).toBe(150.1235); + expect(result.source).toBe(NormalizedSource.ALPHA_VANTAGE); + expect(result.metadata.originalSymbol).toBe('AAPL.US'); + expect(result.metadata.wasTransformed).toBe(true); + }); + }); + + describe('normalizeMany', () => { + it('should normalize multiple prices', () => { + const prices = [ + createMockPrice({ symbol: 'AAPL.US' }), + createMockPrice({ symbol: 'MSFT.NYSE' }), + ]; + + const results = normalizer.normalizeMany(prices); + + expect(results.length).toBe(2); + expect(results[0].symbol).toBe('AAPL'); + expect(results[1].symbol).toBe('MSFT'); + }); + + it('should skip prices from other sources', () => { + const prices = [ + createMockPrice({ symbol: 'AAPL.US' }), + createMockPrice({ symbol: 'GOOGL', source: 'Finnhub' }), + ]; + + const results = normalizer.normalizeMany(prices); + + expect(results.length).toBe(1); + }); + }); +}); diff --git a/apps/aggregator/src/normalizers/alpha-vantage.normalizer.ts b/apps/aggregator/src/normalizers/alpha-vantage.normalizer.ts new file mode 100644 index 0000000..0e60ca6 --- /dev/null +++ b/apps/aggregator/src/normalizers/alpha-vantage.normalizer.ts @@ -0,0 +1,40 @@ +import { RawPrice } from '@oracle-stocks/shared'; +import { NormalizedSource } from '../interfaces/normalized-price.interface'; +import { BaseNormalizer } from './base.normalizer'; + +/** + * Normalizer for Alpha Vantage data source. + * + * Handles quirks: + * - Exchange suffixes like ".US", ".NYSE", ".NASDAQ" + * - Case variations in source name + */ +export class AlphaVantageNormalizer extends BaseNormalizer { + readonly name = 'AlphaVantageNormalizer'; + readonly source = NormalizedSource.ALPHA_VANTAGE; + readonly version = '1.0.0'; + + private readonly SOURCE_IDENTIFIERS = [ + 'alphavantage', + 'alpha_vantage', + 'alpha-vantage', + ]; + + canNormalize(rawPrice: RawPrice): boolean { + const sourceLower = rawPrice.source.toLowerCase().replace(/[\s_-]/g, ''); + return this.SOURCE_IDENTIFIERS.some((id) => + sourceLower.includes(id.replace(/[\s_-]/g, '')), + ); + } + + normalizeSymbol(symbol: string): string { + let normalized = this.cleanSymbol(symbol); + + // Remove common Alpha Vantage exchange suffixes + // e.g., "AAPL.US" -> "AAPL", "MSFT.NYSE" -> "MSFT" + const suffixPattern = /\.(US|NYSE|NASDAQ|LSE|TSX|ASX|HK|LON)$/i; + normalized = normalized.replace(suffixPattern, ''); + + return normalized; + } +} diff --git a/apps/aggregator/src/normalizers/base.normalizer.ts b/apps/aggregator/src/normalizers/base.normalizer.ts new file mode 100644 index 0000000..79114f5 --- /dev/null +++ b/apps/aggregator/src/normalizers/base.normalizer.ts @@ -0,0 +1,169 @@ +import { Logger } from '@nestjs/common'; +import { RawPrice } from '@oracle-stocks/shared'; +import { + NormalizedPriceRecord, + NormalizedSource, + NormalizationMetadata, +} from '../interfaces/normalized-price.interface'; +import { Normalizer } from '../interfaces/normalizer.interface'; +import { ValidationException } from '../exceptions'; + +/** + * Abstract base class for all normalizers providing common functionality. + * Subclasses must implement source-specific symbol normalization. + */ +export abstract class BaseNormalizer implements Normalizer { + protected readonly logger: Logger; + + abstract readonly name: string; + abstract readonly source: NormalizedSource; + abstract readonly version: string; + + constructor() { + this.logger = new Logger(this.constructor.name); + } + + /** + * Check if this normalizer can handle the given raw price + */ + abstract canNormalize(rawPrice: RawPrice): boolean; + + /** + * Source-specific symbol normalization - must be implemented by subclasses + */ + abstract normalizeSymbol(symbol: string): string; + + /** + * Normalize a single raw price record + */ + normalize(rawPrice: RawPrice): NormalizedPriceRecord { + this.validateRawPrice(rawPrice); + + const transformations: string[] = []; + + // Normalize symbol + const normalizedSymbol = this.normalizeSymbol(rawPrice.symbol); + if (normalizedSymbol !== rawPrice.symbol) { + transformations.push(`symbol: ${rawPrice.symbol} -> ${normalizedSymbol}`); + } + + // Normalize price to 4 decimal places + const normalizedPrice = this.normalizePrice(rawPrice.price); + if (normalizedPrice !== rawPrice.price) { + transformations.push(`price: ${rawPrice.price} -> ${normalizedPrice}`); + } + + // Normalize timestamp to ISO 8601 UTC + const isoTimestamp = this.normalizeTimestamp(rawPrice.timestamp); + + const metadata: NormalizationMetadata = { + originalSource: rawPrice.source, + originalSymbol: rawPrice.symbol, + normalizedAt: new Date().toISOString(), + normalizerVersion: this.version, + wasTransformed: transformations.length > 0, + transformations, + }; + + return { + symbol: normalizedSymbol, + price: normalizedPrice, + timestamp: isoTimestamp, + originalTimestamp: rawPrice.timestamp, + source: this.source, + metadata, + }; + } + + /** + * Normalize multiple raw price records, skipping failures + */ + normalizeMany(rawPrices: RawPrice[]): NormalizedPriceRecord[] { + const results: NormalizedPriceRecord[] = []; + + for (const rawPrice of rawPrices) { + try { + if (this.canNormalize(rawPrice)) { + results.push(this.normalize(rawPrice)); + } + } catch (error) { + this.logger.warn( + `Failed to normalize price for ${rawPrice.symbol}: ${(error as Error).message}`, + ); + } + } + + return results; + } + + /** + * Validate that raw price has all required fields + */ + protected validateRawPrice(rawPrice: RawPrice): void { + if (!rawPrice) { + throw new ValidationException('Raw price cannot be null or undefined'); + } + if (!rawPrice.symbol || typeof rawPrice.symbol !== 'string') { + throw new ValidationException( + 'Symbol is required and must be a string', + rawPrice, + ); + } + if ( + rawPrice.price === null || + rawPrice.price === undefined || + typeof rawPrice.price !== 'number' + ) { + throw new ValidationException( + 'Price is required and must be a number', + rawPrice, + ); + } + if (isNaN(rawPrice.price) || !isFinite(rawPrice.price)) { + throw new ValidationException( + 'Price must be a valid finite number', + rawPrice, + ); + } + if (rawPrice.price < 0) { + throw new ValidationException('Price cannot be negative', rawPrice); + } + if (!rawPrice.timestamp || typeof rawPrice.timestamp !== 'number') { + throw new ValidationException( + 'Timestamp is required and must be a number', + rawPrice, + ); + } + if (!rawPrice.source || typeof rawPrice.source !== 'string') { + throw new ValidationException( + 'Source is required and must be a string', + rawPrice, + ); + } + } + + /** + * Normalize price to 4 decimal places for financial precision + */ + protected normalizePrice(price: number): number { + return Math.round(price * 10000) / 10000; + } + + /** + * Convert Unix timestamp (milliseconds) to ISO 8601 UTC string + */ + protected normalizeTimestamp(timestamp: number): string { + const date = new Date(timestamp); + if (isNaN(date.getTime())) { + throw new ValidationException(`Invalid timestamp: ${timestamp}`); + } + return date.toISOString(); + } + + /** + * Common symbol cleaning: trim whitespace, uppercase + */ + protected cleanSymbol(symbol: string): string { + return symbol.trim().toUpperCase(); + } +} diff --git a/apps/aggregator/src/normalizers/finnhub.normalizer.spec.ts b/apps/aggregator/src/normalizers/finnhub.normalizer.spec.ts new file mode 100644 index 0000000..c7d3c9b --- /dev/null +++ b/apps/aggregator/src/normalizers/finnhub.normalizer.spec.ts @@ -0,0 +1,112 @@ +import { FinnhubNormalizer } from './finnhub.normalizer'; +import { NormalizedSource } from '../interfaces/normalized-price.interface'; +import { RawPrice } from '@oracle-stocks/shared'; + +describe('FinnhubNormalizer', () => { + let normalizer: FinnhubNormalizer; + + const createMockPrice = (overrides: Partial = {}): RawPrice => ({ + symbol: 'AAPL', + price: 150.0, + timestamp: Date.now(), + source: 'Finnhub', + ...overrides, + }); + + beforeEach(() => { + normalizer = new FinnhubNormalizer(); + }); + + describe('properties', () => { + it('should have correct name', () => { + expect(normalizer.name).toBe('FinnhubNormalizer'); + }); + + it('should have correct source', () => { + expect(normalizer.source).toBe(NormalizedSource.FINNHUB); + }); + + it('should have version', () => { + expect(normalizer.version).toBe('1.0.0'); + }); + }); + + describe('canNormalize', () => { + it('should return true for Finnhub source', () => { + expect(normalizer.canNormalize(createMockPrice({ source: 'Finnhub' }))).toBe(true); + }); + + it('should return true for finnhub lowercase', () => { + expect(normalizer.canNormalize(createMockPrice({ source: 'finnhub' }))).toBe(true); + }); + + it('should return true for FINNHUB uppercase', () => { + expect(normalizer.canNormalize(createMockPrice({ source: 'FINNHUB' }))).toBe(true); + }); + + it('should return false for AlphaVantage source', () => { + expect(normalizer.canNormalize(createMockPrice({ source: 'AlphaVantage' }))).toBe(false); + }); + + it('should return false for Yahoo source', () => { + expect(normalizer.canNormalize(createMockPrice({ source: 'Yahoo Finance' }))).toBe(false); + }); + }); + + describe('normalizeSymbol', () => { + it('should remove US- prefix', () => { + expect(normalizer.normalizeSymbol('US-AAPL')).toBe('AAPL'); + }); + + it('should remove CRYPTO- prefix', () => { + expect(normalizer.normalizeSymbol('CRYPTO-BTC')).toBe('BTC'); + }); + + it('should remove FX- prefix', () => { + expect(normalizer.normalizeSymbol('FX-EURUSD')).toBe('EURUSD'); + }); + + it('should remove INDICES- prefix', () => { + expect(normalizer.normalizeSymbol('INDICES-SPX')).toBe('SPX'); + }); + + it('should handle already clean symbols', () => { + expect(normalizer.normalizeSymbol('AAPL')).toBe('AAPL'); + }); + + it('should uppercase symbols', () => { + expect(normalizer.normalizeSymbol('us-aapl')).toBe('AAPL'); + }); + + it('should trim whitespace', () => { + expect(normalizer.normalizeSymbol(' US-AAPL ')).toBe('AAPL'); + }); + }); + + describe('normalize', () => { + it('should produce correct normalized price', () => { + const rawPrice = createMockPrice({ symbol: 'US-GOOGL', price: 140.5678 }); + const result = normalizer.normalize(rawPrice); + + expect(result.symbol).toBe('GOOGL'); + expect(result.price).toBe(140.5678); + expect(result.source).toBe(NormalizedSource.FINNHUB); + expect(result.metadata.originalSymbol).toBe('US-GOOGL'); + }); + }); + + describe('normalizeMany', () => { + it('should normalize multiple prices', () => { + const prices = [ + createMockPrice({ symbol: 'US-AAPL' }), + createMockPrice({ symbol: 'CRYPTO-ETH' }), + ]; + + const results = normalizer.normalizeMany(prices); + + expect(results.length).toBe(2); + expect(results[0].symbol).toBe('AAPL'); + expect(results[1].symbol).toBe('ETH'); + }); + }); +}); diff --git a/apps/aggregator/src/normalizers/finnhub.normalizer.ts b/apps/aggregator/src/normalizers/finnhub.normalizer.ts new file mode 100644 index 0000000..5a982c3 --- /dev/null +++ b/apps/aggregator/src/normalizers/finnhub.normalizer.ts @@ -0,0 +1,33 @@ +import { RawPrice } from '@oracle-stocks/shared'; +import { NormalizedSource } from '../interfaces/normalized-price.interface'; +import { BaseNormalizer } from './base.normalizer'; + +/** + * Normalizer for Finnhub data source. + * + * Handles quirks: + * - Exchange prefix format like "US-AAPL", "CRYPTO-BTC" + */ +export class FinnhubNormalizer extends BaseNormalizer { + readonly name = 'FinnhubNormalizer'; + readonly source = NormalizedSource.FINNHUB; + readonly version = '1.0.0'; + + private readonly SOURCE_IDENTIFIERS = ['finnhub']; + + canNormalize(rawPrice: RawPrice): boolean { + const sourceLower = rawPrice.source.toLowerCase(); + return this.SOURCE_IDENTIFIERS.some((id) => sourceLower.includes(id)); + } + + normalizeSymbol(symbol: string): string { + let normalized = this.cleanSymbol(symbol); + + // Remove Finnhub exchange prefix format + // e.g., "US-AAPL" -> "AAPL", "CRYPTO-BTC" -> "BTC" + const prefixPattern = /^(US|CRYPTO|FX|INDICES)-/i; + normalized = normalized.replace(prefixPattern, ''); + + return normalized; + } +} diff --git a/apps/aggregator/src/normalizers/index.ts b/apps/aggregator/src/normalizers/index.ts new file mode 100644 index 0000000..6fdc2db --- /dev/null +++ b/apps/aggregator/src/normalizers/index.ts @@ -0,0 +1,5 @@ +export * from './base.normalizer'; +export * from './alpha-vantage.normalizer'; +export * from './finnhub.normalizer'; +export * from './yahoo-finance.normalizer'; +export * from './mock.normalizer'; diff --git a/apps/aggregator/src/normalizers/mock.normalizer.spec.ts b/apps/aggregator/src/normalizers/mock.normalizer.spec.ts new file mode 100644 index 0000000..ebdd412 --- /dev/null +++ b/apps/aggregator/src/normalizers/mock.normalizer.spec.ts @@ -0,0 +1,116 @@ +import { MockNormalizer } from './mock.normalizer'; +import { NormalizedSource } from '../interfaces/normalized-price.interface'; +import { RawPrice } from '@oracle-stocks/shared'; + +describe('MockNormalizer', () => { + let normalizer: MockNormalizer; + + const createMockPrice = (overrides: Partial = {}): RawPrice => ({ + symbol: 'AAPL', + price: 150.0, + timestamp: Date.now(), + source: 'MockProvider', + ...overrides, + }); + + beforeEach(() => { + normalizer = new MockNormalizer(); + }); + + describe('properties', () => { + it('should have correct name', () => { + expect(normalizer.name).toBe('MockNormalizer'); + }); + + it('should have correct source', () => { + expect(normalizer.source).toBe(NormalizedSource.MOCK); + }); + + it('should have version', () => { + expect(normalizer.version).toBe('1.0.0'); + }); + }); + + describe('canNormalize', () => { + it('should return true for MockProvider source', () => { + expect(normalizer.canNormalize(createMockPrice({ source: 'MockProvider' }))).toBe(true); + }); + + it('should return true for mock source', () => { + expect(normalizer.canNormalize(createMockPrice({ source: 'mock' }))).toBe(true); + }); + + it('should return true for MOCK uppercase', () => { + expect(normalizer.canNormalize(createMockPrice({ source: 'MOCK' }))).toBe(true); + }); + + it('should return true for MockData source', () => { + expect(normalizer.canNormalize(createMockPrice({ source: 'MockData' }))).toBe(true); + }); + + it('should return false for AlphaVantage source', () => { + expect(normalizer.canNormalize(createMockPrice({ source: 'AlphaVantage' }))).toBe(false); + }); + + it('should return false for Finnhub source', () => { + expect(normalizer.canNormalize(createMockPrice({ source: 'Finnhub' }))).toBe(false); + }); + }); + + describe('normalizeSymbol', () => { + it('should uppercase symbols', () => { + expect(normalizer.normalizeSymbol('aapl')).toBe('AAPL'); + }); + + it('should trim whitespace', () => { + expect(normalizer.normalizeSymbol(' AAPL ')).toBe('AAPL'); + }); + + it('should handle already clean symbols', () => { + expect(normalizer.normalizeSymbol('AAPL')).toBe('AAPL'); + }); + }); + + describe('normalize', () => { + it('should produce correct normalized price', () => { + const rawPrice = createMockPrice({ symbol: 'tsla', price: 250.5 }); + const result = normalizer.normalize(rawPrice); + + expect(result.symbol).toBe('TSLA'); + expect(result.price).toBe(250.5); + expect(result.source).toBe(NormalizedSource.MOCK); + }); + + it('should track transformation when symbol is uppercased', () => { + const rawPrice = createMockPrice({ symbol: 'aapl' }); + const result = normalizer.normalize(rawPrice); + + expect(result.metadata.wasTransformed).toBe(true); + expect(result.metadata.transformations.length).toBeGreaterThan(0); + }); + }); + + describe('normalizeMany', () => { + it('should normalize multiple prices', () => { + const prices = [ + createMockPrice({ symbol: 'AAPL' }), + createMockPrice({ symbol: 'GOOGL' }), + ]; + + const results = normalizer.normalizeMany(prices); + + expect(results.length).toBe(2); + }); + + it('should skip prices from other sources', () => { + const prices = [ + createMockPrice({ symbol: 'AAPL' }), + createMockPrice({ symbol: 'GOOGL', source: 'AlphaVantage' }), + ]; + + const results = normalizer.normalizeMany(prices); + + expect(results.length).toBe(1); + }); + }); +}); diff --git a/apps/aggregator/src/normalizers/mock.normalizer.ts b/apps/aggregator/src/normalizers/mock.normalizer.ts new file mode 100644 index 0000000..52f1d97 --- /dev/null +++ b/apps/aggregator/src/normalizers/mock.normalizer.ts @@ -0,0 +1,22 @@ +import { RawPrice } from '@oracle-stocks/shared'; +import { NormalizedSource } from '../interfaces/normalized-price.interface'; +import { BaseNormalizer } from './base.normalizer'; + +/** + * Normalizer for Mock data source (used for testing/development). + * + * Performs basic pass-through normalization with standard cleaning. + */ +export class MockNormalizer extends BaseNormalizer { + readonly name = 'MockNormalizer'; + readonly source = NormalizedSource.MOCK; + readonly version = '1.0.0'; + + canNormalize(rawPrice: RawPrice): boolean { + return rawPrice.source.toLowerCase().includes('mock'); + } + + normalizeSymbol(symbol: string): string { + return this.cleanSymbol(symbol); + } +} diff --git a/apps/aggregator/src/normalizers/yahoo-finance.normalizer.spec.ts b/apps/aggregator/src/normalizers/yahoo-finance.normalizer.spec.ts new file mode 100644 index 0000000..51d6a9a --- /dev/null +++ b/apps/aggregator/src/normalizers/yahoo-finance.normalizer.spec.ts @@ -0,0 +1,166 @@ +import { YahooFinanceNormalizer } from './yahoo-finance.normalizer'; +import { NormalizedSource } from '../interfaces/normalized-price.interface'; +import { RawPrice } from '@oracle-stocks/shared'; + +describe('YahooFinanceNormalizer', () => { + let normalizer: YahooFinanceNormalizer; + + const createMockPrice = (overrides: Partial = {}): RawPrice => ({ + symbol: 'AAPL', + price: 150.0, + timestamp: Date.now(), + source: 'Yahoo Finance', + ...overrides, + }); + + beforeEach(() => { + normalizer = new YahooFinanceNormalizer(); + }); + + describe('properties', () => { + it('should have correct name', () => { + expect(normalizer.name).toBe('YahooFinanceNormalizer'); + }); + + it('should have correct source', () => { + expect(normalizer.source).toBe(NormalizedSource.YAHOO_FINANCE); + }); + + it('should have version', () => { + expect(normalizer.version).toBe('1.0.0'); + }); + }); + + describe('canNormalize', () => { + it('should return true for Yahoo Finance source', () => { + expect(normalizer.canNormalize(createMockPrice({ source: 'Yahoo Finance' }))).toBe(true); + }); + + it('should return true for yahoo_finance source', () => { + expect(normalizer.canNormalize(createMockPrice({ source: 'yahoo_finance' }))).toBe(true); + }); + + it('should return true for yahoo-finance source', () => { + expect(normalizer.canNormalize(createMockPrice({ source: 'yahoo-finance' }))).toBe(true); + }); + + it('should return true for YahooFinance source', () => { + expect(normalizer.canNormalize(createMockPrice({ source: 'YahooFinance' }))).toBe(true); + }); + + it('should return true for yahoo source', () => { + expect(normalizer.canNormalize(createMockPrice({ source: 'yahoo' }))).toBe(true); + }); + + it('should return false for AlphaVantage source', () => { + expect(normalizer.canNormalize(createMockPrice({ source: 'AlphaVantage' }))).toBe(false); + }); + + it('should return false for Finnhub source', () => { + expect(normalizer.canNormalize(createMockPrice({ source: 'Finnhub' }))).toBe(false); + }); + }); + + describe('normalizeSymbol', () => { + it('should remove .L suffix (London)', () => { + expect(normalizer.normalizeSymbol('BP.L')).toBe('BP'); + }); + + it('should remove .T suffix (Tokyo)', () => { + expect(normalizer.normalizeSymbol('7203.T')).toBe('7203'); + }); + + it('should remove .AX suffix (Australia)', () => { + expect(normalizer.normalizeSymbol('BHP.AX')).toBe('BHP'); + }); + + it('should remove .HK suffix (Hong Kong)', () => { + expect(normalizer.normalizeSymbol('0005.HK')).toBe('0005'); + }); + + it('should remove .SI suffix (Singapore)', () => { + expect(normalizer.normalizeSymbol('D05.SI')).toBe('D05'); + }); + + it('should remove .KS suffix (Korea)', () => { + expect(normalizer.normalizeSymbol('005930.KS')).toBe('005930'); + }); + + it('should remove .TW suffix (Taiwan)', () => { + expect(normalizer.normalizeSymbol('2330.TW')).toBe('2330'); + }); + + it('should remove .NS suffix (India NSE)', () => { + expect(normalizer.normalizeSymbol('RELIANCE.NS')).toBe('RELIANCE'); + }); + + it('should remove .BO suffix (India BSE)', () => { + expect(normalizer.normalizeSymbol('RELIANCE.BO')).toBe('RELIANCE'); + }); + + it('should remove .TO suffix (Toronto)', () => { + expect(normalizer.normalizeSymbol('RY.TO')).toBe('RY'); + }); + + it('should remove .DE suffix (Germany)', () => { + expect(normalizer.normalizeSymbol('SAP.DE')).toBe('SAP'); + }); + + it('should remove .PA suffix (Paris)', () => { + expect(normalizer.normalizeSymbol('MC.PA')).toBe('MC'); + }); + + it('should remove ^ prefix for indices', () => { + expect(normalizer.normalizeSymbol('^DJI')).toBe('DJI'); + }); + + it('should remove ^ prefix for S&P 500', () => { + expect(normalizer.normalizeSymbol('^GSPC')).toBe('GSPC'); + }); + + it('should remove ^ prefix for NASDAQ', () => { + expect(normalizer.normalizeSymbol('^IXIC')).toBe('IXIC'); + }); + + it('should handle already clean symbols', () => { + expect(normalizer.normalizeSymbol('AAPL')).toBe('AAPL'); + }); + + it('should uppercase symbols', () => { + expect(normalizer.normalizeSymbol('aapl.l')).toBe('AAPL'); + }); + + it('should trim whitespace', () => { + expect(normalizer.normalizeSymbol(' AAPL.L ')).toBe('AAPL'); + }); + }); + + describe('normalize', () => { + it('should produce correct normalized price', () => { + const rawPrice = createMockPrice({ symbol: '^DJI', price: 37500.5 }); + const result = normalizer.normalize(rawPrice); + + expect(result.symbol).toBe('DJI'); + expect(result.price).toBe(37500.5); + expect(result.source).toBe(NormalizedSource.YAHOO_FINANCE); + expect(result.metadata.originalSymbol).toBe('^DJI'); + }); + }); + + describe('normalizeMany', () => { + it('should normalize multiple prices', () => { + const prices = [ + createMockPrice({ symbol: 'AAPL.L' }), + createMockPrice({ symbol: '^GSPC' }), + createMockPrice({ symbol: 'BHP.AX' }), + ]; + + const results = normalizer.normalizeMany(prices); + + expect(results.length).toBe(3); + expect(results[0].symbol).toBe('AAPL'); + expect(results[1].symbol).toBe('GSPC'); + expect(results[2].symbol).toBe('BHP'); + }); + }); +}); diff --git a/apps/aggregator/src/normalizers/yahoo-finance.normalizer.ts b/apps/aggregator/src/normalizers/yahoo-finance.normalizer.ts new file mode 100644 index 0000000..25cd7c5 --- /dev/null +++ b/apps/aggregator/src/normalizers/yahoo-finance.normalizer.ts @@ -0,0 +1,47 @@ +import { RawPrice } from '@oracle-stocks/shared'; +import { NormalizedSource } from '../interfaces/normalized-price.interface'; +import { BaseNormalizer } from './base.normalizer'; + +/** + * Normalizer for Yahoo Finance data source. + * + * Handles quirks: + * - Exchange suffixes like ".L" (London), ".T" (Tokyo), ".AX" (Australia) + * - Index prefix "^" (e.g., "^DJI", "^GSPC") + */ +export class YahooFinanceNormalizer extends BaseNormalizer { + readonly name = 'YahooFinanceNormalizer'; + readonly source = NormalizedSource.YAHOO_FINANCE; + readonly version = '1.0.0'; + + private readonly SOURCE_IDENTIFIERS = [ + 'yahoo', + 'yahoofinance', + 'yahoo_finance', + 'yahoo-finance', + ]; + + canNormalize(rawPrice: RawPrice): boolean { + const sourceLower = rawPrice.source.toLowerCase().replace(/[\s_-]/g, ''); + return this.SOURCE_IDENTIFIERS.some((id) => + sourceLower.includes(id.replace(/[\s_-]/g, '')), + ); + } + + normalizeSymbol(symbol: string): string { + let normalized = this.cleanSymbol(symbol); + + // Remove Yahoo Finance exchange suffixes + // e.g., "AAPL.L" -> "AAPL", "BHP.AX" -> "BHP", "7203.T" -> "7203" + const suffixPattern = + /\.(L|T|AX|HK|SI|KS|TW|NS|BO|TO|V|F|DE|PA|AS|BR|MC|MI|SW|CO|MX|SA|JK|KL)$/i; + normalized = normalized.replace(suffixPattern, ''); + + // Remove index prefix (^DJI -> DJI, ^GSPC -> GSPC) + if (normalized.startsWith('^')) { + normalized = normalized.substring(1); + } + + return normalized; + } +} diff --git a/apps/aggregator/src/services/normalization.service.spec.ts b/apps/aggregator/src/services/normalization.service.spec.ts new file mode 100644 index 0000000..f069411 --- /dev/null +++ b/apps/aggregator/src/services/normalization.service.spec.ts @@ -0,0 +1,378 @@ +import { Test, TestingModule } from '@nestjs/testing'; +import { NormalizationService } from './normalization.service'; +import { + mockRawPrices, + malformedPrices, + validRawPrices, + mixedRawPrices, +} from '../__mocks__/raw-price.fixtures'; +import { NormalizedSource } from '../interfaces/normalized-price.interface'; +import { NormalizationException } from '../exceptions'; +import { RawPrice } from '@oracle-stocks/shared'; + +describe('NormalizationService', () => { + let service: NormalizationService; + + beforeEach(async () => { + const module: TestingModule = await Test.createTestingModule({ + providers: [NormalizationService], + }).compile(); + + service = module.get(NormalizationService); + service.onModuleInit(); + }); + + describe('initialization', () => { + it('should be defined', () => { + expect(service).toBeDefined(); + }); + + it('should register default normalizers', () => { + const normalizers = service.getNormalizers(); + expect(normalizers.length).toBeGreaterThanOrEqual(4); + }); + + it('should have normalizers for all expected sources', () => { + const normalizers = service.getNormalizers(); + const sources = normalizers.map((n) => n.source); + + expect(sources).toContain(NormalizedSource.ALPHA_VANTAGE); + expect(sources).toContain(NormalizedSource.FINNHUB); + expect(sources).toContain(NormalizedSource.YAHOO_FINANCE); + expect(sources).toContain(NormalizedSource.MOCK); + }); + }); + + describe('findNormalizer', () => { + it('should find Alpha Vantage normalizer', () => { + const normalizer = service.findNormalizer(mockRawPrices.alphaVantage); + expect(normalizer).not.toBeNull(); + expect(normalizer?.source).toBe(NormalizedSource.ALPHA_VANTAGE); + }); + + it('should find Finnhub normalizer', () => { + const normalizer = service.findNormalizer(mockRawPrices.finnhub); + expect(normalizer).not.toBeNull(); + expect(normalizer?.source).toBe(NormalizedSource.FINNHUB); + }); + + it('should find Yahoo Finance normalizer', () => { + const normalizer = service.findNormalizer(mockRawPrices.yahooFinance); + expect(normalizer).not.toBeNull(); + expect(normalizer?.source).toBe(NormalizedSource.YAHOO_FINANCE); + }); + + it('should find Mock normalizer', () => { + const normalizer = service.findNormalizer(mockRawPrices.mock); + expect(normalizer).not.toBeNull(); + expect(normalizer?.source).toBe(NormalizedSource.MOCK); + }); + + it('should return null for unknown source', () => { + const normalizer = service.findNormalizer(mockRawPrices.unknown); + expect(normalizer).toBeNull(); + }); + }); + + describe('normalize - Alpha Vantage', () => { + it('should normalize Alpha Vantage price with .US suffix', () => { + const result = service.normalize(mockRawPrices.alphaVantage); + + expect(result.symbol).toBe('AAPL'); + expect(result.source).toBe(NormalizedSource.ALPHA_VANTAGE); + expect(result.metadata.originalSymbol).toBe('AAPL.US'); + expect(result.metadata.originalSource).toBe('AlphaVantage'); + }); + + it('should normalize Alpha Vantage price with .NYSE suffix', () => { + const result = service.normalize(mockRawPrices.alphaVantageNYSE); + + expect(result.symbol).toBe('MSFT'); + expect(result.source).toBe(NormalizedSource.ALPHA_VANTAGE); + }); + + it('should handle alpha_vantage source name variation', () => { + const result = service.normalize(mockRawPrices.alphaVantageNYSE); + expect(result.source).toBe(NormalizedSource.ALPHA_VANTAGE); + }); + }); + + describe('normalize - Finnhub', () => { + it('should normalize Finnhub price with US- prefix', () => { + const result = service.normalize(mockRawPrices.finnhub); + + expect(result.symbol).toBe('GOOGL'); + expect(result.source).toBe(NormalizedSource.FINNHUB); + expect(result.metadata.originalSymbol).toBe('US-GOOGL'); + }); + + it('should normalize Finnhub price with CRYPTO- prefix', () => { + const result = service.normalize(mockRawPrices.finnhubCrypto); + + expect(result.symbol).toBe('BTC'); + expect(result.source).toBe(NormalizedSource.FINNHUB); + }); + }); + + describe('normalize - Yahoo Finance', () => { + it('should normalize Yahoo Finance price with .L suffix', () => { + const result = service.normalize(mockRawPrices.yahooFinance); + + expect(result.symbol).toBe('MSFT'); + expect(result.source).toBe(NormalizedSource.YAHOO_FINANCE); + }); + + it('should normalize Yahoo Finance index with ^ prefix', () => { + const result = service.normalize(mockRawPrices.yahooFinanceIndex); + + expect(result.symbol).toBe('DJI'); + expect(result.source).toBe(NormalizedSource.YAHOO_FINANCE); + }); + + it('should normalize Yahoo Finance price with .AX suffix', () => { + const result = service.normalize(mockRawPrices.yahooFinanceAustralia); + + expect(result.symbol).toBe('BHP'); + expect(result.source).toBe(NormalizedSource.YAHOO_FINANCE); + }); + + it('should handle yahoo_finance source name variation', () => { + const result = service.normalize(mockRawPrices.yahooFinanceIndex); + expect(result.source).toBe(NormalizedSource.YAHOO_FINANCE); + }); + }); + + describe('normalize - Mock', () => { + it('should normalize Mock price', () => { + const result = service.normalize(mockRawPrices.mock); + + expect(result.symbol).toBe('TSLA'); + expect(result.source).toBe(NormalizedSource.MOCK); + }); + + it('should uppercase and trim symbols', () => { + const result = service.normalize(mockRawPrices.mockLowercase); + + expect(result.symbol).toBe('AAPL'); + }); + }); + + describe('normalize - timestamp', () => { + it('should convert timestamp to ISO 8601 UTC format', () => { + const result = service.normalize(mockRawPrices.alphaVantage); + + expect(result.timestamp).toMatch( + /^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}.\d{3}Z$/, + ); + expect(result.timestamp).toBe('2024-01-15T14:30:00.000Z'); + }); + + it('should preserve original timestamp', () => { + const result = service.normalize(mockRawPrices.alphaVantage); + + expect(result.originalTimestamp).toBe(1705329000000); + }); + }); + + describe('normalize - price', () => { + it('should round price to 4 decimal places', () => { + const result = service.normalize(mockRawPrices.alphaVantage); + + expect(result.price).toBe(150.1235); + const decimalPlaces = (result.price.toString().split('.')[1] || '').length; + expect(decimalPlaces).toBeLessThanOrEqual(4); + }); + + it('should handle prices with fewer decimals', () => { + const result = service.normalize(mockRawPrices.mock); + + expect(result.price).toBe(250.5); + }); + }); + + describe('normalize - metadata', () => { + it('should include normalization metadata', () => { + const result = service.normalize(mockRawPrices.alphaVantage); + + expect(result.metadata).toBeDefined(); + expect(result.metadata.originalSource).toBe('AlphaVantage'); + expect(result.metadata.originalSymbol).toBe('AAPL.US'); + expect(result.metadata.normalizedAt).toBeDefined(); + expect(result.metadata.normalizerVersion).toBe('1.0.0'); + }); + + it('should track transformations when symbol changes', () => { + const result = service.normalize(mockRawPrices.alphaVantage); + + expect(result.metadata.wasTransformed).toBe(true); + expect(result.metadata.transformations.length).toBeGreaterThan(0); + expect(result.metadata.transformations[0]).toContain('symbol'); + }); + + it('should track transformations when price is rounded', () => { + const result = service.normalize(mockRawPrices.alphaVantage); + + const priceTransformation = result.metadata.transformations.find((t) => + t.includes('price'), + ); + expect(priceTransformation).toBeDefined(); + }); + }); + + describe('normalize - error handling', () => { + it('should throw for unknown source', () => { + expect(() => service.normalize(mockRawPrices.unknown)).toThrow( + NormalizationException, + ); + }); + + it('should include source in error message', () => { + expect(() => service.normalize(mockRawPrices.unknown)).toThrow( + /UnknownSource/, + ); + }); + }); + + describe('normalizeMany', () => { + it('should normalize multiple prices', () => { + const results = service.normalizeMany(validRawPrices); + + expect(results.length).toBe(4); + }); + + it('should skip invalid prices without throwing', () => { + const results = service.normalizeMany(mixedRawPrices); + + expect(results.length).toBe(2); + }); + + it('should return empty array for all invalid prices', () => { + const results = service.normalizeMany([mockRawPrices.unknown]); + + expect(results.length).toBe(0); + }); + }); + + describe('normalizeManyWithErrors', () => { + it('should return both successful and failed normalizations', () => { + const result = service.normalizeManyWithErrors(mixedRawPrices); + + expect(result.successful.length).toBe(2); + expect(result.failed.length).toBe(1); + }); + + it('should include error details for failures', () => { + const result = service.normalizeManyWithErrors(mixedRawPrices); + + expect(result.failed[0].error).toContain('No normalizer found'); + expect(result.failed[0].rawPrice).toEqual(mockRawPrices.unknown); + expect(result.failed[0].timestamp).toBeDefined(); + }); + + it('should handle all valid prices', () => { + const result = service.normalizeManyWithErrors(validRawPrices); + + expect(result.successful.length).toBe(4); + expect(result.failed.length).toBe(0); + }); + }); + + describe('normalizeBySource', () => { + it('should group normalized prices by source', () => { + const result = service.normalizeBySource(validRawPrices); + + expect(result.get(NormalizedSource.ALPHA_VANTAGE)?.length).toBe(1); + expect(result.get(NormalizedSource.FINNHUB)?.length).toBe(1); + expect(result.get(NormalizedSource.YAHOO_FINANCE)?.length).toBe(1); + expect(result.get(NormalizedSource.MOCK)?.length).toBe(1); + }); + + it('should skip invalid prices', () => { + const result = service.normalizeBySource(mixedRawPrices); + + expect(result.size).toBe(2); + expect(result.get(NormalizedSource.UNKNOWN)).toBeUndefined(); + }); + }); + + describe('validation', () => { + malformedPrices.forEach((malformed, index) => { + // Skip test case #5 (empty source) since it gets overwritten by 'MockProvider' + // That case is tested separately below + if (index === 4) return; + + it(`should reject malformed price #${index + 1}`, () => { + // Create a mock price that will match a normalizer + const testPrice = { + ...malformed, + source: 'MockProvider', + } as RawPrice; + + expect(() => service.normalize(testPrice)).toThrow(); + }); + }); + + it('should reject empty symbol', () => { + const badPrice: RawPrice = { + symbol: '', + price: 100, + timestamp: Date.now(), + source: 'MockProvider', + }; + + expect(() => service.normalize(badPrice)).toThrow(/Symbol/); + }); + + it('should reject NaN price', () => { + const badPrice: RawPrice = { + symbol: 'TEST', + price: NaN, + timestamp: Date.now(), + source: 'MockProvider', + }; + + expect(() => service.normalize(badPrice)).toThrow(/number/); + }); + + it('should reject negative price', () => { + const badPrice: RawPrice = { + symbol: 'TEST', + price: -100, + timestamp: Date.now(), + source: 'MockProvider', + }; + + expect(() => service.normalize(badPrice)).toThrow(/negative/); + }); + + it('should reject invalid timestamp', () => { + const badPrice: RawPrice = { + symbol: 'TEST', + price: 100, + timestamp: NaN, + source: 'MockProvider', + }; + + expect(() => service.normalize(badPrice)).toThrow(); + }); + }); + + describe('registerNormalizer', () => { + it('should allow registering custom normalizers', () => { + const initialCount = service.getNormalizers().length; + + const customNormalizer = { + name: 'CustomNormalizer', + source: NormalizedSource.UNKNOWN, + version: '1.0.0', + canNormalize: () => false, + normalize: jest.fn(), + normalizeMany: jest.fn(), + }; + + service.registerNormalizer(customNormalizer); + + expect(service.getNormalizers().length).toBe(initialCount + 1); + }); + }); +}); diff --git a/apps/aggregator/src/services/normalization.service.ts b/apps/aggregator/src/services/normalization.service.ts new file mode 100644 index 0000000..4789f89 --- /dev/null +++ b/apps/aggregator/src/services/normalization.service.ts @@ -0,0 +1,166 @@ +import { Injectable, Logger, OnModuleInit } from '@nestjs/common'; +import { RawPrice } from '@oracle-stocks/shared'; +import { + NormalizedPriceRecord, + NormalizedSource, +} from '../interfaces/normalized-price.interface'; +import { + Normalizer, + NormalizationResult, + NormalizationFailure, +} from '../interfaces/normalizer.interface'; +import { + AlphaVantageNormalizer, + FinnhubNormalizer, + YahooFinanceNormalizer, + MockNormalizer, +} from '../normalizers'; +import { NormalizationException } from '../exceptions'; + +/** + * Service for normalizing raw price data from multiple sources. + * Uses the Strategy pattern to delegate normalization to source-specific normalizers. + */ +@Injectable() +export class NormalizationService implements OnModuleInit { + private readonly logger = new Logger(NormalizationService.name); + private readonly normalizers: Map = new Map(); + + onModuleInit(): void { + this.registerDefaultNormalizers(); + this.logger.log( + `NormalizationService initialized with ${this.normalizers.size} normalizers`, + ); + } + + /** + * Register all default normalizers + */ + private registerDefaultNormalizers(): void { + this.registerNormalizer(new AlphaVantageNormalizer()); + this.registerNormalizer(new FinnhubNormalizer()); + this.registerNormalizer(new YahooFinanceNormalizer()); + this.registerNormalizer(new MockNormalizer()); + } + + /** + * Register a new normalizer (for extensibility) + */ + registerNormalizer(normalizer: Normalizer): void { + this.normalizers.set(normalizer.name, normalizer); + this.logger.debug(`Registered normalizer: ${normalizer.name}`); + } + + /** + * Get all registered normalizers + */ + getNormalizers(): Normalizer[] { + return Array.from(this.normalizers.values()); + } + + /** + * Find the appropriate normalizer for a raw price + */ + findNormalizer(rawPrice: RawPrice): Normalizer | null { + for (const normalizer of this.normalizers.values()) { + if (normalizer.canNormalize(rawPrice)) { + return normalizer; + } + } + return null; + } + + /** + * Normalize a single raw price + * @throws NormalizationException if no suitable normalizer found + */ + normalize(rawPrice: RawPrice): NormalizedPriceRecord { + const normalizer = this.findNormalizer(rawPrice); + + if (!normalizer) { + throw new NormalizationException( + `No normalizer found for source: ${rawPrice.source}`, + rawPrice, + ); + } + + this.logger.debug( + `Normalizing ${rawPrice.symbol} from ${rawPrice.source} using ${normalizer.name}`, + ); + + return normalizer.normalize(rawPrice); + } + + /** + * Normalize multiple raw prices (skips failures) + */ + normalizeMany(rawPrices: RawPrice[]): NormalizedPriceRecord[] { + const results: NormalizedPriceRecord[] = []; + + for (const rawPrice of rawPrices) { + try { + results.push(this.normalize(rawPrice)); + } catch (error) { + this.logger.warn( + `Failed to normalize ${rawPrice.symbol} from ${rawPrice.source}: ${(error as Error).message}`, + ); + } + } + + this.logger.log( + `Normalized ${results.length}/${rawPrices.length} prices successfully`, + ); + + return results; + } + + /** + * Normalize multiple raw prices with detailed error reporting + */ + normalizeManyWithErrors(rawPrices: RawPrice[]): NormalizationResult { + const successful: NormalizedPriceRecord[] = []; + const failed: NormalizationFailure[] = []; + + for (const rawPrice of rawPrices) { + try { + successful.push(this.normalize(rawPrice)); + } catch (error) { + failed.push({ + rawPrice, + error: (error as Error).message, + timestamp: new Date().toISOString(), + }); + } + } + + this.logger.log( + `Normalization complete: ${successful.length} successful, ${failed.length} failed`, + ); + + return { successful, failed }; + } + + /** + * Normalize prices grouped by source + */ + normalizeBySource( + rawPrices: RawPrice[], + ): Map { + const result = new Map(); + + for (const rawPrice of rawPrices) { + try { + const normalized = this.normalize(rawPrice); + const existing = result.get(normalized.source) || []; + existing.push(normalized); + result.set(normalized.source, existing); + } catch (error) { + this.logger.warn( + `Failed to normalize ${rawPrice.symbol}: ${(error as Error).message}`, + ); + } + } + + return result; + } +} diff --git a/package-lock.json b/package-lock.json index edb3ef7..3184c48 100644 --- a/package-lock.json +++ b/package-lock.json @@ -30,6 +30,7 @@ "@nestjs/core": "^10.0.0", "@nestjs/event-emitter": "^3.0.1", "@nestjs/platform-express": "^10.0.0", + "@oracle-stocks/shared": "*", "@nestjs/terminus": "^10.0.0", "axios": "^1.6.0", "ioredis": "^5.3.2",