import { logger } from '../logger'; import * as memCache from './cache/memory'; import { parseUrl } from './url'; type LookupStatsData = Record; interface TimingStatsReport { count: number; avgMs: number; medianMs: number; maxMs: number; totalMs: number; } export function makeTimingReport(data: number[]): TimingStatsReport { const count = data.length; const totalMs = data.reduce((a, c) => a + c, 0); const avgMs = count ? Math.round(totalMs / count) : 0; const maxMs = Math.max(0, ...data); const sorted = data.sort((a, b) => a - b); const medianMs = count ? sorted[Math.floor(count / 2)] : 0; return { count, avgMs, medianMs, maxMs, totalMs }; } export class LookupStats { static write(datasource: string, duration: number): void { const data = memCache.get('lookup-stats') ?? {}; data[datasource] ??= []; data[datasource].push(duration); memCache.set('lookup-stats', data); } static async wrap( datasource: string, callback: () => Promise, ): Promise { const start = Date.now(); const result = await callback(); const duration = Date.now() - start; LookupStats.write(datasource, duration); return result; } static getReport(): Record { const report: Record = {}; const data = memCache.get('lookup-stats') ?? {}; for (const [datasource, durations] of Object.entries(data)) { report[datasource] = makeTimingReport(durations); } return report; } static report(): void { const report = LookupStats.getReport(); logger.debug(report, 'Lookup statistics'); } } type PackageCacheData = number[]; export class PackageCacheStats { static writeSet(duration: number): void { const data = memCache.get('package-cache-sets') ?? []; data.push(duration); memCache.set('package-cache-sets', data); } static async wrapSet(callback: () => Promise): Promise { const start = Date.now(); const result = await callback(); const duration = Date.now() - start; PackageCacheStats.writeSet(duration); return result; } static writeGet(duration: number): void { const data = memCache.get('package-cache-gets') ?? []; data.push(duration); memCache.set('package-cache-gets', data); } static async wrapGet(callback: () => Promise): Promise { const start = Date.now(); const result = await callback(); const duration = Date.now() - start; PackageCacheStats.writeGet(duration); return result; } static getReport(): { get: TimingStatsReport; set: TimingStatsReport } { const packageCacheGets = memCache.get('package-cache-gets') ?? []; const get = makeTimingReport(packageCacheGets); const packageCacheSets = memCache.get('package-cache-sets') ?? []; const set = makeTimingReport(packageCacheSets); return { get, set }; } static report(): void { const report = PackageCacheStats.getReport(); logger.debug(report, 'Package cache statistics'); } } interface DatasourceCacheDataPoint { datasource: string; registryUrl: string; packageName: string; action: 'hit' | 'miss' | 'set' | 'skip'; } export interface DatasourceCacheReport { long: { [datasource in string]: { [registryUrl in string]: { [packageName in string]: { read?: 'hit' | 'miss'; write?: 'set' | 'skip'; }; }; }; }; short: { [datasource in string]: { [registryUrl in string]: { hit: number; miss: number; set: number; skip: number; }; }; }; } export class DatasourceCacheStats { private static getData(): DatasourceCacheDataPoint[] { return ( memCache.get('datasource-cache-stats') ?? [] ); } private static setData(data: DatasourceCacheDataPoint[]): void { memCache.set('datasource-cache-stats', data); } static hit( datasource: string, registryUrl: string, packageName: string, ): void { const data = this.getData(); data.push({ datasource, registryUrl, packageName, action: 'hit' }); this.setData(data); } static miss( datasource: string, registryUrl: string, packageName: string, ): void { const data = this.getData(); data.push({ datasource, registryUrl, packageName, action: 'miss' }); this.setData(data); } static set( datasource: string, registryUrl: string, packageName: string, ): void { const data = this.getData(); data.push({ datasource, registryUrl, packageName, action: 'set' }); this.setData(data); } static skip( datasource: string, registryUrl: string, packageName: string, ): void { const data = this.getData(); data.push({ datasource, registryUrl, packageName, action: 'skip' }); this.setData(data); } static getReport(): DatasourceCacheReport { const data = this.getData(); const result: DatasourceCacheReport = { long: {}, short: {} }; for (const { datasource, registryUrl, packageName, action } of data) { result.long[datasource] ??= {}; result.long[datasource][registryUrl] ??= {}; result.long[datasource][registryUrl] ??= {}; result.long[datasource][registryUrl][packageName] ??= {}; result.short[datasource] ??= {}; result.short[datasource][registryUrl] ??= { hit: 0, miss: 0, set: 0, skip: 0, }; if (action === 'hit') { result.long[datasource][registryUrl][packageName].read = 'hit'; result.short[datasource][registryUrl].hit += 1; continue; } if (action === 'miss') { result.long[datasource][registryUrl][packageName].read = 'miss'; result.short[datasource][registryUrl].miss += 1; continue; } if (action === 'set') { result.long[datasource][registryUrl][packageName].write = 'set'; result.short[datasource][registryUrl].set += 1; continue; } if (action === 'skip') { result.long[datasource][registryUrl][packageName].write = 'skip'; result.short[datasource][registryUrl].skip += 1; continue; } } return result; } static report(): void { const { long, short } = this.getReport(); if (Object.keys(short).length > 0) { logger.debug(short, 'Datasource cache statistics'); } if (Object.keys(long).length > 0) { logger.trace(long, 'Datasource cache detailed statistics'); } } } export interface HttpRequestStatsDataPoint { method: string; url: string; reqMs: number; queueMs: number; status: number; } interface HostStatsData { count: number; reqAvgMs: number; reqMedianMs: number; reqMaxMs: number; queueAvgMs: number; queueMedianMs: number; queueMaxMs: number; } // url -> method -> status -> count type UrlHttpStat = Record>>; interface HttpStatsCollection { // debug data urls: UrlHttpStat; hosts: Record; requests: number; // trace data rawRequests: string[]; hostRequests: Record; } export class HttpStats { static write(data: HttpRequestStatsDataPoint): void { const httpRequests = memCache.get('http-requests') ?? []; httpRequests.push(data); memCache.set('http-requests', httpRequests); } static getDataPoints(): HttpRequestStatsDataPoint[] { const httpRequests = memCache.get('http-requests') ?? []; // istanbul ignore next: sorting is hard and not worth testing httpRequests.sort((a, b) => { if (a.url < b.url) { return -1; } if (a.url > b.url) { return 1; } return 0; }); return httpRequests; } static getReport(): HttpStatsCollection { const dataPoints = HttpStats.getDataPoints(); const requests = dataPoints.length; const urls: UrlHttpStat = {}; const rawRequests: string[] = []; const hostRequests: Record = {}; for (const dataPoint of dataPoints) { const { url, reqMs, queueMs, status } = dataPoint; const method = dataPoint.method.toUpperCase(); const parsedUrl = parseUrl(url); if (!parsedUrl) { logger.debug({ url }, 'Failed to parse URL during stats reporting'); continue; } const { hostname, origin, pathname } = parsedUrl; const baseUrl = `${origin}${pathname}`; urls[baseUrl] ??= {}; urls[baseUrl][method] ??= {}; urls[baseUrl][method][status] ??= 0; urls[baseUrl][method][status] += 1; rawRequests.push(`${method} ${url} ${status} ${reqMs} ${queueMs}`); hostRequests[hostname] ??= []; hostRequests[hostname].push(dataPoint); } const hosts: Record = {}; for (const [hostname, dataPoints] of Object.entries(hostRequests)) { const count = dataPoints.length; const reqTimes = dataPoints.map((r) => r.reqMs); const queueTimes = dataPoints.map((r) => r.queueMs); const reqReport = makeTimingReport(reqTimes); const queueReport = makeTimingReport(queueTimes); hosts[hostname] = { count, reqAvgMs: reqReport.avgMs, reqMedianMs: reqReport.medianMs, reqMaxMs: reqReport.maxMs, queueAvgMs: queueReport.avgMs, queueMedianMs: queueReport.medianMs, queueMaxMs: queueReport.maxMs, }; } return { urls, rawRequests, hostRequests, hosts, requests, }; } static report(): void { const { urls, rawRequests, hostRequests, hosts, requests } = HttpStats.getReport(); logger.trace({ rawRequests, hostRequests }, 'HTTP full statistics'); logger.debug({ hosts, requests }, 'HTTP statistics'); logger.trace({ urls }, 'HTTP URL statistics'); } } interface HttpCacheHostStatsData { hit: number; miss: number; localHit?: number; localMiss?: number; } type HttpCacheStatsData = Record; function sortObject(obj: Record): Record { const result: Record = {}; for (const key of Object.keys(obj).sort()) { result[key] = obj[key]; } return result; } export class HttpCacheStats { static getData(): HttpCacheStatsData { return memCache.get('http-cache-stats') ?? {}; } static read(key: string): HttpCacheHostStatsData { return ( this.getData()?.[key] ?? { hit: 0, miss: 0, } ); } static write(key: string, data: HttpCacheHostStatsData): void { const stats = memCache.get('http-cache-stats') ?? {}; stats[key] = data; memCache.set('http-cache-stats', stats); } static getBaseUrl(url: string): string | null { const parsedUrl = parseUrl(url); if (!parsedUrl) { logger.debug({ url }, 'Failed to parse URL during cache stats'); return null; } const { origin, pathname } = parsedUrl; const baseUrl = `${origin}${pathname}`; return baseUrl; } static incLocalHits(url: string): void { const baseUrl = HttpCacheStats.getBaseUrl(url); if (baseUrl) { const host = baseUrl; const stats = HttpCacheStats.read(host); stats.localHit ??= 0; stats.localHit += 1; HttpCacheStats.write(host, stats); } } static incLocalMisses(url: string): void { const baseUrl = HttpCacheStats.getBaseUrl(url); if (baseUrl) { const host = baseUrl; const stats = HttpCacheStats.read(host); stats.localMiss ??= 0; stats.localMiss += 1; HttpCacheStats.write(host, stats); } } static incRemoteHits(url: string): void { const baseUrl = HttpCacheStats.getBaseUrl(url); if (baseUrl) { const host = baseUrl; const stats = HttpCacheStats.read(host); stats.hit += 1; HttpCacheStats.write(host, stats); } } static incRemoteMisses(url: string): void { const baseUrl = HttpCacheStats.getBaseUrl(url); if (baseUrl) { const host = baseUrl; const stats = HttpCacheStats.read(host); stats.miss += 1; HttpCacheStats.write(host, stats); } } static report(): void { const data = HttpCacheStats.getData(); let report: Record> = {}; for (const [url, stats] of Object.entries(data)) { const parsedUrl = parseUrl(url); if (parsedUrl) { const { origin, pathname } = parsedUrl; report[origin] ??= {}; report[origin][pathname] = stats; } } for (const [host, hostStats] of Object.entries(report)) { report[host] = sortObject(hostStats); } report = sortObject(report); logger.debug(report, 'HTTP cache statistics'); } }