前端日志系统的架构设计：采集、传输、存储与查询的性能平衡

前端日志系统的架构设计采集、传输、存储与查询的性能平衡一、前端日志的定位与边界前端日志不是后端日志的延伸。后端日志关注请求链路与服务状态前端日志关注用户行为、渲染性能与异常捕获。两者的采集量、传输频率与存储策略完全不同。一个出行平台的前端日志系统日均采集量在 800 万条左右。如果全量上报带宽成本与查询延迟会超出可接受范围。核心矛盾采集粒度越细信息越完整但传输和存储成本越高。架构设计的本质是在这个矛盾中找到平衡点。二、采集层设计2.1 日志分类与优先级日志按优先级分为三级不同级别的采集策略不同。// log-classification.ts — 日志分类定义 enum LogLevel { ERROR error, // 异常、崩溃 — 必须全量采集 PERFORMANCE perf, // 渲染耗时、接口延迟 — 采样采集 BEHAVIOR behavior, // 用户点击、滚动、导航 — 极低比例采样 } interface LogEntry { level: LogLevel; timestamp: number; sessionId: string; userId?: string; page: string; payload: Recordstring, unknown; } // 各级别的采集策略 interface CollectionPolicy { level: LogLevel; sampleRate: number; // 采集比例 0-1 maxPerMinute: number; // 单用户每分钟上限 batchInterval: number; // 批量上报间隔(ms) } const policies: CollectionPolicy[] [ { level: LogLevel.ERROR, sampleRate: 1.0, maxPerMinute: 30, batchInterval: 1000 }, { level: LogLevel.PERFORMANCE, sampleRate: 0.1, maxPerMinute: 10, batchInterval: 5000 }, { level: LogLevel.BEHAVIOR, sampleRate: 0.01, maxPerMinute: 3, batchInterval: 10000 }, ];2.2 采集 SDK 核心采集 SDK 需要轻量、无侵入、不影响页面渲染性能。核心设计监听而非拦截异步而非同步。// logger-sdk.ts — 前端日志采集 SDK interface LoggerConfig { endpoint: string; // 上报地址 appId: string; // 应用标识 sampleRates: RecordLogLevel, number; maxBatchSize: number; // 单批次最大条数 flushInterval: number; // 定时上报间隔(ms) maxQueueSize: number; // 本地队列最大容量 } class FrontendLogger { private queue: LogEntry[] []; private flushTimer: number | null null; private sessionId: string; private config: LoggerConfig; private droppedCount: number 0; // 因队列满而丢弃的数量 constructor(config: LoggerConfig) { this.config config; this.sessionId this.generateSessionId(); // 注册全局错误监听 this.registerErrorListeners(); // 注册性能指标监听 this.registerPerformanceListeners(); // 注册行为监听 this.registerBehaviorListeners(); // 启动定时上报 this.startFlushTimer(); // 页面卸载时强制上报 window.addEventListener(visibilitychange, () { if (document.visibilityState hidden) { this.flush(); // 立即上报剩余日志 } }); } // 手动记录日志对外接口 log(level: LogLevel, payload: Recordstring, unknown): void { if (!this.shouldSample(level)) return; const entry: LogEntry { level, timestamp: Date.now(), sessionId: this.sessionId, userId: this.tryGetUserId(), page: window.location.pathname, payload, }; this.enqueue(entry); } // 采样判断 private shouldSample(level: LogLevel): boolean { const rate this.config.sampleRates[level] ?? 1; return Math.random() rate; } // 入队超出容量时丢弃低优先级日志 private enqueue(entry: LogEntry): void { if (this.queue.length this.config.maxQueueSize) { // 丢弃行为日志保留错误日志 const behaviorIdx this.queue.findIndex((e) e.level LogLevel.BEHAVIOR); if (behaviorIdx ! -1) { this.queue.splice(behaviorIdx, 1); } else { this.droppedCount; return; // 队列满且无可丢弃项直接跳过 } } this.queue.push(entry); // 达到批次上限时立即上报 if (this.queue.length this.config.maxBatchSize) { this.flush(); } } // 批量上报 private async flush(): Promisevoid { if (this.queue.length 0) return; // 取出待上报日志清空队列 const batch this.queue.splice(0, this.config.maxBatchSize); try { const response await fetch(this.config.endpoint, { method: POST, headers: { Content-Type: application/json }, body: JSON.stringify({ appId: this.config.appId, entries: batch, dropped: this.droppedCount, }), keepalive: true, // 页面卸载时也能完成请求 }); if (!response.ok) { // 上报失败重新入队最多重试一次 this.queue.unshift(...batch); console.warn([Logger] 上报失败 HTTP ${response.status}${batch.length} 条日志重入队列); } this.droppedCount 0; } catch (err) { // 网络异常时重新入队 this.queue.unshift(...batch); console.warn([Logger] 上报异常: ${(err as Error).message}); } } // 定时上报 private startFlushTimer(): void { this.flushTimer window.setInterval(() { this.flush(); }, this.config.flushInterval); } // 全局错误监听 private registerErrorListeners(): void { window.addEventListener(error, (event) { this.log(LogLevel.ERROR, { type: runtime_error, message: event.message, filename: event.filename, lineno: event.lineno, stack: event.error?.stack?.substring(0, 500) ?? , // 截断避免超长 }); }); window.addEventListener(unhandledrejection, (event) { this.log(LogLevel.ERROR, { type: unhandled_rejection, reason: String(event.reason).substring(0, 300), }); }); } // 性能指标监听 private registerPerformanceListeners(): void { // 页面加载性能 window.addEventListener(load, () { setTimeout(() { const perf performance.getEntriesByType(navigation)[0] as PerformanceNavigationTiming; if (!perf) return; this.log(LogLevel.PERFORMANCE, { type: page_load, dns: perf.domainLookupEnd - perf.domainLookupStart, tcp: perf.connectEnd - perf.connectStart, ttfb: perf.responseStart - perf.requestStart, domReady: perf.domContentLoadedEventEnd - perf.fetchStart, fullLoad: perf.loadEventEnd - perf.fetchStart, }); }, 100); // 延迟获取确保指标完整 }); // 长任务监听 if (PerformanceObserver in window) { try { const observer new PerformanceObserver((list) { for (const entry of list.getEntries()) { if (entry.duration 50) { this.log(LogLevel.PERFORMANCE, { type: long_task, duration: entry.duration, name: entry.name, startTime: entry.startTime, }); } } }); observer.observe({ entryTypes: [longtask] }); } catch { // 浏览器不支持 longtask静默忽略 } } } // 行为监听关键点击事件 private registerBehaviorListeners(): void { // 仅监听关键操作区域的点击 const actionElements document.querySelectorAll([data-track]); actionElements.forEach((el) { el.addEventListener(click, () { this.log(LogLevel.BEHAVIOR, { type: click, target: el.getAttribute(data-track) ?? el.tagName, text: (el.textContent ?? ).substring(0, 50), }); }); }); } private generateSessionId(): string { return ${Date.now()}-${Math.random().toString(36).substring(2, 8)}; } private tryGetUserId(): string | undefined { // 从本地存储获取用户标识未登录时为 undefined try { return localStorage.getItem(user_id) ?? undefined; } catch { return undefined; } } destroy(): void { if (this.flushTimer) clearInterval(this.flushTimer); this.flush(); } }三、传输层设计传输层的关键指标上报成功率、带宽占用、对页面性能的影响。3.1 批量与压缩策略// transport.ts — 传输层核心逻辑 interface TransportConfig { endpoint: string; maxBatchSize: number; compressionThreshold: number; // 触发压缩的最小条数 retryMaxAttempts: number; retryBaseDelay: number; // 重试基础延迟(ms) } class LogTransport { private config: TransportConfig; private retryAttempts: number 0; constructor(config: TransportConfig) { this.config config; } async sendBatch(entries: LogEntry[]): Promiseboolean { const payload JSON.stringify(entries); // 大批次时压缩小批次直接发送 let body: string | Blob; let headers: Recordstring, string; if (entries.length this.config.compressionThreshold) { try { const compressed await this.compress(payload); body compressed; headers { Content-Type: application/json, Content-Encoding: gzip, }; } catch { // 压缩失败降级为未压缩发送 body payload; headers { Content-Type: application/json }; } } else { body payload; headers { Content-Type: application/json }; } try { const response await fetch(this.config.endpoint, { method: POST, headers, body, keepalive: true, }); if (response.status 429) { // 服务端限流指数退避 this.retryAttempts; const delay this.config.retryBaseDelay * Math.pow(2, this.retryAttempts); console.warn([Transport] 限流${delay}ms 后重试); await this.sleep(delay); return false; } if (response.ok) { this.retryAttempts 0; return true; } return false; } catch (err) { console.warn([Transport] 发送失败: ${(err as Error).message}); return false; } } private async compress(data: string): PromiseBlob { // 使用 CompressionStream API现代浏览器支持 if (CompressionStream in window) { const stream new Blob([data]).stream(); const compressedStream stream.pipeThrough(new CompressionStream(gzip)); return new Response(compressedStream).blob(); } // 不支持压缩时返回原始数据 return new Blob([data], { type: application/json }); } private sleep(ms: number): Promisevoid { return new Promise((resolve) setTimeout(resolve, Math.min(ms, 30000))); } }3.2 离线缓存与降级上报用户网络不稳定时日志需要暂存本地网络恢复后批量上报。// offline-buffer.ts — 离线缓存策略 class OfflineLogBuffer { private storageKey: string; private maxStorageEntries: number 500; constructor(appId: string) { this.storageKey log_buffer_${appId}; } // 暂存到 IndexedDB容量比 localStorage 大 async store(entries: LogEntry[]): Promisevoid { try { const db await this.openDB(); const tx db.transaction(logs, readwrite); const store tx.objectStore(logs); for (const entry of entries) { // 超出容量时删除最旧的日志 const count await store.count(); if (count this.maxStorageEntries) { const oldest await store.openCursor(); if (oldest) await store.delete(oldest.value.id); } await store.add({ ...entry, id: ${entry.timestamp}-${Math.random().toString(36).slice(2)} }); } await tx.done; } catch (err) { // IndexedDB 不可用时降级到 localStorage this.storeToLocalStorage(entries); } } // 恢复暂存日志 async retrieve(maxCount: number): PromiseLogEntry[] { try { const db await this.openDB(); const tx db.transaction(logs, readonly); const store tx.objectStore(logs); const all await store.getAll(); // 按时间排序取最新 maxCount 条 const sorted all.sort((a: LogEntry, b: LogEntry) b.timestamp - a.timestamp); return sorted.slice(0, maxCount); } catch { return this.retrieveFromLocalStorage(maxCount); } } // 清除已上报的日志 async clear(): Promisevoid { try { const db await this.openDB(); const tx db.transaction(logs, readwrite); tx.objectStore(logs).clear(); await tx.done; } catch { localStorage.removeItem(this.storageKey); } } // 降级存储localStorage private storeToLocalStorage(entries: LogEntry[]): void { try { const existing JSON.parse(localStorage.getItem(this.storageKey) ?? []); const merged [...existing, ...entries].slice(-this.maxStorageEntries); localStorage.setItem(this.storageKey, JSON.stringify(merged)); } catch { // localStorage 不可用时彻底降级丢弃日志 console.warn([OfflineBuffer] 降级存储失败日志丢弃); } } private retrieveFromLocalStorage(maxCount: number): LogEntry[] { try { const stored JSON.parse(localStorage.getItem(this.storageKey) ?? []); return stored.slice(0, maxCount); } catch { return []; } } private async openDB(): PromiseIDBDatabase { return new Promise((resolve, reject) { const request indexedDB.open(frontend_logs, 1); request.onupgradeneeded () { const db request.result; if (!db.objectStoreNames.contains(logs)) { db.createObjectStore(logs, { keyPath: id }); } }; request.onsuccess () resolve(request.result); request.onerror () reject(new Error(IndexedDB 打开失败)); }); } }四、存储与查询层设计4.1 时序数据库选型前端日志的时间属性强查询模式固定按时间范围 页面 日志级别筛选。时序数据库比关系型数据库更适合。存储策略热数据7天内保留在 InfluxDB冷数据归档到对象存储。查询优先走热数据历史分析走归档。// storage-policy.ts — 存储策略配置 interface StorageTier { name: string; retention: string; // 数据保留时长 backend: string; // 存储引擎 queryLatency: string; // 预估查询延迟 costPerGB: string; // 每GB存储成本 } const tiers: StorageTier[] [ { name: 热数据, retention: 7d, backend: InfluxDB, queryLatency: 100ms, costPerGB: ¥2.5/月 }, { name: 温数据, retention: 30d, backend: ClickHouse, queryLatency: 500ms, costPerGB: ¥1.0/月 }, { name: 冷数据, retention: 365d, backend: 对象存储Parquet, queryLatency: 1-5s, costPerGB: ¥0.12/月 }, ]; // 数据路由根据查询时间范围选择存储层 function selectStorageTier(queryStart: Date): StorageTier { const ageInDays (Date.now() - queryStart.getTime()) / 86400000; if (ageInDays 7) return tiers[0]; if (ageInDays 30) return tiers[1]; return tiers[2]; }4.2 查询性能优化// query-service.ts — 查询服务 interface LogQuery { start: Date; end: Date; level?: LogLevel; page?: string; userId?: string; keyword?: string; limit: number; } interface QueryResult { total: number; entries: LogEntry[]; latencyMs: number; source: string; // 来自哪个存储层 } // 查询缓存LRU 策略 const queryCache new Mapstring, { result: QueryResult; expiresAt: number }(); const CACHE_TTL 5 * 60 * 1000; // 5分钟缓存 async function queryLogs(params: LogQuery): PromiseQueryResult { const cacheKey JSON.stringify(params); const cached queryCache.get(cacheKey); if (cached cached.expiresAt Date.now()) { return { ...cached.result, latencyMs: 0, source: cache }; } // 选择存储层 const tier selectStorageTier(params.start); const startTime Date.now(); let entries: LogEntry[]; try { entries await fetchFromStorage(tier, params); } catch (err) { // 查询失败时返回空结果不抛异常 console.error([QueryService] ${tier.backend} 查询失败: ${(err as Error).message}); entries []; } const result: QueryResult { total: entries.length, entries: entries.slice(0, params.limit), latencyMs: Date.now() - startTime, source: tier.backend, }; // 缓存结果 queryCache.set(cacheKey, { result, expiresAt: Date.now() CACHE_TTL }); // 缓存容量控制 if (queryCache.size 200) { const oldest queryCache.keys().next().value; if (oldest) queryCache.delete(oldest); } return result; } async function fetchFromStorage(tier: StorageTier, params: LogQuery): PromiseLogEntry[] { // 根据存储层调用不同接口 const url tier.backend InfluxDB ? /api/logs/influx : tier.backend ClickHouse ? /api/logs/clickhouse : /api/logs/archive; const response await fetch(url, { method: POST, headers: { Content-Type: application/json }, body: JSON.stringify(params), }); if (!response.ok) { throw new Error(查询接口返回 HTTP ${response.status}); } return response.json(); }五、总结前端日志系统的架构设计核心是在采集粒度与成本之间做取舍。出行平台的实践数据表明分级采样策略将日均传输量从 800 万条压降至 120 万条带宽成本降低 85%而错误覆盖率保持在 100%。关键设计原则分级采样错误全量、性能 10%、行为 1%信息价值决定采集比例。批量压缩≥30 条触发 Gzip小批次直接发送避免压缩本身成为开销。离线缓存IndexedDB 优先、localStorage 降级网络恢复后自动上报。分层存储热数据 InfluxDB、温数据 ClickHouse、冷数据对象存储查询延迟与成本分层优化。队列降级队列满时丢弃行为日志保留错误日志保证核心信息不丢。日志系统的有效性不只取决于技术架构更取决于采集策略是否匹配业务需求。每一条日志都有存储成本只有对业务决策有实际支撑作用的日志才值得全量采集。其余的用采样就够了。