diff --git a/.github/workflows/eslint-ci.yml b/.github/workflows/eslint-ci.yml index 4c5b52a53f..3ab8528b04 100644 --- a/.github/workflows/eslint-ci.yml +++ b/.github/workflows/eslint-ci.yml @@ -119,6 +119,10 @@ jobs: echo "" echo "::error::Import order drift detected. Fix locally with:" echo "::error:: npm run sort-imports" + echo "::error::For specific files:" + echo "::error:: npm run sort-imports -- packages/api/src/app/metrics.ts packages/api/src/rum/proxy.ts" + echo "::error::To check without writing files:" + echo "::error:: npm run sort-imports:check" echo "::error::Or rely on the lint-staged pre-commit hook (do not bypass with --no-verify)." exit 1 fi diff --git a/api/server/middleware/__tests__/requireJwtAuth.spec.js b/api/server/middleware/__tests__/requireJwtAuth.spec.js index 873a815874..b70f371a94 100644 --- a/api/server/middleware/__tests__/requireJwtAuth.spec.js +++ b/api/server/middleware/__tests__/requireJwtAuth.spec.js @@ -213,6 +213,7 @@ jest.mock('@librechat/api', () => { normalizeContextValue(req.headers?.['x-correlation-id']); return { isEnabled: jest.fn(() => false), + recordRumProxyRequest: jest.fn(), getAuthFailureReason, getAuthFailureErrorName, buildSafeAuthLogContext, @@ -235,8 +236,13 @@ jest.mock('@librechat/api', () => { // ── Helpers ───────────────────────────────────────────────────────────── const requireJwtAuth = require('../requireJwtAuth'); +const { requireRumProxyAuth } = requireJwtAuth; const { getTenantId, getUserId, logger } = require('@librechat/data-schemas'); -const { isEnabled, maybeRefreshCloudFrontAuthCookiesMiddleware } = require('@librechat/api'); +const { + isEnabled, + maybeRefreshCloudFrontAuthCookiesMiddleware, + recordRumProxyRequest, +} = require('@librechat/api'); const passport = require('passport'); const jwtSecret = 'test-refresh-secret'; @@ -250,7 +256,11 @@ function signedOpenIdUserCookie(userId = 'user-openid') { } function mockRes() { - return { status: jest.fn().mockReturnThis(), json: jest.fn().mockReturnThis() }; + return { + status: jest.fn().mockReturnThis(), + json: jest.fn().mockReturnThis(), + end: jest.fn().mockReturnThis(), + }; } /** Runs requireJwtAuth and returns the tenantId observed inside next(). */ @@ -282,6 +292,7 @@ describe('requireJwtAuth tenant context chaining', () => { logger.info.mockClear(); logger.warn.mockClear(); logger.error.mockClear(); + recordRumProxyRequest.mockClear(); passport.authenticate.mockClear(); passport._strategy.mockClear(); if (originalJwtSecret === undefined) { @@ -836,3 +847,141 @@ describe('requireJwtAuth tenant context chaining', () => { expect(getTenantId()).toBeUndefined(); }); }); + +describe('requireRumProxyAuth', () => { + const originalJwtSecret = process.env.JWT_REFRESH_SECRET; + + beforeEach(() => { + process.env.JWT_REFRESH_SECRET = jwtSecret; + }); + + afterEach(() => { + mockPassportError = null; + mockRegisteredStrategies = new Set(['jwt']); + isEnabled.mockReturnValue(false); + maybeRefreshCloudFrontAuthCookiesMiddleware.mockClear(); + logger.debug.mockClear(); + logger.info.mockClear(); + logger.warn.mockClear(); + logger.error.mockClear(); + recordRumProxyRequest.mockClear(); + passport.authenticate.mockClear(); + passport._strategy.mockClear(); + if (originalJwtSecret === undefined) { + delete process.env.JWT_REFRESH_SECRET; + } else { + process.env.JWT_REFRESH_SECRET = originalJwtSecret; + } + }); + + it('authenticates telemetry with the LibreChat JWT strategy without tenant or cookie refresh middleware', () => { + const req = mockReq({ id: 'user-jwt', tenantId: 'tenant-jwt', role: 'user' }); + const res = mockRes(); + const next = jest.fn(); + + requireRumProxyAuth(req, res, next); + + expect(passport.authenticate).toHaveBeenCalledWith( + 'jwt', + { session: false }, + expect.any(Function), + ); + expect(req.authStrategy).toBe('jwt'); + expect(maybeRefreshCloudFrontAuthCookiesMiddleware).not.toHaveBeenCalled(); + // Success is recorded by the proxy. + expect(recordRumProxyRequest).not.toHaveBeenCalled(); + expect(res.status).not.toHaveBeenCalled(); + expect(next).toHaveBeenCalled(); + }); + + it('authenticates telemetry with OpenID JWT reuse when the reuse cookie is present', () => { + isEnabled.mockReturnValue(true); + mockRegisteredStrategies.add('openidJwt'); + const req = mockReq(undefined, { + headers: { cookie: `token_provider=openid; openid_user_id=${signedOpenIdUserCookie()}` }, + _mockStrategies: { + openidJwt: { user: { id: 'user-openid', tenantId: 'tenant-openid', role: 'user' } }, + jwt: { user: false, info: { message: 'invalid signature' }, status: 401 }, + }, + }); + const res = mockRes(); + const next = jest.fn(); + + requireRumProxyAuth(req, res, next); + + expect(passport.authenticate).toHaveBeenCalledWith( + 'openidJwt', + { session: false }, + expect.any(Function), + ); + expect(req.authStrategy).toBe('openidJwt'); + expect(maybeRefreshCloudFrontAuthCookiesMiddleware).not.toHaveBeenCalled(); + expect(recordRumProxyRequest).not.toHaveBeenCalled(); + expect(res.status).not.toHaveBeenCalled(); + expect(next).toHaveBeenCalled(); + }); + + it('falls back to LibreChat JWT when OpenID JWT telemetry auth fails', () => { + isEnabled.mockReturnValue(true); + mockRegisteredStrategies.add('openidJwt'); + const req = mockReq(undefined, { + headers: { cookie: `token_provider=openid; openid_user_id=${signedOpenIdUserCookie()}` }, + _mockStrategies: { + openidJwt: { + user: false, + info: { message: 'jwt expired', name: 'TokenExpiredError' }, + status: 401, + }, + jwt: { user: { id: 'user-openid', tenantId: 'tenant-jwt', role: 'user' } }, + }, + }); + const res = mockRes(); + const next = jest.fn(); + + requireRumProxyAuth(req, res, next); + + expect(passport.authenticate).toHaveBeenCalledTimes(2); + expect(req.authStrategy).toBe('jwt'); + expect(recordRumProxyRequest).not.toHaveBeenCalled(); + expect(res.status).not.toHaveBeenCalled(); + expect(next).toHaveBeenCalled(); + }); + + it('drops invalid telemetry auth with 204 instead of returning an app auth error', () => { + const req = mockReq(undefined, { + path: '/v1/traces', + _mockStrategies: { + jwt: { + user: false, + info: { message: 'invalid signature', name: 'JsonWebTokenError' }, + status: 401, + }, + }, + }); + const res = mockRes(); + const next = jest.fn(); + + requireRumProxyAuth(req, res, next); + + expect(next).not.toHaveBeenCalled(); + expect(maybeRefreshCloudFrontAuthCookiesMiddleware).not.toHaveBeenCalled(); + expect(recordRumProxyRequest).toHaveBeenCalledWith('traces', 'auth_drop'); + expect(res.status).toHaveBeenCalledWith(204); + expect(res.end).toHaveBeenCalled(); + }); + + it('records passport errors separately from ordinary telemetry auth drops', () => { + mockPassportError = new Error('passport unavailable'); + const req = mockReq(undefined, { path: '/v1/logs' }); + const res = mockRes(); + const next = jest.fn(); + + requireRumProxyAuth(req, res, next); + + expect(next).not.toHaveBeenCalled(); + expect(logger.warn).not.toHaveBeenCalled(); + expect(recordRumProxyRequest).toHaveBeenCalledWith('logs', 'auth_error'); + expect(res.status).toHaveBeenCalledWith(204); + expect(res.end).toHaveBeenCalled(); + }); +}); diff --git a/api/server/middleware/index.js b/api/server/middleware/index.js index 3aa52e3349..bc523ff166 100644 --- a/api/server/middleware/index.js +++ b/api/server/middleware/index.js @@ -11,6 +11,7 @@ const requireLdapAuth = require('./requireLdapAuth'); const abortMiddleware = require('./abortMiddleware'); const checkInviteUser = require('./checkInviteUser'); const requireJwtAuth = require('./requireJwtAuth'); +const { requireRumProxyAuth } = require('./requireJwtAuth'); const configMiddleware = require('./config/app'); const validateModel = require('./validateModel'); const moderateText = require('./moderateText'); @@ -37,6 +38,7 @@ module.exports = { moderateText, validateModel, requireJwtAuth, + requireRumProxyAuth, setTwoFactorTempUser, checkInviteUser, requireLdapAuth, diff --git a/api/server/middleware/requireJwtAuth.js b/api/server/middleware/requireJwtAuth.js index a34dcd6983..9c4d1ca47c 100644 --- a/api/server/middleware/requireJwtAuth.js +++ b/api/server/middleware/requireJwtAuth.js @@ -10,6 +10,7 @@ const { buildSafeAuthLogContext, formatAuthLogMessage, maybeRefreshCloudFrontAuthCookiesMiddleware, + recordRumProxyRequest, } = require('@librechat/api'); const hasPassportStrategy = (strategy) => @@ -35,6 +36,45 @@ const getAuthenticatedUserId = (user) => user?.id?.toString?.() ?? user?._id?.to const refreshCloudFrontCookies = maybeRefreshCloudFrontAuthCookiesMiddleware ?? ((_req, _res, next) => next()); +const getAuthStrategies = (req) => { + const cookieHeader = req.headers.cookie; + const parsedCookies = cookieHeader ? cookies.parse(cookieHeader) : {}; + const tokenProvider = parsedCookies.token_provider; + const openidReuseEnabled = isEnabled(process.env.OPENID_REUSE_TOKENS); + const openidJwtAvailable = openidReuseEnabled && hasPassportStrategy('openidJwt'); + const openIdReuseUserId = getValidOpenIdReuseUserId(parsedCookies); + const useOpenIdJwt = + tokenProvider === 'openid' && openidJwtAvailable && openIdReuseUserId != null; + + return { + tokenProvider, + openidReuseEnabled, + openidJwtAvailable, + openIdReuseUserId, + strategies: useOpenIdJwt ? ['openidJwt', 'jwt'] : ['jwt'], + }; +}; + +const dropRumTelemetry = (res) => { + if (!res.headersSent) { + res.status(204).end(); + } +}; + +// Keep in sync with packages/api/src/rum/proxy.ts; auth drops are recorded before proxy code runs. +const getRumProxyEndpoint = (req) => { + if (req.path === '/v1/traces') { + return 'traces'; + } + if (req.path === '/v1/logs') { + return 'logs'; + } + return 'unknown'; +}; + +const isOpenIdReuseUser = (strategy, user, openIdReuseUserId) => + strategy !== 'openidJwt' || getAuthenticatedUserId(user) === openIdReuseUserId; + /** * Custom Middleware to handle JWT authentication, with support for OpenID token reuse. * Switches between JWT and OpenID authentication based on cookies and environment settings. @@ -44,15 +84,8 @@ const refreshCloudFrontCookies = * for downstream Mongoose tenant isolation and structured logging. */ const requireJwtAuth = (req, res, next) => { - const cookieHeader = req.headers.cookie; - const parsedCookies = cookieHeader ? cookies.parse(cookieHeader) : {}; - const tokenProvider = parsedCookies.token_provider; - const openidReuseEnabled = isEnabled(process.env.OPENID_REUSE_TOKENS); - const openidJwtAvailable = openidReuseEnabled && hasPassportStrategy('openidJwt'); - const openIdReuseUserId = getValidOpenIdReuseUserId(parsedCookies); - const useOpenIdJwt = - tokenProvider === 'openid' && openidJwtAvailable && openIdReuseUserId != null; - const strategies = useOpenIdJwt ? ['openidJwt', 'jwt'] : ['jwt']; + const { tokenProvider, openidReuseEnabled, openidJwtAvailable, openIdReuseUserId, strategies } = + getAuthStrategies(req); const authLogState = { tokenProvider, openidReuseEnabled, @@ -162,4 +195,45 @@ const requireJwtAuth = (req, res, next) => { authenticateWithStrategy(0); }; +const requireRumProxyAuth = (req, res, next) => { + const { openIdReuseUserId, strategies } = getAuthStrategies(req); + const endpoint = getRumProxyEndpoint(req); + let authErrorSeen = false; + + const dropTelemetry = () => { + recordRumProxyRequest(endpoint, authErrorSeen ? 'auth_error' : 'auth_drop'); + dropRumTelemetry(res); + }; + + const finishAuthentication = (strategy, user) => { + req.user = user; + req.authStrategy = strategy; + next(); + }; + + let nextStrategyIndex = 0; + const tryNextStrategy = () => { + const strategy = strategies[nextStrategyIndex]; + nextStrategyIndex += 1; + + if (!strategy) { + dropTelemetry(); + return; + } + + passport.authenticate(strategy, { session: false }, (err, user) => { + authErrorSeen = authErrorSeen || err != null; + if (err || !user || !isOpenIdReuseUser(strategy, user, openIdReuseUserId)) { + tryNextStrategy(); + return; + } + + finishAuthentication(strategy, user); + })(req, res, next); + }; + + tryNextStrategy(); +}; + module.exports = requireJwtAuth; +module.exports.requireRumProxyAuth = requireRumProxyAuth; diff --git a/api/server/routes/__tests__/rum.spec.js b/api/server/routes/__tests__/rum.spec.js index ad161a75eb..cdd0ec3e76 100644 --- a/api/server/routes/__tests__/rum.spec.js +++ b/api/server/routes/__tests__/rum.spec.js @@ -1,12 +1,12 @@ const express = require('express'); const request = require('supertest'); -const mockRequireJwtAuth = jest.fn((_req, _res, next) => next()); +const mockRequireRumProxyAuth = jest.fn((_req, _res, next) => next()); const mockIsRumProxyEnabled = jest.fn(); const mockProxyRumRequest = jest.fn((_req, res) => res.status(202).send()); jest.mock('~/server/middleware', () => ({ - requireJwtAuth: (...args) => mockRequireJwtAuth(...args), + requireRumProxyAuth: (...args) => mockRequireRumProxyAuth(...args), })); jest.mock('@librechat/api', () => ({ @@ -26,7 +26,7 @@ describe('RUM proxy routes', () => { }); beforeEach(() => { - mockRequireJwtAuth.mockClear(); + mockRequireRumProxyAuth.mockClear(); mockIsRumProxyEnabled.mockReset(); mockProxyRumRequest.mockClear(); }); @@ -41,7 +41,7 @@ describe('RUM proxy routes', () => { expect(response.status).toBe(404); expect(response.body).toEqual({ message: 'RUM proxy is not configured' }); - expect(mockRequireJwtAuth).not.toHaveBeenCalled(); + expect(mockRequireRumProxyAuth).not.toHaveBeenCalled(); expect(mockProxyRumRequest).not.toHaveBeenCalled(); }); @@ -54,7 +54,20 @@ describe('RUM proxy routes', () => { .send(Buffer.from('payload')); expect(response.status).toBe(202); - expect(mockRequireJwtAuth).toHaveBeenCalledTimes(1); + expect(mockRequireRumProxyAuth).toHaveBeenCalledTimes(1); + expect(mockProxyRumRequest).toHaveBeenCalledTimes(1); + }); + + it('uses RUM-specific auth for logs as well as traces', async () => { + mockIsRumProxyEnabled.mockReturnValue(true); + + const response = await request(app) + .post('/api/rum/v1/logs') + .set('Content-Type', 'application/x-protobuf') + .send(Buffer.from('payload')); + + expect(response.status).toBe(202); + expect(mockRequireRumProxyAuth).toHaveBeenCalledTimes(1); expect(mockProxyRumRequest).toHaveBeenCalledTimes(1); }); }); diff --git a/api/server/routes/rum.js b/api/server/routes/rum.js index b407992e1e..cc5c2f281a 100644 --- a/api/server/routes/rum.js +++ b/api/server/routes/rum.js @@ -1,6 +1,6 @@ const express = require('express'); const { getRumProxyBodyLimit, isRumProxyEnabled, proxyRumRequest } = require('@librechat/api'); -const { requireJwtAuth } = require('~/server/middleware'); +const { requireRumProxyAuth } = require('~/server/middleware'); const router = express.Router(); const rawOtlpBody = express.raw({ @@ -16,7 +16,13 @@ function requireRumProxyEnabled(_req, res, next) { return next(); } -router.post('/v1/traces', requireRumProxyEnabled, requireJwtAuth, rawOtlpBody, proxyRumRequest); -router.post('/v1/logs', requireRumProxyEnabled, requireJwtAuth, rawOtlpBody, proxyRumRequest); +router.post( + '/v1/traces', + requireRumProxyEnabled, + requireRumProxyAuth, + rawOtlpBody, + proxyRumRequest, +); +router.post('/v1/logs', requireRumProxyEnabled, requireRumProxyAuth, rawOtlpBody, proxyRumRequest); module.exports = router; diff --git a/package-lock.json b/package-lock.json index 8268a7ef58..d6b9f2c031 100644 --- a/package-lock.json +++ b/package-lock.json @@ -19851,6 +19851,13 @@ "@types/node": "*" } }, + "node_modules/@types/cookiejar": { + "version": "2.1.5", + "resolved": "https://registry.npmjs.org/@types/cookiejar/-/cookiejar-2.1.5.tgz", + "integrity": "sha512-he+DHOWReW0nghN24E1WUqM0efK4kI9oTqDm6XmK8ZPe2djZ90BSNdGnIyCLzCPw7/pogPlGbzI2wHGGmi4O/Q==", + "dev": true, + "license": "MIT" + }, "node_modules/@types/css-font-loading-module": { "version": "0.0.7", "resolved": "https://registry.npmjs.org/@types/css-font-loading-module/-/css-font-loading-module-0.0.7.tgz", @@ -20303,6 +20310,13 @@ "@types/unist": "*" } }, + "node_modules/@types/methods": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/@types/methods/-/methods-1.1.4.tgz", + "integrity": "sha512-ymXWVrDiCxTBE3+RIrrP533E70eA+9qu7zdWoHuOmGujkYtzf4HQF96b8nwHLqhuf4ykX61IGRIB38CC6/sImQ==", + "dev": true, + "license": "MIT" + }, "node_modules/@types/mime": { "version": "1.3.5", "resolved": "https://registry.npmjs.org/@types/mime/-/mime-1.3.5.tgz", @@ -20475,6 +20489,30 @@ "integrity": "sha512-9aEbYZ3TbYMznPdcdr3SmIrLXwC/AKZXQeCf9Pgao5CKb8CyHuEX5jzWPTkvregvhRJHcpRO6BFoGW9ycaOkYw==", "dev": true }, + "node_modules/@types/superagent": { + "version": "8.1.10", + "resolved": "https://registry.npmjs.org/@types/superagent/-/superagent-8.1.10.tgz", + "integrity": "sha512-nbt4IWXABhW0jGmmpRzCFNlbmwCTzZ2gTUsNIr+X+ItdqPms+PAJZbWsNzpS2USqXjcoNLQcO6nXo60zcPQiIg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/cookiejar": "^2.1.5", + "@types/methods": "^1.1.4", + "@types/node": "*", + "form-data": "^4.0.0" + } + }, + "node_modules/@types/supertest": { + "version": "7.2.0", + "resolved": "https://registry.npmjs.org/@types/supertest/-/supertest-7.2.0.tgz", + "integrity": "sha512-uh2Lv57xvggst6lCqNdFAmDSvoMG7M/HDtX4iUCquxQ5EGPtaPM5PL5Hmi7LCvOG8db7YaCPNJEeoI8s/WzIQw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/methods": "^1.1.4", + "@types/superagent": "^8.1.0" + } + }, "node_modules/@types/testing-library__jest-dom": { "version": "5.14.9", "resolved": "https://registry.npmjs.org/@types/testing-library__jest-dom/-/testing-library__jest-dom-5.14.9.tgz", @@ -44214,6 +44252,7 @@ "@types/node-fetch": "^2.6.13", "@types/react": "^18.2.18", "@types/sanitize-html": "^2.13.0", + "@types/supertest": "^7.2.0", "@types/winston": "^2.4.4", "@types/yauzl": "^2.10.3", "aws-sdk-client-mock": "^4.1.0", diff --git a/packages/api/package.json b/packages/api/package.json index 096d844098..8018c03f59 100644 --- a/packages/api/package.json +++ b/packages/api/package.json @@ -72,6 +72,7 @@ "@types/node-fetch": "^2.6.13", "@types/react": "^18.2.18", "@types/sanitize-html": "^2.13.0", + "@types/supertest": "^7.2.0", "@types/winston": "^2.4.4", "@types/yauzl": "^2.10.3", "aws-sdk-client-mock": "^4.1.0", diff --git a/packages/api/src/app/metrics.spec.ts b/packages/api/src/app/metrics.spec.ts index 80984dcb8b..ebb1267504 100644 --- a/packages/api/src/app/metrics.spec.ts +++ b/packages/api/src/app/metrics.spec.ts @@ -1,6 +1,7 @@ /// -import { EventEmitter } from 'events'; import express from 'express'; +import request from 'supertest'; +import { EventEmitter } from 'events'; import type { Request, Response } from 'express'; import { createMetrics, @@ -10,11 +11,10 @@ import { recordGenerationStreamResumePendingEvents, recordGenerationStreamSubscription, recordOpenIDUserLookup, + recordRumProxyRequest, setGenerationJobsInFlight, } from './metrics'; -const request = require('supertest') as (app: express.Express) => any; - describe('normalizePath', () => { it.each([ // Known high-cardinality routes @@ -272,6 +272,36 @@ describe('createMetrics', () => { expect(response.text).toMatch(/openid_user_lookup_duration_seconds_sum\{result="found"\} 0.2/); }); + it('tracks RUM proxy request outcomes', async () => { + const app = express(); + process.env.METRICS_SECRET = 'test-secret'; + const { metricsRouter } = createMetrics(); + app.use('/metrics', metricsRouter); + + recordRumProxyRequest('traces', 'success'); + recordRumProxyRequest('traces', 'auth_drop'); + recordRumProxyRequest('logs', 'auth_error'); + recordRumProxyRequest('logs', 'collector_5xx'); + + const response = await request(app) + .get('/metrics') + .set('Authorization', 'Bearer test-secret') + .expect(200); + + expect(response.text).toMatch( + /rum_proxy_requests_total\{endpoint="traces",result="success"\} 1/, + ); + expect(response.text).toMatch( + /rum_proxy_requests_total\{endpoint="traces",result="auth_drop"\} 1/, + ); + expect(response.text).toMatch( + /rum_proxy_requests_total\{endpoint="logs",result="auth_error"\} 1/, + ); + expect(response.text).toMatch( + /rum_proxy_requests_total\{endpoint="logs",result="collector_5xx"\} 1/, + ); + }); + it('tracks mongoose query counts and latency by model and operation', async () => { class FakeQuery { model = { modelName: 'User' }; diff --git a/packages/api/src/app/metrics.ts b/packages/api/src/app/metrics.ts index 4dd9fddac2..da2ce635d3 100644 --- a/packages/api/src/app/metrics.ts +++ b/packages/api/src/app/metrics.ts @@ -1,7 +1,7 @@ -import { timingSafeEqual } from 'crypto'; import { Router } from 'express'; -import { Registry, collectDefaultMetrics, Counter, Gauge, Histogram } from 'prom-client'; +import { timingSafeEqual } from 'crypto'; import { logger } from '@librechat/data-schemas'; +import { Registry, collectDefaultMetrics, Counter, Gauge, Histogram } from 'prom-client'; import type { Request, Response, NextFunction, RequestHandler } from 'express'; import type { Mongoose } from 'mongoose'; @@ -137,6 +137,17 @@ export type GenerationStreamSubscriptionResult = | 'error' | 'found' | 'missing'; +export type RumProxyEndpoint = 'traces' | 'logs' | 'unknown'; +export type RumProxyResult = + | 'success' + | 'auth_drop' + | 'auth_error' + | 'bad_request' + | 'not_configured' + | 'collector_4xx' + | 'collector_5xx' + | 'collector_error' + | 'collector_timeout'; type OpenIDUserLookupMetrics = { recordLookup: (result: OpenIDUserLookupResult, durationSeconds: number) => void; @@ -179,6 +190,14 @@ let generationJobMetrics: GenerationJobMetrics = { recordResumePendingEvents: () => undefined, }; +type RumProxyMetrics = { + recordRequest: (endpoint: RumProxyEndpoint, result: RumProxyResult) => void; +}; + +let rumProxyMetrics: RumProxyMetrics = { + recordRequest: () => undefined, +}; + const resetMetricRecorders = (): void => { openIDUserLookupMetrics = { recordLookup: () => undefined, @@ -192,6 +211,9 @@ const resetMetricRecorders = (): void => { recordSubscription: () => undefined, recordResumePendingEvents: () => undefined, }; + rumProxyMetrics = { + recordRequest: () => undefined, + }; }; export function recordGenerationJob(store: GenerationJobStore, result: GenerationJobResult): void { @@ -217,6 +239,10 @@ export function recordGenerationStreamResumePendingEvents( generationJobMetrics.recordResumePendingEvents(store, count); } +export function recordRumProxyRequest(endpoint: RumProxyEndpoint, result: RumProxyResult): void { + rumProxyMetrics.recordRequest(endpoint, result); +} + const getElapsedSeconds = (startedAt: bigint): number => Number(process.hrtime.bigint() - startedAt) / 1_000_000_000; @@ -493,6 +519,13 @@ export function createMetrics(): PrometheusMetrics { registers: [registry], }); + const rumProxyRequests = new Counter({ + name: 'rum_proxy_requests_total', + help: 'RUM proxy requests by endpoint and result', + labelNames: ['endpoint', 'result'] as const, + registers: [registry], + }); + generationJobMetrics = { recordJob: (store, result) => generationJobs.inc({ store, result }), setJobsInFlight: (store, count) => generationJobsInFlight.set({ store }, count), @@ -502,6 +535,10 @@ export function createMetrics(): PrometheusMetrics { generationStreamResumePendingEvents.inc({ store }, count), }; + rumProxyMetrics = { + recordRequest: (endpoint, result) => rumProxyRequests.inc({ endpoint, result }), + }; + const metricsMiddleware = (req: Request, res: Response, next: NextFunction): void => { const end = httpDuration.startTimer(); const labels = { method: req.method, path: normalizePath(req.path) }; diff --git a/packages/api/src/rum/proxy.spec.ts b/packages/api/src/rum/proxy.spec.ts index 989dea46a5..22aeeb9374 100644 --- a/packages/api/src/rum/proxy.spec.ts +++ b/packages/api/src/rum/proxy.spec.ts @@ -1,3 +1,8 @@ +jest.mock('~/app/metrics', () => ({ + recordRumProxyRequest: jest.fn(), +})); + +import { recordRumProxyRequest } from '~/app/metrics'; import { getRumProxyBodyLimit, getRumProxyClientUrl, @@ -24,6 +29,7 @@ describe('RUM proxy configuration', () => { beforeEach(() => { process.env = { ...originalEnv }; + jest.mocked(recordRumProxyRequest).mockClear(); }); afterEach(() => { @@ -114,6 +120,7 @@ describe('RUM proxy configuration', () => { expect(res.set).toHaveBeenCalledWith('content-type', 'application/json'); expect(res.status).toHaveBeenCalledWith(202); expect(res.send).toHaveBeenCalledWith(Buffer.from('ok')); + expect(recordRumProxyRequest).toHaveBeenCalledWith('traces', 'success'); fetchMock.mockRestore(); }); @@ -133,6 +140,8 @@ describe('RUM proxy configuration', () => { expect(missingBodyRes.status).toHaveBeenCalledWith(400); expect(unsupportedPathRes.status).toHaveBeenCalledWith(404); + expect(recordRumProxyRequest).toHaveBeenCalledWith('traces', 'bad_request'); + expect(recordRumProxyRequest).toHaveBeenCalledWith('unknown', 'not_configured'); }); it('returns 502 when the collector request fails', async () => { @@ -148,6 +157,26 @@ describe('RUM proxy configuration', () => { ); expect(res.status).toHaveBeenCalledWith(502); + expect(recordRumProxyRequest).toHaveBeenCalledWith('traces', 'collector_error'); + fetchMock.mockRestore(); + }); + + it('records collector error status classes', async () => { + process.env.RUM_ENABLED = 'true'; + process.env.RUM_AUTH_MODE = 'proxy'; + process.env.RUM_PROXY_TARGET_URL = 'http://otel-collector:4318'; + const fetchMock = jest + .spyOn(global, 'fetch') + .mockResolvedValue(new Response('nope', { status: 503 })); + const res = makeResponse(); + + await proxyRumRequest( + { path: '/v1/logs', body: Buffer.from('payload'), headers: {} } as never, + res as never, + ); + + expect(res.status).toHaveBeenCalledWith(503); + expect(recordRumProxyRequest).toHaveBeenCalledWith('logs', 'collector_5xx'); fetchMock.mockRestore(); }); }); diff --git a/packages/api/src/rum/proxy.ts b/packages/api/src/rum/proxy.ts index 3cb177a11a..856457be5f 100644 --- a/packages/api/src/rum/proxy.ts +++ b/packages/api/src/rum/proxy.ts @@ -1,5 +1,7 @@ import { logger } from '@librechat/data-schemas'; import type { Request, Response } from 'express'; +import type { RumProxyEndpoint, RumProxyResult } from '~/app/metrics'; +import { recordRumProxyRequest } from '~/app/metrics'; import { isEnabled } from '~/utils'; const DEFAULT_PROXY_PATH = '/api/rum'; @@ -75,6 +77,27 @@ export function resolveRumProxyTarget(path: string): string | undefined { return targetUrl.href; } +// Keep in sync with api/server/middleware/requireJwtAuth.js; auth drops are recorded there. +function getRumProxyEndpoint(path: string): RumProxyEndpoint { + if (path === '/v1/traces') { + return 'traces'; + } + if (path === '/v1/logs') { + return 'logs'; + } + return 'unknown'; +} + +function getRumCollectorResult(status: number): RumProxyResult { + if (status >= 500) { + return 'collector_5xx'; + } + if (status >= 400) { + return 'collector_4xx'; + } + return 'success'; +} + function getRequestBody(req: Request): Buffer | string | undefined { const body = req.body as unknown; @@ -109,14 +132,17 @@ function getProxyHeaders(req: Request, body: Buffer | string): Record { + const endpoint = getRumProxyEndpoint(req.path); const target = resolveRumProxyTarget(req.path); if (!target) { + recordRumProxyRequest(endpoint, 'not_configured'); res.status(404).json({ message: 'RUM proxy is not configured' }); return; } const body = getRequestBody(req); if (!body) { + recordRumProxyRequest(endpoint, 'bad_request'); res.status(400).json({ message: 'RUM payload is required' }); return; } @@ -139,8 +165,13 @@ export async function proxyRumRequest(req: Request, res: Response): Promise