📈 fix: Isolate RUM Telemetry Proxy Auth from App Auth (#13765)

* fix(rum): isolate telemetry proxy auth

* feat(rum): track proxy error metrics

* refactor(rum): simplify proxy auth strategy flow

* test(rum): clarify proxy success metric assertion

* test(metrics): use typed supertest import

* test(metrics): add local supertest types

* test(metrics): keep supertest types local

* test(metrics): use official supertest types

* fix(rum): log proxy auth strategy errors

* fix(rum): classify proxy auth errors in metrics

* style(rum): sort telemetry metric imports

* ci: mention import sort check command

* ci: show targeted import sort example
This commit is contained in:
Ravi Kumar L
2026-06-15 12:49:44 -04:00
committed by GitHub
parent bc5a3f502f
commit fbc990f684
12 changed files with 439 additions and 24 deletions

View File

@@ -119,6 +119,10 @@ jobs:
echo ""
echo "::error::Import order drift detected. Fix locally with:"
echo "::error:: npm run sort-imports"
echo "::error::For specific files:"
echo "::error:: npm run sort-imports -- packages/api/src/app/metrics.ts packages/api/src/rum/proxy.ts"
echo "::error::To check without writing files:"
echo "::error:: npm run sort-imports:check"
echo "::error::Or rely on the lint-staged pre-commit hook (do not bypass with --no-verify)."
exit 1
fi

View File

@@ -213,6 +213,7 @@ jest.mock('@librechat/api', () => {
normalizeContextValue(req.headers?.['x-correlation-id']);
return {
isEnabled: jest.fn(() => false),
recordRumProxyRequest: jest.fn(),
getAuthFailureReason,
getAuthFailureErrorName,
buildSafeAuthLogContext,
@@ -235,8 +236,13 @@ jest.mock('@librechat/api', () => {
// ── Helpers ─────────────────────────────────────────────────────────────
const requireJwtAuth = require('../requireJwtAuth');
const { requireRumProxyAuth } = requireJwtAuth;
const { getTenantId, getUserId, logger } = require('@librechat/data-schemas');
const { isEnabled, maybeRefreshCloudFrontAuthCookiesMiddleware } = require('@librechat/api');
const {
isEnabled,
maybeRefreshCloudFrontAuthCookiesMiddleware,
recordRumProxyRequest,
} = require('@librechat/api');
const passport = require('passport');
const jwtSecret = 'test-refresh-secret';
@@ -250,7 +256,11 @@ function signedOpenIdUserCookie(userId = 'user-openid') {
}
function mockRes() {
return { status: jest.fn().mockReturnThis(), json: jest.fn().mockReturnThis() };
return {
status: jest.fn().mockReturnThis(),
json: jest.fn().mockReturnThis(),
end: jest.fn().mockReturnThis(),
};
}
/** Runs requireJwtAuth and returns the tenantId observed inside next(). */
@@ -282,6 +292,7 @@ describe('requireJwtAuth tenant context chaining', () => {
logger.info.mockClear();
logger.warn.mockClear();
logger.error.mockClear();
recordRumProxyRequest.mockClear();
passport.authenticate.mockClear();
passport._strategy.mockClear();
if (originalJwtSecret === undefined) {
@@ -836,3 +847,141 @@ describe('requireJwtAuth tenant context chaining', () => {
expect(getTenantId()).toBeUndefined();
});
});
describe('requireRumProxyAuth', () => {
const originalJwtSecret = process.env.JWT_REFRESH_SECRET;
beforeEach(() => {
process.env.JWT_REFRESH_SECRET = jwtSecret;
});
afterEach(() => {
mockPassportError = null;
mockRegisteredStrategies = new Set(['jwt']);
isEnabled.mockReturnValue(false);
maybeRefreshCloudFrontAuthCookiesMiddleware.mockClear();
logger.debug.mockClear();
logger.info.mockClear();
logger.warn.mockClear();
logger.error.mockClear();
recordRumProxyRequest.mockClear();
passport.authenticate.mockClear();
passport._strategy.mockClear();
if (originalJwtSecret === undefined) {
delete process.env.JWT_REFRESH_SECRET;
} else {
process.env.JWT_REFRESH_SECRET = originalJwtSecret;
}
});
it('authenticates telemetry with the LibreChat JWT strategy without tenant or cookie refresh middleware', () => {
const req = mockReq({ id: 'user-jwt', tenantId: 'tenant-jwt', role: 'user' });
const res = mockRes();
const next = jest.fn();
requireRumProxyAuth(req, res, next);
expect(passport.authenticate).toHaveBeenCalledWith(
'jwt',
{ session: false },
expect.any(Function),
);
expect(req.authStrategy).toBe('jwt');
expect(maybeRefreshCloudFrontAuthCookiesMiddleware).not.toHaveBeenCalled();
// Success is recorded by the proxy.
expect(recordRumProxyRequest).not.toHaveBeenCalled();
expect(res.status).not.toHaveBeenCalled();
expect(next).toHaveBeenCalled();
});
it('authenticates telemetry with OpenID JWT reuse when the reuse cookie is present', () => {
isEnabled.mockReturnValue(true);
mockRegisteredStrategies.add('openidJwt');
const req = mockReq(undefined, {
headers: { cookie: `token_provider=openid; openid_user_id=${signedOpenIdUserCookie()}` },
_mockStrategies: {
openidJwt: { user: { id: 'user-openid', tenantId: 'tenant-openid', role: 'user' } },
jwt: { user: false, info: { message: 'invalid signature' }, status: 401 },
},
});
const res = mockRes();
const next = jest.fn();
requireRumProxyAuth(req, res, next);
expect(passport.authenticate).toHaveBeenCalledWith(
'openidJwt',
{ session: false },
expect.any(Function),
);
expect(req.authStrategy).toBe('openidJwt');
expect(maybeRefreshCloudFrontAuthCookiesMiddleware).not.toHaveBeenCalled();
expect(recordRumProxyRequest).not.toHaveBeenCalled();
expect(res.status).not.toHaveBeenCalled();
expect(next).toHaveBeenCalled();
});
it('falls back to LibreChat JWT when OpenID JWT telemetry auth fails', () => {
isEnabled.mockReturnValue(true);
mockRegisteredStrategies.add('openidJwt');
const req = mockReq(undefined, {
headers: { cookie: `token_provider=openid; openid_user_id=${signedOpenIdUserCookie()}` },
_mockStrategies: {
openidJwt: {
user: false,
info: { message: 'jwt expired', name: 'TokenExpiredError' },
status: 401,
},
jwt: { user: { id: 'user-openid', tenantId: 'tenant-jwt', role: 'user' } },
},
});
const res = mockRes();
const next = jest.fn();
requireRumProxyAuth(req, res, next);
expect(passport.authenticate).toHaveBeenCalledTimes(2);
expect(req.authStrategy).toBe('jwt');
expect(recordRumProxyRequest).not.toHaveBeenCalled();
expect(res.status).not.toHaveBeenCalled();
expect(next).toHaveBeenCalled();
});
it('drops invalid telemetry auth with 204 instead of returning an app auth error', () => {
const req = mockReq(undefined, {
path: '/v1/traces',
_mockStrategies: {
jwt: {
user: false,
info: { message: 'invalid signature', name: 'JsonWebTokenError' },
status: 401,
},
},
});
const res = mockRes();
const next = jest.fn();
requireRumProxyAuth(req, res, next);
expect(next).not.toHaveBeenCalled();
expect(maybeRefreshCloudFrontAuthCookiesMiddleware).not.toHaveBeenCalled();
expect(recordRumProxyRequest).toHaveBeenCalledWith('traces', 'auth_drop');
expect(res.status).toHaveBeenCalledWith(204);
expect(res.end).toHaveBeenCalled();
});
it('records passport errors separately from ordinary telemetry auth drops', () => {
mockPassportError = new Error('passport unavailable');
const req = mockReq(undefined, { path: '/v1/logs' });
const res = mockRes();
const next = jest.fn();
requireRumProxyAuth(req, res, next);
expect(next).not.toHaveBeenCalled();
expect(logger.warn).not.toHaveBeenCalled();
expect(recordRumProxyRequest).toHaveBeenCalledWith('logs', 'auth_error');
expect(res.status).toHaveBeenCalledWith(204);
expect(res.end).toHaveBeenCalled();
});
});

View File

@@ -11,6 +11,7 @@ const requireLdapAuth = require('./requireLdapAuth');
const abortMiddleware = require('./abortMiddleware');
const checkInviteUser = require('./checkInviteUser');
const requireJwtAuth = require('./requireJwtAuth');
const { requireRumProxyAuth } = require('./requireJwtAuth');
const configMiddleware = require('./config/app');
const validateModel = require('./validateModel');
const moderateText = require('./moderateText');
@@ -37,6 +38,7 @@ module.exports = {
moderateText,
validateModel,
requireJwtAuth,
requireRumProxyAuth,
setTwoFactorTempUser,
checkInviteUser,
requireLdapAuth,

View File

@@ -10,6 +10,7 @@ const {
buildSafeAuthLogContext,
formatAuthLogMessage,
maybeRefreshCloudFrontAuthCookiesMiddleware,
recordRumProxyRequest,
} = require('@librechat/api');
const hasPassportStrategy = (strategy) =>
@@ -35,6 +36,45 @@ const getAuthenticatedUserId = (user) => user?.id?.toString?.() ?? user?._id?.to
const refreshCloudFrontCookies =
maybeRefreshCloudFrontAuthCookiesMiddleware ?? ((_req, _res, next) => next());
const getAuthStrategies = (req) => {
const cookieHeader = req.headers.cookie;
const parsedCookies = cookieHeader ? cookies.parse(cookieHeader) : {};
const tokenProvider = parsedCookies.token_provider;
const openidReuseEnabled = isEnabled(process.env.OPENID_REUSE_TOKENS);
const openidJwtAvailable = openidReuseEnabled && hasPassportStrategy('openidJwt');
const openIdReuseUserId = getValidOpenIdReuseUserId(parsedCookies);
const useOpenIdJwt =
tokenProvider === 'openid' && openidJwtAvailable && openIdReuseUserId != null;
return {
tokenProvider,
openidReuseEnabled,
openidJwtAvailable,
openIdReuseUserId,
strategies: useOpenIdJwt ? ['openidJwt', 'jwt'] : ['jwt'],
};
};
const dropRumTelemetry = (res) => {
if (!res.headersSent) {
res.status(204).end();
}
};
// Keep in sync with packages/api/src/rum/proxy.ts; auth drops are recorded before proxy code runs.
const getRumProxyEndpoint = (req) => {
if (req.path === '/v1/traces') {
return 'traces';
}
if (req.path === '/v1/logs') {
return 'logs';
}
return 'unknown';
};
const isOpenIdReuseUser = (strategy, user, openIdReuseUserId) =>
strategy !== 'openidJwt' || getAuthenticatedUserId(user) === openIdReuseUserId;
/**
* Custom Middleware to handle JWT authentication, with support for OpenID token reuse.
* Switches between JWT and OpenID authentication based on cookies and environment settings.
@@ -44,15 +84,8 @@ const refreshCloudFrontCookies =
* for downstream Mongoose tenant isolation and structured logging.
*/
const requireJwtAuth = (req, res, next) => {
const cookieHeader = req.headers.cookie;
const parsedCookies = cookieHeader ? cookies.parse(cookieHeader) : {};
const tokenProvider = parsedCookies.token_provider;
const openidReuseEnabled = isEnabled(process.env.OPENID_REUSE_TOKENS);
const openidJwtAvailable = openidReuseEnabled && hasPassportStrategy('openidJwt');
const openIdReuseUserId = getValidOpenIdReuseUserId(parsedCookies);
const useOpenIdJwt =
tokenProvider === 'openid' && openidJwtAvailable && openIdReuseUserId != null;
const strategies = useOpenIdJwt ? ['openidJwt', 'jwt'] : ['jwt'];
const { tokenProvider, openidReuseEnabled, openidJwtAvailable, openIdReuseUserId, strategies } =
getAuthStrategies(req);
const authLogState = {
tokenProvider,
openidReuseEnabled,
@@ -162,4 +195,45 @@ const requireJwtAuth = (req, res, next) => {
authenticateWithStrategy(0);
};
const requireRumProxyAuth = (req, res, next) => {
const { openIdReuseUserId, strategies } = getAuthStrategies(req);
const endpoint = getRumProxyEndpoint(req);
let authErrorSeen = false;
const dropTelemetry = () => {
recordRumProxyRequest(endpoint, authErrorSeen ? 'auth_error' : 'auth_drop');
dropRumTelemetry(res);
};
const finishAuthentication = (strategy, user) => {
req.user = user;
req.authStrategy = strategy;
next();
};
let nextStrategyIndex = 0;
const tryNextStrategy = () => {
const strategy = strategies[nextStrategyIndex];
nextStrategyIndex += 1;
if (!strategy) {
dropTelemetry();
return;
}
passport.authenticate(strategy, { session: false }, (err, user) => {
authErrorSeen = authErrorSeen || err != null;
if (err || !user || !isOpenIdReuseUser(strategy, user, openIdReuseUserId)) {
tryNextStrategy();
return;
}
finishAuthentication(strategy, user);
})(req, res, next);
};
tryNextStrategy();
};
module.exports = requireJwtAuth;
module.exports.requireRumProxyAuth = requireRumProxyAuth;

View File

@@ -1,12 +1,12 @@
const express = require('express');
const request = require('supertest');
const mockRequireJwtAuth = jest.fn((_req, _res, next) => next());
const mockRequireRumProxyAuth = jest.fn((_req, _res, next) => next());
const mockIsRumProxyEnabled = jest.fn();
const mockProxyRumRequest = jest.fn((_req, res) => res.status(202).send());
jest.mock('~/server/middleware', () => ({
requireJwtAuth: (...args) => mockRequireJwtAuth(...args),
requireRumProxyAuth: (...args) => mockRequireRumProxyAuth(...args),
}));
jest.mock('@librechat/api', () => ({
@@ -26,7 +26,7 @@ describe('RUM proxy routes', () => {
});
beforeEach(() => {
mockRequireJwtAuth.mockClear();
mockRequireRumProxyAuth.mockClear();
mockIsRumProxyEnabled.mockReset();
mockProxyRumRequest.mockClear();
});
@@ -41,7 +41,7 @@ describe('RUM proxy routes', () => {
expect(response.status).toBe(404);
expect(response.body).toEqual({ message: 'RUM proxy is not configured' });
expect(mockRequireJwtAuth).not.toHaveBeenCalled();
expect(mockRequireRumProxyAuth).not.toHaveBeenCalled();
expect(mockProxyRumRequest).not.toHaveBeenCalled();
});
@@ -54,7 +54,20 @@ describe('RUM proxy routes', () => {
.send(Buffer.from('payload'));
expect(response.status).toBe(202);
expect(mockRequireJwtAuth).toHaveBeenCalledTimes(1);
expect(mockRequireRumProxyAuth).toHaveBeenCalledTimes(1);
expect(mockProxyRumRequest).toHaveBeenCalledTimes(1);
});
it('uses RUM-specific auth for logs as well as traces', async () => {
mockIsRumProxyEnabled.mockReturnValue(true);
const response = await request(app)
.post('/api/rum/v1/logs')
.set('Content-Type', 'application/x-protobuf')
.send(Buffer.from('payload'));
expect(response.status).toBe(202);
expect(mockRequireRumProxyAuth).toHaveBeenCalledTimes(1);
expect(mockProxyRumRequest).toHaveBeenCalledTimes(1);
});
});

View File

@@ -1,6 +1,6 @@
const express = require('express');
const { getRumProxyBodyLimit, isRumProxyEnabled, proxyRumRequest } = require('@librechat/api');
const { requireJwtAuth } = require('~/server/middleware');
const { requireRumProxyAuth } = require('~/server/middleware');
const router = express.Router();
const rawOtlpBody = express.raw({
@@ -16,7 +16,13 @@ function requireRumProxyEnabled(_req, res, next) {
return next();
}
router.post('/v1/traces', requireRumProxyEnabled, requireJwtAuth, rawOtlpBody, proxyRumRequest);
router.post('/v1/logs', requireRumProxyEnabled, requireJwtAuth, rawOtlpBody, proxyRumRequest);
router.post(
'/v1/traces',
requireRumProxyEnabled,
requireRumProxyAuth,
rawOtlpBody,
proxyRumRequest,
);
router.post('/v1/logs', requireRumProxyEnabled, requireRumProxyAuth, rawOtlpBody, proxyRumRequest);
module.exports = router;

39
package-lock.json generated
View File

@@ -19851,6 +19851,13 @@
"@types/node": "*"
}
},
"node_modules/@types/cookiejar": {
"version": "2.1.5",
"resolved": "https://registry.npmjs.org/@types/cookiejar/-/cookiejar-2.1.5.tgz",
"integrity": "sha512-he+DHOWReW0nghN24E1WUqM0efK4kI9oTqDm6XmK8ZPe2djZ90BSNdGnIyCLzCPw7/pogPlGbzI2wHGGmi4O/Q==",
"dev": true,
"license": "MIT"
},
"node_modules/@types/css-font-loading-module": {
"version": "0.0.7",
"resolved": "https://registry.npmjs.org/@types/css-font-loading-module/-/css-font-loading-module-0.0.7.tgz",
@@ -20303,6 +20310,13 @@
"@types/unist": "*"
}
},
"node_modules/@types/methods": {
"version": "1.1.4",
"resolved": "https://registry.npmjs.org/@types/methods/-/methods-1.1.4.tgz",
"integrity": "sha512-ymXWVrDiCxTBE3+RIrrP533E70eA+9qu7zdWoHuOmGujkYtzf4HQF96b8nwHLqhuf4ykX61IGRIB38CC6/sImQ==",
"dev": true,
"license": "MIT"
},
"node_modules/@types/mime": {
"version": "1.3.5",
"resolved": "https://registry.npmjs.org/@types/mime/-/mime-1.3.5.tgz",
@@ -20475,6 +20489,30 @@
"integrity": "sha512-9aEbYZ3TbYMznPdcdr3SmIrLXwC/AKZXQeCf9Pgao5CKb8CyHuEX5jzWPTkvregvhRJHcpRO6BFoGW9ycaOkYw==",
"dev": true
},
"node_modules/@types/superagent": {
"version": "8.1.10",
"resolved": "https://registry.npmjs.org/@types/superagent/-/superagent-8.1.10.tgz",
"integrity": "sha512-nbt4IWXABhW0jGmmpRzCFNlbmwCTzZ2gTUsNIr+X+ItdqPms+PAJZbWsNzpS2USqXjcoNLQcO6nXo60zcPQiIg==",
"dev": true,
"license": "MIT",
"dependencies": {
"@types/cookiejar": "^2.1.5",
"@types/methods": "^1.1.4",
"@types/node": "*",
"form-data": "^4.0.0"
}
},
"node_modules/@types/supertest": {
"version": "7.2.0",
"resolved": "https://registry.npmjs.org/@types/supertest/-/supertest-7.2.0.tgz",
"integrity": "sha512-uh2Lv57xvggst6lCqNdFAmDSvoMG7M/HDtX4iUCquxQ5EGPtaPM5PL5Hmi7LCvOG8db7YaCPNJEeoI8s/WzIQw==",
"dev": true,
"license": "MIT",
"dependencies": {
"@types/methods": "^1.1.4",
"@types/superagent": "^8.1.0"
}
},
"node_modules/@types/testing-library__jest-dom": {
"version": "5.14.9",
"resolved": "https://registry.npmjs.org/@types/testing-library__jest-dom/-/testing-library__jest-dom-5.14.9.tgz",
@@ -44214,6 +44252,7 @@
"@types/node-fetch": "^2.6.13",
"@types/react": "^18.2.18",
"@types/sanitize-html": "^2.13.0",
"@types/supertest": "^7.2.0",
"@types/winston": "^2.4.4",
"@types/yauzl": "^2.10.3",
"aws-sdk-client-mock": "^4.1.0",

View File

@@ -72,6 +72,7 @@
"@types/node-fetch": "^2.6.13",
"@types/react": "^18.2.18",
"@types/sanitize-html": "^2.13.0",
"@types/supertest": "^7.2.0",
"@types/winston": "^2.4.4",
"@types/yauzl": "^2.10.3",
"aws-sdk-client-mock": "^4.1.0",

View File

@@ -1,6 +1,7 @@
/// <reference types="jest" />
import { EventEmitter } from 'events';
import express from 'express';
import request from 'supertest';
import { EventEmitter } from 'events';
import type { Request, Response } from 'express';
import {
createMetrics,
@@ -10,11 +11,10 @@ import {
recordGenerationStreamResumePendingEvents,
recordGenerationStreamSubscription,
recordOpenIDUserLookup,
recordRumProxyRequest,
setGenerationJobsInFlight,
} from './metrics';
const request = require('supertest') as (app: express.Express) => any;
describe('normalizePath', () => {
it.each([
// Known high-cardinality routes
@@ -272,6 +272,36 @@ describe('createMetrics', () => {
expect(response.text).toMatch(/openid_user_lookup_duration_seconds_sum\{result="found"\} 0.2/);
});
it('tracks RUM proxy request outcomes', async () => {
const app = express();
process.env.METRICS_SECRET = 'test-secret';
const { metricsRouter } = createMetrics();
app.use('/metrics', metricsRouter);
recordRumProxyRequest('traces', 'success');
recordRumProxyRequest('traces', 'auth_drop');
recordRumProxyRequest('logs', 'auth_error');
recordRumProxyRequest('logs', 'collector_5xx');
const response = await request(app)
.get('/metrics')
.set('Authorization', 'Bearer test-secret')
.expect(200);
expect(response.text).toMatch(
/rum_proxy_requests_total\{endpoint="traces",result="success"\} 1/,
);
expect(response.text).toMatch(
/rum_proxy_requests_total\{endpoint="traces",result="auth_drop"\} 1/,
);
expect(response.text).toMatch(
/rum_proxy_requests_total\{endpoint="logs",result="auth_error"\} 1/,
);
expect(response.text).toMatch(
/rum_proxy_requests_total\{endpoint="logs",result="collector_5xx"\} 1/,
);
});
it('tracks mongoose query counts and latency by model and operation', async () => {
class FakeQuery {
model = { modelName: 'User' };

View File

@@ -1,7 +1,7 @@
import { timingSafeEqual } from 'crypto';
import { Router } from 'express';
import { Registry, collectDefaultMetrics, Counter, Gauge, Histogram } from 'prom-client';
import { timingSafeEqual } from 'crypto';
import { logger } from '@librechat/data-schemas';
import { Registry, collectDefaultMetrics, Counter, Gauge, Histogram } from 'prom-client';
import type { Request, Response, NextFunction, RequestHandler } from 'express';
import type { Mongoose } from 'mongoose';
@@ -137,6 +137,17 @@ export type GenerationStreamSubscriptionResult =
| 'error'
| 'found'
| 'missing';
export type RumProxyEndpoint = 'traces' | 'logs' | 'unknown';
export type RumProxyResult =
| 'success'
| 'auth_drop'
| 'auth_error'
| 'bad_request'
| 'not_configured'
| 'collector_4xx'
| 'collector_5xx'
| 'collector_error'
| 'collector_timeout';
type OpenIDUserLookupMetrics = {
recordLookup: (result: OpenIDUserLookupResult, durationSeconds: number) => void;
@@ -179,6 +190,14 @@ let generationJobMetrics: GenerationJobMetrics = {
recordResumePendingEvents: () => undefined,
};
type RumProxyMetrics = {
recordRequest: (endpoint: RumProxyEndpoint, result: RumProxyResult) => void;
};
let rumProxyMetrics: RumProxyMetrics = {
recordRequest: () => undefined,
};
const resetMetricRecorders = (): void => {
openIDUserLookupMetrics = {
recordLookup: () => undefined,
@@ -192,6 +211,9 @@ const resetMetricRecorders = (): void => {
recordSubscription: () => undefined,
recordResumePendingEvents: () => undefined,
};
rumProxyMetrics = {
recordRequest: () => undefined,
};
};
export function recordGenerationJob(store: GenerationJobStore, result: GenerationJobResult): void {
@@ -217,6 +239,10 @@ export function recordGenerationStreamResumePendingEvents(
generationJobMetrics.recordResumePendingEvents(store, count);
}
export function recordRumProxyRequest(endpoint: RumProxyEndpoint, result: RumProxyResult): void {
rumProxyMetrics.recordRequest(endpoint, result);
}
const getElapsedSeconds = (startedAt: bigint): number =>
Number(process.hrtime.bigint() - startedAt) / 1_000_000_000;
@@ -493,6 +519,13 @@ export function createMetrics(): PrometheusMetrics {
registers: [registry],
});
const rumProxyRequests = new Counter({
name: 'rum_proxy_requests_total',
help: 'RUM proxy requests by endpoint and result',
labelNames: ['endpoint', 'result'] as const,
registers: [registry],
});
generationJobMetrics = {
recordJob: (store, result) => generationJobs.inc({ store, result }),
setJobsInFlight: (store, count) => generationJobsInFlight.set({ store }, count),
@@ -502,6 +535,10 @@ export function createMetrics(): PrometheusMetrics {
generationStreamResumePendingEvents.inc({ store }, count),
};
rumProxyMetrics = {
recordRequest: (endpoint, result) => rumProxyRequests.inc({ endpoint, result }),
};
const metricsMiddleware = (req: Request, res: Response, next: NextFunction): void => {
const end = httpDuration.startTimer();
const labels = { method: req.method, path: normalizePath(req.path) };

View File

@@ -1,3 +1,8 @@
jest.mock('~/app/metrics', () => ({
recordRumProxyRequest: jest.fn(),
}));
import { recordRumProxyRequest } from '~/app/metrics';
import {
getRumProxyBodyLimit,
getRumProxyClientUrl,
@@ -24,6 +29,7 @@ describe('RUM proxy configuration', () => {
beforeEach(() => {
process.env = { ...originalEnv };
jest.mocked(recordRumProxyRequest).mockClear();
});
afterEach(() => {
@@ -114,6 +120,7 @@ describe('RUM proxy configuration', () => {
expect(res.set).toHaveBeenCalledWith('content-type', 'application/json');
expect(res.status).toHaveBeenCalledWith(202);
expect(res.send).toHaveBeenCalledWith(Buffer.from('ok'));
expect(recordRumProxyRequest).toHaveBeenCalledWith('traces', 'success');
fetchMock.mockRestore();
});
@@ -133,6 +140,8 @@ describe('RUM proxy configuration', () => {
expect(missingBodyRes.status).toHaveBeenCalledWith(400);
expect(unsupportedPathRes.status).toHaveBeenCalledWith(404);
expect(recordRumProxyRequest).toHaveBeenCalledWith('traces', 'bad_request');
expect(recordRumProxyRequest).toHaveBeenCalledWith('unknown', 'not_configured');
});
it('returns 502 when the collector request fails', async () => {
@@ -148,6 +157,26 @@ describe('RUM proxy configuration', () => {
);
expect(res.status).toHaveBeenCalledWith(502);
expect(recordRumProxyRequest).toHaveBeenCalledWith('traces', 'collector_error');
fetchMock.mockRestore();
});
it('records collector error status classes', async () => {
process.env.RUM_ENABLED = 'true';
process.env.RUM_AUTH_MODE = 'proxy';
process.env.RUM_PROXY_TARGET_URL = 'http://otel-collector:4318';
const fetchMock = jest
.spyOn(global, 'fetch')
.mockResolvedValue(new Response('nope', { status: 503 }));
const res = makeResponse();
await proxyRumRequest(
{ path: '/v1/logs', body: Buffer.from('payload'), headers: {} } as never,
res as never,
);
expect(res.status).toHaveBeenCalledWith(503);
expect(recordRumProxyRequest).toHaveBeenCalledWith('logs', 'collector_5xx');
fetchMock.mockRestore();
});
});

View File

@@ -1,5 +1,7 @@
import { logger } from '@librechat/data-schemas';
import type { Request, Response } from 'express';
import type { RumProxyEndpoint, RumProxyResult } from '~/app/metrics';
import { recordRumProxyRequest } from '~/app/metrics';
import { isEnabled } from '~/utils';
const DEFAULT_PROXY_PATH = '/api/rum';
@@ -75,6 +77,27 @@ export function resolveRumProxyTarget(path: string): string | undefined {
return targetUrl.href;
}
// Keep in sync with api/server/middleware/requireJwtAuth.js; auth drops are recorded there.
function getRumProxyEndpoint(path: string): RumProxyEndpoint {
if (path === '/v1/traces') {
return 'traces';
}
if (path === '/v1/logs') {
return 'logs';
}
return 'unknown';
}
function getRumCollectorResult(status: number): RumProxyResult {
if (status >= 500) {
return 'collector_5xx';
}
if (status >= 400) {
return 'collector_4xx';
}
return 'success';
}
function getRequestBody(req: Request): Buffer | string | undefined {
const body = req.body as unknown;
@@ -109,14 +132,17 @@ function getProxyHeaders(req: Request, body: Buffer | string): Record<string, st
}
export async function proxyRumRequest(req: Request, res: Response): Promise<void> {
const endpoint = getRumProxyEndpoint(req.path);
const target = resolveRumProxyTarget(req.path);
if (!target) {
recordRumProxyRequest(endpoint, 'not_configured');
res.status(404).json({ message: 'RUM proxy is not configured' });
return;
}
const body = getRequestBody(req);
if (!body) {
recordRumProxyRequest(endpoint, 'bad_request');
res.status(400).json({ message: 'RUM payload is required' });
return;
}
@@ -139,8 +165,13 @@ export async function proxyRumRequest(req: Request, res: Response): Promise<void
}
const responseBody = Buffer.from(await response.arrayBuffer());
recordRumProxyRequest(endpoint, getRumCollectorResult(response.status));
res.status(response.status).send(responseBody);
} catch (error) {
recordRumProxyRequest(
endpoint,
controller.signal.aborted ? 'collector_timeout' : 'collector_error',
);
logger.warn('[rumProxy] Failed to proxy RUM telemetry', {
error: error instanceof Error ? error.message : String(error),
target,