mirror of
https://github.com/danny-avila/LibreChat.git
synced 2026-06-16 07:51:32 +03:00
📈 fix: Isolate RUM Telemetry Proxy Auth from App Auth (#13765)
* fix(rum): isolate telemetry proxy auth * feat(rum): track proxy error metrics * refactor(rum): simplify proxy auth strategy flow * test(rum): clarify proxy success metric assertion * test(metrics): use typed supertest import * test(metrics): add local supertest types * test(metrics): keep supertest types local * test(metrics): use official supertest types * fix(rum): log proxy auth strategy errors * fix(rum): classify proxy auth errors in metrics * style(rum): sort telemetry metric imports * ci: mention import sort check command * ci: show targeted import sort example
This commit is contained in:
4
.github/workflows/eslint-ci.yml
vendored
4
.github/workflows/eslint-ci.yml
vendored
@@ -119,6 +119,10 @@ jobs:
|
||||
echo ""
|
||||
echo "::error::Import order drift detected. Fix locally with:"
|
||||
echo "::error:: npm run sort-imports"
|
||||
echo "::error::For specific files:"
|
||||
echo "::error:: npm run sort-imports -- packages/api/src/app/metrics.ts packages/api/src/rum/proxy.ts"
|
||||
echo "::error::To check without writing files:"
|
||||
echo "::error:: npm run sort-imports:check"
|
||||
echo "::error::Or rely on the lint-staged pre-commit hook (do not bypass with --no-verify)."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
@@ -213,6 +213,7 @@ jest.mock('@librechat/api', () => {
|
||||
normalizeContextValue(req.headers?.['x-correlation-id']);
|
||||
return {
|
||||
isEnabled: jest.fn(() => false),
|
||||
recordRumProxyRequest: jest.fn(),
|
||||
getAuthFailureReason,
|
||||
getAuthFailureErrorName,
|
||||
buildSafeAuthLogContext,
|
||||
@@ -235,8 +236,13 @@ jest.mock('@librechat/api', () => {
|
||||
// ── Helpers ─────────────────────────────────────────────────────────────
|
||||
|
||||
const requireJwtAuth = require('../requireJwtAuth');
|
||||
const { requireRumProxyAuth } = requireJwtAuth;
|
||||
const { getTenantId, getUserId, logger } = require('@librechat/data-schemas');
|
||||
const { isEnabled, maybeRefreshCloudFrontAuthCookiesMiddleware } = require('@librechat/api');
|
||||
const {
|
||||
isEnabled,
|
||||
maybeRefreshCloudFrontAuthCookiesMiddleware,
|
||||
recordRumProxyRequest,
|
||||
} = require('@librechat/api');
|
||||
const passport = require('passport');
|
||||
|
||||
const jwtSecret = 'test-refresh-secret';
|
||||
@@ -250,7 +256,11 @@ function signedOpenIdUserCookie(userId = 'user-openid') {
|
||||
}
|
||||
|
||||
function mockRes() {
|
||||
return { status: jest.fn().mockReturnThis(), json: jest.fn().mockReturnThis() };
|
||||
return {
|
||||
status: jest.fn().mockReturnThis(),
|
||||
json: jest.fn().mockReturnThis(),
|
||||
end: jest.fn().mockReturnThis(),
|
||||
};
|
||||
}
|
||||
|
||||
/** Runs requireJwtAuth and returns the tenantId observed inside next(). */
|
||||
@@ -282,6 +292,7 @@ describe('requireJwtAuth tenant context chaining', () => {
|
||||
logger.info.mockClear();
|
||||
logger.warn.mockClear();
|
||||
logger.error.mockClear();
|
||||
recordRumProxyRequest.mockClear();
|
||||
passport.authenticate.mockClear();
|
||||
passport._strategy.mockClear();
|
||||
if (originalJwtSecret === undefined) {
|
||||
@@ -836,3 +847,141 @@ describe('requireJwtAuth tenant context chaining', () => {
|
||||
expect(getTenantId()).toBeUndefined();
|
||||
});
|
||||
});
|
||||
|
||||
describe('requireRumProxyAuth', () => {
|
||||
const originalJwtSecret = process.env.JWT_REFRESH_SECRET;
|
||||
|
||||
beforeEach(() => {
|
||||
process.env.JWT_REFRESH_SECRET = jwtSecret;
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
mockPassportError = null;
|
||||
mockRegisteredStrategies = new Set(['jwt']);
|
||||
isEnabled.mockReturnValue(false);
|
||||
maybeRefreshCloudFrontAuthCookiesMiddleware.mockClear();
|
||||
logger.debug.mockClear();
|
||||
logger.info.mockClear();
|
||||
logger.warn.mockClear();
|
||||
logger.error.mockClear();
|
||||
recordRumProxyRequest.mockClear();
|
||||
passport.authenticate.mockClear();
|
||||
passport._strategy.mockClear();
|
||||
if (originalJwtSecret === undefined) {
|
||||
delete process.env.JWT_REFRESH_SECRET;
|
||||
} else {
|
||||
process.env.JWT_REFRESH_SECRET = originalJwtSecret;
|
||||
}
|
||||
});
|
||||
|
||||
it('authenticates telemetry with the LibreChat JWT strategy without tenant or cookie refresh middleware', () => {
|
||||
const req = mockReq({ id: 'user-jwt', tenantId: 'tenant-jwt', role: 'user' });
|
||||
const res = mockRes();
|
||||
const next = jest.fn();
|
||||
|
||||
requireRumProxyAuth(req, res, next);
|
||||
|
||||
expect(passport.authenticate).toHaveBeenCalledWith(
|
||||
'jwt',
|
||||
{ session: false },
|
||||
expect.any(Function),
|
||||
);
|
||||
expect(req.authStrategy).toBe('jwt');
|
||||
expect(maybeRefreshCloudFrontAuthCookiesMiddleware).not.toHaveBeenCalled();
|
||||
// Success is recorded by the proxy.
|
||||
expect(recordRumProxyRequest).not.toHaveBeenCalled();
|
||||
expect(res.status).not.toHaveBeenCalled();
|
||||
expect(next).toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('authenticates telemetry with OpenID JWT reuse when the reuse cookie is present', () => {
|
||||
isEnabled.mockReturnValue(true);
|
||||
mockRegisteredStrategies.add('openidJwt');
|
||||
const req = mockReq(undefined, {
|
||||
headers: { cookie: `token_provider=openid; openid_user_id=${signedOpenIdUserCookie()}` },
|
||||
_mockStrategies: {
|
||||
openidJwt: { user: { id: 'user-openid', tenantId: 'tenant-openid', role: 'user' } },
|
||||
jwt: { user: false, info: { message: 'invalid signature' }, status: 401 },
|
||||
},
|
||||
});
|
||||
const res = mockRes();
|
||||
const next = jest.fn();
|
||||
|
||||
requireRumProxyAuth(req, res, next);
|
||||
|
||||
expect(passport.authenticate).toHaveBeenCalledWith(
|
||||
'openidJwt',
|
||||
{ session: false },
|
||||
expect.any(Function),
|
||||
);
|
||||
expect(req.authStrategy).toBe('openidJwt');
|
||||
expect(maybeRefreshCloudFrontAuthCookiesMiddleware).not.toHaveBeenCalled();
|
||||
expect(recordRumProxyRequest).not.toHaveBeenCalled();
|
||||
expect(res.status).not.toHaveBeenCalled();
|
||||
expect(next).toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('falls back to LibreChat JWT when OpenID JWT telemetry auth fails', () => {
|
||||
isEnabled.mockReturnValue(true);
|
||||
mockRegisteredStrategies.add('openidJwt');
|
||||
const req = mockReq(undefined, {
|
||||
headers: { cookie: `token_provider=openid; openid_user_id=${signedOpenIdUserCookie()}` },
|
||||
_mockStrategies: {
|
||||
openidJwt: {
|
||||
user: false,
|
||||
info: { message: 'jwt expired', name: 'TokenExpiredError' },
|
||||
status: 401,
|
||||
},
|
||||
jwt: { user: { id: 'user-openid', tenantId: 'tenant-jwt', role: 'user' } },
|
||||
},
|
||||
});
|
||||
const res = mockRes();
|
||||
const next = jest.fn();
|
||||
|
||||
requireRumProxyAuth(req, res, next);
|
||||
|
||||
expect(passport.authenticate).toHaveBeenCalledTimes(2);
|
||||
expect(req.authStrategy).toBe('jwt');
|
||||
expect(recordRumProxyRequest).not.toHaveBeenCalled();
|
||||
expect(res.status).not.toHaveBeenCalled();
|
||||
expect(next).toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('drops invalid telemetry auth with 204 instead of returning an app auth error', () => {
|
||||
const req = mockReq(undefined, {
|
||||
path: '/v1/traces',
|
||||
_mockStrategies: {
|
||||
jwt: {
|
||||
user: false,
|
||||
info: { message: 'invalid signature', name: 'JsonWebTokenError' },
|
||||
status: 401,
|
||||
},
|
||||
},
|
||||
});
|
||||
const res = mockRes();
|
||||
const next = jest.fn();
|
||||
|
||||
requireRumProxyAuth(req, res, next);
|
||||
|
||||
expect(next).not.toHaveBeenCalled();
|
||||
expect(maybeRefreshCloudFrontAuthCookiesMiddleware).not.toHaveBeenCalled();
|
||||
expect(recordRumProxyRequest).toHaveBeenCalledWith('traces', 'auth_drop');
|
||||
expect(res.status).toHaveBeenCalledWith(204);
|
||||
expect(res.end).toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('records passport errors separately from ordinary telemetry auth drops', () => {
|
||||
mockPassportError = new Error('passport unavailable');
|
||||
const req = mockReq(undefined, { path: '/v1/logs' });
|
||||
const res = mockRes();
|
||||
const next = jest.fn();
|
||||
|
||||
requireRumProxyAuth(req, res, next);
|
||||
|
||||
expect(next).not.toHaveBeenCalled();
|
||||
expect(logger.warn).not.toHaveBeenCalled();
|
||||
expect(recordRumProxyRequest).toHaveBeenCalledWith('logs', 'auth_error');
|
||||
expect(res.status).toHaveBeenCalledWith(204);
|
||||
expect(res.end).toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
|
||||
@@ -11,6 +11,7 @@ const requireLdapAuth = require('./requireLdapAuth');
|
||||
const abortMiddleware = require('./abortMiddleware');
|
||||
const checkInviteUser = require('./checkInviteUser');
|
||||
const requireJwtAuth = require('./requireJwtAuth');
|
||||
const { requireRumProxyAuth } = require('./requireJwtAuth');
|
||||
const configMiddleware = require('./config/app');
|
||||
const validateModel = require('./validateModel');
|
||||
const moderateText = require('./moderateText');
|
||||
@@ -37,6 +38,7 @@ module.exports = {
|
||||
moderateText,
|
||||
validateModel,
|
||||
requireJwtAuth,
|
||||
requireRumProxyAuth,
|
||||
setTwoFactorTempUser,
|
||||
checkInviteUser,
|
||||
requireLdapAuth,
|
||||
|
||||
@@ -10,6 +10,7 @@ const {
|
||||
buildSafeAuthLogContext,
|
||||
formatAuthLogMessage,
|
||||
maybeRefreshCloudFrontAuthCookiesMiddleware,
|
||||
recordRumProxyRequest,
|
||||
} = require('@librechat/api');
|
||||
|
||||
const hasPassportStrategy = (strategy) =>
|
||||
@@ -35,6 +36,45 @@ const getAuthenticatedUserId = (user) => user?.id?.toString?.() ?? user?._id?.to
|
||||
const refreshCloudFrontCookies =
|
||||
maybeRefreshCloudFrontAuthCookiesMiddleware ?? ((_req, _res, next) => next());
|
||||
|
||||
const getAuthStrategies = (req) => {
|
||||
const cookieHeader = req.headers.cookie;
|
||||
const parsedCookies = cookieHeader ? cookies.parse(cookieHeader) : {};
|
||||
const tokenProvider = parsedCookies.token_provider;
|
||||
const openidReuseEnabled = isEnabled(process.env.OPENID_REUSE_TOKENS);
|
||||
const openidJwtAvailable = openidReuseEnabled && hasPassportStrategy('openidJwt');
|
||||
const openIdReuseUserId = getValidOpenIdReuseUserId(parsedCookies);
|
||||
const useOpenIdJwt =
|
||||
tokenProvider === 'openid' && openidJwtAvailable && openIdReuseUserId != null;
|
||||
|
||||
return {
|
||||
tokenProvider,
|
||||
openidReuseEnabled,
|
||||
openidJwtAvailable,
|
||||
openIdReuseUserId,
|
||||
strategies: useOpenIdJwt ? ['openidJwt', 'jwt'] : ['jwt'],
|
||||
};
|
||||
};
|
||||
|
||||
const dropRumTelemetry = (res) => {
|
||||
if (!res.headersSent) {
|
||||
res.status(204).end();
|
||||
}
|
||||
};
|
||||
|
||||
// Keep in sync with packages/api/src/rum/proxy.ts; auth drops are recorded before proxy code runs.
|
||||
const getRumProxyEndpoint = (req) => {
|
||||
if (req.path === '/v1/traces') {
|
||||
return 'traces';
|
||||
}
|
||||
if (req.path === '/v1/logs') {
|
||||
return 'logs';
|
||||
}
|
||||
return 'unknown';
|
||||
};
|
||||
|
||||
const isOpenIdReuseUser = (strategy, user, openIdReuseUserId) =>
|
||||
strategy !== 'openidJwt' || getAuthenticatedUserId(user) === openIdReuseUserId;
|
||||
|
||||
/**
|
||||
* Custom Middleware to handle JWT authentication, with support for OpenID token reuse.
|
||||
* Switches between JWT and OpenID authentication based on cookies and environment settings.
|
||||
@@ -44,15 +84,8 @@ const refreshCloudFrontCookies =
|
||||
* for downstream Mongoose tenant isolation and structured logging.
|
||||
*/
|
||||
const requireJwtAuth = (req, res, next) => {
|
||||
const cookieHeader = req.headers.cookie;
|
||||
const parsedCookies = cookieHeader ? cookies.parse(cookieHeader) : {};
|
||||
const tokenProvider = parsedCookies.token_provider;
|
||||
const openidReuseEnabled = isEnabled(process.env.OPENID_REUSE_TOKENS);
|
||||
const openidJwtAvailable = openidReuseEnabled && hasPassportStrategy('openidJwt');
|
||||
const openIdReuseUserId = getValidOpenIdReuseUserId(parsedCookies);
|
||||
const useOpenIdJwt =
|
||||
tokenProvider === 'openid' && openidJwtAvailable && openIdReuseUserId != null;
|
||||
const strategies = useOpenIdJwt ? ['openidJwt', 'jwt'] : ['jwt'];
|
||||
const { tokenProvider, openidReuseEnabled, openidJwtAvailable, openIdReuseUserId, strategies } =
|
||||
getAuthStrategies(req);
|
||||
const authLogState = {
|
||||
tokenProvider,
|
||||
openidReuseEnabled,
|
||||
@@ -162,4 +195,45 @@ const requireJwtAuth = (req, res, next) => {
|
||||
authenticateWithStrategy(0);
|
||||
};
|
||||
|
||||
const requireRumProxyAuth = (req, res, next) => {
|
||||
const { openIdReuseUserId, strategies } = getAuthStrategies(req);
|
||||
const endpoint = getRumProxyEndpoint(req);
|
||||
let authErrorSeen = false;
|
||||
|
||||
const dropTelemetry = () => {
|
||||
recordRumProxyRequest(endpoint, authErrorSeen ? 'auth_error' : 'auth_drop');
|
||||
dropRumTelemetry(res);
|
||||
};
|
||||
|
||||
const finishAuthentication = (strategy, user) => {
|
||||
req.user = user;
|
||||
req.authStrategy = strategy;
|
||||
next();
|
||||
};
|
||||
|
||||
let nextStrategyIndex = 0;
|
||||
const tryNextStrategy = () => {
|
||||
const strategy = strategies[nextStrategyIndex];
|
||||
nextStrategyIndex += 1;
|
||||
|
||||
if (!strategy) {
|
||||
dropTelemetry();
|
||||
return;
|
||||
}
|
||||
|
||||
passport.authenticate(strategy, { session: false }, (err, user) => {
|
||||
authErrorSeen = authErrorSeen || err != null;
|
||||
if (err || !user || !isOpenIdReuseUser(strategy, user, openIdReuseUserId)) {
|
||||
tryNextStrategy();
|
||||
return;
|
||||
}
|
||||
|
||||
finishAuthentication(strategy, user);
|
||||
})(req, res, next);
|
||||
};
|
||||
|
||||
tryNextStrategy();
|
||||
};
|
||||
|
||||
module.exports = requireJwtAuth;
|
||||
module.exports.requireRumProxyAuth = requireRumProxyAuth;
|
||||
|
||||
@@ -1,12 +1,12 @@
|
||||
const express = require('express');
|
||||
const request = require('supertest');
|
||||
|
||||
const mockRequireJwtAuth = jest.fn((_req, _res, next) => next());
|
||||
const mockRequireRumProxyAuth = jest.fn((_req, _res, next) => next());
|
||||
const mockIsRumProxyEnabled = jest.fn();
|
||||
const mockProxyRumRequest = jest.fn((_req, res) => res.status(202).send());
|
||||
|
||||
jest.mock('~/server/middleware', () => ({
|
||||
requireJwtAuth: (...args) => mockRequireJwtAuth(...args),
|
||||
requireRumProxyAuth: (...args) => mockRequireRumProxyAuth(...args),
|
||||
}));
|
||||
|
||||
jest.mock('@librechat/api', () => ({
|
||||
@@ -26,7 +26,7 @@ describe('RUM proxy routes', () => {
|
||||
});
|
||||
|
||||
beforeEach(() => {
|
||||
mockRequireJwtAuth.mockClear();
|
||||
mockRequireRumProxyAuth.mockClear();
|
||||
mockIsRumProxyEnabled.mockReset();
|
||||
mockProxyRumRequest.mockClear();
|
||||
});
|
||||
@@ -41,7 +41,7 @@ describe('RUM proxy routes', () => {
|
||||
|
||||
expect(response.status).toBe(404);
|
||||
expect(response.body).toEqual({ message: 'RUM proxy is not configured' });
|
||||
expect(mockRequireJwtAuth).not.toHaveBeenCalled();
|
||||
expect(mockRequireRumProxyAuth).not.toHaveBeenCalled();
|
||||
expect(mockProxyRumRequest).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
@@ -54,7 +54,20 @@ describe('RUM proxy routes', () => {
|
||||
.send(Buffer.from('payload'));
|
||||
|
||||
expect(response.status).toBe(202);
|
||||
expect(mockRequireJwtAuth).toHaveBeenCalledTimes(1);
|
||||
expect(mockRequireRumProxyAuth).toHaveBeenCalledTimes(1);
|
||||
expect(mockProxyRumRequest).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it('uses RUM-specific auth for logs as well as traces', async () => {
|
||||
mockIsRumProxyEnabled.mockReturnValue(true);
|
||||
|
||||
const response = await request(app)
|
||||
.post('/api/rum/v1/logs')
|
||||
.set('Content-Type', 'application/x-protobuf')
|
||||
.send(Buffer.from('payload'));
|
||||
|
||||
expect(response.status).toBe(202);
|
||||
expect(mockRequireRumProxyAuth).toHaveBeenCalledTimes(1);
|
||||
expect(mockProxyRumRequest).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
const express = require('express');
|
||||
const { getRumProxyBodyLimit, isRumProxyEnabled, proxyRumRequest } = require('@librechat/api');
|
||||
const { requireJwtAuth } = require('~/server/middleware');
|
||||
const { requireRumProxyAuth } = require('~/server/middleware');
|
||||
|
||||
const router = express.Router();
|
||||
const rawOtlpBody = express.raw({
|
||||
@@ -16,7 +16,13 @@ function requireRumProxyEnabled(_req, res, next) {
|
||||
return next();
|
||||
}
|
||||
|
||||
router.post('/v1/traces', requireRumProxyEnabled, requireJwtAuth, rawOtlpBody, proxyRumRequest);
|
||||
router.post('/v1/logs', requireRumProxyEnabled, requireJwtAuth, rawOtlpBody, proxyRumRequest);
|
||||
router.post(
|
||||
'/v1/traces',
|
||||
requireRumProxyEnabled,
|
||||
requireRumProxyAuth,
|
||||
rawOtlpBody,
|
||||
proxyRumRequest,
|
||||
);
|
||||
router.post('/v1/logs', requireRumProxyEnabled, requireRumProxyAuth, rawOtlpBody, proxyRumRequest);
|
||||
|
||||
module.exports = router;
|
||||
|
||||
39
package-lock.json
generated
39
package-lock.json
generated
@@ -19851,6 +19851,13 @@
|
||||
"@types/node": "*"
|
||||
}
|
||||
},
|
||||
"node_modules/@types/cookiejar": {
|
||||
"version": "2.1.5",
|
||||
"resolved": "https://registry.npmjs.org/@types/cookiejar/-/cookiejar-2.1.5.tgz",
|
||||
"integrity": "sha512-he+DHOWReW0nghN24E1WUqM0efK4kI9oTqDm6XmK8ZPe2djZ90BSNdGnIyCLzCPw7/pogPlGbzI2wHGGmi4O/Q==",
|
||||
"dev": true,
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/@types/css-font-loading-module": {
|
||||
"version": "0.0.7",
|
||||
"resolved": "https://registry.npmjs.org/@types/css-font-loading-module/-/css-font-loading-module-0.0.7.tgz",
|
||||
@@ -20303,6 +20310,13 @@
|
||||
"@types/unist": "*"
|
||||
}
|
||||
},
|
||||
"node_modules/@types/methods": {
|
||||
"version": "1.1.4",
|
||||
"resolved": "https://registry.npmjs.org/@types/methods/-/methods-1.1.4.tgz",
|
||||
"integrity": "sha512-ymXWVrDiCxTBE3+RIrrP533E70eA+9qu7zdWoHuOmGujkYtzf4HQF96b8nwHLqhuf4ykX61IGRIB38CC6/sImQ==",
|
||||
"dev": true,
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/@types/mime": {
|
||||
"version": "1.3.5",
|
||||
"resolved": "https://registry.npmjs.org/@types/mime/-/mime-1.3.5.tgz",
|
||||
@@ -20475,6 +20489,30 @@
|
||||
"integrity": "sha512-9aEbYZ3TbYMznPdcdr3SmIrLXwC/AKZXQeCf9Pgao5CKb8CyHuEX5jzWPTkvregvhRJHcpRO6BFoGW9ycaOkYw==",
|
||||
"dev": true
|
||||
},
|
||||
"node_modules/@types/superagent": {
|
||||
"version": "8.1.10",
|
||||
"resolved": "https://registry.npmjs.org/@types/superagent/-/superagent-8.1.10.tgz",
|
||||
"integrity": "sha512-nbt4IWXABhW0jGmmpRzCFNlbmwCTzZ2gTUsNIr+X+ItdqPms+PAJZbWsNzpS2USqXjcoNLQcO6nXo60zcPQiIg==",
|
||||
"dev": true,
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"@types/cookiejar": "^2.1.5",
|
||||
"@types/methods": "^1.1.4",
|
||||
"@types/node": "*",
|
||||
"form-data": "^4.0.0"
|
||||
}
|
||||
},
|
||||
"node_modules/@types/supertest": {
|
||||
"version": "7.2.0",
|
||||
"resolved": "https://registry.npmjs.org/@types/supertest/-/supertest-7.2.0.tgz",
|
||||
"integrity": "sha512-uh2Lv57xvggst6lCqNdFAmDSvoMG7M/HDtX4iUCquxQ5EGPtaPM5PL5Hmi7LCvOG8db7YaCPNJEeoI8s/WzIQw==",
|
||||
"dev": true,
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"@types/methods": "^1.1.4",
|
||||
"@types/superagent": "^8.1.0"
|
||||
}
|
||||
},
|
||||
"node_modules/@types/testing-library__jest-dom": {
|
||||
"version": "5.14.9",
|
||||
"resolved": "https://registry.npmjs.org/@types/testing-library__jest-dom/-/testing-library__jest-dom-5.14.9.tgz",
|
||||
@@ -44214,6 +44252,7 @@
|
||||
"@types/node-fetch": "^2.6.13",
|
||||
"@types/react": "^18.2.18",
|
||||
"@types/sanitize-html": "^2.13.0",
|
||||
"@types/supertest": "^7.2.0",
|
||||
"@types/winston": "^2.4.4",
|
||||
"@types/yauzl": "^2.10.3",
|
||||
"aws-sdk-client-mock": "^4.1.0",
|
||||
|
||||
@@ -72,6 +72,7 @@
|
||||
"@types/node-fetch": "^2.6.13",
|
||||
"@types/react": "^18.2.18",
|
||||
"@types/sanitize-html": "^2.13.0",
|
||||
"@types/supertest": "^7.2.0",
|
||||
"@types/winston": "^2.4.4",
|
||||
"@types/yauzl": "^2.10.3",
|
||||
"aws-sdk-client-mock": "^4.1.0",
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
/// <reference types="jest" />
|
||||
import { EventEmitter } from 'events';
|
||||
import express from 'express';
|
||||
import request from 'supertest';
|
||||
import { EventEmitter } from 'events';
|
||||
import type { Request, Response } from 'express';
|
||||
import {
|
||||
createMetrics,
|
||||
@@ -10,11 +11,10 @@ import {
|
||||
recordGenerationStreamResumePendingEvents,
|
||||
recordGenerationStreamSubscription,
|
||||
recordOpenIDUserLookup,
|
||||
recordRumProxyRequest,
|
||||
setGenerationJobsInFlight,
|
||||
} from './metrics';
|
||||
|
||||
const request = require('supertest') as (app: express.Express) => any;
|
||||
|
||||
describe('normalizePath', () => {
|
||||
it.each([
|
||||
// Known high-cardinality routes
|
||||
@@ -272,6 +272,36 @@ describe('createMetrics', () => {
|
||||
expect(response.text).toMatch(/openid_user_lookup_duration_seconds_sum\{result="found"\} 0.2/);
|
||||
});
|
||||
|
||||
it('tracks RUM proxy request outcomes', async () => {
|
||||
const app = express();
|
||||
process.env.METRICS_SECRET = 'test-secret';
|
||||
const { metricsRouter } = createMetrics();
|
||||
app.use('/metrics', metricsRouter);
|
||||
|
||||
recordRumProxyRequest('traces', 'success');
|
||||
recordRumProxyRequest('traces', 'auth_drop');
|
||||
recordRumProxyRequest('logs', 'auth_error');
|
||||
recordRumProxyRequest('logs', 'collector_5xx');
|
||||
|
||||
const response = await request(app)
|
||||
.get('/metrics')
|
||||
.set('Authorization', 'Bearer test-secret')
|
||||
.expect(200);
|
||||
|
||||
expect(response.text).toMatch(
|
||||
/rum_proxy_requests_total\{endpoint="traces",result="success"\} 1/,
|
||||
);
|
||||
expect(response.text).toMatch(
|
||||
/rum_proxy_requests_total\{endpoint="traces",result="auth_drop"\} 1/,
|
||||
);
|
||||
expect(response.text).toMatch(
|
||||
/rum_proxy_requests_total\{endpoint="logs",result="auth_error"\} 1/,
|
||||
);
|
||||
expect(response.text).toMatch(
|
||||
/rum_proxy_requests_total\{endpoint="logs",result="collector_5xx"\} 1/,
|
||||
);
|
||||
});
|
||||
|
||||
it('tracks mongoose query counts and latency by model and operation', async () => {
|
||||
class FakeQuery {
|
||||
model = { modelName: 'User' };
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
import { timingSafeEqual } from 'crypto';
|
||||
import { Router } from 'express';
|
||||
import { Registry, collectDefaultMetrics, Counter, Gauge, Histogram } from 'prom-client';
|
||||
import { timingSafeEqual } from 'crypto';
|
||||
import { logger } from '@librechat/data-schemas';
|
||||
import { Registry, collectDefaultMetrics, Counter, Gauge, Histogram } from 'prom-client';
|
||||
import type { Request, Response, NextFunction, RequestHandler } from 'express';
|
||||
import type { Mongoose } from 'mongoose';
|
||||
|
||||
@@ -137,6 +137,17 @@ export type GenerationStreamSubscriptionResult =
|
||||
| 'error'
|
||||
| 'found'
|
||||
| 'missing';
|
||||
export type RumProxyEndpoint = 'traces' | 'logs' | 'unknown';
|
||||
export type RumProxyResult =
|
||||
| 'success'
|
||||
| 'auth_drop'
|
||||
| 'auth_error'
|
||||
| 'bad_request'
|
||||
| 'not_configured'
|
||||
| 'collector_4xx'
|
||||
| 'collector_5xx'
|
||||
| 'collector_error'
|
||||
| 'collector_timeout';
|
||||
|
||||
type OpenIDUserLookupMetrics = {
|
||||
recordLookup: (result: OpenIDUserLookupResult, durationSeconds: number) => void;
|
||||
@@ -179,6 +190,14 @@ let generationJobMetrics: GenerationJobMetrics = {
|
||||
recordResumePendingEvents: () => undefined,
|
||||
};
|
||||
|
||||
type RumProxyMetrics = {
|
||||
recordRequest: (endpoint: RumProxyEndpoint, result: RumProxyResult) => void;
|
||||
};
|
||||
|
||||
let rumProxyMetrics: RumProxyMetrics = {
|
||||
recordRequest: () => undefined,
|
||||
};
|
||||
|
||||
const resetMetricRecorders = (): void => {
|
||||
openIDUserLookupMetrics = {
|
||||
recordLookup: () => undefined,
|
||||
@@ -192,6 +211,9 @@ const resetMetricRecorders = (): void => {
|
||||
recordSubscription: () => undefined,
|
||||
recordResumePendingEvents: () => undefined,
|
||||
};
|
||||
rumProxyMetrics = {
|
||||
recordRequest: () => undefined,
|
||||
};
|
||||
};
|
||||
|
||||
export function recordGenerationJob(store: GenerationJobStore, result: GenerationJobResult): void {
|
||||
@@ -217,6 +239,10 @@ export function recordGenerationStreamResumePendingEvents(
|
||||
generationJobMetrics.recordResumePendingEvents(store, count);
|
||||
}
|
||||
|
||||
export function recordRumProxyRequest(endpoint: RumProxyEndpoint, result: RumProxyResult): void {
|
||||
rumProxyMetrics.recordRequest(endpoint, result);
|
||||
}
|
||||
|
||||
const getElapsedSeconds = (startedAt: bigint): number =>
|
||||
Number(process.hrtime.bigint() - startedAt) / 1_000_000_000;
|
||||
|
||||
@@ -493,6 +519,13 @@ export function createMetrics(): PrometheusMetrics {
|
||||
registers: [registry],
|
||||
});
|
||||
|
||||
const rumProxyRequests = new Counter({
|
||||
name: 'rum_proxy_requests_total',
|
||||
help: 'RUM proxy requests by endpoint and result',
|
||||
labelNames: ['endpoint', 'result'] as const,
|
||||
registers: [registry],
|
||||
});
|
||||
|
||||
generationJobMetrics = {
|
||||
recordJob: (store, result) => generationJobs.inc({ store, result }),
|
||||
setJobsInFlight: (store, count) => generationJobsInFlight.set({ store }, count),
|
||||
@@ -502,6 +535,10 @@ export function createMetrics(): PrometheusMetrics {
|
||||
generationStreamResumePendingEvents.inc({ store }, count),
|
||||
};
|
||||
|
||||
rumProxyMetrics = {
|
||||
recordRequest: (endpoint, result) => rumProxyRequests.inc({ endpoint, result }),
|
||||
};
|
||||
|
||||
const metricsMiddleware = (req: Request, res: Response, next: NextFunction): void => {
|
||||
const end = httpDuration.startTimer();
|
||||
const labels = { method: req.method, path: normalizePath(req.path) };
|
||||
|
||||
@@ -1,3 +1,8 @@
|
||||
jest.mock('~/app/metrics', () => ({
|
||||
recordRumProxyRequest: jest.fn(),
|
||||
}));
|
||||
|
||||
import { recordRumProxyRequest } from '~/app/metrics';
|
||||
import {
|
||||
getRumProxyBodyLimit,
|
||||
getRumProxyClientUrl,
|
||||
@@ -24,6 +29,7 @@ describe('RUM proxy configuration', () => {
|
||||
|
||||
beforeEach(() => {
|
||||
process.env = { ...originalEnv };
|
||||
jest.mocked(recordRumProxyRequest).mockClear();
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
@@ -114,6 +120,7 @@ describe('RUM proxy configuration', () => {
|
||||
expect(res.set).toHaveBeenCalledWith('content-type', 'application/json');
|
||||
expect(res.status).toHaveBeenCalledWith(202);
|
||||
expect(res.send).toHaveBeenCalledWith(Buffer.from('ok'));
|
||||
expect(recordRumProxyRequest).toHaveBeenCalledWith('traces', 'success');
|
||||
|
||||
fetchMock.mockRestore();
|
||||
});
|
||||
@@ -133,6 +140,8 @@ describe('RUM proxy configuration', () => {
|
||||
|
||||
expect(missingBodyRes.status).toHaveBeenCalledWith(400);
|
||||
expect(unsupportedPathRes.status).toHaveBeenCalledWith(404);
|
||||
expect(recordRumProxyRequest).toHaveBeenCalledWith('traces', 'bad_request');
|
||||
expect(recordRumProxyRequest).toHaveBeenCalledWith('unknown', 'not_configured');
|
||||
});
|
||||
|
||||
it('returns 502 when the collector request fails', async () => {
|
||||
@@ -148,6 +157,26 @@ describe('RUM proxy configuration', () => {
|
||||
);
|
||||
|
||||
expect(res.status).toHaveBeenCalledWith(502);
|
||||
expect(recordRumProxyRequest).toHaveBeenCalledWith('traces', 'collector_error');
|
||||
fetchMock.mockRestore();
|
||||
});
|
||||
|
||||
it('records collector error status classes', async () => {
|
||||
process.env.RUM_ENABLED = 'true';
|
||||
process.env.RUM_AUTH_MODE = 'proxy';
|
||||
process.env.RUM_PROXY_TARGET_URL = 'http://otel-collector:4318';
|
||||
const fetchMock = jest
|
||||
.spyOn(global, 'fetch')
|
||||
.mockResolvedValue(new Response('nope', { status: 503 }));
|
||||
const res = makeResponse();
|
||||
|
||||
await proxyRumRequest(
|
||||
{ path: '/v1/logs', body: Buffer.from('payload'), headers: {} } as never,
|
||||
res as never,
|
||||
);
|
||||
|
||||
expect(res.status).toHaveBeenCalledWith(503);
|
||||
expect(recordRumProxyRequest).toHaveBeenCalledWith('logs', 'collector_5xx');
|
||||
fetchMock.mockRestore();
|
||||
});
|
||||
});
|
||||
|
||||
@@ -1,5 +1,7 @@
|
||||
import { logger } from '@librechat/data-schemas';
|
||||
import type { Request, Response } from 'express';
|
||||
import type { RumProxyEndpoint, RumProxyResult } from '~/app/metrics';
|
||||
import { recordRumProxyRequest } from '~/app/metrics';
|
||||
import { isEnabled } from '~/utils';
|
||||
|
||||
const DEFAULT_PROXY_PATH = '/api/rum';
|
||||
@@ -75,6 +77,27 @@ export function resolveRumProxyTarget(path: string): string | undefined {
|
||||
return targetUrl.href;
|
||||
}
|
||||
|
||||
// Keep in sync with api/server/middleware/requireJwtAuth.js; auth drops are recorded there.
|
||||
function getRumProxyEndpoint(path: string): RumProxyEndpoint {
|
||||
if (path === '/v1/traces') {
|
||||
return 'traces';
|
||||
}
|
||||
if (path === '/v1/logs') {
|
||||
return 'logs';
|
||||
}
|
||||
return 'unknown';
|
||||
}
|
||||
|
||||
function getRumCollectorResult(status: number): RumProxyResult {
|
||||
if (status >= 500) {
|
||||
return 'collector_5xx';
|
||||
}
|
||||
if (status >= 400) {
|
||||
return 'collector_4xx';
|
||||
}
|
||||
return 'success';
|
||||
}
|
||||
|
||||
function getRequestBody(req: Request): Buffer | string | undefined {
|
||||
const body = req.body as unknown;
|
||||
|
||||
@@ -109,14 +132,17 @@ function getProxyHeaders(req: Request, body: Buffer | string): Record<string, st
|
||||
}
|
||||
|
||||
export async function proxyRumRequest(req: Request, res: Response): Promise<void> {
|
||||
const endpoint = getRumProxyEndpoint(req.path);
|
||||
const target = resolveRumProxyTarget(req.path);
|
||||
if (!target) {
|
||||
recordRumProxyRequest(endpoint, 'not_configured');
|
||||
res.status(404).json({ message: 'RUM proxy is not configured' });
|
||||
return;
|
||||
}
|
||||
|
||||
const body = getRequestBody(req);
|
||||
if (!body) {
|
||||
recordRumProxyRequest(endpoint, 'bad_request');
|
||||
res.status(400).json({ message: 'RUM payload is required' });
|
||||
return;
|
||||
}
|
||||
@@ -139,8 +165,13 @@ export async function proxyRumRequest(req: Request, res: Response): Promise<void
|
||||
}
|
||||
|
||||
const responseBody = Buffer.from(await response.arrayBuffer());
|
||||
recordRumProxyRequest(endpoint, getRumCollectorResult(response.status));
|
||||
res.status(response.status).send(responseBody);
|
||||
} catch (error) {
|
||||
recordRumProxyRequest(
|
||||
endpoint,
|
||||
controller.signal.aborted ? 'collector_timeout' : 'collector_error',
|
||||
);
|
||||
logger.warn('[rumProxy] Failed to proxy RUM telemetry', {
|
||||
error: error instanceof Error ? error.message : String(error),
|
||||
target,
|
||||
|
||||
Reference in New Issue
Block a user