wip(benchmark)
This commit is contained in:
parent
eac768bafe
commit
e0200d036a
14 changed files with 1062 additions and 61 deletions
97
client/src/components/EditModal.tsx
Normal file
97
client/src/components/EditModal.tsx
Normal file
|
|
@ -0,0 +1,97 @@
|
|||
import { Component, For, createSignal } from 'solid-js';
|
||||
|
||||
type FieldType = 'text' | 'email' | 'checkbox';
|
||||
|
||||
interface FieldConfig {
|
||||
name: string;
|
||||
label: string;
|
||||
type: FieldType;
|
||||
required?: boolean;
|
||||
placeholder?: string;
|
||||
}
|
||||
|
||||
interface EditModalProps {
|
||||
isOpen: boolean;
|
||||
onClose: () => void;
|
||||
onSubmit: (data: Record<string, any>) => Promise<void>;
|
||||
title: string;
|
||||
fields: FieldConfig[];
|
||||
initialValues: Record<string, any>;
|
||||
}
|
||||
|
||||
export const EditModal: Component<EditModalProps> = (props) => {
|
||||
const [formData, setFormData] = createSignal(props.initialValues);
|
||||
|
||||
const handleSubmit = async (e: Event) => {
|
||||
e.preventDefault();
|
||||
const data = formData();
|
||||
|
||||
for (const field of props.fields) {
|
||||
if (field.required && !data[field.name]) {
|
||||
alert(`${field.label} is required`);
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
try {
|
||||
await props.onSubmit(data);
|
||||
props.onClose();
|
||||
} catch (error) {
|
||||
const message = error instanceof Error ? error.message : 'Update failed';
|
||||
alert(message);
|
||||
}
|
||||
};
|
||||
|
||||
if (!props.isOpen) return null;
|
||||
|
||||
return (
|
||||
<div style={{ position: 'fixed', inset: 0, background: 'rgba(0,0,0,0.5)', display: 'flex', 'align-items': 'center', 'justify-content': 'center', 'z-index': 1000 }}>
|
||||
<div style={{ background: 'white', padding: '30px', 'border-radius': '8px', width: '400px' }}>
|
||||
<h3 style={{ margin: '0 0 20px 0' }}>{props.title}</h3>
|
||||
<form onSubmit={handleSubmit}>
|
||||
<For each={props.fields}>{(field) => (
|
||||
<div style={{ 'margin-bottom': field.type === 'checkbox' ? '20px' : '15px' }}>
|
||||
<label style={{ display: field.type === 'checkbox' ? 'flex' : 'block', 'align-items': field.type === 'checkbox' ? 'center' : 'flex-start', gap: '8px', 'margin-bottom': field.type === 'checkbox' ? 0 : '5px', 'font-weight': 'bold' }}>
|
||||
{field.type === 'checkbox' ? (
|
||||
<input
|
||||
type="checkbox"
|
||||
checked={formData()[field.name] || false}
|
||||
onChange={(e) => setFormData({ ...formData(), [field.name]: e.target.checked })}
|
||||
style={{ width: '18px', height: '18px' }}
|
||||
/>
|
||||
) : null}
|
||||
{field.label}
|
||||
{field.required && <span style={{ color: 'red' }}>*</span>}
|
||||
</label>
|
||||
{field.type !== 'checkbox' && (
|
||||
<input
|
||||
type={field.type}
|
||||
value={formData()[field.name] || ''}
|
||||
onInput={(e) => setFormData({ ...formData(), [field.name]: e.target.value })}
|
||||
placeholder={field.placeholder}
|
||||
style={{ width: '100%', padding: '8px', border: '1px solid #cbd5e1', 'border-radius': '4px', 'box-sizing': 'border-box' }}
|
||||
required={field.required}
|
||||
/>
|
||||
)}
|
||||
</div>
|
||||
)}</For>
|
||||
<div style={{ display: 'flex', gap: '10px', 'justify-content': 'flex-end' }}>
|
||||
<button
|
||||
type="button"
|
||||
onClick={props.onClose}
|
||||
style={{ padding: '8px 16px', background: '#e2e8f0', border: 'none', 'border-radius': '4px', cursor: 'pointer' }}
|
||||
>
|
||||
Cancel
|
||||
</button>
|
||||
<button
|
||||
type="submit"
|
||||
style={{ padding: '8px 16px', background: '#3b82f6', color: 'white', border: 'none', 'border-radius': '4px', cursor: 'pointer' }}
|
||||
>
|
||||
Update
|
||||
</button>
|
||||
</div>
|
||||
</form>
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
};
|
||||
|
|
@ -2,11 +2,13 @@ import { Component, createResource, For, createSignal } from 'solid-js';
|
|||
import { api } from '../api/client';
|
||||
import type { Backend } from '../types';
|
||||
import { Layout } from '../components/Layout';
|
||||
import { EditModal } from '../components/EditModal';
|
||||
|
||||
export const Backends: Component = () => {
|
||||
const [backends, { refetch }] = createResource(() => api.backends.getAll());
|
||||
const [showModal, setShowModal] = createSignal(false);
|
||||
const [formData, setFormData] = createSignal({ name: '', base_url: '', api_key: '' });
|
||||
const [editingBackend, setEditingBackend] = createSignal<Backend | null>(null);
|
||||
|
||||
const handleSubmit = async (e: Event) => {
|
||||
e.preventDefault();
|
||||
|
|
@ -25,6 +27,26 @@ export const Backends: Component = () => {
|
|||
refetch();
|
||||
};
|
||||
|
||||
const handleEdit = (backend: Backend) => {
|
||||
setEditingBackend(backend);
|
||||
};
|
||||
|
||||
const handleUpdate = async (data: Record<string, any>) => {
|
||||
if (!editingBackend()) return;
|
||||
|
||||
if (!confirm('Are you sure you want to update this backend?')) return;
|
||||
|
||||
const updateData: Partial<Backend> = {};
|
||||
if (data.name) updateData.name = data.name.trim();
|
||||
if (data.base_url) updateData.base_url = data.base_url.trim();
|
||||
if (data.api_key !== undefined) updateData.api_key = data.api_key.trim() || undefined;
|
||||
if (data.is_active !== undefined) updateData.is_active = data.is_active;
|
||||
|
||||
await api.backends.update(editingBackend()!.id, updateData);
|
||||
setEditingBackend(null);
|
||||
refetch();
|
||||
};
|
||||
|
||||
return (
|
||||
<Layout>
|
||||
<div style={{ padding: '30px' }}>
|
||||
|
|
@ -60,14 +82,20 @@ export const Backends: Component = () => {
|
|||
<td style={{ padding: '12px', color: backend.is_active ? '#22c55e' : '#ef4444' }}>
|
||||
{backend.is_active ? 'Active' : 'Inactive'}
|
||||
</td>
|
||||
<td style={{ padding: '12px' }}>
|
||||
<button
|
||||
onClick={() => handleDelete(backend.id)}
|
||||
style={{ padding: '4px 8px', background: '#ef4444', color: 'white', border: 'none', 'border-radius': '4px', cursor: 'pointer', 'font-size': '0.8rem' }}
|
||||
>
|
||||
Delete
|
||||
</button>
|
||||
</td>
|
||||
<td style={{ padding: '12px', display: 'flex', gap: '8px' }}>
|
||||
<button
|
||||
onClick={() => handleEdit(backend)}
|
||||
style={{ padding: '4px 8px', background: '#3b82f6', color: 'white', border: 'none', 'border-radius': '4px', cursor: 'pointer', 'font-size': '0.8rem' }}
|
||||
>
|
||||
Edit
|
||||
</button>
|
||||
<button
|
||||
onClick={() => handleDelete(backend.id)}
|
||||
style={{ padding: '4px 8px', background: '#ef4444', color: 'white', border: 'none', 'border-radius': '4px', cursor: 'pointer', 'font-size': '0.8rem' }}
|
||||
>
|
||||
Delete
|
||||
</button>
|
||||
</td>
|
||||
</tr>
|
||||
)}</For>
|
||||
</tbody>
|
||||
|
|
@ -129,6 +157,27 @@ export const Backends: Component = () => {
|
|||
</div>
|
||||
</div>
|
||||
)}
|
||||
|
||||
{editingBackend() && (
|
||||
<EditModal
|
||||
isOpen={!!editingBackend()}
|
||||
onClose={() => setEditingBackend(null)}
|
||||
onSubmit={handleUpdate}
|
||||
title="Edit Backend"
|
||||
fields={[
|
||||
{ name: 'name', label: 'Name', type: 'text', required: true },
|
||||
{ name: 'base_url', label: 'Base URL', type: 'text', required: true },
|
||||
{ name: 'api_key', label: 'API Key', type: 'text', required: false },
|
||||
{ name: 'is_active', label: 'Active', type: 'checkbox', required: false },
|
||||
]}
|
||||
initialValues={{
|
||||
name: editingBackend()!.name,
|
||||
base_url: editingBackend()!.base_url,
|
||||
api_key: editingBackend()!.api_key || '',
|
||||
is_active: editingBackend()!.is_active,
|
||||
}}
|
||||
/>
|
||||
)}
|
||||
</div>
|
||||
</Layout>
|
||||
);
|
||||
|
|
|
|||
|
|
@ -2,11 +2,13 @@ import { Component, createResource, For, createSignal } from 'solid-js';
|
|||
import { api } from '../api/client';
|
||||
import type { User } from '../types';
|
||||
import { Layout } from '../components/Layout';
|
||||
import { EditModal } from '../components/EditModal';
|
||||
|
||||
export const Users: Component = () => {
|
||||
const [users, { refetch }] = createResource(() => api.users.getAll());
|
||||
const [showModal, setShowModal] = createSignal(false);
|
||||
const [formData, setFormData] = createSignal({ name: '', email: '' });
|
||||
const [editingUser, setEditingUser] = createSignal<User | null>(null);
|
||||
|
||||
const handleSubmit = async (e: Event) => {
|
||||
e.preventDefault();
|
||||
|
|
@ -30,6 +32,25 @@ export const Users: Component = () => {
|
|||
refetch();
|
||||
};
|
||||
|
||||
const handleEdit = (user: User) => {
|
||||
setEditingUser(user);
|
||||
};
|
||||
|
||||
const handleUpdate = async (data: Record<string, any>) => {
|
||||
if (!editingUser()) return;
|
||||
|
||||
if (!confirm('Are you sure you want to update this user?')) return;
|
||||
|
||||
const updateData: Partial<User> = {};
|
||||
if (data.name) updateData.name = data.name.trim();
|
||||
if (data.email !== undefined) updateData.email = data.email.trim() || undefined;
|
||||
if (data.is_active !== undefined) updateData.is_active = data.is_active;
|
||||
|
||||
await api.users.update(editingUser()!.id, updateData);
|
||||
setEditingUser(null);
|
||||
refetch();
|
||||
};
|
||||
|
||||
const copyToClipboard = async (apiKey: string) => {
|
||||
try {
|
||||
await navigator.clipboard.writeText(apiKey);
|
||||
|
|
@ -85,20 +106,26 @@ export const Users: Component = () => {
|
|||
<td style={{ padding: '12px', color: user.is_active ? '#22c55e' : '#ef4444' }}>
|
||||
{user.is_active ? 'Active' : 'Inactive'}
|
||||
</td>
|
||||
<td style={{ padding: '12px', display: 'flex', gap: '8px' }}>
|
||||
<button
|
||||
onClick={() => handleRegenerateApiKey(user.id)}
|
||||
style={{ padding: '4px 8px', background: '#64748b', color: 'white', border: 'none', 'border-radius': '4px', cursor: 'pointer', 'font-size': '0.8rem' }}
|
||||
>
|
||||
Regenerate Key
|
||||
</button>
|
||||
<button
|
||||
onClick={() => handleDelete(user.id)}
|
||||
style={{ padding: '4px 8px', background: '#ef4444', color: 'white', border: 'none', 'border-radius': '4px', cursor: 'pointer', 'font-size': '0.8rem' }}
|
||||
>
|
||||
Delete
|
||||
</button>
|
||||
</td>
|
||||
<td style={{ padding: '12px', display: 'flex', gap: '8px' }}>
|
||||
<button
|
||||
onClick={() => handleRegenerateApiKey(user.id)}
|
||||
style={{ padding: '4px 8px', background: '#64748b', color: 'white', border: 'none', 'border-radius': '4px', cursor: 'pointer', 'font-size': '0.8rem' }}
|
||||
>
|
||||
Regenerate Key
|
||||
</button>
|
||||
<button
|
||||
onClick={() => handleEdit(user)}
|
||||
style={{ padding: '4px 8px', background: '#3b82f6', color: 'white', border: 'none', 'border-radius': '4px', cursor: 'pointer', 'font-size': '0.8rem' }}
|
||||
>
|
||||
Edit
|
||||
</button>
|
||||
<button
|
||||
onClick={() => handleDelete(user.id)}
|
||||
style={{ padding: '4px 8px', background: '#ef4444', color: 'white', border: 'none', 'border-radius': '4px', cursor: 'pointer', 'font-size': '0.8rem' }}
|
||||
>
|
||||
Delete
|
||||
</button>
|
||||
</td>
|
||||
</tr>
|
||||
)}</For>
|
||||
</tbody>
|
||||
|
|
@ -148,6 +175,25 @@ export const Users: Component = () => {
|
|||
</div>
|
||||
</div>
|
||||
)}
|
||||
|
||||
{editingUser() && (
|
||||
<EditModal
|
||||
isOpen={!!editingUser()}
|
||||
onClose={() => setEditingUser(null)}
|
||||
onSubmit={handleUpdate}
|
||||
title="Edit User"
|
||||
fields={[
|
||||
{ name: 'name', label: 'Name', type: 'text', required: true },
|
||||
{ name: 'email', label: 'Email', type: 'email', required: false },
|
||||
{ name: 'is_active', label: 'Active', type: 'checkbox', required: false },
|
||||
]}
|
||||
initialValues={{
|
||||
name: editingUser()!.name,
|
||||
email: editingUser()!.email || '',
|
||||
is_active: editingUser()!.is_active,
|
||||
}}
|
||||
/>
|
||||
)}
|
||||
</div>
|
||||
</Layout>
|
||||
);
|
||||
|
|
|
|||
|
|
@ -6,7 +6,8 @@
|
|||
"dev": "concurrently \"pnpm exec tsx watch server/src/index.ts\" \"pnpm -r --filter client run dev\"",
|
||||
"build": "pnpm -r build",
|
||||
"start": "pnpm -r start",
|
||||
"test": "pnpm -r --filter server test"
|
||||
"test": "pnpm -r --filter server test",
|
||||
"bench": "pnpm -r --filter server run bench"
|
||||
},
|
||||
"keywords": [
|
||||
"llm",
|
||||
|
|
|
|||
38
pnpm-lock.yaml
generated
38
pnpm-lock.yaml
generated
|
|
@ -73,6 +73,15 @@ importers:
|
|||
'@types/supertest':
|
||||
specifier: ^7.2.0
|
||||
version: 7.2.0
|
||||
chalk:
|
||||
specifier: ^5.6.0
|
||||
version: 5.6.2
|
||||
cli-table3:
|
||||
specifier: ^0.6.5
|
||||
version: 0.6.5
|
||||
commander:
|
||||
specifier: ^14.0.0
|
||||
version: 14.0.3
|
||||
supertest:
|
||||
specifier: ^7.2.2
|
||||
version: 7.2.2
|
||||
|
|
@ -169,6 +178,10 @@ packages:
|
|||
resolution: {integrity: sha512-LwdZHpScM4Qz8Xw2iKSzS+cfglZzJGvofQICy7W7v4caru4EaAmyUuO6BGrbyQ2mYV11W0U8j5mBhd14dd3B0A==}
|
||||
engines: {node: '>=6.9.0'}
|
||||
|
||||
'@colors/colors@1.5.0':
|
||||
resolution: {integrity: sha512-ooWCrlZP11i8GImSjTHYHLkvFDP48nS4+204nGb1RiX/WXYHmJA2III9/e2DWVabCESdW7hBAEzHRqUn9OUVvQ==}
|
||||
engines: {node: '>=0.1.90'}
|
||||
|
||||
'@esbuild/aix-ppc64@0.27.3':
|
||||
resolution: {integrity: sha512-9fJMTNFTWZMh5qwrBItuziu834eOCUcEqymSH7pY+zoMVEZg3gcPuBNxH1EvfVYe9h0x/Ptw8KBzv7qxb7l8dg==}
|
||||
engines: {node: '>=18'}
|
||||
|
|
@ -681,9 +694,17 @@ packages:
|
|||
resolution: {integrity: sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA==}
|
||||
engines: {node: '>=10'}
|
||||
|
||||
chalk@5.6.2:
|
||||
resolution: {integrity: sha512-7NzBL0rN6fMUW+f7A6Io4h40qQlG+xGmtMxfbnH/K7TAtt8JQWVQK+6g0UXKMeVJoyV5EkkNsErQ8pVD3bLHbA==}
|
||||
engines: {node: ^12.17.0 || ^14.13 || >=16.0.0}
|
||||
|
||||
chownr@1.1.4:
|
||||
resolution: {integrity: sha512-jJ0bqzaylmJtVnNgzTeSOs8DPavpbYgEr/b0YL8/2GO3xJEhInFmhKMUnEJQjZumK7KXGFhUy89PrsJWlakBVg==}
|
||||
|
||||
cli-table3@0.6.5:
|
||||
resolution: {integrity: sha512-+W/5efTR7y5HRD7gACw9yQjqMVvEMLBHmboM/kPWam+H+Hmyrgjh6YncVKK122YZkXrLudzTuAukUw9FnMf7IQ==}
|
||||
engines: {node: 10.* || >= 12.*}
|
||||
|
||||
cliui@8.0.1:
|
||||
resolution: {integrity: sha512-BSeNnyus75C4//NQ9gQt1/csTXyo/8Sb+afLAkzAptFuMsod9HFokGNudZpi/oQV73hnVK+sR+5PVRMd+Dr7YQ==}
|
||||
engines: {node: '>=12'}
|
||||
|
|
@ -699,6 +720,10 @@ packages:
|
|||
resolution: {integrity: sha512-FQN4MRfuJeHf7cBbBMJFXhKSDq+2kAArBlmRBvcvFE5BB1HZKXtSFASDhdlz9zOYwxh8lDdnvmMOe/+5cdoEdg==}
|
||||
engines: {node: '>= 0.8'}
|
||||
|
||||
commander@14.0.3:
|
||||
resolution: {integrity: sha512-H+y0Jo/T1RZ9qPP4Eh1pkcQcLRglraJaSLoyOtHxu6AapkjWVCy2Sit1QQ4x3Dng8qDlSsZEet7g5Pq06MvTgw==}
|
||||
engines: {node: '>=20'}
|
||||
|
||||
component-emitter@1.3.1:
|
||||
resolution: {integrity: sha512-T0+barUSQRTUQASh8bx02dl+DhF54GtIDY13Y3m9oWTklKbb3Wv974meRpeZ3lp1JpLVECWWNHC4vaG2XHXouQ==}
|
||||
|
||||
|
|
@ -1603,6 +1628,9 @@ snapshots:
|
|||
'@babel/helper-string-parser': 7.27.1
|
||||
'@babel/helper-validator-identifier': 7.28.5
|
||||
|
||||
'@colors/colors@1.5.0':
|
||||
optional: true
|
||||
|
||||
'@esbuild/aix-ppc64@0.27.3':
|
||||
optional: true
|
||||
|
||||
|
|
@ -2021,8 +2049,16 @@ snapshots:
|
|||
ansi-styles: 4.3.0
|
||||
supports-color: 7.2.0
|
||||
|
||||
chalk@5.6.2: {}
|
||||
|
||||
chownr@1.1.4: {}
|
||||
|
||||
cli-table3@0.6.5:
|
||||
dependencies:
|
||||
string-width: 4.2.3
|
||||
optionalDependencies:
|
||||
'@colors/colors': 1.5.0
|
||||
|
||||
cliui@8.0.1:
|
||||
dependencies:
|
||||
string-width: 4.2.3
|
||||
|
|
@ -2039,6 +2075,8 @@ snapshots:
|
|||
dependencies:
|
||||
delayed-stream: 1.0.0
|
||||
|
||||
commander@14.0.3: {}
|
||||
|
||||
component-emitter@1.3.1: {}
|
||||
|
||||
concurrently@9.2.1:
|
||||
|
|
|
|||
203
server/benchmarks/index.ts
Normal file
203
server/benchmarks/index.ts
Normal file
|
|
@ -0,0 +1,203 @@
|
|||
#!/usr/bin/env node
|
||||
import { Command } from 'commander';
|
||||
import chalk from 'chalk';
|
||||
import { BenchmarkConfig, BenchmarkEnv, setupBenchmark, runBenchmark } from './runner';
|
||||
import { Scenarios, createRealBackendPayload } from './scenarios';
|
||||
import { calculateStats, BenchmarkResult } from './stats';
|
||||
import { printReport, exportToJSON } from './report';
|
||||
|
||||
// Utility: Normalize backend URL (remove trailing slash and /v1 prefix)
|
||||
function normalizeBackendUrl(url: string): string {
|
||||
let normalized = url || '';
|
||||
if (normalized.endsWith('/')) {
|
||||
normalized = normalized.slice(0, -1);
|
||||
}
|
||||
if (normalized.endsWith('/v1')) {
|
||||
normalized = normalized.slice(0, -3);
|
||||
}
|
||||
return normalized;
|
||||
}
|
||||
|
||||
// Utility: Build request headers with optional authentication
|
||||
function buildHeaders(authToken?: string): Record<string, string> {
|
||||
const headers: Record<string, string> = {
|
||||
'Content-Type': 'application/json'
|
||||
};
|
||||
if (authToken) {
|
||||
headers['Authorization'] = `Bearer ${authToken}`;
|
||||
}
|
||||
return headers;
|
||||
}
|
||||
|
||||
// Utility: Build benchmark URLs for direct and routed requests
|
||||
function buildUrls(
|
||||
backendType: 'mock' | 'real',
|
||||
backendBaseUrl: string | undefined,
|
||||
routerPort: number | undefined,
|
||||
mockBackendPort: number | undefined,
|
||||
endpoint: string
|
||||
): { directUrl: string; routeUrl: string } {
|
||||
if (backendType === 'real') {
|
||||
const normalizedUrl = normalizeBackendUrl(backendBaseUrl || '');
|
||||
return {
|
||||
directUrl: `${normalizedUrl}${endpoint}`,
|
||||
routeUrl: `http://localhost:${routerPort || 3099}${endpoint}`
|
||||
};
|
||||
} else {
|
||||
return {
|
||||
directUrl: `http://localhost:${mockBackendPort}${endpoint}`,
|
||||
routeUrl: `http://localhost:${routerPort || 3099}${endpoint}`
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
// Utility: Run benchmark for both direct and routed requests
|
||||
async function runScenarioBenchmark(
|
||||
backendType: 'mock' | 'real',
|
||||
scenario: any,
|
||||
config: BenchmarkConfig,
|
||||
env: BenchmarkEnv | null,
|
||||
backendApiKey?: string
|
||||
): Promise<{ directResults: any[]; routeResults: any[] }> {
|
||||
const urls = buildUrls(
|
||||
backendType,
|
||||
config.backendUrl,
|
||||
env?.routerPort,
|
||||
env?.mockBackendPort,
|
||||
scenario.endpoint
|
||||
);
|
||||
|
||||
const directHeaders = buildHeaders(backendApiKey);
|
||||
const routeHeaders = buildHeaders(env?.userApiKey);
|
||||
|
||||
const benchmarkConfig = {
|
||||
concurrent: config.concurrentRequests,
|
||||
total: config.totalRequests,
|
||||
warmup: config.warmupRequests
|
||||
};
|
||||
|
||||
if (backendType === 'real') {
|
||||
console.log(chalk.yellow(' Running direct requests...'));
|
||||
} else {
|
||||
console.log(chalk.yellow(' Running direct requests to mock backend...'));
|
||||
}
|
||||
const directRaw = await runBenchmark(
|
||||
urls.directUrl,
|
||||
scenario.method,
|
||||
scenario.payload,
|
||||
directHeaders,
|
||||
benchmarkConfig
|
||||
);
|
||||
|
||||
console.log(chalk.yellow(' Running routed requests...'));
|
||||
const routeRaw = await runBenchmark(
|
||||
urls.routeUrl,
|
||||
scenario.method,
|
||||
scenario.payload,
|
||||
routeHeaders,
|
||||
benchmarkConfig
|
||||
);
|
||||
|
||||
return { directResults: directRaw, routeResults: routeRaw };
|
||||
}
|
||||
|
||||
const program = new Command();
|
||||
|
||||
program
|
||||
.name('bench')
|
||||
.description('LLM Router Benchmark Tool')
|
||||
.version('1.0.0')
|
||||
.option('-c, --concurrent <number>', 'Number of concurrent requests', '10')
|
||||
.option('-r, --requests <number>', 'Total number of requests', '100')
|
||||
.option('-w, --warmup <number>', 'Number of warmup requests', '5')
|
||||
.option('-b, --backend <type>', 'Backend type (mock|real)', 'mock')
|
||||
.option('-u, --backend-url <url>', 'Real backend URL (required for real backend)')
|
||||
.option('-k, --backend-key <key>', 'Real backend API key (optional)')
|
||||
.option('-o, --output <file>', 'Export results to JSON file')
|
||||
.parse(process.argv);
|
||||
|
||||
const options = program.opts();
|
||||
|
||||
async function main() {
|
||||
console.log(chalk.bold.cyan('\n🚀 LLM Router Benchmark Tool\n'));
|
||||
|
||||
const config: BenchmarkConfig = {
|
||||
concurrentRequests: parseInt(options.concurrent),
|
||||
totalRequests: parseInt(options.requests),
|
||||
warmupRequests: parseInt(options.warmup),
|
||||
backendType: options.backend as 'mock' | 'real',
|
||||
backendUrl: options.backendUrl,
|
||||
backendApiKey: options.backendKey
|
||||
};
|
||||
|
||||
// Validate real backend options
|
||||
if (config.backendType === 'real' && !config.backendUrl) {
|
||||
console.error(chalk.red('Error: --backend-url is required for real backend'));
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
let env: BenchmarkEnv | null = null;
|
||||
const allResults: BenchmarkResult[] = [];
|
||||
|
||||
try {
|
||||
// Setup benchmark environment
|
||||
console.log(chalk.yellow('Setting up benchmark environment...'));
|
||||
env = await setupBenchmark(config);
|
||||
|
||||
// Define scenarios to run
|
||||
const scenarios = [
|
||||
Scenarios.smallPayload(),
|
||||
Scenarios.largePayload(),
|
||||
Scenarios.modelsEndpoint()
|
||||
];
|
||||
|
||||
if (config.backendType === 'real') {
|
||||
scenarios.push(createRealBackendPayload());
|
||||
}
|
||||
|
||||
// Run benchmarks for each scenario
|
||||
for (const scenario of scenarios) {
|
||||
console.log(chalk.bold(`\n📊 Running: ${scenario.name}`));
|
||||
console.log(chalk.gray(` ${scenario.description}`));
|
||||
|
||||
const { directResults, routeResults } = await runScenarioBenchmark(
|
||||
config.backendType,
|
||||
scenario,
|
||||
config,
|
||||
env,
|
||||
config.backendApiKey
|
||||
);
|
||||
|
||||
// Calculate statistics
|
||||
const directStats = calculateStats(directResults, scenario.name, 'direct');
|
||||
const routeStats = calculateStats(routeResults, scenario.name, 'route');
|
||||
|
||||
allResults.push(directStats, routeStats);
|
||||
}
|
||||
|
||||
// Print report
|
||||
printReport(allResults, {
|
||||
concurrent: config.concurrentRequests,
|
||||
total: config.totalRequests,
|
||||
warmup: config.warmupRequests,
|
||||
backend: config.backendType
|
||||
});
|
||||
|
||||
// Export to JSON if requested
|
||||
if (options.output) {
|
||||
exportToJSON(allResults, options.output);
|
||||
}
|
||||
|
||||
} catch (error) {
|
||||
console.error(chalk.red(`Error: ${error instanceof Error ? error.message : 'Unknown error'}`));
|
||||
process.exit(1);
|
||||
} finally {
|
||||
// Cleanup
|
||||
if (env) {
|
||||
env.cleanup();
|
||||
console.log(chalk.gray('Cleanup completed.\n'));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
main();
|
||||
96
server/benchmarks/report.ts
Normal file
96
server/benchmarks/report.ts
Normal file
|
|
@ -0,0 +1,96 @@
|
|||
import Table from 'cli-table3';
|
||||
import chalk from 'chalk';
|
||||
import { BenchmarkResult, calculateOverhead } from './stats';
|
||||
|
||||
export function printReport(results: BenchmarkResult[], config: any) {
|
||||
console.log('\n' + '='.repeat(80));
|
||||
console.log(chalk.bold.cyan(' BENCHMARK RESULTS'));
|
||||
console.log('='.repeat(80));
|
||||
console.log(`\nConfiguration:`);
|
||||
console.log(` Concurrent Requests: ${config.concurrent}`);
|
||||
console.log(` Total Requests: ${config.total}`);
|
||||
console.log(` Warmup Requests: ${config.warmup}`);
|
||||
console.log(` Backend Type: ${config.backend}`);
|
||||
|
||||
// Group results by scenario
|
||||
const scenarios = [...new Set(results.map(r => r.scenario))];
|
||||
|
||||
for (const scenario of scenarios) {
|
||||
const scenarioResults = results.filter(r => r.scenario === scenario);
|
||||
const direct = scenarioResults.find(r => r.mode === 'direct');
|
||||
const route = scenarioResults.find(r => r.mode === 'route');
|
||||
|
||||
console.log(`\n${chalk.bold.yellow(`\n${scenario}`)}`);
|
||||
console.log('-'.repeat(80));
|
||||
|
||||
const table = new Table({
|
||||
head: [
|
||||
chalk.cyan('Mode'),
|
||||
chalk.cyan('Success'),
|
||||
chalk.cyan('Avg (ms)'),
|
||||
chalk.cyan('Min (ms)'),
|
||||
chalk.cyan('Max (ms)'),
|
||||
chalk.cyan('P50 (ms)'),
|
||||
chalk.cyan('P95 (ms)'),
|
||||
chalk.cyan('P99 (ms)'),
|
||||
chalk.cyan('Errors'),
|
||||
chalk.cyan('Req/s')
|
||||
],
|
||||
colAligns: [
|
||||
'left', 'right', 'right', 'right', 'right', 'right', 'right', 'right', 'right', 'right'
|
||||
]
|
||||
});
|
||||
|
||||
if (direct) {
|
||||
table.push([
|
||||
chalk.green('Direct'),
|
||||
`${direct.successfulRequests}/${direct.totalRequests}`,
|
||||
direct.avgResponseTime.toFixed(2),
|
||||
direct.minResponseTime.toFixed(2),
|
||||
direct.maxResponseTime.toFixed(2),
|
||||
direct.p50ResponseTime.toFixed(2),
|
||||
direct.p95ResponseTime.toFixed(2),
|
||||
direct.p99ResponseTime.toFixed(2),
|
||||
direct.errors,
|
||||
direct.throughput.toFixed(2)
|
||||
]);
|
||||
}
|
||||
|
||||
if (route) {
|
||||
const overhead = direct ? calculateOverhead(direct, route) : 0;
|
||||
const overheadColor = overhead > 20 ? chalk.red : overhead > 10 ? chalk.yellow : chalk.green;
|
||||
|
||||
table.push([
|
||||
chalk.blue('Route'),
|
||||
`${route.successfulRequests}/${route.totalRequests}`,
|
||||
route.avgResponseTime.toFixed(2),
|
||||
route.minResponseTime.toFixed(2),
|
||||
route.maxResponseTime.toFixed(2),
|
||||
route.p50ResponseTime.toFixed(2),
|
||||
route.p95ResponseTime.toFixed(2),
|
||||
route.p99ResponseTime.toFixed(2),
|
||||
route.errors,
|
||||
route.throughput.toFixed(2)
|
||||
]);
|
||||
|
||||
console.log(table.toString());
|
||||
console.log(`\n${overheadColor(` Overhead: ${overhead.toFixed(2)}%`)}`);
|
||||
} else {
|
||||
console.log(table.toString());
|
||||
}
|
||||
}
|
||||
|
||||
console.log('\n' + '='.repeat(80));
|
||||
console.log(chalk.bold.green(' Benchmark completed!'));
|
||||
console.log('='.repeat(80) + '\n');
|
||||
}
|
||||
|
||||
export function exportToJSON(results: BenchmarkResult[], outputPath: string) {
|
||||
const fs = require('fs');
|
||||
const data = {
|
||||
timestamp: new Date().toISOString(),
|
||||
results
|
||||
};
|
||||
fs.writeFileSync(outputPath, JSON.stringify(data, null, 2));
|
||||
console.log(chalk.green(`Results exported to ${outputPath}`));
|
||||
}
|
||||
248
server/benchmarks/runner.ts
Normal file
248
server/benchmarks/runner.ts
Normal file
|
|
@ -0,0 +1,248 @@
|
|||
import { createMockBackend } from '../tests/utils/mockBackend';
|
||||
import express from 'express';
|
||||
import { BackendModel } from '../src/models/Backend';
|
||||
import { UserModel } from '../src/models/User';
|
||||
import { PermissionModel } from '../src/models/Permission';
|
||||
import { createServer } from '../src/index';
|
||||
import { RawResult } from './stats';
|
||||
|
||||
export interface BenchmarkConfig {
|
||||
concurrentRequests: number;
|
||||
totalRequests: number;
|
||||
warmupRequests: number;
|
||||
backendType: 'mock' | 'real';
|
||||
backendUrl?: string;
|
||||
backendApiKey?: string;
|
||||
}
|
||||
|
||||
export interface BenchmarkEnv {
|
||||
mockBackendPort?: number;
|
||||
routerPort?: number;
|
||||
userApiKey?: string;
|
||||
backendApiKey?: string;
|
||||
cleanup: () => void;
|
||||
}
|
||||
|
||||
export async function setupBenchmark(config: BenchmarkConfig): Promise<BenchmarkEnv> {
|
||||
let mockBackendPort: number | undefined;
|
||||
let routerPort: number | undefined;
|
||||
let userApiKey: string | undefined;
|
||||
let backendApiKey: string | undefined;
|
||||
let mockServer: any = null;
|
||||
let routerServer: any = null;
|
||||
let backendId: number | undefined;
|
||||
|
||||
// Always start router server for both mock and real backend
|
||||
if (config.backendType === 'mock') {
|
||||
// Cleanup existing benchmark data
|
||||
const existingBackend = BackendModel.findAll().find(b => b.name === 'benchmark-backend');
|
||||
if (existingBackend) {
|
||||
BackendModel.delete(existingBackend.id);
|
||||
}
|
||||
const existingUser = UserModel.findAll().find(u => u.name === 'benchmark-user');
|
||||
if (existingUser) {
|
||||
const permissions = PermissionModel.findAll().filter(p => p.user_id === existingUser.id);
|
||||
permissions.forEach(p => PermissionModel.delete(p.user_id, p.backend_id));
|
||||
UserModel.delete(existingUser.id);
|
||||
}
|
||||
|
||||
const { server, port } = createMockBackend();
|
||||
mockServer = server;
|
||||
mockBackendPort = port;
|
||||
console.log(`Mock backend started on port ${port}`);
|
||||
|
||||
// Check if benchmark backend already exists
|
||||
let backend = BackendModel.findAll().find(b => b.name === 'benchmark-backend');
|
||||
let backendId: number;
|
||||
|
||||
if (backend) {
|
||||
backendId = backend.id;
|
||||
// Update the backend URL to point to new mock server (without /v1 prefix)
|
||||
BackendModel.update(backendId, { base_url: `http://localhost:${port}` });
|
||||
} else {
|
||||
const newBackend = BackendModel.create({
|
||||
name: 'benchmark-backend',
|
||||
base_url: `http://localhost:${port}`,
|
||||
api_key: 'mock-backend-key'
|
||||
});
|
||||
backendId = newBackend.id;
|
||||
}
|
||||
backendApiKey = 'mock-backend-key';
|
||||
|
||||
// Check if benchmark user already exists
|
||||
let user = UserModel.findAll().find(u => u.name === 'benchmark-user');
|
||||
let userId: number;
|
||||
|
||||
if (user) {
|
||||
userId = user.id;
|
||||
userApiKey = user.api_key;
|
||||
console.log(` Using existing user: ${user.id}, API key: ${userApiKey?.substring(0, 15)}...`);
|
||||
} else {
|
||||
user = UserModel.create({ name: 'benchmark-user', email: 'benchmark@test.com' });
|
||||
userId = user.id;
|
||||
const newApiKey = UserModel.regenerateApiKey(userId);
|
||||
if (newApiKey) {
|
||||
userApiKey = newApiKey;
|
||||
}
|
||||
console.log(` Created new user: ${userId}, API key: ${userApiKey?.substring(0, 15)}...`);
|
||||
}
|
||||
|
||||
// Check if permission already exists
|
||||
const existingPermission = PermissionModel.findAll().find(
|
||||
p => p.user_id === userId && p.backend_id === backendId
|
||||
);
|
||||
|
||||
if (!existingPermission) {
|
||||
PermissionModel.create({ user_id: userId, backend_id: backendId });
|
||||
}
|
||||
|
||||
routerPort = 3099;
|
||||
const app = createServer();
|
||||
routerServer = app.listen(routerPort, () => {
|
||||
console.log(`Router server listening on port ${routerPort}`);
|
||||
}).on('error', (err: Error) => {
|
||||
console.error(`Router server error on port ${routerPort}:`, err.message);
|
||||
});
|
||||
|
||||
// Wait for server to be ready
|
||||
await new Promise(resolve => setTimeout(resolve, 500));
|
||||
} else if (config.backendType === 'real') {
|
||||
// For real backend, still need router server and a test user
|
||||
routerPort = 3099;
|
||||
const app = createServer();
|
||||
routerServer = app.listen(routerPort, () => {
|
||||
console.log(`Router server listening on port ${routerPort}`);
|
||||
}).on('error', (err: Error) => {
|
||||
console.error(`Router server error on port ${routerPort}:`, err.message);
|
||||
});
|
||||
|
||||
// Wait for server to be ready
|
||||
await new Promise(resolve => setTimeout(resolve, 500));
|
||||
|
||||
// Create a test user for router authentication
|
||||
let user = UserModel.findAll().find(u => u.name === 'benchmark-user');
|
||||
if (user) {
|
||||
userApiKey = user.api_key;
|
||||
} else {
|
||||
user = UserModel.create({ name: 'benchmark-user', email: 'benchmark@test.com' });
|
||||
const newApiKey = UserModel.regenerateApiKey(user.id);
|
||||
if (newApiKey) {
|
||||
userApiKey = newApiKey;
|
||||
}
|
||||
}
|
||||
|
||||
// Create backend entry for the real backend
|
||||
let backend = BackendModel.findAll().find(b => b.name === 'real-backend');
|
||||
if (backend) {
|
||||
BackendModel.update(backend.id, {
|
||||
base_url: config.backendUrl?.replace(/\/v1$/, '') || '',
|
||||
api_key: config.backendApiKey || '',
|
||||
is_active: true
|
||||
});
|
||||
backendId = backend.id;
|
||||
} else {
|
||||
const newBackend = BackendModel.create({
|
||||
name: 'real-backend',
|
||||
base_url: config.backendUrl?.replace(/\/v1$/, '') || '',
|
||||
api_key: config.backendApiKey || ''
|
||||
});
|
||||
backendId = newBackend.id;
|
||||
// Ensure backend is active
|
||||
BackendModel.update(backendId, { is_active: true });
|
||||
}
|
||||
|
||||
// Grant permission to the user
|
||||
const existingPermission = PermissionModel.findAll().find(
|
||||
p => p.user_id === user!.id && p.backend_id === backendId!
|
||||
);
|
||||
if (!existingPermission && user && backendId) {
|
||||
PermissionModel.create({ user_id: user.id, backend_id: backendId });
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
mockBackendPort,
|
||||
routerPort,
|
||||
userApiKey,
|
||||
backendApiKey,
|
||||
cleanup: () => {
|
||||
if (mockServer) mockServer.close();
|
||||
if (routerServer) routerServer.close();
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
export async function runBenchmark(
|
||||
url: string,
|
||||
method: string,
|
||||
payload: any,
|
||||
headers: Record<string, string>,
|
||||
config: { concurrent: number; total: number; warmup: number }
|
||||
): Promise<RawResult[]> {
|
||||
const allResults: RawResult[] = [];
|
||||
|
||||
// Warmup
|
||||
console.log(` Warmup: ${config.warmup} requests...`);
|
||||
for (let i = 0; i < config.warmup; i++) {
|
||||
await makeRequest(url, method, payload, headers);
|
||||
}
|
||||
|
||||
// Benchmark
|
||||
console.log(` Running: ${config.total} requests (concurrent: ${config.concurrent})...`);
|
||||
|
||||
const batches = Math.ceil(config.total / config.concurrent);
|
||||
|
||||
for (let batch = 0; batch < batches; batch++) {
|
||||
const batchPromises = [];
|
||||
for (let i = 0; i < config.concurrent && (batch * config.concurrent + i) < config.total; i++) {
|
||||
batchPromises.push(makeRequest(url, method, payload, headers));
|
||||
}
|
||||
|
||||
const batchResults = await Promise.all(batchPromises);
|
||||
allResults.push(...batchResults);
|
||||
}
|
||||
|
||||
return allResults;
|
||||
}
|
||||
|
||||
async function makeRequest(
|
||||
url: string,
|
||||
method: string,
|
||||
payload: any,
|
||||
headers: Record<string, string>
|
||||
): Promise<RawResult> {
|
||||
const startTime = Date.now();
|
||||
|
||||
try {
|
||||
const options: RequestInit = {
|
||||
method,
|
||||
headers,
|
||||
signal: AbortSignal.timeout(30000)
|
||||
};
|
||||
|
||||
if (method !== 'GET' && payload && Object.keys(payload).length > 0) {
|
||||
options.body = JSON.stringify(payload);
|
||||
}
|
||||
|
||||
const response = await fetch(url, options);
|
||||
const responseTime = Date.now() - startTime;
|
||||
|
||||
if (response.ok) {
|
||||
return { responseTime, success: true };
|
||||
} else {
|
||||
const errorText = await response.text().catch(() => 'Unknown error');
|
||||
return {
|
||||
responseTime,
|
||||
success: false,
|
||||
error: `HTTP ${response.status}: ${errorText.substring(0, 100)}`
|
||||
};
|
||||
}
|
||||
} catch (error) {
|
||||
const responseTime = Date.now() - startTime;
|
||||
return {
|
||||
responseTime,
|
||||
success: false,
|
||||
error: error instanceof Error ? error.message : 'Unknown error'
|
||||
};
|
||||
}
|
||||
}
|
||||
79
server/benchmarks/scenarios.ts
Normal file
79
server/benchmarks/scenarios.ts
Normal file
|
|
@ -0,0 +1,79 @@
|
|||
export interface Scenario {
|
||||
name: string;
|
||||
description: string;
|
||||
payload: ChatCompletionPayload;
|
||||
endpoint: string;
|
||||
method: string;
|
||||
}
|
||||
|
||||
export interface ChatCompletionPayload {
|
||||
model: string;
|
||||
messages: Message[];
|
||||
temperature?: number;
|
||||
max_tokens?: number;
|
||||
}
|
||||
|
||||
export interface Message {
|
||||
role: 'system' | 'user' | 'assistant';
|
||||
content: string;
|
||||
}
|
||||
|
||||
export const Scenarios = {
|
||||
smallPayload: (): Scenario => ({
|
||||
name: 'Small Payload',
|
||||
description: 'Single short message',
|
||||
endpoint: '/v1/chat/completions',
|
||||
method: 'POST',
|
||||
payload: {
|
||||
model: 'test-model',
|
||||
messages: [
|
||||
{ role: 'user', content: 'Hello' }
|
||||
],
|
||||
temperature: 0.7,
|
||||
max_tokens: 100
|
||||
}
|
||||
}),
|
||||
|
||||
largePayload: (): Scenario => ({
|
||||
name: 'Large Payload',
|
||||
description: 'Multiple long messages',
|
||||
endpoint: '/v1/chat/completions',
|
||||
method: 'POST',
|
||||
payload: {
|
||||
model: 'test-model',
|
||||
messages: [
|
||||
{ role: 'system', content: 'You are a helpful assistant that provides detailed and accurate information to users. Always respond in a clear and concise manner.' },
|
||||
{ role: 'user', content: 'Can you explain the difference between supervised and unsupervised learning in machine learning? Please provide examples of each and discuss their use cases.' },
|
||||
{ role: 'assistant', content: 'Supervised learning uses labeled data to train models, while unsupervised learning finds patterns in unlabeled data.' },
|
||||
{ role: 'user', content: 'That is helpful. Can you also explain reinforcement learning and how it differs from these approaches? What are some practical applications?' }
|
||||
],
|
||||
temperature: 0.7,
|
||||
max_tokens: 500
|
||||
}
|
||||
}),
|
||||
|
||||
modelsEndpoint: (): Scenario => ({
|
||||
name: 'Models Endpoint',
|
||||
description: 'GET /models request',
|
||||
endpoint: '/v1/models',
|
||||
method: 'GET',
|
||||
payload: {} as ChatCompletionPayload
|
||||
})
|
||||
};
|
||||
|
||||
export function createRealBackendPayload(): Scenario {
|
||||
return {
|
||||
name: 'Real Backend',
|
||||
description: 'Test with real OAI-compatible API',
|
||||
endpoint: '/v1/chat/completions',
|
||||
method: 'POST',
|
||||
payload: {
|
||||
model: process.env.REAL_MODEL || 'default-model',
|
||||
messages: [
|
||||
{ role: 'user', content: 'Hello, this is a benchmark test.' }
|
||||
],
|
||||
temperature: 0.7,
|
||||
max_tokens: 100
|
||||
}
|
||||
};
|
||||
}
|
||||
68
server/benchmarks/stats.ts
Normal file
68
server/benchmarks/stats.ts
Normal file
|
|
@ -0,0 +1,68 @@
|
|||
export interface BenchmarkResult {
|
||||
scenario: string;
|
||||
mode: 'direct' | 'route';
|
||||
totalRequests: number;
|
||||
successfulRequests: number;
|
||||
avgResponseTime: number;
|
||||
minResponseTime: number;
|
||||
maxResponseTime: number;
|
||||
p50ResponseTime: number;
|
||||
p95ResponseTime: number;
|
||||
p99ResponseTime: number;
|
||||
errors: number;
|
||||
throughput: number;
|
||||
totalDuration: number;
|
||||
}
|
||||
|
||||
export interface RawResult {
|
||||
responseTime: number;
|
||||
success: boolean;
|
||||
error?: string;
|
||||
}
|
||||
|
||||
export function calculateStats(results: RawResult[], scenario: string, mode: 'direct' | 'route'): BenchmarkResult {
|
||||
const responseTimes = results.filter(r => r.success).map(r => r.responseTime);
|
||||
const errors = results.filter(r => !r.success).length;
|
||||
const successfulRequests = responseTimes.length;
|
||||
const totalDuration = Math.max(...responseTimes, 0);
|
||||
|
||||
const sortedTimes = [...responseTimes].sort((a, b) => a - b);
|
||||
|
||||
const avgResponseTime = sortedTimes.length > 0
|
||||
? sortedTimes.reduce((a, b) => a + b, 0) / sortedTimes.length
|
||||
: 0;
|
||||
|
||||
const minResponseTime = sortedTimes.length > 0 ? sortedTimes[0] : 0;
|
||||
const maxResponseTime = sortedTimes.length > 0 ? sortedTimes[sortedTimes.length - 1] : 0;
|
||||
|
||||
const p50Index = Math.floor(sortedTimes.length * 0.5);
|
||||
const p95Index = Math.floor(sortedTimes.length * 0.95);
|
||||
const p99Index = Math.floor(sortedTimes.length * 0.99);
|
||||
|
||||
const p50ResponseTime = sortedTimes.length > 0 ? sortedTimes[p50Index] || 0 : 0;
|
||||
const p95ResponseTime = sortedTimes.length > 0 ? sortedTimes[Math.min(p95Index, sortedTimes.length - 1)] || 0 : 0;
|
||||
const p99ResponseTime = sortedTimes.length > 0 ? sortedTimes[Math.min(p99Index, sortedTimes.length - 1)] || 0 : 0;
|
||||
|
||||
const throughput = totalDuration > 0 ? (successfulRequests / totalDuration) * 1000 : 0;
|
||||
|
||||
return {
|
||||
scenario,
|
||||
mode,
|
||||
totalRequests: results.length,
|
||||
successfulRequests,
|
||||
avgResponseTime: Math.round(avgResponseTime * 100) / 100,
|
||||
minResponseTime: Math.round(minResponseTime * 100) / 100,
|
||||
maxResponseTime: Math.round(maxResponseTime * 100) / 100,
|
||||
p50ResponseTime: Math.round(p50ResponseTime * 100) / 100,
|
||||
p95ResponseTime: Math.round(p95ResponseTime * 100) / 100,
|
||||
p99ResponseTime: Math.round(p99ResponseTime * 100) / 100,
|
||||
errors,
|
||||
throughput: Math.round(throughput * 100) / 100,
|
||||
totalDuration,
|
||||
};
|
||||
}
|
||||
|
||||
export function calculateOverhead(direct: BenchmarkResult, route: BenchmarkResult): number {
|
||||
if (direct.avgResponseTime === 0) return 0;
|
||||
return ((route.avgResponseTime - direct.avgResponseTime) / direct.avgResponseTime) * 100;
|
||||
}
|
||||
|
|
@ -9,7 +9,8 @@
|
|||
"dev": "tsx watch src/index.ts",
|
||||
"typecheck": "tsc --noEmit",
|
||||
"test": "vitest run",
|
||||
"test:watch": "vitest"
|
||||
"test:watch": "vitest",
|
||||
"bench": "tsx benchmarks/index.ts"
|
||||
},
|
||||
"keywords": [],
|
||||
"author": "",
|
||||
|
|
@ -28,6 +29,9 @@
|
|||
"@types/express": "^5.0.6",
|
||||
"@types/node": "^25.3.3",
|
||||
"@types/supertest": "^7.2.0",
|
||||
"chalk": "^5.6.0",
|
||||
"cli-table3": "^0.6.5",
|
||||
"commander": "^14.0.0",
|
||||
"supertest": "^7.2.2",
|
||||
"tsx": "^4.21.0",
|
||||
"typescript": "^5.9.3",
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
import express from 'express';
|
||||
import express, { Application } from 'express';
|
||||
import cors from 'cors';
|
||||
import dotenv from 'dotenv';
|
||||
import path from 'path';
|
||||
|
|
@ -10,33 +10,44 @@ import { logger } from './utils/logger';
|
|||
|
||||
dotenv.config();
|
||||
|
||||
const app = express();
|
||||
export function createServer(): Application {
|
||||
const app = express();
|
||||
|
||||
const corsOrigins = process.env.CORS_ORIGINS
|
||||
? process.env.CORS_ORIGINS.split(',').map(origin => origin.trim())
|
||||
: ['http://localhost:5173', 'http://localhost:3001'];
|
||||
|
||||
app.use(cors({
|
||||
origin: corsOrigins,
|
||||
credentials: true,
|
||||
}));
|
||||
app.use(express.json());
|
||||
|
||||
app.use('/admin', adminRoutes);
|
||||
app.use('/v1', apiRoutes);
|
||||
app.use('/admin/analytics', analyticsRoutes);
|
||||
|
||||
app.get('/health', (req, res) => {
|
||||
res.json({ status: 'ok', timestamp: new Date().toISOString() });
|
||||
});
|
||||
|
||||
app.use((req, res) => {
|
||||
res.status(404).json({ error: 'Not found' });
|
||||
});
|
||||
|
||||
return app;
|
||||
}
|
||||
|
||||
const app = createServer();
|
||||
const PORT = process.env.SERVER_PORT || 3000;
|
||||
|
||||
const corsOrigins = process.env.CORS_ORIGINS
|
||||
? process.env.CORS_ORIGINS.split(',').map(origin => origin.trim())
|
||||
: ['http://localhost:5173', 'http://localhost:3001'];
|
||||
// Only start server if this is the main module (not imported)
|
||||
if (require.main === module) {
|
||||
app.listen(PORT, () => {
|
||||
logger.info(`Server running on port ${PORT}`);
|
||||
logger.info(`Admin API: http://localhost:${PORT}/admin`);
|
||||
logger.info(`OpenAI API: http://localhost:${PORT}/v1`);
|
||||
});
|
||||
}
|
||||
|
||||
app.use(cors({
|
||||
origin: corsOrigins,
|
||||
credentials: true,
|
||||
}));
|
||||
app.use(express.json());
|
||||
|
||||
app.use('/admin', adminRoutes);
|
||||
app.use('/v1', apiRoutes);
|
||||
app.use('/admin/analytics', analyticsRoutes);
|
||||
|
||||
app.get('/health', (req, res) => {
|
||||
res.json({ status: 'ok', timestamp: new Date().toISOString() });
|
||||
});
|
||||
|
||||
app.use((req, res) => {
|
||||
res.status(404).json({ error: 'Not found' });
|
||||
});
|
||||
|
||||
app.listen(PORT, () => {
|
||||
logger.info(`Server running on port ${PORT}`);
|
||||
logger.info(`Admin API: http://localhost:${PORT}/admin`);
|
||||
logger.info(`OpenAI API: http://localhost:${PORT}/v1`);
|
||||
});
|
||||
export default app;
|
||||
|
|
|
|||
|
|
@ -52,13 +52,19 @@ router.post('/chat/completions', async (req: AuthenticatedRequest, res: Response
|
|||
});
|
||||
|
||||
if (response.status >= 400) {
|
||||
logger.error(`Backend error for user ${user.id}: ${JSON.stringify(response.data)}`);
|
||||
const errorDetails = response.data as any;
|
||||
const errorInfo = errorDetails.error || 'Unknown error';
|
||||
const causeInfo = errorDetails.cause ? ` (Cause: ${errorDetails.cause})` : '';
|
||||
const backendInfo = errorDetails.backend ? ` [Backend: ${errorDetails.backend}]` : '';
|
||||
logger.error(`Backend error for user ${user.id}: ${errorInfo}${causeInfo}${backendInfo}`);
|
||||
}
|
||||
|
||||
res.status(response.status).json(response.data);
|
||||
} catch (error) {
|
||||
const responseTime = Date.now() - startTime;
|
||||
|
||||
const errorMsg = error instanceof Error ? error.message : 'Unknown error';
|
||||
|
||||
AnalyticsService.logRequest({
|
||||
user_id: user.id,
|
||||
backend_id: backend.id,
|
||||
|
|
@ -66,11 +72,11 @@ router.post('/chat/completions', async (req: AuthenticatedRequest, res: Response
|
|||
request_model: req.body.model,
|
||||
status_code: 502,
|
||||
response_time_ms: responseTime,
|
||||
error_message: error instanceof Error ? error.message : 'Unknown error',
|
||||
error_message: errorMsg,
|
||||
});
|
||||
|
||||
logger.error(`Request failed for user ${user.id}: ${error instanceof Error ? error.message : 'Unknown error'}`);
|
||||
res.status(502).json({ error: 'Backend request failed' });
|
||||
logger.error(`Request failed for user ${user.id}: ${errorMsg}`);
|
||||
res.status(502).json({ error: 'Backend request failed', details: errorMsg });
|
||||
}
|
||||
});
|
||||
|
||||
|
|
@ -98,8 +104,9 @@ router.get('/models', async (req: AuthenticatedRequest, res: Response) => {
|
|||
|
||||
res.status(response.status).json(response.data);
|
||||
} catch (error) {
|
||||
logger.error(`Models request failed for user ${req.user!.id}: ${error instanceof Error ? error.message : 'Unknown error'}`);
|
||||
res.status(502).json({ error: 'Failed to fetch models from backend' });
|
||||
const errorMsg = error instanceof Error ? error.message : 'Unknown error';
|
||||
logger.error(`Models request failed for user ${req.user!.id}: ${errorMsg}`);
|
||||
res.status(502).json({ error: 'Failed to fetch models from backend', details: errorMsg });
|
||||
}
|
||||
});
|
||||
|
||||
|
|
|
|||
|
|
@ -26,7 +26,12 @@ export class RouterService {
|
|||
headers: Record<string, string>,
|
||||
body?: unknown
|
||||
): Promise<{ status: number; data: unknown }> {
|
||||
const backendUrl = backend.base_url.replace(/\/$/, '') + path;
|
||||
let backendPath = path;
|
||||
if (backend.base_url.includes('/v1')) {
|
||||
backendPath = path.replace(/^\/v1/, '');
|
||||
}
|
||||
|
||||
const backendUrl = backend.base_url.replace(/\/$/, '') + backendPath;
|
||||
|
||||
const fetchHeaders: Record<string, string> = {
|
||||
'Content-Type': 'application/json',
|
||||
|
|
@ -51,9 +56,58 @@ export class RouterService {
|
|||
data,
|
||||
};
|
||||
} catch (error) {
|
||||
const errorMsg = error instanceof Error ? error.message : 'Unknown error';
|
||||
|
||||
// Extract detailed error information from error cause
|
||||
let cause: string | undefined;
|
||||
let errorType: string;
|
||||
|
||||
if (error instanceof Error && error.cause) {
|
||||
const causeError = error.cause as any;
|
||||
const causeCode = causeError.code || causeError.errno;
|
||||
const causeSyscall = causeError.syscall;
|
||||
const causeAddress = causeError.address || causeError.hostname;
|
||||
const causePort = causeError.port;
|
||||
|
||||
if (causeCode === 'ECONNREFUSED') {
|
||||
errorType = 'Backend connection refused';
|
||||
cause = causeAddress && causePort
|
||||
? `Backend server at ${causeAddress}:${causePort} is not accepting connections`
|
||||
: 'Backend server is not accepting connections';
|
||||
} else if (causeCode === 'ETIMEDOUT' || causeCode === 'ECONNABORTED') {
|
||||
errorType = 'Backend request timeout';
|
||||
cause = 'Connection to backend timed out';
|
||||
} else if (causeCode === 'ENOTFOUND') {
|
||||
errorType = 'Backend unreachable';
|
||||
cause = causeAddress ? `Could not resolve hostname: ${causeAddress}` : 'Could not resolve backend hostname';
|
||||
} else if (causeCode === 'EPIPE' || causeSyscall === 'write') {
|
||||
errorType = 'Backend connection lost';
|
||||
cause = causeSyscall ? `Connection broken during ${causeSyscall} operation` : 'Connection broken during operation';
|
||||
} else {
|
||||
errorType = 'Backend connection error';
|
||||
cause = `${causeCode || 'Unknown error'} during ${causeSyscall || 'connection'}`;
|
||||
}
|
||||
} else if (errorMsg.includes('ETIMEDOUT') || errorMsg.includes('ECONNABORTED')) {
|
||||
errorType = 'Backend request timeout';
|
||||
cause = 'Connection timed out after 30s';
|
||||
} else if (errorMsg.includes('aborted')) {
|
||||
errorType = 'Request aborted';
|
||||
cause = 'Request was aborted before completion';
|
||||
} else {
|
||||
errorType = 'Failed to forward request to backend';
|
||||
cause = errorMsg;
|
||||
}
|
||||
|
||||
const detailedError = {
|
||||
error: errorType,
|
||||
cause: cause,
|
||||
backend: backend.base_url,
|
||||
path: path,
|
||||
};
|
||||
|
||||
return {
|
||||
status: 502,
|
||||
data: { error: 'Failed to forward request to backend' },
|
||||
data: detailedError,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue