mirrors/misskey
2
0
Fork 1
mirror of https://github.com/misskey-dev/misskey.git synced 2026-06-25 17:10:43 +00:00
Code Issues Projects Releases Packages Wiki Activity

Merge branch 'develop' into fix-font-mfm-inline-block

Browse source
This commit is contained in:
momizi06 2026-06-24 19:05:29 +09:00 • committed by GitHub
commit 3d12163f3a
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
14 changed files with 1386 additions and 101 deletions
Download patch file Download diff file Expand all files Collapse all files

46
.github/scripts/backend-js-footprint-loader.mjs vendored Normal file
View file

@ -0,0 +1,46 @@
/*
* SPDX-FileCopyrightText: syuilo and misskey-project
* SPDX-License-Identifier: AGPL-3.0-only
*/
import { appendFileSync, statSync } from 'node:fs';
import { extname } from 'node:path';
import { fileURLToPath } from 'node:url';
const traceFile = process.env.MK_BACKEND_JS_FOOTPRINT_TRACE;
const jsExtensions = new Set(['.js', '.mjs', '.cjs']);
function recordLoadedFile(kind, url, format) {
if (traceFile == null || !url.startsWith('file:')) return;
let filePath;
try {
filePath = fileURLToPath(url);
} catch {
return;
}
const extension = extname(filePath);
if (!jsExtensions.has(extension)) return;
let size = null;
try {
size = statSync(filePath).size;
} catch {
return;
}
appendFileSync(traceFile, `${JSON.stringify({
kind,
format,
path: filePath,
size,
timestamp: Date.now(),
})}\n`);
}
export async function load(url, context, nextLoad) {
const result = await nextLoad(url, context);
recordLoadedFile('esm', url, result.format ?? context.format ?? null);
return result;
}

46
.github/scripts/backend-js-footprint-require.cjs vendored Normal file
View file

@ -0,0 +1,46 @@
/*
* SPDX-FileCopyrightText: syuilo and misskey-project
* SPDX-License-Identifier: AGPL-3.0-only
*/
'use strict';
const { appendFileSync, statSync } = require('node:fs');
const Module = require('node:module');
const { extname } = require('node:path');
const traceFile = process.env.MK_BACKEND_JS_FOOTPRINT_TRACE;
const jsExtensions = new Set(['.js', '.mjs', '.cjs']);
function recordLoadedFile(kind, filePath, request) {
if (traceFile == null || typeof filePath !== 'string') return;
const extension = extname(filePath);
if (!jsExtensions.has(extension) && extension !== '.node') return;
let size = null;
try {
size = statSync(filePath).size;
} catch {
return;
}
appendFileSync(traceFile, `${JSON.stringify({
kind,
format: extension === '.node' ? 'native' : 'commonjs',
path: filePath,
request,
size,
timestamp: Date.now(),
})}\n`);
}
const originalLoad = Module._load;
const originalResolveFilename = Module._resolveFilename;
Module._load = function load(request, parent, isMain) {
const resolved = originalResolveFilename.call(this, request, parent, isMain);
const result = originalLoad.apply(this, arguments);
recordLoadedFile('cjs', resolved, request);
return result;
};

530
.github/scripts/backend-js-footprint.mjs vendored Normal file
View file

@ -0,0 +1,530 @@
/*
* SPDX-FileCopyrightText: syuilo and misskey-project
* SPDX-License-Identifier: AGPL-3.0-only
*/
import { fork, spawn } from 'node:child_process';
import { createRequire } from 'node:module';
import { cpus, tmpdir } from 'node:os';
import { dirname, extname, join, relative, resolve, sep } from 'node:path';
import { setTimeout } from 'node:timers/promises';
import { fileURLToPath, pathToFileURL } from 'node:url';
import { gzipSync } from 'node:zlib';
import * as fs from 'node:fs/promises';
import * as fsSync from 'node:fs';
import * as http from 'node:http';
const __filename = fileURLToPath(import.meta.url);
const __dirname = dirname(__filename);
const [repoDirArg, outputFileArg] = process.argv.slice(2);
if (repoDirArg == null || outputFileArg == null) {
console.error('Usage: node .github/scripts/backend-js-footprint.mjs <repo-dir> <output.json>');
process.exit(1);
}
const STARTUP_TIMEOUT = readIntegerEnv('MK_JS_FOOTPRINT_STARTUP_TIMEOUT_MS', 120000, 1);
const SETTLE_TIME = readIntegerEnv('MK_JS_FOOTPRINT_SETTLE_TIME_MS', 10000, 0);
const REQUEST_COUNT = readIntegerEnv('MK_JS_FOOTPRINT_REQUEST_COUNT', 10, 0);
const MAX_TABLE_ITEMS = readIntegerEnv('MK_JS_FOOTPRINT_MAX_ITEMS', 20, 1);
const repoDir = resolve(repoDirArg);
const outputFile = resolve(outputFileArg);
const backendDir = join(repoDir, 'packages/backend');
const backendBuiltDir = join(backendDir, 'built');
const traceFile = join(tmpdir(), `misskey-backend-js-footprint-${process.pid}-${Date.now()}.jsonl`);
const require = createRequire(join(repoDir, 'package.json'));
const ts = require('typescript');
const jsExtensions = new Set(['.js', '.mjs', '.cjs']);
const fileMetricCache = new Map();
const packageInfoCache = new Map();
const nativePackageNames = new Set();
function readIntegerEnv(name, defaultValue, min) {
const rawValue = process.env[name];
if (rawValue == null || rawValue === '') return defaultValue;
if (!/^\d+$/.test(rawValue)) throw new Error(`${name} must be an integer`);
const value = Number(rawValue);
if (!Number.isSafeInteger(value) || value < min) throw new Error(`${name} must be >= ${min}`);
return value;
}
function commandName(command) {
if (process.platform !== 'win32') return command;
if (command === 'pnpm') return 'pnpm.cmd';
return command;
}
function isInside(parent, child) {
const rel = relative(parent, child);
return rel === '' || (!rel.startsWith('..') && !rel.includes(`..${sep}`));
}
function normalizePath(filePath) {
return filePath.split(sep).join('/');
}
function bytesToKiB(value) {
return Math.round(value / 1024);
}
function run(command, args, options = {}) {
return new Promise((resolvePromise, reject) => {
const child = spawn(commandName(command), args, {
cwd: options.cwd,
env: options.env,
stdio: ['ignore', 'pipe', 'pipe'],
});
let stdout = '';
let stderr = '';
child.stdout.on('data', data => {
stdout += data;
if (options.logStdout) process.stderr.write(data);
});
child.stderr.on('data', data => {
stderr += data;
process.stderr.write(data);
});
child.on('error', reject);
child.on('close', code => {
if (code === 0) {
resolvePromise(stdout);
} else {
reject(new Error(`${command} ${args.join(' ')} failed with exit code ${code}\n${stderr}`));
}
});
});
}
async function resetState() {
const backendRequire = createRequire(join(backendDir, 'package.json'));
const pg = backendRequire('pg');
const Redis = backendRequire('ioredis');
const postgres = new pg.Client({
host: '127.0.0.1',
port: 54312,
database: 'postgres',
user: 'postgres',
});
await postgres.connect();
try {
await postgres.query('DROP DATABASE IF EXISTS "test-misskey" WITH (FORCE)');
await postgres.query('CREATE DATABASE "test-misskey"');
} finally {
await postgres.end();
}
const redis = new Redis({ host: '127.0.0.1', port: 56312 });
try {
await redis.flushall();
} finally {
redis.disconnect();
}
}
function createRequest() {
return new Promise((resolvePromise, reject) => {
const req = http.request({
host: 'localhost',
port: 61812,
path: '/api/meta',
method: 'POST',
}, res => {
res.on('data', () => { });
res.on('end', () => resolvePromise());
});
req.on('error', reject);
req.end();
});
}
async function waitForServerReady(serverProcess) {
let serverReady = false;
serverProcess.on('message', message => {
if (message === 'ok') serverReady = true;
});
const startupStartTime = Date.now();
while (!serverReady) {
if (Date.now() - startupStartTime > STARTUP_TIMEOUT) {
serverProcess.kill('SIGTERM');
throw new Error('Server startup timeout');
}
await setTimeout(100);
}
}
async function stopServer(serverProcess) {
serverProcess.kill('SIGTERM');
let exited = false;
await new Promise(resolvePromise => {
serverProcess.on('exit', () => {
exited = true;
resolvePromise(undefined);
});
setTimeout(10000).then(() => {
if (!exited) serverProcess.kill('SIGKILL');
resolvePromise(undefined);
});
});
}
function getPackageNameFromPath(filePath) {
const normalized = normalizePath(filePath);
const marker = '/node_modules/';
const index = normalized.lastIndexOf(marker);
if (index === -1) return null;
const rest = normalized.slice(index + marker.length).split('/');
if (rest[0] === '.pnpm') {
const nestedNodeModulesIndex = rest.indexOf('node_modules');
if (nestedNodeModulesIndex === -1) return null;
const packageParts = rest.slice(nestedNodeModulesIndex + 1);
if (packageParts.length === 0) return null;
return packageParts[0].startsWith('@') ? packageParts.slice(0, 2).join('/') : packageParts[0];
}
return rest[0]?.startsWith('@') ? rest.slice(0, 2).join('/') : rest[0] ?? null;
}
function findPackageDir(filePath, packageName) {
const normalizedPackageName = packageName.split('/').join(sep);
let current = dirname(filePath);
while (current !== dirname(current)) {
if (current.endsWith(`${sep}${normalizedPackageName}`) && fsSync.existsSync(join(current, 'package.json'))) {
return current;
}
const parent = dirname(current);
if (parent === current) break;
current = parent;
}
return null;
}
function readPackageInfo(filePath) {
const externalPackageName = getPackageNameFromPath(filePath);
if (externalPackageName != null) {
const packageDir = findPackageDir(filePath, externalPackageName);
const cacheKey = packageDir ?? externalPackageName;
if (packageInfoCache.has(cacheKey)) return packageInfoCache.get(cacheKey);
let version = null;
if (packageDir != null) {
try {
const packageJson = JSON.parse(fsSync.readFileSync(join(packageDir, 'package.json'), 'utf8'));
version = typeof packageJson.version === 'string' ? packageJson.version : null;
} catch { }
}
const info = {
category: 'external',
name: externalPackageName,
version,
dir: packageDir,
};
packageInfoCache.set(cacheKey, info);
return info;
}
if (isInside(backendBuiltDir, filePath)) {
return {
category: 'internal',
name: 'backend',
version: null,
dir: backendDir,
};
}
return {
category: 'internal',
name: 'workspace',
version: null,
dir: repoDir,
};
}
function analyzeSource(filePath, source) {
const sourceFile = ts.createSourceFile(filePath, source, ts.ScriptTarget.Latest, true, ts.ScriptKind.JS);
const metrics = {
astNodeCount: 0,
functionCount: 0,
classCount: 0,
stringLiteralBytes: 0,
};
function visit(node) {
metrics.astNodeCount += 1;
if (
ts.isFunctionDeclaration(node) ||
ts.isFunctionExpression(node) ||
ts.isArrowFunction(node) ||
ts.isMethodDeclaration(node) ||
ts.isConstructorDeclaration(node) ||
ts.isGetAccessorDeclaration(node) ||
ts.isSetAccessorDeclaration(node)
) {
metrics.functionCount += 1;
} else if (ts.isClassDeclaration(node) || ts.isClassExpression(node)) {
metrics.classCount += 1;
} else if (ts.isStringLiteral(node) || ts.isNoSubstitutionTemplateLiteral(node)) {
metrics.stringLiteralBytes += Buffer.byteLength(node.text);
}
ts.forEachChild(node, visit);
}
visit(sourceFile);
return metrics;
}
function readFileMetrics(filePath) {
if (fileMetricCache.has(filePath)) return fileMetricCache.get(filePath);
const source = fsSync.readFileSync(filePath);
const sourceText = source.toString('utf8');
const astMetrics = analyzeSource(filePath, sourceText);
const packageInfo = readPackageInfo(filePath);
const metric = {
path: filePath,
displayPath: normalizePath(relative(repoDir, filePath)),
sourceBytes: source.byteLength,
gzipBytes: gzipSync(source).byteLength,
...astMetrics,
package: packageInfo,
};
fileMetricCache.set(filePath, metric);
return metric;
}
async function readTraceRecords() {
let content = '';
try {
content = await fs.readFile(traceFile, 'utf8');
} catch (err) {
if (err.code === 'ENOENT') return [];
throw err;
}
const records = [];
for (const line of content.split('\n')) {
if (line.trim() === '') continue;
try {
records.push(JSON.parse(line));
} catch { }
}
return records;
}
function emptyTotals() {
return {
loadedJsModules: 0,
loadedJsSourceBytes: 0,
loadedJsGzipBytes: 0,
astNodeCount: 0,
functionCount: 0,
classCount: 0,
stringLiteralBytes: 0,
externalPackageCount: 0,
nativeAddonPackageCount: 0,
};
}
function addFileMetrics(target, metric) {
target.loadedJsModules += 1;
target.loadedJsSourceBytes += metric.sourceBytes;
target.loadedJsGzipBytes += metric.gzipBytes;
target.astNodeCount += metric.astNodeCount;
target.functionCount += metric.functionCount;
target.classCount += metric.classCount;
target.stringLiteralBytes += metric.stringLiteralBytes;
}
function summarizeRecords(records, phase) {
const jsPaths = new Set();
const nativePaths = new Set();
for (const record of records) {
if (typeof record.path !== 'string') continue;
const extension = extname(record.path);
if (jsExtensions.has(extension)) {
jsPaths.add(resolve(record.path));
} else if (extension === '.node') {
nativePaths.add(resolve(record.path));
}
}
for (const nativePath of nativePaths) {
const packageInfo = readPackageInfo(nativePath);
if (packageInfo.category === 'external') nativePackageNames.add(packageInfo.name);
}
const totals = emptyTotals();
const packages = new Map();
const modules = [];
for (const filePath of [...jsPaths].toSorted()) {
let metric;
try {
metric = readFileMetrics(filePath);
} catch (err) {
process.stderr.write(`Failed to analyze ${filePath}: ${err.message}\n`);
continue;
}
addFileMetrics(totals, metric);
const packageKey = metric.package.name;
if (!packages.has(packageKey)) {
packages.set(packageKey, {
name: metric.package.name,
version: metric.package.version,
category: metric.package.category,
sourceBytes: 0,
gzipBytes: 0,
modules: 0,
astNodeCount: 0,
functionCount: 0,
classCount: 0,
stringLiteralBytes: 0,
nativeAddon: false,
});
}
const packageSummary = packages.get(packageKey);
packageSummary.sourceBytes += metric.sourceBytes;
packageSummary.gzipBytes += metric.gzipBytes;
packageSummary.modules += 1;
packageSummary.astNodeCount += metric.astNodeCount;
packageSummary.functionCount += metric.functionCount;
packageSummary.classCount += metric.classCount;
packageSummary.stringLiteralBytes += metric.stringLiteralBytes;
modules.push({
path: metric.displayPath,
package: metric.package.name,
category: metric.package.category,
sourceBytes: metric.sourceBytes,
gzipBytes: metric.gzipBytes,
astNodeCount: metric.astNodeCount,
functionCount: metric.functionCount,
classCount: metric.classCount,
stringLiteralBytes: metric.stringLiteralBytes,
});
}
for (const packageName of nativePackageNames) {
const packageSummary = packages.get(packageName);
if (packageSummary != null) packageSummary.nativeAddon = true;
}
const externalPackages = [...packages.values()].filter(packageSummary => packageSummary.category === 'external');
totals.externalPackageCount = externalPackages.length;
totals.nativeAddonPackageCount = externalPackages.filter(packageSummary => packageSummary.nativeAddon).length;
return {
phase,
totals: {
...totals,
loadedJsSourceKiB: bytesToKiB(totals.loadedJsSourceBytes),
loadedJsGzipKiB: bytesToKiB(totals.loadedJsGzipBytes),
stringLiteralKiB: bytesToKiB(totals.stringLiteralBytes),
},
packages: [...packages.values()].toSorted((a, b) => b.sourceBytes - a.sourceBytes),
modules: modules.toSorted((a, b) => b.sourceBytes - a.sourceBytes).slice(0, MAX_TABLE_ITEMS),
};
}
async function measureFootprint() {
await fs.writeFile(traceFile, '');
process.stderr.write('Resetting database and Redis\n');
await resetState();
process.stderr.write('Running migrations\n');
await run('pnpm', ['--filter', 'backend', 'migrate'], {
cwd: repoDir,
env: process.env,
logStdout: true,
});
const serverProcess = fork(join(backendBuiltDir, 'entry.js'), [], {
cwd: backendDir,
env: {
...process.env,
NODE_ENV: 'production',
MK_DISABLE_CLUSTERING: '1',
MK_ONLY_SERVER: '1',
MK_NO_DAEMONS: '1',
MK_BACKEND_JS_FOOTPRINT_TRACE: traceFile,
},
stdio: ['pipe', 'pipe', 'pipe', 'ipc'],
execArgv: [
'--require',
join(__dirname, 'backend-js-footprint-require.cjs'),
'--experimental-loader',
pathToFileURL(join(__dirname, 'backend-js-footprint-loader.mjs')).href,
],
});
serverProcess.stdout?.on('data', data => {
process.stderr.write(`[server stdout] ${data}`);
});
serverProcess.stderr?.on('data', data => {
process.stderr.write(`[server stderr] ${data}`);
});
serverProcess.on('error', err => {
process.stderr.write(`[server error] ${err}\n`);
});
try {
await waitForServerReady(serverProcess);
await setTimeout(SETTLE_TIME);
const startup = summarizeRecords(await readTraceRecords(), 'startup');
await Promise.all(
Array.from({ length: REQUEST_COUNT }).map(() => createRequest()),
);
await setTimeout(1000);
const afterRequest = summarizeRecords(await readTraceRecords(), 'afterRequest');
return {
timestamp: new Date().toISOString(),
measurement: {
strategy: 'runtime-loader-trace',
startupTimeoutMs: STARTUP_TIMEOUT,
settleTimeMs: SETTLE_TIME,
requestCount: REQUEST_COUNT,
cpus: cpus().length,
},
startup,
afterRequest,
};
} finally {
await stopServer(serverProcess);
await fs.rm(traceFile, { force: true });
}
}
const result = await measureFootprint();
await fs.writeFile(outputFile, `${JSON.stringify(result, null, 2)}\n`);

475
.github/scripts/backend-memory-report.mjs vendored
View file

@ -1,9 +1,9 @@
import { readFile, writeFile } from 'node:fs/promises';
const [baseFile, headFile, outputFile] = process.argv.slice(2);
const [baseFile, headFile, outputFile, baseJsFootprintFile, headJsFootprintFile] = process.argv.slice(2);
if (baseFile == null || headFile == null || outputFile == null) {
console.error('Usage: node .github/scripts/backend-memory-report.mjs <base-memory.json> <head-memory.json> <report.md>');
console.error('Usage: node .github/scripts/backend-memory-report.mjs <base-memory.json> <head-memory.json> <report.md> [base-js-footprint.json head-js-footprint.json]');
process.exit(1);
}
@ -26,6 +26,17 @@ const metrics = [
'External',
];
const heapSnapshotCategories = [
'Code',
'Strings',
'JS arrays',
'Typed arrays',
'System objects',
'Other JS objects',
'Other non-JS objects',
'Total',
];
function formatNumber(value) {
return numberFormatter.format(value);
}
@ -34,6 +45,13 @@ function formatMemory(valueKiB) {
return `${formatNumber(valueKiB / 1024)} MB`;
}
function formatBytes(value) {
if (!Number.isFinite(value)) return '-';
if (value < 1024) return `${formatNumber(value)} B`;
if (value < 1024 * 1024) return `${formatNumber(value / 1024)} KiB`;
return `${formatNumber(value / 1024 / 1024)} MiB`;
}
function formatPercent(value) {
return `${formatNumber(value)}%`;
}
@ -96,6 +114,64 @@ function getSampleSpread(report, phase, metric) {
return median(values.map(value => Math.abs(value - center)));
}
function mad(values) {
if (values.length < 2) return null;
const center = median(values);
return median(values.map(value => Math.abs(value - center)));
}
function getSamplesByRound(report) {
const samplesByRound = new Map();
if (!Array.isArray(report?.samples)) return samplesByRound;
for (const sample of report.samples) {
if (!Number.isInteger(sample?.round) || sample.round <= 0) continue;
samplesByRound.set(sample.round, sample);
}
return samplesByRound;
}
function getPairedDeltaValues(base, head, phase, metric) {
const baseSamplesByRound = getSamplesByRound(base);
const headSamplesByRound = getSamplesByRound(head);
const values = [];
for (const [round, baseSample] of baseSamplesByRound) {
const headSample = headSamplesByRound.get(round);
if (headSample == null) continue;
const baseValue = getMemoryValue(baseSample, phase, metric);
const headValue = getMemoryValue(headSample, phase, metric);
if (baseValue == null || headValue == null) continue;
values.push(headValue - baseValue);
}
return values;
}
function formatDeltaMemory(diffKiB) {
if (diffKiB === 0) return formatMemory(0);
const sign = diffKiB > 0 ? '+' : '-';
return formatColoredDiff(`${sign}${formatMemory(Math.abs(diffKiB))}`, diffKiB);
}
function pairedDeltaSummary(base, head, phase, metric) {
const values = getPairedDeltaValues(base, head, phase, metric);
if (values.length === 0) return null;
return {
median: median(values),
mad: mad(values),
min: Math.min(...values),
max: Math.max(...values),
samples: values.length,
};
}
function renderTable(base, head, phase) {
const lines = [
'| Metric | Base | Head | Δ | Δ (%) |',
@ -116,6 +192,23 @@ function renderTable(base, head, phase) {
return lines.join('\n');
}
function renderPairedDeltaTable(base, head, phase) {
const lines = [
'| Metric | Δ median | Δ MAD | Δ min | Δ max |',
'| --- | ---: | ---: | ---: | ---: |',
];
for (const metric of metrics) {
const summary = pairedDeltaSummary(base, head, phase, metric);
if (summary == null) continue;
lines.push(`| ${metric} | ${formatDeltaMemory(summary.median)} | ${summary.mad == null ? '-' : formatMemory(summary.mad)} | ${formatDeltaMemory(summary.min)} | ${formatDeltaMemory(summary.max)} |`);
}
if (lines.length === 2) return null;
return lines.join('\n');
}
function getDiffPercent(base, head, phase, metric) {
const baseValue = getMemoryValue(base, phase, metric);
const headValue = getMemoryValue(head, phase, metric);
@ -174,23 +267,391 @@ function measurementSummary(base, head) {
return `_Sample count: base ${baseCount}, head ${headCount}. Values are medians; ± is median absolute deviation._`;
}
function formatPlainDiff(baseValue, headValue, formatter = formatNumber) {
const diff = headValue - baseValue;
if (diff === 0) return formatter(0);
const sign = diff > 0 ? '+' : '-';
return `${sign}${formatter(Math.abs(diff))}`;
}
function formatPlainDiffPercent(baseValue, headValue) {
const diff = headValue - baseValue;
if (diff === 0) return '0%';
if (baseValue <= 0) return '-';
const sign = diff > 0 ? '+' : '-';
return `${sign}${formatPercent(Math.abs((diff * 100) / baseValue))}`;
}
function getHeapSnapshotCategoryValue(report, phase, category) {
const value = report?.[phase]?.heapSnapshot?.categories?.[category];
return Number.isFinite(value) ? value : null;
}
function getHeapSnapshotSampleValues(report, phase, category) {
if (!Array.isArray(report?.samples)) return [];
return report.samples
.map(sample => getHeapSnapshotCategoryValue(sample, phase, category))
.filter(value => Number.isFinite(value));
}
function getHeapSnapshotSampleSpread(report, phase, category) {
const values = getHeapSnapshotSampleValues(report, phase, category);
if (values.length < 2) return null;
const center = median(values);
return median(values.map(value => Math.abs(value - center)));
}
function formatDiffBytes(baseBytes, headBytes) {
const diff = headBytes - baseBytes;
if (diff === 0) return formatBytes(0);
const sign = diff > 0 ? '+' : '-';
return formatColoredDiff(`${sign}${formatBytes(Math.abs(diff))}`, diff);
}
function formatDiffBytesPercent(baseBytes, headBytes) {
const diff = headBytes - baseBytes;
if (diff === 0) return '0%';
if (baseBytes <= 0) return '-';
const sign = diff > 0 ? '+' : '-';
return formatColoredDiff(`${sign}${formatPercent(Math.abs((diff * 100) / baseBytes))}`, diff);
}
function getPairedHeapSnapshotDeltaValues(base, head, phase, category) {
const baseSamplesByRound = getSamplesByRound(base);
const headSamplesByRound = getSamplesByRound(head);
const values = [];
for (const [round, baseSample] of baseSamplesByRound) {
const headSample = headSamplesByRound.get(round);
if (headSample == null) continue;
const baseValue = getHeapSnapshotCategoryValue(baseSample, phase, category);
const headValue = getHeapSnapshotCategoryValue(headSample, phase, category);
if (baseValue == null || headValue == null) continue;
values.push(headValue - baseValue);
}
return values;
}
function formatDeltaBytes(diffBytes) {
if (diffBytes === 0) return formatBytes(0);
const sign = diffBytes > 0 ? '+' : '-';
return formatColoredDiff(`${sign}${formatBytes(Math.abs(diffBytes))}`, diffBytes);
}
function pairedHeapSnapshotDeltaSummary(base, head, phase, category) {
const values = getPairedHeapSnapshotDeltaValues(base, head, phase, category);
if (values.length === 0) return null;
return {
median: median(values),
mad: mad(values),
min: Math.min(...values),
max: Math.max(...values),
samples: values.length,
};
}
function renderHeapSnapshotTable(base, head, phase) {
const lines = [
'| Category | Base | Head | Δ | Δ (%) |',
'| --- | ---: | ---: | ---: | ---: |',
];
for (const category of heapSnapshotCategories) {
const baseValue = getHeapSnapshotCategoryValue(base, phase, category);
const headValue = getHeapSnapshotCategoryValue(head, phase, category);
if (baseValue == null || headValue == null) continue;
const baseSpread = getHeapSnapshotSampleSpread(base, phase, category);
const headSpread = getHeapSnapshotSampleSpread(head, phase, category);
lines.push(`| ${category} | ${formatBytes(baseValue)} <br> ± ${baseSpread == null ? '-' : formatBytes(baseSpread)} | ${formatBytes(headValue)} <br> ± ${headSpread == null ? '-' : formatBytes(headSpread)} | ${formatDiffBytes(baseValue, headValue)} | ${formatDiffBytesPercent(baseValue, headValue)} |`);
}
if (lines.length === 2) return null;
return lines.join('\n');
}
function renderHeapSnapshotPairedDeltaTable(base, head, phase) {
const lines = [
'| Category | Δ median | Δ MAD | Δ min | Δ max |',
'| --- | ---: | ---: | ---: | ---: |',
];
for (const category of heapSnapshotCategories) {
const summary = pairedHeapSnapshotDeltaSummary(base, head, phase, category);
if (summary == null) continue;
lines.push(`| ${category} | ${formatDeltaBytes(summary.median)} | ${summary.mad == null ? '-' : formatBytes(summary.mad)} | ${formatDeltaBytes(summary.min)} | ${formatDeltaBytes(summary.max)} |`);
}
if (lines.length === 2) return null;
return lines.join('\n');
}
function renderHeapSnapshotSection(base, head) {
const table = renderHeapSnapshotTable(base, head, 'afterRequest');
if (table == null) return null;
const lines = [
'### V8 Heap Snapshot Statistics',
'',
table,
'',
];
const pairedDeltaTable = renderHeapSnapshotPairedDeltaTable(base, head, 'afterRequest');
if (pairedDeltaTable != null) {
lines.push('#### Paired Delta Summary');
lines.push('');
lines.push(pairedDeltaTable);
lines.push('');
}
return lines.join('\n');
}
function getJsFootprintValue(report, phase, key) {
const value = report?.[phase]?.totals?.[key];
return Number.isFinite(value) ? value : null;
}
function renderJsFootprintMetricTable(base, head) {
const metricRows = [
['Loaded JS modules', 'loadedJsModules', formatNumber],
['Loaded JS source', 'loadedJsSourceBytes', formatBytes],
//['Loaded JS gzip estimate', 'loadedJsGzipBytes', formatBytes],
//['AST nodes', 'astNodeCount', formatNumber],
//['Functions', 'functionCount', formatNumber],
//['Classes', 'classCount', formatNumber],
//['String literals', 'stringLiteralBytes', formatBytes],
['External packages loaded', 'externalPackageCount', formatNumber],
['Native addon packages', 'nativeAddonPackageCount', formatNumber],
];
const lines = [
'| Metric | Base | Head | Δ | Δ (%) |',
'| --- | ---: | ---: | ---: | ---: |',
];
for (const [title, key, formatter] of metricRows) {
const baseValue = getJsFootprintValue(base, 'afterRequest', key);
const headValue = getJsFootprintValue(head, 'afterRequest', key);
if (baseValue == null || headValue == null) continue;
lines.push(`| ${title} | ${formatter(baseValue)} | ${formatter(headValue)} | ${formatPlainDiff(baseValue, headValue, formatter)} | ${formatPlainDiffPercent(baseValue, headValue)} |`);
}
return lines.join('\n');
}
function renderJsFootprintPhaseTable(base, head) {
const lines = [
'| Phase | Base modules | Head modules | Δ modules | Base source | Head source | Δ source |',
'| --- | ---: | ---: | ---: | ---: | ---: | ---: |',
];
for (const [phase, title] of [['startup', 'Startup'], ['afterRequest', 'After warmup requests']]) {
const baseModules = getJsFootprintValue(base, phase, 'loadedJsModules');
const headModules = getJsFootprintValue(head, phase, 'loadedJsModules');
const baseSource = getJsFootprintValue(base, phase, 'loadedJsSourceBytes');
const headSource = getJsFootprintValue(head, phase, 'loadedJsSourceBytes');
if (baseModules == null || headModules == null || baseSource == null || headSource == null) continue;
lines.push(`| ${title} | ${formatNumber(baseModules)} | ${formatNumber(headModules)} | ${formatPlainDiff(baseModules, headModules)} | ${formatBytes(baseSource)} | ${formatBytes(headSource)} | ${formatPlainDiff(baseSource, headSource, formatBytes)} |`);
}
return lines.join('\n');
}
function packageMap(report) {
const map = new Map();
for (const packageSummary of report?.afterRequest?.packages ?? []) {
if (packageSummary?.category !== 'external' || typeof packageSummary.name !== 'string') continue;
map.set(packageSummary.name, packageSummary);
}
return map;
}
function packageDisplayName(packageSummary) {
if (packageSummary.version == null) return packageSummary.name;
return `${packageSummary.name} ${packageSummary.version}`;
}
function renderNewExternalPackages(base, head) {
const basePackages = packageMap(base);
const headPackages = packageMap(head);
const newPackages = [...headPackages.values()]
.filter(packageSummary => !basePackages.has(packageSummary.name))
.toSorted((a, b) => b.sourceBytes - a.sourceBytes)
.slice(0, 10);
if (newPackages.length === 0) return null;
const lines = [
'#### Newly Loaded External Packages',
'',
'| Package | Loaded JS | Modules | Notes |',
'| --- | ---: | ---: | --- |',
];
for (const packageSummary of newPackages) {
lines.push(`| ${packageDisplayName(packageSummary)} | ${formatBytes(packageSummary.sourceBytes)} | ${formatNumber(packageSummary.modules)} | ${packageSummary.nativeAddon ? 'native addon' : ''} |`);
}
return lines.join('\n');
}
function renderLargestPackageIncreases(base, head) {
const basePackages = packageMap(base);
const headPackages = packageMap(head);
const increases = [...headPackages.values()]
.map(headPackage => {
const basePackage = basePackages.get(headPackage.name);
const baseSourceBytes = basePackage?.sourceBytes ?? 0;
const baseModules = basePackage?.modules ?? 0;
return {
...headPackage,
baseSourceBytes,
baseModules,
sourceDiff: headPackage.sourceBytes - baseSourceBytes,
moduleDiff: headPackage.modules - baseModules,
};
})
.filter(packageSummary => packageSummary.sourceDiff > 0)
.toSorted((a, b) => b.sourceDiff - a.sourceDiff)
.slice(0, 10);
if (increases.length === 0) return null;
const lines = [
'#### Largest Package Increases',
'',
'| Package | Base | Head | Δ | Modules Δ |',
'| --- | ---: | ---: | ---: | ---: |',
];
for (const packageSummary of increases) {
lines.push(`| ${packageDisplayName(packageSummary)} | ${formatBytes(packageSummary.baseSourceBytes)} | ${formatBytes(packageSummary.sourceBytes)} | ${formatPlainDiff(packageSummary.baseSourceBytes, packageSummary.sourceBytes, formatBytes)} | ${formatPlainDiff(packageSummary.baseModules, packageSummary.modules)} |`);
}
return lines.join('\n');
}
function moduleMap(report) {
const map = new Map();
for (const moduleSummary of report?.afterRequest?.modules ?? []) {
if (typeof moduleSummary.path !== 'string') continue;
map.set(moduleSummary.path, moduleSummary);
}
return map;
}
function renderNewLoadedModules(base, head) {
const baseModules = moduleMap(base);
const headModules = moduleMap(head);
const newModules = [...headModules.values()]
.filter(moduleSummary => !baseModules.has(moduleSummary.path))
.toSorted((a, b) => b.sourceBytes - a.sourceBytes)
.slice(0, 10);
if (newModules.length === 0) return null;
const lines = [
'#### Largest Newly Loaded Modules',
'',
'| Module | Package | Loaded JS |',
'| --- | --- | ---: |',
];
for (const moduleSummary of newModules) {
lines.push(`| \`${moduleSummary.path}\` | ${moduleSummary.package} | ${formatBytes(moduleSummary.sourceBytes)} |`);
}
return lines.join('\n');
}
function renderJsFootprintSection(base, head) {
if (base == null || head == null) return null;
const lines = [
'### Runtime Loaded JS Footprint',
'',
'<details><summary>Click to show</summary>',
'',
renderJsFootprintMetricTable(base, head),
'',
//'#### Load Phase Breakdown',
//'',
//renderJsFootprintPhaseTable(base, head),
//'',
];
for (const block of [
renderNewExternalPackages(base, head),
renderLargestPackageIncreases(base, head),
renderNewLoadedModules(base, head),
]) {
if (block == null) continue;
lines.push(block);
lines.push('');
}
lines.push('</details>');
lines.push('');
return lines.join('\n');
}
const base = JSON.parse(await readFile(baseFile, 'utf8'));
const head = JSON.parse(await readFile(headFile, 'utf8'));
const baseJsFootprint = baseJsFootprintFile == null ? null : JSON.parse(await readFile(baseJsFootprintFile, 'utf8'));
const headJsFootprint = headJsFootprintFile == null ? null : JSON.parse(await readFile(headJsFootprintFile, 'utf8'));
const lines = [
'## Backend Memory Usage Report',
'',
];
const summary = measurementSummary(base, head);
if (summary != null) {
lines.push(summary);
lines.push('');
}
//const summary = measurementSummary(base, head);
//if (summary != null) {
// lines.push(summary);
// lines.push('');
//}
for (const phase of phases) {
lines.push(`### ${phase.title}`);
lines.push(renderTable(base, head, phase.key));
lines.push('');
const pairedDeltaTable = renderPairedDeltaTable(base, head, phase.key);
if (pairedDeltaTable != null) {
lines.push('#### Paired Delta Summary');
lines.push('');
lines.push(pairedDeltaTable);
lines.push('');
}
}
const heapSnapshotSection = renderHeapSnapshotSection(base, head);
if (heapSnapshotSection != null) {
lines.push(heapSnapshotSection);
lines.push('');
}
const jsFootprintSection = renderJsFootprintSection(baseJsFootprint, headJsFootprint);
if (jsFootprintSection != null) {
lines.push(jsFootprintSection);
lines.push('');
}
const warningMetric = getWarningMetric(base, head);

5
.github/scripts/frontend-js-size.mjs vendored
View file

@ -91,7 +91,7 @@ function formatDiffPercent(before, after) {
if (before == null || before === 0 || after == null || after === 0) return '-';
const diff = after - before;
if (diff === 0) return `0%`;
const percent = Math.round(diff / before * 100);
const percent = Math.abs(Math.round(diff / before * 100));
return formatColoredDiff(`${percent}%`, diff);
}
@ -536,6 +536,7 @@ const before = await collectReport(beforeDir);
const after = await collectReport(afterDir);
const beforeStats = JSON.parse(await fs.readFile(beforeStatsFile, 'utf8'));
const afterStats = JSON.parse(await fs.readFile(afterStatsFile, 'utf8'));
const visualizerArtifactLink = `[Download detailed HTML](${process.env.FRONTEND_BUNDLE_REPORT_ARTIFACT_URL})`;
const body = [
marker,
@ -547,6 +548,8 @@ const body = [
'## Bundle Stats',
'',
renderFrontendBundleReport(collectVisualizerReport(beforeStats), collectVisualizerReport(afterStats)),
'',
visualizerArtifactLink,
].join('\n');
await fs.writeFile(outFile, body);

46
.github/scripts/measure-backend-memory-comparison.mjs vendored
View file

@ -9,6 +9,16 @@ import { writeFile } from 'node:fs/promises';
import { join, resolve } from 'node:path';
const phases = ['beforeGc', 'afterGc', 'afterRequest'];
const heapSnapshotCategories = [
'Code',
'Strings',
'JS arrays',
'Typed arrays',
'System objects',
'Other JS objects',
'Other non-JS objects',
'Total',
];
const [baseDirArg, headDirArg, baseOutputArg, headOutputArg] = process.argv.slice(2);
@ -121,6 +131,31 @@ function summarizeSamples(samples) {
if (values.length > 0) summary[phase][key] = median(values);
}
const heapSnapshotCategoryValues = {};
for (const category of heapSnapshotCategories) {
const values = samples
.map(sample => sample[phase]?.heapSnapshot?.categories?.[category])
.filter(value => Number.isFinite(value));
if (values.length > 0) heapSnapshotCategoryValues[category] = median(values);
}
const heapSnapshotNodeCountValues = {};
for (const category of heapSnapshotCategories) {
const values = samples
.map(sample => sample[phase]?.heapSnapshot?.nodeCounts?.[category])
.filter(value => Number.isFinite(value));
if (values.length > 0) heapSnapshotNodeCountValues[category] = median(values);
}
if (Object.keys(heapSnapshotCategoryValues).length > 0) {
summary[phase].heapSnapshot = {
categories: heapSnapshotCategoryValues,
nodeCounts: heapSnapshotNodeCountValues,
};
}
}
return summary;
@ -138,12 +173,15 @@ async function measureRepo(label, repoDir, round, orderIndex) {
});
process.stderr.write(`[${label}] Measuring memory\n`);
const measureEnv = {
...process.env,
MK_MEMORY_SAMPLE_COUNT: '1',
};
if (round <= 0) measureEnv.MK_MEMORY_HEAP_SNAPSHOT = '0';
const stdout = await run('node', ['packages/backend/scripts/measure-memory.mjs'], {
cwd: repoDir,
env: {
...process.env,
MK_MEMORY_SAMPLE_COUNT: '1',
},
env: measureEnv,
});
const report = JSON.parse(stdout);

14
.github/workflows/frontend-bundle-report.yml vendored
View file

@ -101,7 +101,6 @@ jobs:
working-directory: before
env:
FRONTEND_BUNDLE_VISUALIZER: 'true'
FRONTEND_BUNDLE_VISUALIZER_TEMPLATE: raw-data
FRONTEND_BUNDLE_VISUALIZER_FILE: ${{ runner.temp }}/frontend-bundle-report/before-stats.json
run: pnpm --filter frontend run build
@ -120,10 +119,20 @@ jobs:
working-directory: after
env:
FRONTEND_BUNDLE_VISUALIZER: 'true'
FRONTEND_BUNDLE_VISUALIZER_TEMPLATE: raw-data
FRONTEND_BUNDLE_VISUALIZER_FILE: ${{ runner.temp }}/frontend-bundle-report/after-stats.json
FRONTEND_BUNDLE_VISUALIZER_HTML_FILE: ${{ runner.temp }}/frontend-bundle-report/frontend-bundle-visualizer.html
run: pnpm --filter frontend run build
- name: Upload bundle visualizer
if: steps.check-base-visualizer.outputs.supported == 'true'
id: upload-bundle-visualizer
uses: actions/upload-artifact@v7
with:
name: frontend-bundle-visualizer
path: ${{ runner.temp }}/frontend-bundle-report/frontend-bundle-visualizer.html
if-no-files-found: error
retention-days: 7
- name: Generate report markdown
if: steps.check-base-visualizer.outputs.supported == 'true'
shell: bash
@ -131,6 +140,7 @@ jobs:
BASE_SHA: ${{ github.event.pull_request.base.sha }}
HEAD_SHA: ${{ github.event.pull_request.head.sha }}
PR_NUMBER: ${{ github.event.pull_request.number }}
FRONTEND_BUNDLE_REPORT_ARTIFACT_URL: ${{ steps.upload-bundle-visualizer.outputs.artifact-url }}
run: |
REPORT_DIR="$RUNNER_TEMP/frontend-bundle-report"
node after/.github/scripts/frontend-js-size.mjs before after "$REPORT_DIR/before-stats.json" "$REPORT_DIR/after-stats.json" "$REPORT_DIR/frontend-js-size-report.md"

11
.github/workflows/get-backend-memory.yml vendored
View file

@ -10,6 +10,10 @@ on:
- packages/backend/**
- packages/misskey-js/**
- .github/scripts/backend-memory-report.mjs
- .github/scripts/measure-backend-memory-comparison.mjs
- .github/scripts/backend-js-footprint.mjs
- .github/scripts/backend-js-footprint-loader.mjs
- .github/scripts/backend-js-footprint-require.cjs
- .github/workflows/get-backend-memory.yml
- .github/workflows/report-backend-memory.yml
@ -89,7 +93,12 @@ jobs:
env:
MK_MEMORY_COMPARE_ROUNDS: 5
MK_MEMORY_COMPARE_WARMUP_ROUNDS: 1
MK_MEMORY_HEAP_SNAPSHOT: 1
run: node head/.github/scripts/measure-backend-memory-comparison.mjs base head memory-base.json memory-head.json
- name: Measure backend loaded JS footprint
run: |
node head/.github/scripts/backend-js-footprint.mjs base js-footprint-base.json
node head/.github/scripts/backend-js-footprint.mjs head js-footprint-head.json
- name: Upload Artifact
uses: actions/upload-artifact@v7
with:
@ -97,6 +106,8 @@ jobs:
path: |
memory-base.json
memory-head.json
js-footprint-base.json
js-footprint-head.json
save-pr-number:
runs-on: ubuntu-latest

6
.github/workflows/report-backend-memory.yml vendored
View file

@ -53,9 +53,13 @@ jobs:
run: cat ./artifacts/memory-base.json
- name: Output head
run: cat ./artifacts/memory-head.json
- name: Output base JS footprint
run: cat ./artifacts/js-footprint-base.json
- name: Output head JS footprint
run: cat ./artifacts/js-footprint-head.json
- id: build-comment
name: Build memory comment
run: node .github/scripts/backend-memory-report.mjs ./artifacts/memory-base.json ./artifacts/memory-head.json ./output.md
run: node .github/scripts/backend-memory-report.mjs ./artifacts/memory-base.json ./artifacts/memory-head.json ./output.md ./artifacts/js-footprint-base.json ./artifacts/js-footprint-head.json
- uses: thollander/actions-comment-pull-request@v3
with:
pr-number: ${{ steps.load-pr-num.outputs.pr-number }}

35
.github/workflows/test-backend.yml vendored
View file

@ -19,12 +19,6 @@ on:
- .github/workflows/test-backend.yml
- .github/misskey/test.yml
workflow_dispatch:
inputs:
force_ffmpeg_cache_update:
description: 'Force update ffmpeg cache'
required: false
default: false
type: boolean
jobs:
unit:
@ -62,36 +56,9 @@ jobs:
submodules: true
- name: Setup pnpm
uses: pnpm/action-setup@v6.0.3
- name: Get current date
id: current-date
run: echo "today=$(date +'%Y-%m-%d')" >> $GITHUB_OUTPUT
- name: Setup and Restore ffmpeg/ffprobe Cache
id: cache-ffmpeg
uses: actions/cache@v5
with:
path: |
/usr/local/bin/ffmpeg
/usr/local/bin/ffprobe
# daily cache
key: ${{ runner.os }}-ffmpeg-${{ steps.current-date.outputs.today }}
restore-keys: |
${{ runner.os }}-ffmpeg-${{ steps.current-date.outputs.today }}
- name: Install FFmpeg
if: steps.cache-ffmpeg.outputs.cache-hit != 'true' || github.event.inputs.force_ffmpeg_cache_update == true
run: |
for i in {1..3}; do
echo "Attempt $i: Installing FFmpeg..."
curl -s -L https://johnvansickle.com/ffmpeg/releases/ffmpeg-release-amd64-static.tar.xz -o ffmpeg.tar.xz && \
tar -xf ffmpeg.tar.xz && \
mv ffmpeg-*-static/ffmpeg /usr/local/bin/ && \
mv ffmpeg-*-static/ffprobe /usr/local/bin/ && \
rm -rf ffmpeg.tar.xz ffmpeg-*-static/ && \
break || sleep 10
if [ $i -eq 3 ]; then
echo "Failed to install FFmpeg after 3 attempts"
exit 1
fi
done
sudo apt install -y ffmpeg
- name: Use Node.js
uses: actions/setup-node@v6.4.0
with:

35
.github/workflows/test-federation.yml vendored
View file

@ -15,12 +15,6 @@ on:
- packages/misskey-js/**
- .github/workflows/test-federation.yml
workflow_dispatch:
inputs:
force_ffmpeg_cache_update:
description: 'Force update ffmpeg cache'
required: false
default: false
type: boolean
jobs:
test:
@ -37,36 +31,9 @@ jobs:
submodules: true
- name: Setup pnpm
uses: pnpm/action-setup@v6.0.3
- name: Get current date
id: current-date
run: echo "today=$(date +'%Y-%m-%d')" >> $GITHUB_OUTPUT
- name: Setup and Restore ffmpeg/ffprobe Cache
id: cache-ffmpeg
uses: actions/cache@v5
with:
path: |
/usr/local/bin/ffmpeg
/usr/local/bin/ffprobe
# daily cache
key: ${{ runner.os }}-ffmpeg-${{ steps.current-date.outputs.today }}
restore-keys: |
${{ runner.os }}-ffmpeg-${{ steps.current-date.outputs.today }}
- name: Install FFmpeg
if: steps.cache-ffmpeg.outputs.cache-hit != 'true' || github.event.inputs.force_ffmpeg_cache_update == true
run: |
for i in {1..3}; do
echo "Attempt $i: Installing FFmpeg..."
curl -s -L https://johnvansickle.com/ffmpeg/releases/ffmpeg-release-amd64-static.tar.xz -o ffmpeg.tar.xz && \
tar -xf ffmpeg.tar.xz && \
mv ffmpeg-*-static/ffmpeg /usr/local/bin/ && \
mv ffmpeg-*-static/ffprobe /usr/local/bin/ && \
rm -rf ffmpeg.tar.xz ffmpeg-*-static/ && \
break || sleep 10
if [ $i -eq 3 ]; then
echo "Failed to install FFmpeg after 3 attempts"
exit 1
fi
done
sudo apt install -y ffmpeg
- name: Use Node.js
uses: actions/setup-node@v6.4.0
with:

184
packages/backend/scripts/measure-memory.mjs
View file

@ -14,6 +14,7 @@ import { fork } from 'node:child_process';
import { setTimeout } from 'node:timers/promises';
import { fileURLToPath } from 'node:url';
import { dirname, join } from 'node:path';
import { tmpdir } from 'node:os';
import * as http from 'node:http';
import * as fs from 'node:fs/promises';
@ -30,11 +31,21 @@ function readIntegerEnv(name, defaultValue, min) {
return value;
}
function readBooleanEnv(name, defaultValue) {
const rawValue = process.env[name];
if (rawValue == null || rawValue === '') return defaultValue;
if (rawValue === '1' || rawValue === 'true') return true;
if (rawValue === '0' || rawValue === 'false') return false;
throw new Error(`${name} must be one of: 1, 0, true, false`);
}
const SAMPLE_COUNT = readIntegerEnv('MK_MEMORY_SAMPLE_COUNT', 3, 1); // Number of samples to measure
const STARTUP_TIMEOUT = readIntegerEnv('MK_MEMORY_STARTUP_TIMEOUT_MS', 120000, 1); // Timeout for server startup
const MEMORY_SETTLE_TIME = readIntegerEnv('MK_MEMORY_SETTLE_TIME_MS', 10000, 0); // Wait after startup for memory to settle
const IPC_TIMEOUT = readIntegerEnv('MK_MEMORY_IPC_TIMEOUT_MS', 30000, 1); // Timeout for IPC responses
const REQUEST_COUNT = readIntegerEnv('MK_MEMORY_REQUEST_COUNT', 10, 0);
const HEAP_SNAPSHOT = readBooleanEnv('MK_MEMORY_HEAP_SNAPSHOT', false);
const HEAP_SNAPSHOT_TIMEOUT = readIntegerEnv('MK_MEMORY_HEAP_SNAPSHOT_TIMEOUT_MS', 120000, 1);
const procStatusKeys = {
VmPeak: 0,
@ -74,6 +85,45 @@ const memoryKeys = {
const phases = ['beforeGc', 'afterGc', 'afterRequest'];
const heapSnapshotCategories = [
'Code',
'Strings',
'JS arrays',
'Typed arrays',
'System objects',
'Other JS objects',
'Other non-JS objects',
'Total',
];
const typedArrayNames = new Set([
'ArrayBuffer',
'SharedArrayBuffer',
'DataView',
'Int8Array',
'Uint8Array',
'Uint8ClampedArray',
'Int16Array',
'Uint16Array',
'Int32Array',
'Uint32Array',
'Float16Array',
'Float32Array',
'Float64Array',
'BigInt64Array',
'BigUint64Array',
'system / JSArrayBufferData',
]);
const otherJsNodeTypes = new Set([
'object',
'closure',
'regexp',
'number',
'symbol',
'bigint',
]);
function parseMemoryFile(content, keys, path, required) {
const result = {};
for (const key of Object.keys(keys)) {
@ -91,6 +141,76 @@ function bytesToKiB(value) {
return Math.round(value / 1024);
}
function createEmptyHeapSnapshotCategoryMap() {
return Object.fromEntries(heapSnapshotCategories.map(category => [category, 0]));
}
function isTypedArrayNode(type, name) {
return typedArrayNames.has(name) ||
(type === 'native' && (name.includes('ArrayBuffer') || name.includes('TypedArray')));
}
function isSystemNode(type, name) {
return type === 'hidden' ||
type === 'synthetic' ||
type === 'object shape' ||
name.startsWith('system /') ||
name.startsWith('(system ');
}
function classifyHeapSnapshotNode(type, name) {
if (type === 'code') return 'Code';
if (type === 'string' || type === 'concatenated string' || type === 'sliced string') return 'Strings';
if (isTypedArrayNode(type, name)) return 'Typed arrays';
if (type === 'array' || (type === 'object' && name === 'Array')) return 'JS arrays';
if (isSystemNode(type, name)) return 'System objects';
if (otherJsNodeTypes.has(type)) return 'Other JS objects';
return 'Other non-JS objects';
}
function analyzeHeapSnapshot(snapshot) {
const meta = snapshot?.snapshot?.meta;
const nodes = snapshot?.nodes;
const strings = snapshot?.strings;
if (meta == null || !Array.isArray(nodes) || !Array.isArray(strings)) {
throw new Error('Invalid heap snapshot format');
}
const nodeFields = meta.node_fields;
if (!Array.isArray(nodeFields)) throw new Error('Invalid heap snapshot node fields');
const typeOffset = nodeFields.indexOf('type');
const nameOffset = nodeFields.indexOf('name');
const selfSizeOffset = nodeFields.indexOf('self_size');
if (typeOffset < 0 || nameOffset < 0 || selfSizeOffset < 0) {
throw new Error('Heap snapshot is missing required node fields');
}
const nodeTypeNames = meta.node_types?.[typeOffset];
if (!Array.isArray(nodeTypeNames)) throw new Error('Invalid heap snapshot node types');
const fieldCount = nodeFields.length;
const categories = createEmptyHeapSnapshotCategoryMap();
const nodeCounts = createEmptyHeapSnapshotCategoryMap();
for (let offset = 0; offset < nodes.length; offset += fieldCount) {
const type = nodeTypeNames[nodes[offset + typeOffset]] ?? 'unknown';
const name = strings[nodes[offset + nameOffset]] ?? '';
const selfSize = nodes[offset + selfSizeOffset] ?? 0;
const category = classifyHeapSnapshotNode(type, name);
categories[category] += selfSize;
categories.Total += selfSize;
nodeCounts[category]++;
nodeCounts.Total++;
}
return {
categories,
nodeCounts,
};
}
async function getMemoryUsage(pid) {
const path = `/proc/${pid}/status`;
const status = await fs.readFile(path, 'utf-8');
@ -150,6 +270,39 @@ async function getRuntimeMemoryUsage(serverProcess) {
};
}
async function getHeapSnapshotStatistics(serverProcess) {
if (!HEAP_SNAPSHOT) return null;
const snapshotPath = join(tmpdir(), `misskey-backend-heap-${process.pid}-${serverProcess.pid}-${Date.now()}.heapsnapshot`);
const response = waitForMessage(
serverProcess,
message => message != null && typeof message === 'object' && (message.type === 'heap snapshot' || message.type === 'heap snapshot error'),
'heap snapshot',
HEAP_SNAPSHOT_TIMEOUT,
);
serverProcess.send({
type: 'heap snapshot',
path: snapshotPath,
});
const message = await response;
if (message.type === 'heap snapshot error') {
throw new Error(`Failed to write heap snapshot: ${message.message}`);
}
const writtenPath = typeof message.path === 'string' ? message.path : snapshotPath;
try {
const snapshot = JSON.parse(await fs.readFile(writtenPath, 'utf-8'));
return analyzeHeapSnapshot(snapshot);
} finally {
await fs.unlink(writtenPath).catch(err => {
process.stderr.write(`Failed to delete heap snapshot ${writtenPath}: ${err.message}\n`);
});
}
}
async function getAllMemoryUsage(serverProcess) {
const pid = serverProcess.pid;
return {
@ -180,6 +333,31 @@ function summarizeResults(results) {
summary[phase][key] = median(values);
}
}
const heapSnapshotCategoryValues = {};
for (const category of heapSnapshotCategories) {
const values = results
.map(result => result[phase]?.heapSnapshot?.categories?.[category])
.filter(value => Number.isFinite(value));
if (values.length > 0) heapSnapshotCategoryValues[category] = median(values);
}
const heapSnapshotNodeCountValues = {};
for (const category of heapSnapshotCategories) {
const values = results
.map(result => result[phase]?.heapSnapshot?.nodeCounts?.[category])
.filter(value => Number.isFinite(value));
if (values.length > 0) heapSnapshotNodeCountValues[category] = median(values);
}
if (Object.keys(heapSnapshotCategoryValues).length > 0) {
summary[phase].heapSnapshot = {
categories: heapSnapshotCategoryValues,
nodeCounts: heapSnapshotNodeCountValues,
};
}
}
return summary;
@ -290,6 +468,8 @@ async function measureMemory() {
await triggerGc();
const afterRequest = await getAllMemoryUsage(serverProcess);
const heapSnapshot = await getHeapSnapshotStatistics(serverProcess);
if (heapSnapshot != null) afterRequest.heapSnapshot = heapSnapshot;
// Stop the server
serverProcess.kill('SIGTERM');
@ -339,6 +519,10 @@ async function main() {
memorySettleTimeMs: MEMORY_SETTLE_TIME,
ipcTimeoutMs: IPC_TIMEOUT,
requestCount: REQUEST_COUNT,
heapSnapshot: {
enabled: HEAP_SNAPSHOT,
timeoutMs: HEAP_SNAPSHOT_TIMEOUT,
},
},
...summary,
samples: results,

16
packages/backend/src/boot/entry.ts
View file

@ -9,6 +9,7 @@
import cluster from 'node:cluster';
import { EventEmitter } from 'node:events';
import { writeHeapSnapshot } from 'node:v8';
import chalk from 'chalk';
import Xev from 'xev';
import Logger from '@/logger.js';
@ -106,6 +107,21 @@ process.on('message', msg => {
value: process.memoryUsage(),
});
}
} else if (msg != null && typeof msg === 'object' && 'type' in msg && msg.type === 'heap snapshot' && 'path' in msg && typeof msg.path === 'string') {
if (process.send != null) {
try {
const path = writeHeapSnapshot(msg.path);
process.send({
type: 'heap snapshot',
path,
});
} catch (err) {
process.send({
type: 'heap snapshot error',
message: err instanceof Error ? err.message : String(err),
});
}
}
}
});

38
packages/frontend/vite.config.ts
View file

@ -27,27 +27,29 @@ const extensions = ['.ts', '.tsx', '.js', '.jsx', '.mjs', '.json', '.json5', '.s
function getBundleVisualizerPlugin(): PluginOption[] {
if (process.env.FRONTEND_BUNDLE_VISUALIZER !== 'true') return [];
const template = process.env.FRONTEND_BUNDLE_VISUALIZER_TEMPLATE === 'markdown'
? 'markdown'
: process.env.FRONTEND_BUNDLE_VISUALIZER_TEMPLATE === 'raw-data'
? 'raw-data'
: 'treemap';
const defaultFilename = template === 'markdown'
? path.resolve(__dirname, '../../built/_frontend_bundle_visualizer_/report.md')
: template === 'raw-data'
? path.resolve(__dirname, '../../built/_frontend_bundle_visualizer_/stats.json')
: path.resolve(__dirname, '../../built/_frontend_bundle_visualizer_/stats.html');
return [
const visualizerOptions = {
title: 'Misskey frontend bundle visualizer',
gzipSize: true,
brotliSize: true,
projectRoot: path.resolve(__dirname, '../..'),
};
const plugins = [
visualizer({
filename: process.env.FRONTEND_BUNDLE_VISUALIZER_FILE ?? defaultFilename,
title: 'Misskey frontend bundle visualizer',
template,
gzipSize: true,
brotliSize: true,
projectRoot: path.resolve(__dirname, '../..'),
...visualizerOptions,
filename: process.env.FRONTEND_BUNDLE_VISUALIZER_FILE,
template: 'raw-data',
}) as PluginOption,
];
if (process.env.FRONTEND_BUNDLE_VISUALIZER_HTML_FILE != null) {
plugins.push(visualizer({
...visualizerOptions,
filename: process.env.FRONTEND_BUNDLE_VISUALIZER_HTML_FILE,
template: 'treemap',
}) as PluginOption);
}
return plugins;
}
/**