(test) enhance(dev): improve backend-memory-report

2026-06-24 18:45:58 +09:00 · 2026-06-24 18:45:58 +09:00 · 8dfa900729
commit 8dfa900729
parent 5ab352da11
5 changed files with 397 additions and 4 deletions
--- a/.github/scripts/backend-memory-report.mjs
+++ b/.github/scripts/backend-memory-report.mjs
@ -26,6 +26,17 @@ const metrics = [
 	'External',
 ];

+const heapSnapshotCategories = [
+	'Code',
+	'Strings',
+	'JS arrays',
+	'Typed arrays',
+	'System objects',
+	'Other JS objects',
+	'Other non-JS objects',
+	'Total',
+];
+
 function formatNumber(value) {
 	return numberFormatter.format(value);
 }
@ -273,6 +284,143 @@ function formatPlainDiffPercent(baseValue, headValue) {
 	return `${sign}${formatPercent(Math.abs((diff * 100) / baseValue))}`;
 }

+function getHeapSnapshotCategoryValue(report, phase, category) {
+	const value = report?.[phase]?.heapSnapshot?.categories?.[category];
+	return Number.isFinite(value) ? value : null;
+}
+
+function getHeapSnapshotSampleValues(report, phase, category) {
+	if (!Array.isArray(report?.samples)) return [];
+
+	return report.samples
+		.map(sample => getHeapSnapshotCategoryValue(sample, phase, category))
+		.filter(value => Number.isFinite(value));
+}
+
+function getHeapSnapshotSampleSpread(report, phase, category) {
+	const values = getHeapSnapshotSampleValues(report, phase, category);
+	if (values.length < 2) return null;
+
+	const center = median(values);
+	return median(values.map(value => Math.abs(value - center)));
+}
+
+function formatDiffBytes(baseBytes, headBytes) {
+	const diff = headBytes - baseBytes;
+	if (diff === 0) return formatBytes(0);
+
+	const sign = diff > 0 ? '+' : '-';
+	return formatColoredDiff(`${sign}${formatBytes(Math.abs(diff))}`, diff);
+}
+
+function formatDiffBytesPercent(baseBytes, headBytes) {
+	const diff = headBytes - baseBytes;
+	if (diff === 0) return '0%';
+	if (baseBytes <= 0) return '-';
+
+	const sign = diff > 0 ? '+' : '-';
+	return formatColoredDiff(`${sign}${formatPercent(Math.abs((diff * 100) / baseBytes))}`, diff);
+}
+
+function getPairedHeapSnapshotDeltaValues(base, head, phase, category) {
+	const baseSamplesByRound = getSamplesByRound(base);
+	const headSamplesByRound = getSamplesByRound(head);
+	const values = [];
+
+	for (const [round, baseSample] of baseSamplesByRound) {
+		const headSample = headSamplesByRound.get(round);
+		if (headSample == null) continue;
+
+		const baseValue = getHeapSnapshotCategoryValue(baseSample, phase, category);
+		const headValue = getHeapSnapshotCategoryValue(headSample, phase, category);
+		if (baseValue == null || headValue == null) continue;
+
+		values.push(headValue - baseValue);
+	}
+
+	return values;
+}
+
+function formatDeltaBytes(diffBytes) {
+	if (diffBytes === 0) return formatBytes(0);
+
+	const sign = diffBytes > 0 ? '+' : '-';
+	return formatColoredDiff(`${sign}${formatBytes(Math.abs(diffBytes))}`, diffBytes);
+}
+
+function pairedHeapSnapshotDeltaSummary(base, head, phase, category) {
+	const values = getPairedHeapSnapshotDeltaValues(base, head, phase, category);
+	if (values.length === 0) return null;
+
+	return {
+		median: median(values),
+		mad: mad(values),
+		min: Math.min(...values),
+		max: Math.max(...values),
+		samples: values.length,
+	};
+}
+
+function renderHeapSnapshotTable(base, head, phase) {
+	const lines = [
+		'| Category | Base | Head | Δ | Δ (%) |',
+		'| --- | ---: | ---: | ---: | ---: |',
+	];
+
+	for (const category of heapSnapshotCategories) {
+		const baseValue = getHeapSnapshotCategoryValue(base, phase, category);
+		const headValue = getHeapSnapshotCategoryValue(head, phase, category);
+		if (baseValue == null || headValue == null) continue;
+
+		const baseSpread = getHeapSnapshotSampleSpread(base, phase, category);
+		const headSpread = getHeapSnapshotSampleSpread(head, phase, category);
+
+		lines.push(`| ${category} | ${formatBytes(baseValue)} <br> ± ${baseSpread == null ? '-' : formatBytes(baseSpread)} | ${formatBytes(headValue)} <br> ± ${headSpread == null ? '-' : formatBytes(headSpread)} | ${formatDiffBytes(baseValue, headValue)} | ${formatDiffBytesPercent(baseValue, headValue)} |`);
+	}
+
+	if (lines.length === 2) return null;
+	return lines.join('\n');
+}
+
+function renderHeapSnapshotPairedDeltaTable(base, head, phase) {
+	const lines = [
+		'| Category | Δ median | Δ MAD | Δ min | Δ max |',
+		'| --- | ---: | ---: | ---: | ---: |',
+	];
+
+	for (const category of heapSnapshotCategories) {
+		const summary = pairedHeapSnapshotDeltaSummary(base, head, phase, category);
+		if (summary == null) continue;
+
+		lines.push(`| ${category} | ${formatDeltaBytes(summary.median)} | ${summary.mad == null ? '-' : formatBytes(summary.mad)} | ${formatDeltaBytes(summary.min)} | ${formatDeltaBytes(summary.max)} |`);
+	}
+
+	if (lines.length === 2) return null;
+	return lines.join('\n');
+}
+
+function renderHeapSnapshotSection(base, head) {
+	const table = renderHeapSnapshotTable(base, head, 'afterRequest');
+	if (table == null) return null;
+
+	const lines = [
+		'### V8 Heap Snapshot Statistics',
+		'',
+		table,
+		'',
+	];
+
+	const pairedDeltaTable = renderHeapSnapshotPairedDeltaTable(base, head, 'afterRequest');
+	if (pairedDeltaTable != null) {
+		lines.push('#### Paired Delta Summary');
+		lines.push('');
+		lines.push(pairedDeltaTable);
+		lines.push('');
+	}
+
+	return lines.join('\n');
+}
+
 function getJsFootprintValue(report, phase, key) {
 	const value = report?.[phase]?.totals?.[key];
 	return Number.isFinite(value) ? value : null;
@ -494,6 +642,12 @@ for (const phase of phases) {
 	}
 }

+const heapSnapshotSection = renderHeapSnapshotSection(base, head);
+if (heapSnapshotSection != null) {
+	lines.push(heapSnapshotSection);
+	lines.push('');
+}
+
 const jsFootprintSection = renderJsFootprintSection(baseJsFootprint, headJsFootprint);
 if (jsFootprintSection != null) {
 	lines.push(jsFootprintSection);
--- a/.github/scripts/measure-backend-memory-comparison.mjs
+++ b/.github/scripts/measure-backend-memory-comparison.mjs
@ -9,6 +9,16 @@ import { writeFile } from 'node:fs/promises';
 import { join, resolve } from 'node:path';

 const phases = ['beforeGc', 'afterGc', 'afterRequest'];
+const heapSnapshotCategories = [
+	'Code',
+	'Strings',
+	'JS arrays',
+	'Typed arrays',
+	'System objects',
+	'Other JS objects',
+	'Other non-JS objects',
+	'Total',
+];

 const [baseDirArg, headDirArg, baseOutputArg, headOutputArg] = process.argv.slice(2);

@ -121,6 +131,31 @@ function summarizeSamples(samples) {

 			if (values.length > 0) summary[phase][key] = median(values);
 		}
+
+		const heapSnapshotCategoryValues = {};
+		for (const category of heapSnapshotCategories) {
+			const values = samples
+				.map(sample => sample[phase]?.heapSnapshot?.categories?.[category])
+				.filter(value => Number.isFinite(value));
+
+			if (values.length > 0) heapSnapshotCategoryValues[category] = median(values);
+		}
+
+		const heapSnapshotNodeCountValues = {};
+		for (const category of heapSnapshotCategories) {
+			const values = samples
+				.map(sample => sample[phase]?.heapSnapshot?.nodeCounts?.[category])
+				.filter(value => Number.isFinite(value));
+
+			if (values.length > 0) heapSnapshotNodeCountValues[category] = median(values);
+		}
+
+		if (Object.keys(heapSnapshotCategoryValues).length > 0) {
+			summary[phase].heapSnapshot = {
+				categories: heapSnapshotCategoryValues,
+				nodeCounts: heapSnapshotNodeCountValues,
+			};
+		}
 	}

 	return summary;
@ -138,12 +173,15 @@ async function measureRepo(label, repoDir, round, orderIndex) {
 	});

 	process.stderr.write(`[${label}] Measuring memory\n`);
+	const measureEnv = {
+		...process.env,
+		MK_MEMORY_SAMPLE_COUNT: '1',
+	};
+	if (round <= 0) measureEnv.MK_MEMORY_HEAP_SNAPSHOT = '0';
+
 	const stdout = await run('node', ['packages/backend/scripts/measure-memory.mjs'], {
 		cwd: repoDir,
-		env: {
-			...process.env,
-			MK_MEMORY_SAMPLE_COUNT: '1',
-		},
+		env: measureEnv,
 	});

 	const report = JSON.parse(stdout);
--- a/.github/workflows/get-backend-memory.yml
+++ b/.github/workflows/get-backend-memory.yml
@ -93,6 +93,7 @@ jobs:
      env:
        MK_MEMORY_COMPARE_ROUNDS: 5
        MK_MEMORY_COMPARE_WARMUP_ROUNDS: 1
+        MK_MEMORY_HEAP_SNAPSHOT: 1
      run: node head/.github/scripts/measure-backend-memory-comparison.mjs base head memory-base.json memory-head.json
    - name: Measure backend loaded JS footprint
      run: |
--- a/packages/backend/scripts/measure-memory.mjs
+++ b/packages/backend/scripts/measure-memory.mjs
@ -14,6 +14,7 @@ import { fork } from 'node:child_process';
 import { setTimeout } from 'node:timers/promises';
 import { fileURLToPath } from 'node:url';
 import { dirname, join } from 'node:path';
+import { tmpdir } from 'node:os';
 import * as http from 'node:http';
 import * as fs from 'node:fs/promises';

@ -30,11 +31,21 @@ function readIntegerEnv(name, defaultValue, min) {
 	return value;
 }

+function readBooleanEnv(name, defaultValue) {
+	const rawValue = process.env[name];
+	if (rawValue == null || rawValue === '') return defaultValue;
+	if (rawValue === '1' || rawValue === 'true') return true;
+	if (rawValue === '0' || rawValue === 'false') return false;
+	throw new Error(`${name} must be one of: 1, 0, true, false`);
+}
+
 const SAMPLE_COUNT = readIntegerEnv('MK_MEMORY_SAMPLE_COUNT', 3, 1); // Number of samples to measure
 const STARTUP_TIMEOUT = readIntegerEnv('MK_MEMORY_STARTUP_TIMEOUT_MS', 120000, 1); // Timeout for server startup
 const MEMORY_SETTLE_TIME = readIntegerEnv('MK_MEMORY_SETTLE_TIME_MS', 10000, 0); // Wait after startup for memory to settle
 const IPC_TIMEOUT = readIntegerEnv('MK_MEMORY_IPC_TIMEOUT_MS', 30000, 1); // Timeout for IPC responses
 const REQUEST_COUNT = readIntegerEnv('MK_MEMORY_REQUEST_COUNT', 10, 0);
+const HEAP_SNAPSHOT = readBooleanEnv('MK_MEMORY_HEAP_SNAPSHOT', false);
+const HEAP_SNAPSHOT_TIMEOUT = readIntegerEnv('MK_MEMORY_HEAP_SNAPSHOT_TIMEOUT_MS', 120000, 1);

 const procStatusKeys = {
 	VmPeak: 0,
@ -74,6 +85,45 @@ const memoryKeys = {

 const phases = ['beforeGc', 'afterGc', 'afterRequest'];

+const heapSnapshotCategories = [
+	'Code',
+	'Strings',
+	'JS arrays',
+	'Typed arrays',
+	'System objects',
+	'Other JS objects',
+	'Other non-JS objects',
+	'Total',
+];
+
+const typedArrayNames = new Set([
+	'ArrayBuffer',
+	'SharedArrayBuffer',
+	'DataView',
+	'Int8Array',
+	'Uint8Array',
+	'Uint8ClampedArray',
+	'Int16Array',
+	'Uint16Array',
+	'Int32Array',
+	'Uint32Array',
+	'Float16Array',
+	'Float32Array',
+	'Float64Array',
+	'BigInt64Array',
+	'BigUint64Array',
+	'system / JSArrayBufferData',
+]);
+
+const otherJsNodeTypes = new Set([
+	'object',
+	'closure',
+	'regexp',
+	'number',
+	'symbol',
+	'bigint',
+]);
+
 function parseMemoryFile(content, keys, path, required) {
 	const result = {};
 	for (const key of Object.keys(keys)) {
@ -91,6 +141,76 @@ function bytesToKiB(value) {
 	return Math.round(value / 1024);
 }

+function createEmptyHeapSnapshotCategoryMap() {
+	return Object.fromEntries(heapSnapshotCategories.map(category => [category, 0]));
+}
+
+function isTypedArrayNode(type, name) {
+	return typedArrayNames.has(name) ||
+		(type === 'native' && (name.includes('ArrayBuffer') || name.includes('TypedArray')));
+}
+
+function isSystemNode(type, name) {
+	return type === 'hidden' ||
+		type === 'synthetic' ||
+		type === 'object shape' ||
+		name.startsWith('system /') ||
+		name.startsWith('(system ');
+}
+
+function classifyHeapSnapshotNode(type, name) {
+	if (type === 'code') return 'Code';
+	if (type === 'string' || type === 'concatenated string' || type === 'sliced string') return 'Strings';
+	if (isTypedArrayNode(type, name)) return 'Typed arrays';
+	if (type === 'array' || (type === 'object' && name === 'Array')) return 'JS arrays';
+	if (isSystemNode(type, name)) return 'System objects';
+	if (otherJsNodeTypes.has(type)) return 'Other JS objects';
+	return 'Other non-JS objects';
+}
+
+function analyzeHeapSnapshot(snapshot) {
+	const meta = snapshot?.snapshot?.meta;
+	const nodes = snapshot?.nodes;
+	const strings = snapshot?.strings;
+	if (meta == null || !Array.isArray(nodes) || !Array.isArray(strings)) {
+		throw new Error('Invalid heap snapshot format');
+	}
+
+	const nodeFields = meta.node_fields;
+	if (!Array.isArray(nodeFields)) throw new Error('Invalid heap snapshot node fields');
+
+	const typeOffset = nodeFields.indexOf('type');
+	const nameOffset = nodeFields.indexOf('name');
+	const selfSizeOffset = nodeFields.indexOf('self_size');
+	if (typeOffset < 0 || nameOffset < 0 || selfSizeOffset < 0) {
+		throw new Error('Heap snapshot is missing required node fields');
+	}
+
+	const nodeTypeNames = meta.node_types?.[typeOffset];
+	if (!Array.isArray(nodeTypeNames)) throw new Error('Invalid heap snapshot node types');
+
+	const fieldCount = nodeFields.length;
+	const categories = createEmptyHeapSnapshotCategoryMap();
+	const nodeCounts = createEmptyHeapSnapshotCategoryMap();
+
+	for (let offset = 0; offset < nodes.length; offset += fieldCount) {
+		const type = nodeTypeNames[nodes[offset + typeOffset]] ?? 'unknown';
+		const name = strings[nodes[offset + nameOffset]] ?? '';
+		const selfSize = nodes[offset + selfSizeOffset] ?? 0;
+		const category = classifyHeapSnapshotNode(type, name);
+
+		categories[category] += selfSize;
+		categories.Total += selfSize;
+		nodeCounts[category]++;
+		nodeCounts.Total++;
+	}
+
+	return {
+		categories,
+		nodeCounts,
+	};
+}
+
 async function getMemoryUsage(pid) {
 	const path = `/proc/${pid}/status`;
 	const status = await fs.readFile(path, 'utf-8');
@ -150,6 +270,39 @@ async function getRuntimeMemoryUsage(serverProcess) {
 	};
 }

+async function getHeapSnapshotStatistics(serverProcess) {
+	if (!HEAP_SNAPSHOT) return null;
+
+	const snapshotPath = join(tmpdir(), `misskey-backend-heap-${process.pid}-${serverProcess.pid}-${Date.now()}.heapsnapshot`);
+	const response = waitForMessage(
+		serverProcess,
+		message => message != null && typeof message === 'object' && (message.type === 'heap snapshot' || message.type === 'heap snapshot error'),
+		'heap snapshot',
+		HEAP_SNAPSHOT_TIMEOUT,
+	);
+
+	serverProcess.send({
+		type: 'heap snapshot',
+		path: snapshotPath,
+	});
+
+	const message = await response;
+	if (message.type === 'heap snapshot error') {
+		throw new Error(`Failed to write heap snapshot: ${message.message}`);
+	}
+
+	const writtenPath = typeof message.path === 'string' ? message.path : snapshotPath;
+
+	try {
+		const snapshot = JSON.parse(await fs.readFile(writtenPath, 'utf-8'));
+		return analyzeHeapSnapshot(snapshot);
+	} finally {
+		await fs.unlink(writtenPath).catch(err => {
+			process.stderr.write(`Failed to delete heap snapshot ${writtenPath}: ${err.message}\n`);
+		});
+	}
+}
+
 async function getAllMemoryUsage(serverProcess) {
 	const pid = serverProcess.pid;
 	return {
@ -180,6 +333,31 @@ function summarizeResults(results) {
 				summary[phase][key] = median(values);
 			}
 		}
+
+		const heapSnapshotCategoryValues = {};
+		for (const category of heapSnapshotCategories) {
+			const values = results
+				.map(result => result[phase]?.heapSnapshot?.categories?.[category])
+				.filter(value => Number.isFinite(value));
+
+			if (values.length > 0) heapSnapshotCategoryValues[category] = median(values);
+		}
+
+		const heapSnapshotNodeCountValues = {};
+		for (const category of heapSnapshotCategories) {
+			const values = results
+				.map(result => result[phase]?.heapSnapshot?.nodeCounts?.[category])
+				.filter(value => Number.isFinite(value));
+
+			if (values.length > 0) heapSnapshotNodeCountValues[category] = median(values);
+		}
+
+		if (Object.keys(heapSnapshotCategoryValues).length > 0) {
+			summary[phase].heapSnapshot = {
+				categories: heapSnapshotCategoryValues,
+				nodeCounts: heapSnapshotNodeCountValues,
+			};
+		}
 	}

 	return summary;
@ -290,6 +468,8 @@ async function measureMemory() {
 	await triggerGc();

 	const afterRequest = await getAllMemoryUsage(serverProcess);
+	const heapSnapshot = await getHeapSnapshotStatistics(serverProcess);
+	if (heapSnapshot != null) afterRequest.heapSnapshot = heapSnapshot;

 	// Stop the server
 	serverProcess.kill('SIGTERM');
@ -339,6 +519,10 @@ async function main() {
 			memorySettleTimeMs: MEMORY_SETTLE_TIME,
 			ipcTimeoutMs: IPC_TIMEOUT,
 			requestCount: REQUEST_COUNT,
+			heapSnapshot: {
+				enabled: HEAP_SNAPSHOT,
+				timeoutMs: HEAP_SNAPSHOT_TIMEOUT,
+			},
 		},
 		...summary,
 		samples: results,
--- a/packages/backend/src/boot/entry.ts
+++ b/packages/backend/src/boot/entry.ts
@ -9,6 +9,7 @@

 import cluster from 'node:cluster';
 import { EventEmitter } from 'node:events';
+import { writeHeapSnapshot } from 'node:v8';
 import chalk from 'chalk';
 import Xev from 'xev';
 import Logger from '@/logger.js';
@ -106,6 +107,21 @@ process.on('message', msg => {
 				value: process.memoryUsage(),
 			});
 		}
+	} else if (msg != null && typeof msg === 'object' && 'type' in msg && msg.type === 'heap snapshot' && 'path' in msg && typeof msg.path === 'string') {
+		if (process.send != null) {
+			try {
+				const path = writeHeapSnapshot(msg.path);
+				process.send({
+					type: 'heap snapshot',
+					path,
+				});
+			} catch (err) {
+				process.send({
+					type: 'heap snapshot error',
+					message: err instanceof Error ? err.message : String(err),
+				});
+			}
+		}
 	}
 });