Merge branch 'develop' into sp-reaction

This commit is contained in:
syuilo 2026-06-25 11:06:52 +09:00 committed by GitHub
commit e5d7084634
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
6 changed files with 769 additions and 57 deletions

View file

@ -8,7 +8,7 @@ if (baseFile == null || headFile == null || outputFile == null) {
}
const numberFormatter = new Intl.NumberFormat('en-US', {
maximumFractionDigits: 2,
maximumFractionDigits: 1,
});
const phases = [
@ -26,6 +26,39 @@ const metrics = [
'External',
];
const heapSnapshotCategories = [
'Total',
'Code',
'Strings',
'JS arrays',
'Typed arrays',
'System objects',
'Other JS objects',
'Other non-JS objects',
];
const heapSnapshotCategoriesColors = {
'Total': 'gray',
'Code': 'orange',
'Strings': 'red',
'JS arrays': 'cyan',
'Typed arrays': 'green',
'System objects': 'yellow',
'Other JS objects': 'violet',
'Other non-JS objects': 'pink',
};
const heapSnapshotCategoriesColorsHex = {
'Total': '#888888',
'Code': '#f28e2c',
'Strings': '#e15759',
'JS arrays': '#76b7b2',
'Typed arrays': '#59a14f',
'System objects': '#edc949',
'Other JS objects': '#af7aa1',
'Other non-JS objects': '#ff9da7',
};
function formatNumber(value) {
return numberFormatter.format(value);
}
@ -45,6 +78,14 @@ function formatPercent(value) {
return `${formatNumber(value)}%`;
}
function formatDeltaPercent(diff, baseValue) {
if (diff === 0) return '0%';
if (baseValue <= 0) return '-';
const sign = diff > 0 ? '+' : '-';
return formatColoredDiff(`${sign}${formatPercent(Math.abs((diff * 100) / baseValue))}`, diff);
}
function formatMathText(text) {
return text
.replaceAll('\\', '\\\\')
@ -58,23 +99,6 @@ function formatColoredDiff(text, diff) {
return `$\\color{${color}}{\\text{${formatMathText(text).replaceAll('\\%', '\\\\%')}}}$`;
}
function formatDiff(baseKiB, headKiB) {
const diff = headKiB - baseKiB;
if (diff === 0) return formatMemory(0);
const sign = diff > 0 ? '+' : '-';
return formatColoredDiff(`${sign}${formatMemory(Math.abs(diff))}`, diff);
}
function formatDiffPercent(baseKiB, headKiB) {
const diff = headKiB - baseKiB;
if (diff === 0) return '0%';
if (baseKiB <= 0) return '-';
const sign = diff > 0 ? '+' : '-';
return formatColoredDiff(`${sign}${formatPercent(Math.abs((diff * 100) / baseKiB))}`, diff);
}
function getMemoryValue(report, phase, metric) {
const value = report?.[phase]?.[metric];
return Number.isFinite(value) ? value : null;
@ -163,8 +187,8 @@ function pairedDeltaSummary(base, head, phase, metric) {
function renderTable(base, head, phase) {
const lines = [
'| Metric | Base | Head | Δ | Δ (%) |',
'| --- | ---: | ---: | ---: | ---: |',
'| Metric | Base | Head | Δ median | Δ MAD | Δ min | Δ max |',
'| --- | ---: | ---: | ---: | ---: | ---: | ---: |',
];
for (const metric of metrics) {
@ -174,27 +198,12 @@ function renderTable(base, head, phase) {
const baseSpread = getSampleSpread(base, phase, metric);
const headSpread = getSampleSpread(head, phase, metric);
lines.push(`| ${metric} | ${formatMemory(baseValue)} <br> ± ${formatMemory(baseSpread)} | ${formatMemory(headValue)} <br> ± ${formatMemory(headSpread)} | ${formatDiff(baseValue, headValue)} | ${formatDiffPercent(baseValue, headValue)} |`);
}
return lines.join('\n');
}
function renderPairedDeltaTable(base, head, phase) {
const lines = [
'| Metric | Δ median | Δ MAD | Δ min | Δ max |',
'| --- | ---: | ---: | ---: | ---: |',
];
for (const metric of metrics) {
const summary = pairedDeltaSummary(base, head, phase, metric);
if (summary == null) continue;
const deltaMedian = summary == null ? '-' : `${formatDeltaMemory(summary.median)}<br>${formatDeltaPercent(summary.median, baseValue)}`;
lines.push(`| ${metric} | ${formatDeltaMemory(summary.median)} | ${summary.mad == null ? '-' : formatMemory(summary.mad)} | ${formatDeltaMemory(summary.min)} | ${formatDeltaMemory(summary.max)} |`);
lines.push(`| **${metric}** | ${formatMemory(baseValue)} <br> ± ${formatMemory(baseSpread)} | ${formatMemory(headValue)} <br> ± ${formatMemory(headSpread)} | ${deltaMedian} | ${summary?.mad == null ? '-' : formatMemory(summary.mad)} | ${summary == null ? '-' : formatDeltaMemory(summary.min)} | ${summary == null ? '-' : formatDeltaMemory(summary.max)} |`);
}
if (lines.length === 2) return null;
return lines.join('\n');
}
@ -273,6 +282,241 @@ function formatPlainDiffPercent(baseValue, headValue) {
return `${sign}${formatPercent(Math.abs((diff * 100) / baseValue))}`;
}
function getHeapSnapshotCategoryValue(report, phase, category) {
const value = report?.[phase]?.heapSnapshot?.categories?.[category];
return Number.isFinite(value) ? value : null;
}
function getHeapSnapshotBreakdownEntries(report, phase, category) {
const breakdown = report?.[phase]?.heapSnapshot?.breakdowns?.[category];
if (breakdown == null || typeof breakdown !== 'object') return [];
return Object.entries(breakdown)
.filter(([, value]) => Number.isFinite(value) && value > 0)
.toSorted((a, b) => b[1] - a[1]);
}
const heapSnapshotSankeyChildMinRatio = 0.3;
const heapSnapshotSankeyParentMinPercent = 10;
function escapeCsvValue(value) {
return `"${String(value).replaceAll('"', '""')}"`;
}
function formatSankeyPercentValue(value) {
const rounded = Math.round(value * 100) / 100;
if (rounded === 0 && value > 0) return '0.01';
if (Number.isInteger(rounded)) return String(rounded);
return rounded.toFixed(2).replace(/0+$/, '').replace(/\.$/, '');
}
function formatHeapSnapshotSankeyChildLabel(label) {
return String(label).replace(/^[^:]+:\s*/, '');
}
function renderHeapSnapshotSankey(report, phase, title) {
const total = getHeapSnapshotCategoryValue(report, phase, 'Total');
if (total == null || total <= 0) return null;
const categories = heapSnapshotCategories
.filter(category => category !== 'Total')
.map(category => {
const value = getHeapSnapshotCategoryValue(report, phase, category);
if (value == null || value <= 0) return null;
const breakdownEntries = getHeapSnapshotBreakdownEntries(report, phase, category);
const breakdownTotal = breakdownEntries.reduce((sum, [, childValue]) => sum + childValue, 0);
const percent = (value * 100) / total;
const childEntries = [];
let otherPercent = 0;
if (breakdownTotal > 0 && percent > heapSnapshotSankeyParentMinPercent) {
for (const [childName, childValue] of breakdownEntries) {
const childRatio = childValue / breakdownTotal;
const childPercent = percent * childRatio;
if (childRatio >= heapSnapshotSankeyChildMinRatio) {
childEntries.push([formatHeapSnapshotSankeyChildLabel(childName), childPercent]);
} else {
otherPercent += childPercent;
}
}
if (childEntries.length > 0 && otherPercent > 0) {
childEntries.push(['Other', otherPercent]);
}
}
return {
category,
percent,
childEntries,
};
})
.filter(value => value != null);
if (categories.length === 0) return null;
const nodeColors = {
[title]: heapSnapshotCategoriesColorsHex.Total,
};
for (const { category, childEntries } of categories) {
const categoryColor = heapSnapshotCategoriesColorsHex[category] ?? heapSnapshotCategoriesColorsHex.Total;
nodeColors[category] = categoryColor;
for (const [childName] of childEntries) {
nodeColors[childName] = categoryColor;
}
}
const lines = [
`<details><summary>${title} heap snapshot composition</summary>`,
'',
'```mermaid',
`%%{init: ${JSON.stringify({
sankey: {
showValues: false,
linkColor: 'target',
labelStyle: 'outlined',
nodeAlignment: 'center',
nodePadding: 10,
nodeColors: {
...nodeColors,
'Other': '#888888',
},
},
})}}%%`,
'sankey-beta',
];
for (const { category, percent, childEntries } of categories) {
lines.push(`${escapeCsvValue(title)},${escapeCsvValue(category)},${formatSankeyPercentValue(percent)}`);
for (const [childName, childPercent] of childEntries) {
lines.push(`${escapeCsvValue(category)},${escapeCsvValue(childName)},${formatSankeyPercentValue(childPercent)}`);
}
}
lines.push('```');
lines.push('');
lines.push('</details>');
return lines.join('\n');
}
function formatHeapSnapshotCategoryLabel(category, baseValue, headValue, baseTotal, headTotal) {
if (category === 'Total' || baseTotal == null || headTotal == null || baseTotal <= 0 || headTotal <= 0) return `**${category}**`;
const basePercent = formatPercent((baseValue * 100) / baseTotal);
const headPercent = formatPercent((headValue * 100) / headTotal);
return `**${category}**<br>${basePercent}${headPercent}`;
}
function getHeapSnapshotSampleValues(report, phase, category) {
if (!Array.isArray(report?.samples)) return [];
return report.samples
.map(sample => getHeapSnapshotCategoryValue(sample, phase, category))
.filter(value => Number.isFinite(value));
}
function getHeapSnapshotSampleSpread(report, phase, category) {
const values = getHeapSnapshotSampleValues(report, phase, category);
if (values.length < 2) return null;
const center = median(values);
return median(values.map(value => Math.abs(value - center)));
}
function getPairedHeapSnapshotDeltaValues(base, head, phase, category) {
const baseSamplesByRound = getSamplesByRound(base);
const headSamplesByRound = getSamplesByRound(head);
const values = [];
for (const [round, baseSample] of baseSamplesByRound) {
const headSample = headSamplesByRound.get(round);
if (headSample == null) continue;
const baseValue = getHeapSnapshotCategoryValue(baseSample, phase, category);
const headValue = getHeapSnapshotCategoryValue(headSample, phase, category);
if (baseValue == null || headValue == null) continue;
values.push(headValue - baseValue);
}
return values;
}
function formatDeltaBytes(diffBytes) {
if (diffBytes === 0) return formatBytes(0);
const sign = diffBytes > 0 ? '+' : '-';
return formatColoredDiff(`${sign}${formatBytes(Math.abs(diffBytes))}`, diffBytes);
}
function pairedHeapSnapshotDeltaSummary(base, head, phase, category) {
const values = getPairedHeapSnapshotDeltaValues(base, head, phase, category);
if (values.length === 0) return null;
return {
median: median(values),
mad: mad(values),
min: Math.min(...values),
max: Math.max(...values),
samples: values.length,
};
}
function renderHeapSnapshotTable(base, head, phase) {
const lines = [
'| Metric | Base | Head | Δ median | Δ MAD | Δ min | Δ max |',
'| --- | ---: | ---: | ---: | ---: | ---: | ---: |',
];
const baseTotal = getHeapSnapshotCategoryValue(base, phase, 'Total');
const headTotal = getHeapSnapshotCategoryValue(head, phase, 'Total');
for (const category of heapSnapshotCategories) {
const baseValue = getHeapSnapshotCategoryValue(base, phase, category);
const headValue = getHeapSnapshotCategoryValue(head, phase, category);
if (baseValue == null || headValue == null) continue;
const baseSpread = getHeapSnapshotSampleSpread(base, phase, category);
const headSpread = getHeapSnapshotSampleSpread(head, phase, category);
const summary = pairedHeapSnapshotDeltaSummary(base, head, phase, category);
const deltaMedian = summary == null ? '-' : `${formatDeltaBytes(summary.median)}<br>${formatDeltaPercent(summary.median, baseValue)}`;
const categoryLabel = formatHeapSnapshotCategoryLabel(category, baseValue, headValue, baseTotal, headTotal);
lines.push(`| $\\color{${heapSnapshotCategoriesColors[category]}}{\\rule{8pt}{8pt}}$ ${categoryLabel} | ${formatBytes(baseValue)} <br> ± ${baseSpread == null ? '-' : formatBytes(baseSpread)} | ${formatBytes(headValue)} <br> ± ${headSpread == null ? '-' : formatBytes(headSpread)} | ${deltaMedian} | ${summary?.mad == null ? '-' : formatBytes(summary.mad)} | ${summary == null ? '-' : formatDeltaBytes(summary.min)} | ${summary == null ? '-' : formatDeltaBytes(summary.max)} |`);
if (category === 'Total') {
lines.push('| | | | | | | |');
}
}
if (lines.length === 2) return null;
return lines.join('\n');
}
function renderHeapSnapshotSection(base, head) {
const table = renderHeapSnapshotTable(base, head, 'afterRequest');
if (table == null) return null;
const lines = [
'### V8 Heap Snapshot Statistics',
'',
table,
'',
];
for (const graph of [
renderHeapSnapshotSankey(base, 'afterRequest', 'Base'),
renderHeapSnapshotSankey(head, 'afterRequest', 'Head'),
]) {
if (graph == null) continue;
lines.push(graph);
lines.push('');
}
return lines.join('\n');
}
function getJsFootprintValue(report, phase, key) {
const value = report?.[phase]?.totals?.[key];
return Number.isFinite(value) ? value : null;
@ -301,7 +545,7 @@ function renderJsFootprintMetricTable(base, head) {
const headValue = getJsFootprintValue(head, 'afterRequest', key);
if (baseValue == null || headValue == null) continue;
lines.push(`| ${title} | ${formatter(baseValue)} | ${formatter(headValue)} | ${formatPlainDiff(baseValue, headValue, formatter)} | ${formatPlainDiffPercent(baseValue, headValue)} |`);
lines.push(`| **${title}** | ${formatter(baseValue)} | ${formatter(headValue)} | ${formatPlainDiff(baseValue, headValue, formatter)} | ${formatPlainDiffPercent(baseValue, headValue)} |`);
}
return lines.join('\n');
@ -443,12 +687,10 @@ function renderJsFootprintSection(base, head) {
'',
renderJsFootprintMetricTable(base, head),
'',
'#### Load Phase Breakdown',
'',
renderJsFootprintPhaseTable(base, head),
'',
'</details>',
'',
//'#### Load Phase Breakdown',
//'',
//renderJsFootprintPhaseTable(base, head),
//'',
];
for (const block of [
@ -461,6 +703,9 @@ function renderJsFootprintSection(base, head) {
lines.push('');
}
lines.push('</details>');
lines.push('');
return lines.join('\n');
}
@ -469,7 +714,7 @@ const head = JSON.parse(await readFile(headFile, 'utf8'));
const baseJsFootprint = baseJsFootprintFile == null ? null : JSON.parse(await readFile(baseJsFootprintFile, 'utf8'));
const headJsFootprint = headJsFootprintFile == null ? null : JSON.parse(await readFile(headJsFootprintFile, 'utf8'));
const lines = [
'## Backend Memory Usage Report',
'## ⚙️ Backend Memory Usage Report',
'',
];
@ -483,14 +728,12 @@ for (const phase of phases) {
lines.push(`### ${phase.title}`);
lines.push(renderTable(base, head, phase.key));
lines.push('');
}
const pairedDeltaTable = renderPairedDeltaTable(base, head, phase.key);
if (pairedDeltaTable != null) {
lines.push('#### Paired Delta Summary');
lines.push('');
lines.push(pairedDeltaTable);
lines.push('');
}
const heapSnapshotSection = renderHeapSnapshotSection(base, head);
if (heapSnapshotSection != null) {
lines.push(heapSnapshotSection);
lines.push('');
}
const jsFootprintSection = renderJsFootprintSection(baseJsFootprint, headJsFootprint);

View file

@ -530,12 +530,68 @@ function renderFrontendBundleReport(before, after) {
return lines.join('\n');
}
const visualizerTreemapLimit = 30;
function mermaidTreemapLabel(value) {
const label = String(value)
.replaceAll('\\', '/')
.replaceAll('"', "'")
.replaceAll('`', "'")
.replaceAll('\r', ' ')
.replaceAll('\n', ' ')
.trim();
return label === '' ? '(unknown)' : label;
}
function mermaidTreemapModuleLabel(id) {
const normalizedId = String(id).replaceAll('\\', '/');
const filePath = normalizedId.split(/[?#]/, 1)[0];
const fileName = path.posix.basename(filePath);
return mermaidTreemapLabel(fileName || normalizedId);
}
function renderVisualizerTreemap(label, report) {
const rows = report.hotModules
.filter((row) => row.renderedLength > 0)
.slice(0, visualizerTreemapLimit);
const topRendered = rows.reduce((sum, row) => sum + row.renderedLength, 0);
const otherRendered = Math.max(0, report.metrics.renderedLength - topRendered);
const lines = [
'```mermaid',
'treemap-beta',
`"${mermaidTreemapLabel(label)}"`,
];
for (const row of rows) {
lines.push(` "${mermaidTreemapModuleLabel(row.id)}": ${Math.round(row.renderedLength)}`);
}
if (otherRendered > 0) {
lines.push(` "Other": ${Math.round(otherRendered)}`);
}
lines.push('```');
return lines.join('\n');
}
function renderVisualizerTreemapDetails(label, report, open = false) {
return [
`<details${open ? ' open' : ''}>`,
`<summary>${label} rendered size treemap (top ${visualizerTreemapLimit} + Other)</summary>`,
'',
renderVisualizerTreemap(label, report),
'',
'</details>',
].join('\n');
}
const args = process.argv.slice(2);
const [beforeDir, afterDir, beforeStatsFile, afterStatsFile, outFile] = args;
const before = await collectReport(beforeDir);
const after = await collectReport(afterDir);
const beforeStats = JSON.parse(await fs.readFile(beforeStatsFile, 'utf8'));
const afterStats = JSON.parse(await fs.readFile(afterStatsFile, 'utf8'));
const beforeVisualizerReport = collectVisualizerReport(beforeStats);
const afterVisualizerReport = collectVisualizerReport(afterStats);
const visualizerArtifactLink = `[Download detailed HTML](${process.env.FRONTEND_BUNDLE_REPORT_ARTIFACT_URL})`;
const body = [
@ -547,7 +603,11 @@ const body = [
'',
'## Bundle Stats',
'',
renderFrontendBundleReport(collectVisualizerReport(beforeStats), collectVisualizerReport(afterStats)),
renderFrontendBundleReport(beforeVisualizerReport, afterVisualizerReport),
'',
renderVisualizerTreemapDetails('Before', beforeVisualizerReport),
'',
renderVisualizerTreemapDetails('After', afterVisualizerReport, true),
'',
visualizerArtifactLink,
].join('\n');

View file

@ -9,6 +9,16 @@ import { writeFile } from 'node:fs/promises';
import { join, resolve } from 'node:path';
const phases = ['beforeGc', 'afterGc', 'afterRequest'];
const heapSnapshotCategories = [
'Code',
'Strings',
'JS arrays',
'Typed arrays',
'System objects',
'Other JS objects',
'Other non-JS objects',
'Total',
];
const [baseDirArg, headDirArg, baseOutputArg, headOutputArg] = process.argv.slice(2);
@ -27,6 +37,8 @@ function readIntegerEnv(name, defaultValue, min) {
return value;
}
const HEAP_SNAPSHOT_BREAKDOWN_TOP_N = readIntegerEnv('MK_MEMORY_HEAP_SNAPSHOT_BREAKDOWN_TOP_N', 6, 1);
function commandName(command) {
if (process.platform !== 'win32') return command;
if (command === 'pnpm') return 'pnpm.cmd';
@ -101,6 +113,51 @@ function median(values) {
return Math.round((sorted[center - 1] + sorted[center]) / 2);
}
function summarizeHeapSnapshotBreakdowns(samples, phase) {
const breakdowns = {};
for (const category of heapSnapshotCategories) {
if (category === 'Total') continue;
const childKeys = new Set();
for (const sample of samples) {
for (const childKey of Object.keys(sample[phase]?.heapSnapshot?.breakdowns?.[category] ?? {})) {
childKeys.add(childKey);
}
}
const categoryBreakdown = {};
for (const childKey of childKeys) {
const values = samples
.map(sample => sample[phase]?.heapSnapshot?.breakdowns?.[category]?.[childKey])
.filter(value => Number.isFinite(value));
if (values.length > 0) categoryBreakdown[childKey] = median(values);
}
if (Object.keys(categoryBreakdown).length > 0) {
breakdowns[category] = collapseHeapSnapshotBreakdown(categoryBreakdown);
}
}
return breakdowns;
}
function collapseHeapSnapshotBreakdown(breakdown) {
const entries = Object.entries(breakdown)
.filter(([, value]) => value > 0)
.toSorted((a, b) => b[1] - a[1]);
const topEntries = entries.slice(0, HEAP_SNAPSHOT_BREAKDOWN_TOP_N);
const otherValue = entries
.slice(HEAP_SNAPSHOT_BREAKDOWN_TOP_N)
.reduce((sum, [, value]) => sum + value, 0);
const collapsed = Object.fromEntries(topEntries);
if (otherValue > 0) collapsed.Other = otherValue;
return collapsed;
}
function summarizeSamples(samples) {
const summary = {};
@ -121,6 +178,34 @@ function summarizeSamples(samples) {
if (values.length > 0) summary[phase][key] = median(values);
}
const heapSnapshotCategoryValues = {};
for (const category of heapSnapshotCategories) {
const values = samples
.map(sample => sample[phase]?.heapSnapshot?.categories?.[category])
.filter(value => Number.isFinite(value));
if (values.length > 0) heapSnapshotCategoryValues[category] = median(values);
}
const heapSnapshotNodeCountValues = {};
for (const category of heapSnapshotCategories) {
const values = samples
.map(sample => sample[phase]?.heapSnapshot?.nodeCounts?.[category])
.filter(value => Number.isFinite(value));
if (values.length > 0) heapSnapshotNodeCountValues[category] = median(values);
}
if (Object.keys(heapSnapshotCategoryValues).length > 0) {
const heapSnapshotBreakdowns = summarizeHeapSnapshotBreakdowns(samples, phase);
summary[phase].heapSnapshot = {
categories: heapSnapshotCategoryValues,
nodeCounts: heapSnapshotNodeCountValues,
...(Object.keys(heapSnapshotBreakdowns).length > 0 ? { breakdowns: heapSnapshotBreakdowns } : {}),
};
}
}
return summary;
@ -138,12 +223,15 @@ async function measureRepo(label, repoDir, round, orderIndex) {
});
process.stderr.write(`[${label}] Measuring memory\n`);
const measureEnv = {
...process.env,
MK_MEMORY_SAMPLE_COUNT: '1',
};
if (round <= 0) measureEnv.MK_MEMORY_HEAP_SNAPSHOT = '0';
const stdout = await run('node', ['packages/backend/scripts/measure-memory.mjs'], {
cwd: repoDir,
env: {
...process.env,
MK_MEMORY_SAMPLE_COUNT: '1',
},
env: measureEnv,
});
const report = JSON.parse(stdout);

View file

@ -93,6 +93,7 @@ jobs:
env:
MK_MEMORY_COMPARE_ROUNDS: 5
MK_MEMORY_COMPARE_WARMUP_ROUNDS: 1
MK_MEMORY_HEAP_SNAPSHOT: 1
run: node head/.github/scripts/measure-backend-memory-comparison.mjs base head memory-base.json memory-head.json
- name: Measure backend loaded JS footprint
run: |

View file

@ -14,6 +14,7 @@ import { fork } from 'node:child_process';
import { setTimeout } from 'node:timers/promises';
import { fileURLToPath } from 'node:url';
import { dirname, join } from 'node:path';
import { tmpdir } from 'node:os';
import * as http from 'node:http';
import * as fs from 'node:fs/promises';
@ -30,11 +31,22 @@ function readIntegerEnv(name, defaultValue, min) {
return value;
}
function readBooleanEnv(name, defaultValue) {
const rawValue = process.env[name];
if (rawValue == null || rawValue === '') return defaultValue;
if (rawValue === '1' || rawValue === 'true') return true;
if (rawValue === '0' || rawValue === 'false') return false;
throw new Error(`${name} must be one of: 1, 0, true, false`);
}
const SAMPLE_COUNT = readIntegerEnv('MK_MEMORY_SAMPLE_COUNT', 3, 1); // Number of samples to measure
const STARTUP_TIMEOUT = readIntegerEnv('MK_MEMORY_STARTUP_TIMEOUT_MS', 120000, 1); // Timeout for server startup
const MEMORY_SETTLE_TIME = readIntegerEnv('MK_MEMORY_SETTLE_TIME_MS', 10000, 0); // Wait after startup for memory to settle
const IPC_TIMEOUT = readIntegerEnv('MK_MEMORY_IPC_TIMEOUT_MS', 30000, 1); // Timeout for IPC responses
const REQUEST_COUNT = readIntegerEnv('MK_MEMORY_REQUEST_COUNT', 10, 0);
const HEAP_SNAPSHOT = readBooleanEnv('MK_MEMORY_HEAP_SNAPSHOT', false);
const HEAP_SNAPSHOT_TIMEOUT = readIntegerEnv('MK_MEMORY_HEAP_SNAPSHOT_TIMEOUT_MS', 120000, 1);
const HEAP_SNAPSHOT_BREAKDOWN_TOP_N = readIntegerEnv('MK_MEMORY_HEAP_SNAPSHOT_BREAKDOWN_TOP_N', 6, 1);
const procStatusKeys = {
VmPeak: 0,
@ -74,6 +86,45 @@ const memoryKeys = {
const phases = ['beforeGc', 'afterGc', 'afterRequest'];
const heapSnapshotCategories = [
'Code',
'Strings',
'JS arrays',
'Typed arrays',
'System objects',
'Other JS objects',
'Other non-JS objects',
'Total',
];
const typedArrayNames = new Set([
'ArrayBuffer',
'SharedArrayBuffer',
'DataView',
'Int8Array',
'Uint8Array',
'Uint8ClampedArray',
'Int16Array',
'Uint16Array',
'Int32Array',
'Uint32Array',
'Float16Array',
'Float32Array',
'Float64Array',
'BigInt64Array',
'BigUint64Array',
'system / JSArrayBufferData',
]);
const otherJsNodeTypes = new Set([
'object',
'closure',
'regexp',
'number',
'symbol',
'bigint',
]);
function parseMemoryFile(content, keys, path, required) {
const result = {};
for (const key of Object.keys(keys)) {
@ -91,6 +142,161 @@ function bytesToKiB(value) {
return Math.round(value / 1024);
}
function createEmptyHeapSnapshotCategoryMap() {
return Object.fromEntries(heapSnapshotCategories.map(category => [category, 0]));
}
function isTypedArrayNode(type, name) {
return typedArrayNames.has(name) ||
(type === 'native' && (name.includes('ArrayBuffer') || name.includes('TypedArray')));
}
function isSystemNode(type, name) {
return type === 'hidden' ||
type === 'synthetic' ||
type === 'object shape' ||
name.startsWith('system /') ||
name.startsWith('(system ');
}
function classifyHeapSnapshotNode(type, name) {
if (type === 'code') return 'Code';
if (type === 'string' || type === 'concatenated string' || type === 'sliced string') return 'Strings';
if (isTypedArrayNode(type, name)) return 'Typed arrays';
if (type === 'array' || (type === 'object' && name === 'Array')) return 'JS arrays';
if (isSystemNode(type, name)) return 'System objects';
if (otherJsNodeTypes.has(type)) return 'Other JS objects';
return 'Other non-JS objects';
}
function addValue(map, key, value) {
map[key] = (map[key] ?? 0) + value;
}
function sanitizeHeapSnapshotBreakdownLabel(value, fallback = 'unknown') {
const label = String(value ?? '').replace(/\s+/g, ' ').trim();
if (label === '') return fallback;
if (label.length <= 80) return label;
return `${label.slice(0, 77)}...`;
}
function classifyHeapSnapshotBreakdown(category, type, name) {
if (category === 'Strings') return type;
if (category === 'JS arrays') {
if (type === 'array') return 'array nodes';
if (type === 'object' && name === 'Array') return 'Array objects';
return sanitizeHeapSnapshotBreakdownLabel(`${type}: ${name}`);
}
if (category === 'Typed arrays') {
if (name === 'system / JSArrayBufferData') return 'ArrayBuffer data';
if (name === 'Uint8Array') return 'Uint8Array / Buffer';
if (typedArrayNames.has(name)) return name;
if (type === 'native' && name.includes('ArrayBuffer')) return 'native ArrayBuffer';
if (type === 'native' && name.includes('TypedArray')) return 'native TypedArray';
return sanitizeHeapSnapshotBreakdownLabel(`${type}: ${name}`);
}
if (category === 'System objects') {
if (name.startsWith('system /')) return sanitizeHeapSnapshotBreakdownLabel(name);
if (name.startsWith('(system ')) return sanitizeHeapSnapshotBreakdownLabel(name);
return sanitizeHeapSnapshotBreakdownLabel(`${type}: ${name}`, type);
}
if (category === 'Other JS objects') {
if (type === 'object') return sanitizeHeapSnapshotBreakdownLabel(`object: ${name}`, 'object: unknown');
return type;
}
if (category === 'Other non-JS objects') {
if (type === 'native') return sanitizeHeapSnapshotBreakdownLabel(`native: ${name}`, 'native: unknown');
return sanitizeHeapSnapshotBreakdownLabel(`${type}: ${name}`, type);
}
if (category === 'Code') {
const lowerName = name.toLowerCase();
if (lowerName.includes('bytecode')) return 'bytecode';
if (lowerName.includes('builtin')) return 'builtins';
if (lowerName.includes('regexp')) return 'regexp code';
if (lowerName.includes('stub')) return 'stubs';
return sanitizeHeapSnapshotBreakdownLabel(`code: ${name}`, 'code: unknown');
}
return sanitizeHeapSnapshotBreakdownLabel(`${type}: ${name}`, type);
}
function collapseHeapSnapshotBreakdown(breakdowns) {
const collapsed = {};
for (const [category, children] of Object.entries(breakdowns)) {
const entries = Object.entries(children)
.filter(([, value]) => value > 0)
.toSorted((a, b) => b[1] - a[1]);
const topEntries = entries.slice(0, HEAP_SNAPSHOT_BREAKDOWN_TOP_N);
const otherValue = entries
.slice(HEAP_SNAPSHOT_BREAKDOWN_TOP_N)
.reduce((sum, [, value]) => sum + value, 0);
const categoryBreakdown = Object.fromEntries(topEntries);
if (otherValue > 0) categoryBreakdown.Other = otherValue;
if (Object.keys(categoryBreakdown).length > 0) collapsed[category] = categoryBreakdown;
}
return collapsed;
}
function analyzeHeapSnapshot(snapshot) {
const meta = snapshot?.snapshot?.meta;
const nodes = snapshot?.nodes;
const strings = snapshot?.strings;
if (meta == null || !Array.isArray(nodes) || !Array.isArray(strings)) {
throw new Error('Invalid heap snapshot format');
}
const nodeFields = meta.node_fields;
if (!Array.isArray(nodeFields)) throw new Error('Invalid heap snapshot node fields');
const typeOffset = nodeFields.indexOf('type');
const nameOffset = nodeFields.indexOf('name');
const selfSizeOffset = nodeFields.indexOf('self_size');
if (typeOffset < 0 || nameOffset < 0 || selfSizeOffset < 0) {
throw new Error('Heap snapshot is missing required node fields');
}
const nodeTypeNames = meta.node_types?.[typeOffset];
if (!Array.isArray(nodeTypeNames)) throw new Error('Invalid heap snapshot node types');
const fieldCount = nodeFields.length;
const categories = createEmptyHeapSnapshotCategoryMap();
const nodeCounts = createEmptyHeapSnapshotCategoryMap();
const breakdowns = Object.fromEntries(
heapSnapshotCategories
.filter(category => category !== 'Total')
.map(category => [category, {}]),
);
for (let offset = 0; offset < nodes.length; offset += fieldCount) {
const type = nodeTypeNames[nodes[offset + typeOffset]] ?? 'unknown';
const name = strings[nodes[offset + nameOffset]] ?? '';
const selfSize = nodes[offset + selfSizeOffset] ?? 0;
const category = classifyHeapSnapshotNode(type, name);
categories[category] += selfSize;
categories.Total += selfSize;
nodeCounts[category]++;
nodeCounts.Total++;
addValue(breakdowns[category], classifyHeapSnapshotBreakdown(category, type, name), selfSize);
}
return {
categories,
nodeCounts,
breakdowns: collapseHeapSnapshotBreakdown(breakdowns),
};
}
async function getMemoryUsage(pid) {
const path = `/proc/${pid}/status`;
const status = await fs.readFile(path, 'utf-8');
@ -150,6 +356,39 @@ async function getRuntimeMemoryUsage(serverProcess) {
};
}
async function getHeapSnapshotStatistics(serverProcess) {
if (!HEAP_SNAPSHOT) return null;
const snapshotPath = join(tmpdir(), `misskey-backend-heap-${process.pid}-${serverProcess.pid}-${Date.now()}.heapsnapshot`);
const response = waitForMessage(
serverProcess,
message => message != null && typeof message === 'object' && (message.type === 'heap snapshot' || message.type === 'heap snapshot error'),
'heap snapshot',
HEAP_SNAPSHOT_TIMEOUT,
);
serverProcess.send({
type: 'heap snapshot',
path: snapshotPath,
});
const message = await response;
if (message.type === 'heap snapshot error') {
throw new Error(`Failed to write heap snapshot: ${message.message}`);
}
const writtenPath = typeof message.path === 'string' ? message.path : snapshotPath;
try {
const snapshot = JSON.parse(await fs.readFile(writtenPath, 'utf-8'));
return analyzeHeapSnapshot(snapshot);
} finally {
await fs.unlink(writtenPath).catch(err => {
process.stderr.write(`Failed to delete heap snapshot ${writtenPath}: ${err.message}\n`);
});
}
}
async function getAllMemoryUsage(serverProcess) {
const pid = serverProcess.pid;
return {
@ -166,6 +405,36 @@ function median(values) {
return Math.round((sorted[center - 1] + sorted[center]) / 2);
}
function summarizeHeapSnapshotBreakdowns(results, phase) {
const breakdowns = {};
for (const category of heapSnapshotCategories) {
if (category === 'Total') continue;
const childKeys = new Set();
for (const result of results) {
for (const childKey of Object.keys(result[phase]?.heapSnapshot?.breakdowns?.[category] ?? {})) {
childKeys.add(childKey);
}
}
const categoryBreakdown = {};
for (const childKey of childKeys) {
const values = results
.map(result => result[phase]?.heapSnapshot?.breakdowns?.[category]?.[childKey])
.filter(value => Number.isFinite(value));
if (values.length > 0) categoryBreakdown[childKey] = median(values);
}
if (Object.keys(categoryBreakdown).length > 0) {
breakdowns[category] = collapseHeapSnapshotBreakdown({ [category]: categoryBreakdown })[category] ?? categoryBreakdown;
}
}
return breakdowns;
}
function summarizeResults(results) {
const summary = {};
@ -180,6 +449,34 @@ function summarizeResults(results) {
summary[phase][key] = median(values);
}
}
const heapSnapshotCategoryValues = {};
for (const category of heapSnapshotCategories) {
const values = results
.map(result => result[phase]?.heapSnapshot?.categories?.[category])
.filter(value => Number.isFinite(value));
if (values.length > 0) heapSnapshotCategoryValues[category] = median(values);
}
const heapSnapshotNodeCountValues = {};
for (const category of heapSnapshotCategories) {
const values = results
.map(result => result[phase]?.heapSnapshot?.nodeCounts?.[category])
.filter(value => Number.isFinite(value));
if (values.length > 0) heapSnapshotNodeCountValues[category] = median(values);
}
if (Object.keys(heapSnapshotCategoryValues).length > 0) {
const heapSnapshotBreakdowns = summarizeHeapSnapshotBreakdowns(results, phase);
summary[phase].heapSnapshot = {
categories: heapSnapshotCategoryValues,
nodeCounts: heapSnapshotNodeCountValues,
...(Object.keys(heapSnapshotBreakdowns).length > 0 ? { breakdowns: heapSnapshotBreakdowns } : {}),
};
}
}
return summary;
@ -290,6 +587,8 @@ async function measureMemory() {
await triggerGc();
const afterRequest = await getAllMemoryUsage(serverProcess);
const heapSnapshot = await getHeapSnapshotStatistics(serverProcess);
if (heapSnapshot != null) afterRequest.heapSnapshot = heapSnapshot;
// Stop the server
serverProcess.kill('SIGTERM');
@ -339,6 +638,11 @@ async function main() {
memorySettleTimeMs: MEMORY_SETTLE_TIME,
ipcTimeoutMs: IPC_TIMEOUT,
requestCount: REQUEST_COUNT,
heapSnapshot: {
enabled: HEAP_SNAPSHOT,
timeoutMs: HEAP_SNAPSHOT_TIMEOUT,
breakdownTopN: HEAP_SNAPSHOT_BREAKDOWN_TOP_N,
},
},
...summary,
samples: results,

View file

@ -9,6 +9,7 @@
import cluster from 'node:cluster';
import { EventEmitter } from 'node:events';
import { writeHeapSnapshot } from 'node:v8';
import chalk from 'chalk';
import Xev from 'xev';
import Logger from '@/logger.js';
@ -106,6 +107,21 @@ process.on('message', msg => {
value: process.memoryUsage(),
});
}
} else if (msg != null && typeof msg === 'object' && 'type' in msg && msg.type === 'heap snapshot' && 'path' in msg && typeof msg.path === 'string') {
if (process.send != null) {
try {
const path = writeHeapSnapshot(msg.path);
process.send({
type: 'heap snapshot',
path,
});
} catch (err) {
process.send({
type: 'heap snapshot error',
message: err instanceof Error ? err.message : String(err),
});
}
}
}
});