mirror of
https://github.com/misskey-dev/misskey.git
synced 2026-06-25 17:10:43 +00:00
Compare commits
36 commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
3c83952c48 |
||
|
|
3ac6d287d6 |
||
|
|
42a59b5d76 |
||
|
|
138e66e618 |
||
|
|
4188d68457 |
||
|
|
6391a4e7e2 |
||
|
|
41048638a2 |
||
|
|
9c0e3e7937 |
||
|
|
fe3dd8edb5 |
||
|
|
0d46089f9a |
||
|
|
7420c10a58 |
||
|
|
e40c84f31d |
||
|
|
994fc062cf |
||
|
|
e7681f6c79 |
||
|
|
19053339d9 |
||
|
|
b4e16c83e2 |
||
|
|
56cc89b521 |
||
|
|
1eab314b17 |
||
|
|
ec21336d45 |
||
|
|
e86e9b46b3 |
||
|
|
9b729b3d25 |
||
|
|
3c973e21f2 |
||
|
|
830e2f0a5b |
||
|
|
1620477a1c |
||
|
|
92b9a5218d |
||
|
|
9ed0d5ccec |
||
|
|
a6d1727205 |
||
|
|
3c3982464f |
||
|
|
bef73ff530 |
||
|
|
4d31c0b1de |
||
|
|
a5f28c21e4 |
||
|
|
c93ead7474 |
||
|
|
36880493cb |
||
|
|
e8518de054 |
||
|
|
b99e13e667 |
||
|
|
2518cf36d0 |
22 changed files with 1121 additions and 3032 deletions
46
.github/scripts/backend-js-footprint-loader.mjs
vendored
46
.github/scripts/backend-js-footprint-loader.mjs
vendored
|
|
@ -1,46 +0,0 @@
|
||||||
/*
|
|
||||||
* SPDX-FileCopyrightText: syuilo and misskey-project
|
|
||||||
* SPDX-License-Identifier: AGPL-3.0-only
|
|
||||||
*/
|
|
||||||
|
|
||||||
import { appendFileSync, statSync } from 'node:fs';
|
|
||||||
import { extname } from 'node:path';
|
|
||||||
import { fileURLToPath } from 'node:url';
|
|
||||||
|
|
||||||
const traceFile = process.env.MK_BACKEND_JS_FOOTPRINT_TRACE;
|
|
||||||
const jsExtensions = new Set(['.js', '.mjs', '.cjs']);
|
|
||||||
|
|
||||||
function recordLoadedFile(kind, url, format) {
|
|
||||||
if (traceFile == null || !url.startsWith('file:')) return;
|
|
||||||
|
|
||||||
let filePath;
|
|
||||||
try {
|
|
||||||
filePath = fileURLToPath(url);
|
|
||||||
} catch {
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
const extension = extname(filePath);
|
|
||||||
if (!jsExtensions.has(extension)) return;
|
|
||||||
|
|
||||||
let size = null;
|
|
||||||
try {
|
|
||||||
size = statSync(filePath).size;
|
|
||||||
} catch {
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
appendFileSync(traceFile, `${JSON.stringify({
|
|
||||||
kind,
|
|
||||||
format,
|
|
||||||
path: filePath,
|
|
||||||
size,
|
|
||||||
timestamp: Date.now(),
|
|
||||||
})}\n`);
|
|
||||||
}
|
|
||||||
|
|
||||||
export async function load(url, context, nextLoad) {
|
|
||||||
const result = await nextLoad(url, context);
|
|
||||||
recordLoadedFile('esm', url, result.format ?? context.format ?? null);
|
|
||||||
return result;
|
|
||||||
}
|
|
||||||
46
.github/scripts/backend-js-footprint-require.cjs
vendored
46
.github/scripts/backend-js-footprint-require.cjs
vendored
|
|
@ -1,46 +0,0 @@
|
||||||
/*
|
|
||||||
* SPDX-FileCopyrightText: syuilo and misskey-project
|
|
||||||
* SPDX-License-Identifier: AGPL-3.0-only
|
|
||||||
*/
|
|
||||||
|
|
||||||
'use strict';
|
|
||||||
|
|
||||||
const { appendFileSync, statSync } = require('node:fs');
|
|
||||||
const Module = require('node:module');
|
|
||||||
const { extname } = require('node:path');
|
|
||||||
|
|
||||||
const traceFile = process.env.MK_BACKEND_JS_FOOTPRINT_TRACE;
|
|
||||||
const jsExtensions = new Set(['.js', '.mjs', '.cjs']);
|
|
||||||
|
|
||||||
function recordLoadedFile(kind, filePath, request) {
|
|
||||||
if (traceFile == null || typeof filePath !== 'string') return;
|
|
||||||
|
|
||||||
const extension = extname(filePath);
|
|
||||||
if (!jsExtensions.has(extension) && extension !== '.node') return;
|
|
||||||
|
|
||||||
let size = null;
|
|
||||||
try {
|
|
||||||
size = statSync(filePath).size;
|
|
||||||
} catch {
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
appendFileSync(traceFile, `${JSON.stringify({
|
|
||||||
kind,
|
|
||||||
format: extension === '.node' ? 'native' : 'commonjs',
|
|
||||||
path: filePath,
|
|
||||||
request,
|
|
||||||
size,
|
|
||||||
timestamp: Date.now(),
|
|
||||||
})}\n`);
|
|
||||||
}
|
|
||||||
|
|
||||||
const originalLoad = Module._load;
|
|
||||||
const originalResolveFilename = Module._resolveFilename;
|
|
||||||
|
|
||||||
Module._load = function load(request, parent, isMain) {
|
|
||||||
const resolved = originalResolveFilename.call(this, request, parent, isMain);
|
|
||||||
const result = originalLoad.apply(this, arguments);
|
|
||||||
recordLoadedFile('cjs', resolved, request);
|
|
||||||
return result;
|
|
||||||
};
|
|
||||||
478
.github/scripts/backend-js-footprint.mjs
vendored
478
.github/scripts/backend-js-footprint.mjs
vendored
|
|
@ -1,478 +0,0 @@
|
||||||
/*
|
|
||||||
* SPDX-FileCopyrightText: syuilo and misskey-project
|
|
||||||
* SPDX-License-Identifier: AGPL-3.0-only
|
|
||||||
*/
|
|
||||||
|
|
||||||
import { fork, spawn } from 'node:child_process';
|
|
||||||
import { createRequire } from 'node:module';
|
|
||||||
import { cpus, tmpdir } from 'node:os';
|
|
||||||
import { dirname, extname, join, relative, resolve, sep } from 'node:path';
|
|
||||||
import { setTimeout } from 'node:timers/promises';
|
|
||||||
import { fileURLToPath, pathToFileURL } from 'node:url';
|
|
||||||
import { gzipSync } from 'node:zlib';
|
|
||||||
import * as fs from 'node:fs/promises';
|
|
||||||
import * as fsSync from 'node:fs';
|
|
||||||
import * as http from 'node:http';
|
|
||||||
import * as util from './utility.mts';
|
|
||||||
|
|
||||||
const __filename = fileURLToPath(import.meta.url);
|
|
||||||
const __dirname = dirname(__filename);
|
|
||||||
|
|
||||||
const [repoDirArg, outputFileArg] = process.argv.slice(2);
|
|
||||||
|
|
||||||
const STARTUP_TIMEOUT = util.readIntegerEnv('MK_JS_FOOTPRINT_STARTUP_TIMEOUT_MS', 120000, 1);
|
|
||||||
const SETTLE_TIME = util.readIntegerEnv('MK_JS_FOOTPRINT_SETTLE_TIME_MS', 10000, 0);
|
|
||||||
const REQUEST_COUNT = util.readIntegerEnv('MK_JS_FOOTPRINT_REQUEST_COUNT', 10, 0);
|
|
||||||
const MAX_TABLE_ITEMS = util.readIntegerEnv('MK_JS_FOOTPRINT_MAX_ITEMS', 20, 1);
|
|
||||||
|
|
||||||
const repoDir = resolve(repoDirArg);
|
|
||||||
const outputFile = resolve(outputFileArg);
|
|
||||||
const backendDir = join(repoDir, 'packages/backend');
|
|
||||||
const backendBuiltDir = join(backendDir, 'built');
|
|
||||||
const traceFile = join(tmpdir(), `misskey-backend-js-footprint-${process.pid}-${Date.now()}.jsonl`);
|
|
||||||
const require = createRequire(join(repoDir, 'package.json'));
|
|
||||||
const ts = require('typescript');
|
|
||||||
const jsExtensions = new Set(['.js', '.mjs', '.cjs']);
|
|
||||||
const fileMetricCache = new Map();
|
|
||||||
const packageInfoCache = new Map();
|
|
||||||
const nativePackageNames = new Set();
|
|
||||||
|
|
||||||
function isInside(parent, child) {
|
|
||||||
const rel = relative(parent, child);
|
|
||||||
return rel === '' || (!rel.startsWith('..') && !rel.includes(`..${sep}`));
|
|
||||||
}
|
|
||||||
|
|
||||||
function normalizePath(filePath) {
|
|
||||||
return filePath.split(sep).join('/');
|
|
||||||
}
|
|
||||||
|
|
||||||
function bytesToKiB(value) {
|
|
||||||
return Math.round(value / 1024);
|
|
||||||
}
|
|
||||||
|
|
||||||
async function resetState() {
|
|
||||||
const backendRequire = createRequire(join(backendDir, 'package.json'));
|
|
||||||
const pg = backendRequire('pg');
|
|
||||||
const Redis = backendRequire('ioredis');
|
|
||||||
|
|
||||||
const postgres = new pg.Client({
|
|
||||||
host: '127.0.0.1',
|
|
||||||
port: 54312,
|
|
||||||
database: 'postgres',
|
|
||||||
user: 'postgres',
|
|
||||||
});
|
|
||||||
|
|
||||||
await postgres.connect();
|
|
||||||
try {
|
|
||||||
await postgres.query('DROP DATABASE IF EXISTS "test-misskey" WITH (FORCE)');
|
|
||||||
await postgres.query('CREATE DATABASE "test-misskey"');
|
|
||||||
} finally {
|
|
||||||
await postgres.end();
|
|
||||||
}
|
|
||||||
|
|
||||||
const redis = new Redis({ host: '127.0.0.1', port: 56312 });
|
|
||||||
try {
|
|
||||||
await redis.flushall();
|
|
||||||
} finally {
|
|
||||||
redis.disconnect();
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
function createRequest() {
|
|
||||||
return new Promise((resolvePromise, reject) => {
|
|
||||||
const req = http.request({
|
|
||||||
host: 'localhost',
|
|
||||||
port: 61812,
|
|
||||||
path: '/api/meta',
|
|
||||||
method: 'POST',
|
|
||||||
}, res => {
|
|
||||||
res.on('data', () => { });
|
|
||||||
res.on('end', () => resolvePromise());
|
|
||||||
});
|
|
||||||
req.on('error', reject);
|
|
||||||
req.end();
|
|
||||||
});
|
|
||||||
}
|
|
||||||
|
|
||||||
async function waitForServerReady(serverProcess) {
|
|
||||||
let serverReady = false;
|
|
||||||
serverProcess.on('message', message => {
|
|
||||||
if (message === 'ok') serverReady = true;
|
|
||||||
});
|
|
||||||
|
|
||||||
const startupStartTime = Date.now();
|
|
||||||
while (!serverReady) {
|
|
||||||
if (Date.now() - startupStartTime > STARTUP_TIMEOUT) {
|
|
||||||
serverProcess.kill('SIGTERM');
|
|
||||||
throw new Error('Server startup timeout');
|
|
||||||
}
|
|
||||||
await setTimeout(100);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
async function stopServer(serverProcess) {
|
|
||||||
serverProcess.kill('SIGTERM');
|
|
||||||
|
|
||||||
let exited = false;
|
|
||||||
await new Promise(resolvePromise => {
|
|
||||||
serverProcess.on('exit', () => {
|
|
||||||
exited = true;
|
|
||||||
resolvePromise(undefined);
|
|
||||||
});
|
|
||||||
|
|
||||||
setTimeout(10000).then(() => {
|
|
||||||
if (!exited) serverProcess.kill('SIGKILL');
|
|
||||||
resolvePromise(undefined);
|
|
||||||
});
|
|
||||||
});
|
|
||||||
}
|
|
||||||
|
|
||||||
function getPackageNameFromPath(filePath) {
|
|
||||||
const normalized = normalizePath(filePath);
|
|
||||||
const marker = '/node_modules/';
|
|
||||||
const index = normalized.lastIndexOf(marker);
|
|
||||||
if (index === -1) return null;
|
|
||||||
|
|
||||||
const rest = normalized.slice(index + marker.length).split('/');
|
|
||||||
if (rest[0] === '.pnpm') {
|
|
||||||
const nestedNodeModulesIndex = rest.indexOf('node_modules');
|
|
||||||
if (nestedNodeModulesIndex === -1) return null;
|
|
||||||
const packageParts = rest.slice(nestedNodeModulesIndex + 1);
|
|
||||||
if (packageParts.length === 0) return null;
|
|
||||||
return packageParts[0].startsWith('@') ? packageParts.slice(0, 2).join('/') : packageParts[0];
|
|
||||||
}
|
|
||||||
|
|
||||||
return rest[0]?.startsWith('@') ? rest.slice(0, 2).join('/') : rest[0] ?? null;
|
|
||||||
}
|
|
||||||
|
|
||||||
function findPackageDir(filePath, packageName) {
|
|
||||||
const normalizedPackageName = packageName.split('/').join(sep);
|
|
||||||
let current = dirname(filePath);
|
|
||||||
|
|
||||||
while (current !== dirname(current)) {
|
|
||||||
if (current.endsWith(`${sep}${normalizedPackageName}`) && fsSync.existsSync(join(current, 'package.json'))) {
|
|
||||||
return current;
|
|
||||||
}
|
|
||||||
|
|
||||||
const parent = dirname(current);
|
|
||||||
if (parent === current) break;
|
|
||||||
current = parent;
|
|
||||||
}
|
|
||||||
|
|
||||||
return null;
|
|
||||||
}
|
|
||||||
|
|
||||||
function readPackageInfo(filePath) {
|
|
||||||
const externalPackageName = getPackageNameFromPath(filePath);
|
|
||||||
if (externalPackageName != null) {
|
|
||||||
const packageDir = findPackageDir(filePath, externalPackageName);
|
|
||||||
const cacheKey = packageDir ?? externalPackageName;
|
|
||||||
if (packageInfoCache.has(cacheKey)) return packageInfoCache.get(cacheKey);
|
|
||||||
|
|
||||||
let version = null;
|
|
||||||
if (packageDir != null) {
|
|
||||||
try {
|
|
||||||
const packageJson = JSON.parse(fsSync.readFileSync(join(packageDir, 'package.json'), 'utf8'));
|
|
||||||
version = typeof packageJson.version === 'string' ? packageJson.version : null;
|
|
||||||
} catch { }
|
|
||||||
}
|
|
||||||
|
|
||||||
const info = {
|
|
||||||
category: 'external',
|
|
||||||
name: externalPackageName,
|
|
||||||
version,
|
|
||||||
dir: packageDir,
|
|
||||||
};
|
|
||||||
packageInfoCache.set(cacheKey, info);
|
|
||||||
return info;
|
|
||||||
}
|
|
||||||
|
|
||||||
if (isInside(backendBuiltDir, filePath)) {
|
|
||||||
return {
|
|
||||||
category: 'internal',
|
|
||||||
name: 'backend',
|
|
||||||
version: null,
|
|
||||||
dir: backendDir,
|
|
||||||
};
|
|
||||||
}
|
|
||||||
|
|
||||||
return {
|
|
||||||
category: 'internal',
|
|
||||||
name: 'workspace',
|
|
||||||
version: null,
|
|
||||||
dir: repoDir,
|
|
||||||
};
|
|
||||||
}
|
|
||||||
|
|
||||||
function analyzeSource(filePath, source) {
|
|
||||||
const sourceFile = ts.createSourceFile(filePath, source, ts.ScriptTarget.Latest, true, ts.ScriptKind.JS);
|
|
||||||
const metrics = {
|
|
||||||
astNodeCount: 0,
|
|
||||||
functionCount: 0,
|
|
||||||
classCount: 0,
|
|
||||||
stringLiteralBytes: 0,
|
|
||||||
};
|
|
||||||
|
|
||||||
function visit(node) {
|
|
||||||
metrics.astNodeCount += 1;
|
|
||||||
|
|
||||||
if (
|
|
||||||
ts.isFunctionDeclaration(node) ||
|
|
||||||
ts.isFunctionExpression(node) ||
|
|
||||||
ts.isArrowFunction(node) ||
|
|
||||||
ts.isMethodDeclaration(node) ||
|
|
||||||
ts.isConstructorDeclaration(node) ||
|
|
||||||
ts.isGetAccessorDeclaration(node) ||
|
|
||||||
ts.isSetAccessorDeclaration(node)
|
|
||||||
) {
|
|
||||||
metrics.functionCount += 1;
|
|
||||||
} else if (ts.isClassDeclaration(node) || ts.isClassExpression(node)) {
|
|
||||||
metrics.classCount += 1;
|
|
||||||
} else if (ts.isStringLiteral(node) || ts.isNoSubstitutionTemplateLiteral(node)) {
|
|
||||||
metrics.stringLiteralBytes += Buffer.byteLength(node.text);
|
|
||||||
}
|
|
||||||
|
|
||||||
ts.forEachChild(node, visit);
|
|
||||||
}
|
|
||||||
|
|
||||||
visit(sourceFile);
|
|
||||||
return metrics;
|
|
||||||
}
|
|
||||||
|
|
||||||
function readFileMetrics(filePath) {
|
|
||||||
if (fileMetricCache.has(filePath)) return fileMetricCache.get(filePath);
|
|
||||||
|
|
||||||
const source = fsSync.readFileSync(filePath);
|
|
||||||
const sourceText = source.toString('utf8');
|
|
||||||
const astMetrics = analyzeSource(filePath, sourceText);
|
|
||||||
const packageInfo = readPackageInfo(filePath);
|
|
||||||
const metric = {
|
|
||||||
path: filePath,
|
|
||||||
displayPath: normalizePath(relative(repoDir, filePath)),
|
|
||||||
sourceBytes: source.byteLength,
|
|
||||||
gzipBytes: gzipSync(source).byteLength,
|
|
||||||
...astMetrics,
|
|
||||||
package: packageInfo,
|
|
||||||
};
|
|
||||||
|
|
||||||
fileMetricCache.set(filePath, metric);
|
|
||||||
return metric;
|
|
||||||
}
|
|
||||||
|
|
||||||
async function readTraceRecords() {
|
|
||||||
let content = '';
|
|
||||||
try {
|
|
||||||
content = await fs.readFile(traceFile, 'utf8');
|
|
||||||
} catch (err) {
|
|
||||||
if (err.code === 'ENOENT') return [];
|
|
||||||
throw err;
|
|
||||||
}
|
|
||||||
|
|
||||||
const records = [];
|
|
||||||
for (const line of content.split('\n')) {
|
|
||||||
if (line.trim() === '') continue;
|
|
||||||
try {
|
|
||||||
records.push(JSON.parse(line));
|
|
||||||
} catch { }
|
|
||||||
}
|
|
||||||
return records;
|
|
||||||
}
|
|
||||||
|
|
||||||
function emptyTotals() {
|
|
||||||
return {
|
|
||||||
loadedJsModules: 0,
|
|
||||||
loadedJsSourceBytes: 0,
|
|
||||||
loadedJsGzipBytes: 0,
|
|
||||||
astNodeCount: 0,
|
|
||||||
functionCount: 0,
|
|
||||||
classCount: 0,
|
|
||||||
stringLiteralBytes: 0,
|
|
||||||
externalPackageCount: 0,
|
|
||||||
nativeAddonPackageCount: 0,
|
|
||||||
};
|
|
||||||
}
|
|
||||||
|
|
||||||
function addFileMetrics(target, metric) {
|
|
||||||
target.loadedJsModules += 1;
|
|
||||||
target.loadedJsSourceBytes += metric.sourceBytes;
|
|
||||||
target.loadedJsGzipBytes += metric.gzipBytes;
|
|
||||||
target.astNodeCount += metric.astNodeCount;
|
|
||||||
target.functionCount += metric.functionCount;
|
|
||||||
target.classCount += metric.classCount;
|
|
||||||
target.stringLiteralBytes += metric.stringLiteralBytes;
|
|
||||||
}
|
|
||||||
|
|
||||||
function summarizeRecords(records, phase) {
|
|
||||||
const jsPaths = new Set();
|
|
||||||
const nativePaths = new Set();
|
|
||||||
|
|
||||||
for (const record of records) {
|
|
||||||
if (typeof record.path !== 'string') continue;
|
|
||||||
|
|
||||||
const extension = extname(record.path);
|
|
||||||
if (jsExtensions.has(extension)) {
|
|
||||||
jsPaths.add(resolve(record.path));
|
|
||||||
} else if (extension === '.node') {
|
|
||||||
nativePaths.add(resolve(record.path));
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
for (const nativePath of nativePaths) {
|
|
||||||
const packageInfo = readPackageInfo(nativePath);
|
|
||||||
if (packageInfo.category === 'external') nativePackageNames.add(packageInfo.name);
|
|
||||||
}
|
|
||||||
|
|
||||||
const totals = emptyTotals();
|
|
||||||
const packages = new Map();
|
|
||||||
const modules = [];
|
|
||||||
|
|
||||||
for (const filePath of [...jsPaths].toSorted()) {
|
|
||||||
let metric;
|
|
||||||
try {
|
|
||||||
metric = readFileMetrics(filePath);
|
|
||||||
} catch (err) {
|
|
||||||
process.stderr.write(`Failed to analyze ${filePath}: ${err.message}\n`);
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
|
|
||||||
addFileMetrics(totals, metric);
|
|
||||||
|
|
||||||
const packageKey = metric.package.name;
|
|
||||||
if (!packages.has(packageKey)) {
|
|
||||||
packages.set(packageKey, {
|
|
||||||
name: metric.package.name,
|
|
||||||
version: metric.package.version,
|
|
||||||
category: metric.package.category,
|
|
||||||
sourceBytes: 0,
|
|
||||||
gzipBytes: 0,
|
|
||||||
modules: 0,
|
|
||||||
astNodeCount: 0,
|
|
||||||
functionCount: 0,
|
|
||||||
classCount: 0,
|
|
||||||
stringLiteralBytes: 0,
|
|
||||||
nativeAddon: false,
|
|
||||||
});
|
|
||||||
}
|
|
||||||
|
|
||||||
const packageSummary = packages.get(packageKey);
|
|
||||||
packageSummary.sourceBytes += metric.sourceBytes;
|
|
||||||
packageSummary.gzipBytes += metric.gzipBytes;
|
|
||||||
packageSummary.modules += 1;
|
|
||||||
packageSummary.astNodeCount += metric.astNodeCount;
|
|
||||||
packageSummary.functionCount += metric.functionCount;
|
|
||||||
packageSummary.classCount += metric.classCount;
|
|
||||||
packageSummary.stringLiteralBytes += metric.stringLiteralBytes;
|
|
||||||
|
|
||||||
modules.push({
|
|
||||||
path: metric.displayPath,
|
|
||||||
package: metric.package.name,
|
|
||||||
category: metric.package.category,
|
|
||||||
sourceBytes: metric.sourceBytes,
|
|
||||||
gzipBytes: metric.gzipBytes,
|
|
||||||
astNodeCount: metric.astNodeCount,
|
|
||||||
functionCount: metric.functionCount,
|
|
||||||
classCount: metric.classCount,
|
|
||||||
stringLiteralBytes: metric.stringLiteralBytes,
|
|
||||||
});
|
|
||||||
}
|
|
||||||
|
|
||||||
for (const packageName of nativePackageNames) {
|
|
||||||
const packageSummary = packages.get(packageName);
|
|
||||||
if (packageSummary != null) packageSummary.nativeAddon = true;
|
|
||||||
}
|
|
||||||
|
|
||||||
const externalPackages = [...packages.values()].filter(packageSummary => packageSummary.category === 'external');
|
|
||||||
totals.externalPackageCount = externalPackages.length;
|
|
||||||
totals.nativeAddonPackageCount = externalPackages.filter(packageSummary => packageSummary.nativeAddon).length;
|
|
||||||
|
|
||||||
return {
|
|
||||||
totals: {
|
|
||||||
...totals,
|
|
||||||
loadedJsSourceKiB: bytesToKiB(totals.loadedJsSourceBytes),
|
|
||||||
loadedJsGzipKiB: bytesToKiB(totals.loadedJsGzipBytes),
|
|
||||||
stringLiteralKiB: bytesToKiB(totals.stringLiteralBytes),
|
|
||||||
},
|
|
||||||
packages: [...packages.values()].toSorted((a, b) => b.sourceBytes - a.sourceBytes),
|
|
||||||
modules: modules.toSorted((a, b) => b.sourceBytes - a.sourceBytes).slice(0, MAX_TABLE_ITEMS),
|
|
||||||
};
|
|
||||||
}
|
|
||||||
|
|
||||||
async function measureFootprint() {
|
|
||||||
await fs.writeFile(traceFile, '');
|
|
||||||
|
|
||||||
process.stderr.write('Resetting database and Redis\n');
|
|
||||||
await resetState();
|
|
||||||
|
|
||||||
process.stderr.write('Running migrations\n');
|
|
||||||
await util.run('pnpm', ['--filter', 'backend', 'migrate'], {
|
|
||||||
cwd: repoDir,
|
|
||||||
env: process.env,
|
|
||||||
logStdout: true,
|
|
||||||
});
|
|
||||||
|
|
||||||
const serverProcess = fork(join(backendBuiltDir, 'entry.js'), [], {
|
|
||||||
cwd: backendDir,
|
|
||||||
env: {
|
|
||||||
...process.env,
|
|
||||||
NODE_ENV: 'production',
|
|
||||||
MK_DISABLE_CLUSTERING: '1',
|
|
||||||
MK_ONLY_SERVER: '1',
|
|
||||||
MK_NO_DAEMONS: '1',
|
|
||||||
MK_BACKEND_JS_FOOTPRINT_TRACE: traceFile,
|
|
||||||
},
|
|
||||||
stdio: ['pipe', 'pipe', 'pipe', 'ipc'],
|
|
||||||
execArgv: [
|
|
||||||
'--require',
|
|
||||||
join(__dirname, 'backend-js-footprint-require.cjs'),
|
|
||||||
'--experimental-loader',
|
|
||||||
pathToFileURL(join(__dirname, 'backend-js-footprint-loader.mjs')).href,
|
|
||||||
],
|
|
||||||
});
|
|
||||||
|
|
||||||
serverProcess.stdout?.on('data', data => {
|
|
||||||
process.stderr.write(`[server stdout] ${data}`);
|
|
||||||
});
|
|
||||||
|
|
||||||
serverProcess.stderr?.on('data', data => {
|
|
||||||
process.stderr.write(`[server stderr] ${data}`);
|
|
||||||
});
|
|
||||||
|
|
||||||
serverProcess.on('error', err => {
|
|
||||||
process.stderr.write(`[server error] ${err}\n`);
|
|
||||||
});
|
|
||||||
|
|
||||||
try {
|
|
||||||
await waitForServerReady(serverProcess);
|
|
||||||
await setTimeout(SETTLE_TIME);
|
|
||||||
|
|
||||||
//const startup = summarizeRecords(await readTraceRecords(), 'startup');
|
|
||||||
|
|
||||||
await Promise.all(
|
|
||||||
Array.from({ length: REQUEST_COUNT }).map(() => createRequest()),
|
|
||||||
);
|
|
||||||
await setTimeout(1000);
|
|
||||||
|
|
||||||
const afterRequest = summarizeRecords(await readTraceRecords(), 'afterRequest');
|
|
||||||
|
|
||||||
return {
|
|
||||||
timestamp: new Date().toISOString(),
|
|
||||||
measurement: {
|
|
||||||
strategy: 'runtime-loader-trace',
|
|
||||||
startupTimeoutMs: STARTUP_TIMEOUT,
|
|
||||||
settleTimeMs: SETTLE_TIME,
|
|
||||||
requestCount: REQUEST_COUNT,
|
|
||||||
cpus: cpus().length,
|
|
||||||
},
|
|
||||||
phases: {
|
|
||||||
//startup,
|
|
||||||
afterRequest,
|
|
||||||
},
|
|
||||||
};
|
|
||||||
} finally {
|
|
||||||
await stopServer(serverProcess);
|
|
||||||
await fs.rm(traceFile, { force: true });
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
const result = await measureFootprint();
|
|
||||||
await fs.writeFile(outputFile, `${JSON.stringify(result, null, 2)}\n`);
|
|
||||||
673
.github/scripts/backend-memory-report.mts
vendored
673
.github/scripts/backend-memory-report.mts
vendored
|
|
@ -1,673 +0,0 @@
|
||||||
/*
|
|
||||||
* SPDX-FileCopyrightText: syuilo and misskey-project
|
|
||||||
* SPDX-License-Identifier: AGPL-3.0-only
|
|
||||||
*/
|
|
||||||
|
|
||||||
import { readFile, writeFile } from 'node:fs/promises';
|
|
||||||
import * as util from './utility.mts';
|
|
||||||
import type { MemoryReport } from './measure-backend-memory-comparison.mts';
|
|
||||||
|
|
||||||
const [baseFile, headFile, outputFile, baseJsFootprintFile, headJsFootprintFile] = process.argv.slice(2);
|
|
||||||
|
|
||||||
type RuntimeLoadedJsFootprintReport = {
|
|
||||||
phases: Record<'afterRequest', {
|
|
||||||
totals: {
|
|
||||||
loadedJsModules: number;
|
|
||||||
loadedJsSourceBytes: number;
|
|
||||||
loadedJsGzipBytes: number;
|
|
||||||
astNodeCount: number;
|
|
||||||
functionCount: number;
|
|
||||||
classCount: number;
|
|
||||||
stringLiteralBytes: number;
|
|
||||||
externalPackageCount: number;
|
|
||||||
nativeAddonPackageCount: number;
|
|
||||||
};
|
|
||||||
modules: {
|
|
||||||
path: string;
|
|
||||||
package: string;
|
|
||||||
category: string;
|
|
||||||
sourceBytes: number;
|
|
||||||
gzipBytes: number;
|
|
||||||
astNodeCount: number;
|
|
||||||
functionCount: number;
|
|
||||||
classCount: number;
|
|
||||||
stringLiteralBytes: number;
|
|
||||||
}[];
|
|
||||||
}>;
|
|
||||||
};
|
|
||||||
|
|
||||||
const memoryReportPhases = [
|
|
||||||
{
|
|
||||||
key: 'afterGc',
|
|
||||||
title: 'After GC',
|
|
||||||
},
|
|
||||||
] as const;
|
|
||||||
|
|
||||||
const metrics = [
|
|
||||||
'HeapUsed',
|
|
||||||
'Pss',
|
|
||||||
'Private_Dirty',
|
|
||||||
'VmRSS',
|
|
||||||
'External',
|
|
||||||
] as const;
|
|
||||||
|
|
||||||
const heapSnapshotCategoriesColors = {
|
|
||||||
'Total': 'gray',
|
|
||||||
'Code': 'orange',
|
|
||||||
'Strings': 'red',
|
|
||||||
'JS arrays': 'cyan',
|
|
||||||
'Typed arrays': 'green',
|
|
||||||
'System objects': 'yellow',
|
|
||||||
'Other JS objects': 'violet',
|
|
||||||
'Other non-JS objects': 'pink',
|
|
||||||
} as const;
|
|
||||||
|
|
||||||
const heapSnapshotCategoriesColorsHex = {
|
|
||||||
'Total': '#888888',
|
|
||||||
'Code': '#f28e2c',
|
|
||||||
'Strings': '#e15759',
|
|
||||||
'JS arrays': '#76b7b2',
|
|
||||||
'Typed arrays': '#59a14f',
|
|
||||||
'System objects': '#edc949',
|
|
||||||
'Other JS objects': '#af7aa1',
|
|
||||||
'Other non-JS objects': '#ff9da7',
|
|
||||||
} as const;
|
|
||||||
|
|
||||||
function formatMemoryMb(valueKiB: number | null | undefined) {
|
|
||||||
if (valueKiB == null) return '-';
|
|
||||||
return `${util.formatNumber(valueKiB / 1024)} MB`;
|
|
||||||
}
|
|
||||||
|
|
||||||
function getMemoryValue(report: MemoryReport, phase: typeof memoryReportPhases[number]['key'], metric: typeof metrics[number]) {
|
|
||||||
return report.summary[phase].memoryUsage[metric];
|
|
||||||
}
|
|
||||||
|
|
||||||
function getMemoryValueFromSample(sample: MemoryReport['samples'][number], phase: typeof memoryReportPhases[number]['key'], metric: typeof metrics[number]) {
|
|
||||||
return sample.phases[phase].memoryUsage[metric];
|
|
||||||
}
|
|
||||||
|
|
||||||
function getSampleSpread(report: MemoryReport, phase: typeof memoryReportPhases[number]['key'], metric: typeof metrics[number]) {
|
|
||||||
const values = report.samples.map(sample => getMemoryValueFromSample(sample, phase, metric));
|
|
||||||
if (values.length < 2) return null;
|
|
||||||
|
|
||||||
const center = util.median(values);
|
|
||||||
return util.median(values.map(value => Math.abs(value - center)));
|
|
||||||
}
|
|
||||||
|
|
||||||
function getSamplesByRound(report: MemoryReport) {
|
|
||||||
const samplesByRound = new Map<number, MemoryReport['samples'][number]>();
|
|
||||||
if (!Array.isArray(report.samples)) return samplesByRound;
|
|
||||||
|
|
||||||
for (const sample of report.samples) {
|
|
||||||
if (sample.round <= 0) continue;
|
|
||||||
samplesByRound.set(sample.round, sample);
|
|
||||||
}
|
|
||||||
|
|
||||||
return samplesByRound;
|
|
||||||
}
|
|
||||||
|
|
||||||
function formatDeltaMemory(diffKiB: number) {
|
|
||||||
return util.formatColoredDelta(formatMemoryMb(Math.abs(diffKiB)), diffKiB);
|
|
||||||
}
|
|
||||||
|
|
||||||
function pairedDeltaSummary(base: MemoryReport, head: MemoryReport, phase: typeof memoryReportPhases[number]['key'], metric: typeof metrics[number]) {
|
|
||||||
const baseSamplesByRound = getSamplesByRound(base);
|
|
||||||
const headSamplesByRound = getSamplesByRound(head);
|
|
||||||
const values = [];
|
|
||||||
|
|
||||||
for (const [round, baseSample] of baseSamplesByRound) {
|
|
||||||
const headSample = headSamplesByRound.get(round);
|
|
||||||
if (headSample == null) continue;
|
|
||||||
|
|
||||||
const baseValue = getMemoryValueFromSample(baseSample, phase, metric);
|
|
||||||
const headValue = getMemoryValueFromSample(headSample, phase, metric);
|
|
||||||
if (baseValue == null || headValue == null) continue;
|
|
||||||
|
|
||||||
values.push(headValue - baseValue);
|
|
||||||
}
|
|
||||||
|
|
||||||
return {
|
|
||||||
median: util.median(values),
|
|
||||||
mad: util.mad(values),
|
|
||||||
min: Math.min(...values),
|
|
||||||
max: Math.max(...values),
|
|
||||||
samples: values.length,
|
|
||||||
};
|
|
||||||
}
|
|
||||||
|
|
||||||
function renderMainTableForPhase(base: MemoryReport, head: MemoryReport, phase: typeof memoryReportPhases[number]['key']) {
|
|
||||||
const lines = [
|
|
||||||
'| Metric | Base | Head | Δ median | Δ MAD | Δ min | Δ max |',
|
|
||||||
'| --- | ---: | ---: | ---: | ---: | ---: | ---: |',
|
|
||||||
];
|
|
||||||
|
|
||||||
for (const metric of metrics) {
|
|
||||||
const baseValue = getMemoryValue(base, phase, metric);
|
|
||||||
const headValue = getMemoryValue(head, phase, metric);
|
|
||||||
|
|
||||||
const baseSpread = getSampleSpread(base, phase, metric);
|
|
||||||
const headSpread = getSampleSpread(head, phase, metric);
|
|
||||||
const summary = pairedDeltaSummary(base, head, phase, metric);
|
|
||||||
const percent = summary.median * 100 / baseValue;
|
|
||||||
const deltaMedian = summary == null ? '-' : `${formatDeltaMemory(summary.median)}<br>${util.formatDeltaPercent(percent)}`;
|
|
||||||
|
|
||||||
lines.push(`| **${metric}** | ${formatMemoryMb(baseValue)} <br> ± ${formatMemoryMb(baseSpread)} | ${formatMemoryMb(headValue)} <br> ± ${formatMemoryMb(headSpread)} | ${deltaMedian} | ${summary?.mad == null ? '-' : formatMemoryMb(summary.mad)} | ${summary == null ? '-' : formatDeltaMemory(summary.min)} | ${summary == null ? '-' : formatDeltaMemory(summary.max)} |`);
|
|
||||||
}
|
|
||||||
|
|
||||||
return lines.join('\n');
|
|
||||||
}
|
|
||||||
|
|
||||||
function getDiffPercent(base: MemoryReport, head: MemoryReport, phase: typeof memoryReportPhases[number]['key'], metric: typeof metrics[number]) {
|
|
||||||
const baseValue = getMemoryValue(base, phase, metric);
|
|
||||||
const headValue = getMemoryValue(head, phase, metric);
|
|
||||||
if (baseValue == null || headValue == null || baseValue <= 0) return null;
|
|
||||||
|
|
||||||
return ((headValue - baseValue) * 100) / baseValue;
|
|
||||||
}
|
|
||||||
|
|
||||||
/*
|
|
||||||
function measurementSummary(base, head) {
|
|
||||||
const baseCount = base?.sampleCount;
|
|
||||||
const headCount = head?.sampleCount;
|
|
||||||
const strategy = base?.comparison?.strategy;
|
|
||||||
if (baseCount == null || headCount == null) return null;
|
|
||||||
|
|
||||||
if (strategy === 'interleaved-pairs') {
|
|
||||||
const rounds = base?.comparison?.rounds ?? baseCount;
|
|
||||||
const warmupRounds = base?.comparison?.warmupRounds ?? 0;
|
|
||||||
return `_Measured as ${rounds} interleaved base/head pairs after ${warmupRounds} warmup pair(s). Values are medians; ± is median absolute deviation._`;
|
|
||||||
}
|
|
||||||
|
|
||||||
return `_Sample count: base ${baseCount}, head ${headCount}. Values are medians; ± is median absolute deviation._`;
|
|
||||||
}
|
|
||||||
*/
|
|
||||||
|
|
||||||
function formatPlainDelta(baseValue: number, headValue: number, formatter = util.formatNumber) {
|
|
||||||
const delta = headValue - baseValue;
|
|
||||||
if (delta === 0) return formatter(0);
|
|
||||||
|
|
||||||
const sign = delta > 0 ? '+' : '-';
|
|
||||||
return `${sign}${formatter(Math.abs(delta))}`;
|
|
||||||
}
|
|
||||||
|
|
||||||
function getHeapSnapshotCategoryValue(report: MemoryReport, phase: typeof memoryReportPhases[number]['key'], category: typeof util.heapSnapshotCategories[number]) {
|
|
||||||
const value = report.summary[phase]?.heapSnapshot?.categories?.[category];
|
|
||||||
return Number.isFinite(value) ? value : null;
|
|
||||||
}
|
|
||||||
|
|
||||||
function getHeapSnapshotCategoryValueFromSample(sample: MemoryReport['samples'][number], phase: typeof memoryReportPhases[number]['key'], category: typeof util.heapSnapshotCategories[number]) {
|
|
||||||
const value = sample.phases[phase]?.heapSnapshot?.categories?.[category];
|
|
||||||
return Number.isFinite(value) ? value : null;
|
|
||||||
}
|
|
||||||
|
|
||||||
const heapSnapshotSankeyChildMinRatio = 0.3;
|
|
||||||
const heapSnapshotSankeyParentMinPercent = 10;
|
|
||||||
|
|
||||||
function escapeCsvValue(value: string) {
|
|
||||||
return `"${String(value).replaceAll('"', '""')}"`;
|
|
||||||
}
|
|
||||||
|
|
||||||
function formatSankeyPercentValue(value: number) {
|
|
||||||
const rounded = Math.round(value * 100) / 100;
|
|
||||||
if (rounded === 0 && value > 0) return '0.01';
|
|
||||||
if (Number.isInteger(rounded)) return String(rounded);
|
|
||||||
return rounded.toFixed(2).replace(/0+$/, '').replace(/\.$/, '');
|
|
||||||
}
|
|
||||||
|
|
||||||
function formatHeapSnapshotSankeyChildLabel(label: string) {
|
|
||||||
return String(label).replace(/^[^:]+:\s*/, '');
|
|
||||||
}
|
|
||||||
|
|
||||||
function renderHeapSnapshotSankey(report: MemoryReport, phase: typeof memoryReportPhases[number]['key'], title: string) {
|
|
||||||
const total = getHeapSnapshotCategoryValue(report, phase, 'Total');
|
|
||||||
if (total == null || total <= 0) return null;
|
|
||||||
|
|
||||||
function getHeapSnapshotBreakdownEntries(category: typeof util.heapSnapshotCategories[number]) {
|
|
||||||
const breakdown = report.summary[phase].heapSnapshot?.breakdowns?.[category];
|
|
||||||
if (breakdown == null || typeof breakdown !== 'object') return [];
|
|
||||||
|
|
||||||
return Object.entries(breakdown)
|
|
||||||
.filter(([, value]) => Number.isFinite(value) && value > 0)
|
|
||||||
.toSorted((a, b) => b[1] - a[1]);
|
|
||||||
}
|
|
||||||
|
|
||||||
const categories = util.heapSnapshotCategories
|
|
||||||
.filter(category => category !== 'Total')
|
|
||||||
.map(category => {
|
|
||||||
const value = getHeapSnapshotCategoryValue(report, phase, category);
|
|
||||||
if (value == null || value <= 0) return null;
|
|
||||||
const breakdownEntries = getHeapSnapshotBreakdownEntries(category);
|
|
||||||
const breakdownTotal = breakdownEntries.reduce((sum, [, childValue]) => sum + childValue, 0);
|
|
||||||
const percent = (value * 100) / total;
|
|
||||||
const childEntries = [];
|
|
||||||
let otherPercent = 0;
|
|
||||||
|
|
||||||
if (breakdownTotal > 0 && percent > heapSnapshotSankeyParentMinPercent) {
|
|
||||||
for (const [childName, childValue] of breakdownEntries) {
|
|
||||||
const childRatio = childValue / breakdownTotal;
|
|
||||||
const childPercent = percent * childRatio;
|
|
||||||
if (childRatio >= heapSnapshotSankeyChildMinRatio) {
|
|
||||||
childEntries.push([formatHeapSnapshotSankeyChildLabel(childName), childPercent]);
|
|
||||||
} else {
|
|
||||||
otherPercent += childPercent;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
if (childEntries.length > 0 && otherPercent > 0) {
|
|
||||||
childEntries.push(['Other', otherPercent]);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return {
|
|
||||||
category,
|
|
||||||
percent,
|
|
||||||
childEntries,
|
|
||||||
};
|
|
||||||
})
|
|
||||||
.filter(value => value != null);
|
|
||||||
|
|
||||||
if (categories.length === 0) return null;
|
|
||||||
|
|
||||||
const nodeColors = {
|
|
||||||
[title]: heapSnapshotCategoriesColorsHex.Total,
|
|
||||||
} as Record<string, string>;
|
|
||||||
for (const { category, childEntries } of categories) {
|
|
||||||
const categoryColor = heapSnapshotCategoriesColorsHex[category] ?? heapSnapshotCategoriesColorsHex.Total;
|
|
||||||
nodeColors[category] = categoryColor;
|
|
||||||
|
|
||||||
for (const [childName] of childEntries) {
|
|
||||||
nodeColors[childName] = categoryColor;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
const lines = [
|
|
||||||
`<details><summary>${title} heap snapshot composition</summary>`,
|
|
||||||
'',
|
|
||||||
'```mermaid',
|
|
||||||
`%%{init: ${JSON.stringify({
|
|
||||||
sankey: {
|
|
||||||
showValues: false,
|
|
||||||
linkColor: 'target',
|
|
||||||
labelStyle: 'outlined',
|
|
||||||
nodeAlignment: 'center',
|
|
||||||
nodePadding: 10,
|
|
||||||
nodeColors: {
|
|
||||||
...nodeColors,
|
|
||||||
'Other': '#888888',
|
|
||||||
},
|
|
||||||
},
|
|
||||||
})}}%%`,
|
|
||||||
'sankey-beta',
|
|
||||||
];
|
|
||||||
|
|
||||||
for (const { category, percent, childEntries } of categories) {
|
|
||||||
lines.push(`${escapeCsvValue(title)},${escapeCsvValue(category)},${formatSankeyPercentValue(percent)}`);
|
|
||||||
|
|
||||||
for (const [childName, childPercent] of childEntries) {
|
|
||||||
lines.push(`${escapeCsvValue(category)},${escapeCsvValue(childName)},${formatSankeyPercentValue(childPercent)}`);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
lines.push('```');
|
|
||||||
lines.push('');
|
|
||||||
lines.push('</details>');
|
|
||||||
|
|
||||||
return lines.join('\n');
|
|
||||||
}
|
|
||||||
|
|
||||||
function pairedHeapSnapshotDeltaSummary(base: MemoryReport, head: MemoryReport, phase: typeof memoryReportPhases[number]['key'], category: typeof util.heapSnapshotCategories[number]) {
|
|
||||||
const baseSamplesByRound = getSamplesByRound(base);
|
|
||||||
const headSamplesByRound = getSamplesByRound(head);
|
|
||||||
const values = [] as number[];
|
|
||||||
|
|
||||||
for (const [round, baseSample] of baseSamplesByRound) {
|
|
||||||
const headSample = headSamplesByRound.get(round);
|
|
||||||
if (headSample == null) continue;
|
|
||||||
|
|
||||||
const baseValue = getHeapSnapshotCategoryValueFromSample(baseSample, phase, category);
|
|
||||||
const headValue = getHeapSnapshotCategoryValueFromSample(headSample, phase, category);
|
|
||||||
if (baseValue == null || headValue == null) continue;
|
|
||||||
|
|
||||||
values.push(headValue - baseValue);
|
|
||||||
}
|
|
||||||
|
|
||||||
return {
|
|
||||||
median: util.median(values),
|
|
||||||
mad: util.mad(values),
|
|
||||||
min: Math.min(...values),
|
|
||||||
max: Math.max(...values),
|
|
||||||
samples: values.length,
|
|
||||||
};
|
|
||||||
}
|
|
||||||
|
|
||||||
function renderHeapSnapshotTable(base: MemoryReport, head: MemoryReport, phase: typeof memoryReportPhases[number]['key']) {
|
|
||||||
const lines = [
|
|
||||||
'| Metric | Base | Head | Δ median | Δ MAD | Δ min | Δ max |',
|
|
||||||
'| --- | ---: | ---: | ---: | ---: | ---: | ---: |',
|
|
||||||
];
|
|
||||||
const baseTotal = getHeapSnapshotCategoryValue(base, phase, 'Total');
|
|
||||||
const headTotal = getHeapSnapshotCategoryValue(head, phase, 'Total');
|
|
||||||
|
|
||||||
function formatHeapSnapshotCategoryLabel(category: typeof heapSnapshotCategories[number], baseValue: number, headValue: number, baseTotal: number, headTotal: number) {
|
|
||||||
if (category === 'Total' || baseTotal == null || headTotal == null || baseTotal <= 0 || headTotal <= 0) return `**${category}**`;
|
|
||||||
|
|
||||||
const basePercent = util.formatPercent((baseValue * 100) / baseTotal);
|
|
||||||
const headPercent = util.formatPercent((headValue * 100) / headTotal);
|
|
||||||
return `**${category}**<br>${basePercent} → ${headPercent}`;
|
|
||||||
}
|
|
||||||
|
|
||||||
function getHeapSnapshotSampleSpread(report: MemoryReport, phase: typeof memoryReportPhases[number]['key'], category: typeof util.heapSnapshotCategories[number]) {
|
|
||||||
const values = report.samples
|
|
||||||
.map(sample => getHeapSnapshotCategoryValueFromSample(sample, phase, category))
|
|
||||||
.filter(value => Number.isFinite(value)) as number[];
|
|
||||||
if (values.length < 2) return null;
|
|
||||||
|
|
||||||
const center = util.median(values);
|
|
||||||
return util.median(values.map(value => Math.abs(value - center)));
|
|
||||||
}
|
|
||||||
|
|
||||||
for (const category of util.heapSnapshotCategories) {
|
|
||||||
const baseValue = getHeapSnapshotCategoryValue(base, phase, category);
|
|
||||||
const headValue = getHeapSnapshotCategoryValue(head, phase, category);
|
|
||||||
if (baseValue == null || headValue == null) continue;
|
|
||||||
|
|
||||||
const baseSpread = getHeapSnapshotSampleSpread(base, phase, category);
|
|
||||||
const headSpread = getHeapSnapshotSampleSpread(head, phase, category);
|
|
||||||
const summary = pairedHeapSnapshotDeltaSummary(base, head, phase, category);
|
|
||||||
const percent = summary.median * 100 / baseValue;
|
|
||||||
const deltaMedian = summary == null ? '-' : `${util.formatDeltaBytes(summary.median)}<br>${util.formatDeltaPercent(percent)}`;
|
|
||||||
const categoryLabel = formatHeapSnapshotCategoryLabel(category, baseValue, headValue, baseTotal, headTotal);
|
|
||||||
|
|
||||||
lines.push(`| $\\color{${heapSnapshotCategoriesColors[category]}}{\\rule{8pt}{8pt}}$ ${categoryLabel} | ${util.formatBytes(baseValue)} <br> ± ${baseSpread == null ? '-' : util.formatBytes(baseSpread)} | ${util.formatBytes(headValue)} <br> ± ${headSpread == null ? '-' : util.formatBytes(headSpread)} | ${deltaMedian} | ${summary?.mad == null ? '-' : util.formatBytes(summary.mad)} | ${summary == null ? '-' : util.formatDeltaBytes(summary.min)} | ${summary == null ? '-' : util.formatDeltaBytes(summary.max)} |`);
|
|
||||||
if (category === 'Total') {
|
|
||||||
lines.push('| | | | | | | |');
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
if (lines.length === 2) return null;
|
|
||||||
return lines.join('\n');
|
|
||||||
}
|
|
||||||
|
|
||||||
function renderHeapSnapshotSection(base: MemoryReport, head: MemoryReport) {
|
|
||||||
const table = renderHeapSnapshotTable(base, head, 'afterGc');
|
|
||||||
if (table == null) return null;
|
|
||||||
|
|
||||||
const lines = [
|
|
||||||
'### V8 Heap Snapshot Statistics',
|
|
||||||
'',
|
|
||||||
table,
|
|
||||||
'',
|
|
||||||
];
|
|
||||||
|
|
||||||
for (const graph of [
|
|
||||||
renderHeapSnapshotSankey(base, 'afterGc', 'Base'),
|
|
||||||
renderHeapSnapshotSankey(head, 'afterGc', 'Head'),
|
|
||||||
]) {
|
|
||||||
if (graph == null) continue;
|
|
||||||
lines.push(graph);
|
|
||||||
lines.push('');
|
|
||||||
}
|
|
||||||
|
|
||||||
return lines.join('\n');
|
|
||||||
}
|
|
||||||
|
|
||||||
function getJsFootprintValue(report: RuntimeLoadedJsFootprintReport, phase: 'afterRequest', key: keyof RuntimeLoadedJsFootprintReport['phases'][typeof phase]['totals']) {
|
|
||||||
const value = report.phases[phase].totals[key];
|
|
||||||
return Number.isFinite(value) ? value : null;
|
|
||||||
}
|
|
||||||
|
|
||||||
function renderJsFootprintMetricTable(base: RuntimeLoadedJsFootprintReport, head: RuntimeLoadedJsFootprintReport) {
|
|
||||||
const metricRows = [
|
|
||||||
['Loaded JS modules', 'loadedJsModules', util.formatNumber],
|
|
||||||
['Loaded JS source', 'loadedJsSourceBytes', util.formatBytes],
|
|
||||||
//['Loaded JS gzip estimate', 'loadedJsGzipBytes', util.formatBytes],
|
|
||||||
//['AST nodes', 'astNodeCount', util.formatNumber],
|
|
||||||
//['Functions', 'functionCount', util.formatNumber],
|
|
||||||
//['Classes', 'classCount', util.formatNumber],
|
|
||||||
//['String literals', 'stringLiteralBytes', util.formatBytes],
|
|
||||||
['External packages loaded', 'externalPackageCount', util.formatNumber],
|
|
||||||
['Native addon packages', 'nativeAddonPackageCount', util.formatNumber],
|
|
||||||
] as const;
|
|
||||||
|
|
||||||
const lines = [
|
|
||||||
'| Metric | Base | Head | Δ | Δ (%) |',
|
|
||||||
'| --- | ---: | ---: | ---: | ---: |',
|
|
||||||
];
|
|
||||||
|
|
||||||
for (const [title, key, formatter] of metricRows) {
|
|
||||||
const baseValue = getJsFootprintValue(base, 'afterRequest', key);
|
|
||||||
const headValue = getJsFootprintValue(head, 'afterRequest', key);
|
|
||||||
if (baseValue == null || headValue == null) continue;
|
|
||||||
|
|
||||||
lines.push(`| **${title}** | ${formatter(baseValue)} | ${formatter(headValue)} | ${formatPlainDelta(baseValue, headValue, formatter)} | ${util.calcAndFormatDeltaPercent(baseValue, headValue)} |`);
|
|
||||||
}
|
|
||||||
|
|
||||||
return lines.join('\n');
|
|
||||||
}
|
|
||||||
|
|
||||||
/*
|
|
||||||
function renderJsFootprintPhaseTable(base, head) {
|
|
||||||
const lines = [
|
|
||||||
'| Phase | Base modules | Head modules | Δ modules | Base source | Head source | Δ source |',
|
|
||||||
'| --- | ---: | ---: | ---: | ---: | ---: | ---: |',
|
|
||||||
];
|
|
||||||
|
|
||||||
for (const [phase, title] of [['startup', 'Startup'], ['afterRequest', 'After warmup requests']]) {
|
|
||||||
const baseModules = getJsFootprintValue(base, phase, 'loadedJsModules');
|
|
||||||
const headModules = getJsFootprintValue(head, phase, 'loadedJsModules');
|
|
||||||
const baseSource = getJsFootprintValue(base, phase, 'loadedJsSourceBytes');
|
|
||||||
const headSource = getJsFootprintValue(head, phase, 'loadedJsSourceBytes');
|
|
||||||
if (baseModules == null || headModules == null || baseSource == null || headSource == null) continue;
|
|
||||||
|
|
||||||
lines.push(`| ${title} | ${util.formatNumber(baseModules)} | ${util.formatNumber(headModules)} | ${formatPlainDelta(baseModules, headModules)} | ${util.formatBytes(baseSource)} | ${util.formatBytes(headSource)} | ${formatPlainDelta(baseSource, headSource, util.formatBytes)} |`);
|
|
||||||
}
|
|
||||||
|
|
||||||
return lines.join('\n');
|
|
||||||
}
|
|
||||||
*/
|
|
||||||
|
|
||||||
function packageMap(report: RuntimeLoadedJsFootprintReport) {
|
|
||||||
const map = new Map();
|
|
||||||
for (const packageSummary of report.phases.afterRequest.packages) {
|
|
||||||
if (packageSummary?.category !== 'external' || typeof packageSummary.name !== 'string') continue;
|
|
||||||
map.set(packageSummary.name, packageSummary);
|
|
||||||
}
|
|
||||||
return map;
|
|
||||||
}
|
|
||||||
|
|
||||||
function packageDisplayName(packageSummary: { name: string; version?: string | null }) {
|
|
||||||
if (packageSummary.version == null) return packageSummary.name;
|
|
||||||
return `${packageSummary.name} ${packageSummary.version}`;
|
|
||||||
}
|
|
||||||
|
|
||||||
function renderNewExternalPackages(base: RuntimeLoadedJsFootprintReport, head: RuntimeLoadedJsFootprintReport) {
|
|
||||||
const basePackages = packageMap(base);
|
|
||||||
const headPackages = packageMap(head);
|
|
||||||
const newPackages = [...headPackages.values()]
|
|
||||||
.filter(packageSummary => !basePackages.has(packageSummary.name))
|
|
||||||
.toSorted((a, b) => b.sourceBytes - a.sourceBytes)
|
|
||||||
.slice(0, 10);
|
|
||||||
|
|
||||||
if (newPackages.length === 0) return null;
|
|
||||||
|
|
||||||
const lines = [
|
|
||||||
'#### Newly Loaded External Packages',
|
|
||||||
'',
|
|
||||||
'| Package | Loaded JS | Modules | Notes |',
|
|
||||||
'| --- | ---: | ---: | --- |',
|
|
||||||
];
|
|
||||||
|
|
||||||
for (const packageSummary of newPackages) {
|
|
||||||
lines.push(`| ${packageDisplayName(packageSummary)} | ${util.formatBytes(packageSummary.sourceBytes)} | ${util.formatNumber(packageSummary.modules)} | ${packageSummary.nativeAddon ? 'native addon' : ''} |`);
|
|
||||||
}
|
|
||||||
|
|
||||||
return lines.join('\n');
|
|
||||||
}
|
|
||||||
|
|
||||||
function renderLargestPackageIncreases(base: RuntimeLoadedJsFootprintReport, head: RuntimeLoadedJsFootprintReport) {
|
|
||||||
const basePackages = packageMap(base);
|
|
||||||
const headPackages = packageMap(head);
|
|
||||||
const increases = [...headPackages.values()]
|
|
||||||
.map(headPackage => {
|
|
||||||
const basePackage = basePackages.get(headPackage.name);
|
|
||||||
const baseSourceBytes = basePackage?.sourceBytes ?? 0;
|
|
||||||
const baseModules = basePackage?.modules ?? 0;
|
|
||||||
return {
|
|
||||||
...headPackage,
|
|
||||||
baseSourceBytes,
|
|
||||||
baseModules,
|
|
||||||
sourceDiff: headPackage.sourceBytes - baseSourceBytes,
|
|
||||||
moduleDiff: headPackage.modules - baseModules,
|
|
||||||
};
|
|
||||||
})
|
|
||||||
.filter(packageSummary => packageSummary.sourceDiff > 0)
|
|
||||||
.toSorted((a, b) => b.sourceDiff - a.sourceDiff)
|
|
||||||
.slice(0, 10);
|
|
||||||
|
|
||||||
if (increases.length === 0) return null;
|
|
||||||
|
|
||||||
const lines = [
|
|
||||||
'#### Largest Package Increases',
|
|
||||||
'',
|
|
||||||
'| Package | Base | Head | Δ | Modules Δ |',
|
|
||||||
'| --- | ---: | ---: | ---: | ---: |',
|
|
||||||
];
|
|
||||||
|
|
||||||
for (const packageSummary of increases) {
|
|
||||||
lines.push(`| ${packageDisplayName(packageSummary)} | ${util.formatBytes(packageSummary.baseSourceBytes)} | ${util.formatBytes(packageSummary.sourceBytes)} | ${formatPlainDelta(packageSummary.baseSourceBytes, packageSummary.sourceBytes, util.formatBytes)} | ${formatPlainDelta(packageSummary.baseModules, packageSummary.modules)} |`);
|
|
||||||
}
|
|
||||||
|
|
||||||
return lines.join('\n');
|
|
||||||
}
|
|
||||||
|
|
||||||
function renderNewLoadedModules(base: RuntimeLoadedJsFootprintReport, head: RuntimeLoadedJsFootprintReport) {
|
|
||||||
function moduleMap(report: RuntimeLoadedJsFootprintReport) {
|
|
||||||
const map = new Map();
|
|
||||||
for (const moduleSummary of report.phases.afterRequest.modules) {
|
|
||||||
if (typeof moduleSummary.path !== 'string') continue;
|
|
||||||
map.set(moduleSummary.path, moduleSummary);
|
|
||||||
}
|
|
||||||
return map;
|
|
||||||
}
|
|
||||||
|
|
||||||
const baseModules = moduleMap(base);
|
|
||||||
const headModules = moduleMap(head);
|
|
||||||
const newModules = [...headModules.values()]
|
|
||||||
.filter(moduleSummary => !baseModules.has(moduleSummary.path))
|
|
||||||
.toSorted((a, b) => b.sourceBytes - a.sourceBytes)
|
|
||||||
.slice(0, 10);
|
|
||||||
|
|
||||||
if (newModules.length === 0) return null;
|
|
||||||
|
|
||||||
const lines = [
|
|
||||||
'#### Largest Newly Loaded Modules',
|
|
||||||
'',
|
|
||||||
'| Module | Package | Loaded JS |',
|
|
||||||
'| --- | --- | ---: |',
|
|
||||||
];
|
|
||||||
|
|
||||||
for (const moduleSummary of newModules) {
|
|
||||||
lines.push(`| \`${moduleSummary.path}\` | ${moduleSummary.package} | ${util.formatBytes(moduleSummary.sourceBytes)} |`);
|
|
||||||
}
|
|
||||||
|
|
||||||
return lines.join('\n');
|
|
||||||
}
|
|
||||||
|
|
||||||
function renderJsFootprintSection(base: RuntimeLoadedJsFootprintReport, head: RuntimeLoadedJsFootprintReport) {
|
|
||||||
const lines = [
|
|
||||||
'### Runtime Loaded JS Footprint',
|
|
||||||
'',
|
|
||||||
'<details><summary>Click to show</summary>',
|
|
||||||
'',
|
|
||||||
renderJsFootprintMetricTable(base, head),
|
|
||||||
'',
|
|
||||||
//'#### Load Phase Breakdown',
|
|
||||||
//'',
|
|
||||||
//renderJsFootprintPhaseTable(base, head),
|
|
||||||
//'',
|
|
||||||
];
|
|
||||||
|
|
||||||
for (const block of [
|
|
||||||
renderNewExternalPackages(base, head),
|
|
||||||
renderLargestPackageIncreases(base, head),
|
|
||||||
renderNewLoadedModules(base, head),
|
|
||||||
]) {
|
|
||||||
if (block == null) continue;
|
|
||||||
lines.push(block);
|
|
||||||
lines.push('');
|
|
||||||
}
|
|
||||||
|
|
||||||
lines.push('</details>');
|
|
||||||
lines.push('');
|
|
||||||
|
|
||||||
return lines.join('\n');
|
|
||||||
}
|
|
||||||
|
|
||||||
const base = JSON.parse(await readFile(baseFile, 'utf8')) as MemoryReport;
|
|
||||||
const head = JSON.parse(await readFile(headFile, 'utf8')) as MemoryReport;
|
|
||||||
const baseJsFootprint = JSON.parse(await readFile(baseJsFootprintFile, 'utf8')) as RuntimeLoadedJsFootprintReport;
|
|
||||||
const headJsFootprint = JSON.parse(await readFile(headJsFootprintFile, 'utf8')) as RuntimeLoadedJsFootprintReport;
|
|
||||||
const lines = [
|
|
||||||
'## ⚙️ Backend Memory Usage Report',
|
|
||||||
'',
|
|
||||||
];
|
|
||||||
|
|
||||||
//const summary = measurementSummary(base, head);
|
|
||||||
//if (summary != null) {
|
|
||||||
// lines.push(summary);
|
|
||||||
// lines.push('');
|
|
||||||
//}
|
|
||||||
|
|
||||||
for (const phase of memoryReportPhases) {
|
|
||||||
lines.push(`### ${phase.title}`);
|
|
||||||
lines.push(renderMainTableForPhase(base, head, phase.key));
|
|
||||||
lines.push('');
|
|
||||||
}
|
|
||||||
|
|
||||||
const heapSnapshotSection = renderHeapSnapshotSection(base, head);
|
|
||||||
if (heapSnapshotSection != null) {
|
|
||||||
lines.push(heapSnapshotSection);
|
|
||||||
lines.push('');
|
|
||||||
}
|
|
||||||
|
|
||||||
const jsFootprintSection = renderJsFootprintSection(baseJsFootprint, headJsFootprint);
|
|
||||||
if (jsFootprintSection != null) {
|
|
||||||
lines.push(jsFootprintSection);
|
|
||||||
lines.push('');
|
|
||||||
}
|
|
||||||
|
|
||||||
function getWarningMetric(base: MemoryReport, head: MemoryReport) {
|
|
||||||
for (const metric of ['Pss', 'Private_Dirty', 'VmRSS'] as const) {
|
|
||||||
if (getMemoryValue(base, 'afterGc', metric) != null && getMemoryValue(head, 'afterGc', metric) != null) {
|
|
||||||
return metric;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return null;
|
|
||||||
}
|
|
||||||
|
|
||||||
function isBeyondSampleNoise(base: MemoryReport, head: MemoryReport, phase: typeof memoryReportPhases[number]['key'], metric: typeof metrics[number]) {
|
|
||||||
const baseValue = getMemoryValue(base, phase, metric);
|
|
||||||
const headValue = getMemoryValue(head, phase, metric);
|
|
||||||
if (baseValue == null || headValue == null) return false;
|
|
||||||
|
|
||||||
const diff = headValue - baseValue;
|
|
||||||
if (diff <= 0) return false;
|
|
||||||
|
|
||||||
const baseSpread = getSampleSpread(base, phase, metric);
|
|
||||||
const headSpread = getSampleSpread(head, phase, metric);
|
|
||||||
if (baseSpread == null || headSpread == null) return true;
|
|
||||||
|
|
||||||
const combinedSpread = Math.hypot(baseSpread, headSpread);
|
|
||||||
return diff > combinedSpread * 3;
|
|
||||||
}
|
|
||||||
|
|
||||||
const warningMetric = getWarningMetric(base, head);
|
|
||||||
const warningDiffPercent = warningMetric == null ? null : getDiffPercent(base, head, 'afterGc', warningMetric);
|
|
||||||
if (warningMetric != null && warningDiffPercent != null && warningDiffPercent > 5 && isBeyondSampleNoise(base, head, 'afterGc', warningMetric)) {
|
|
||||||
lines.push(`⚠️ **Warning**: Memory usage (${warningMetric}) has increased by more than 5% and exceeds the observed sample noise. Please verify this is not an unintended change.`);
|
|
||||||
lines.push('');
|
|
||||||
}
|
|
||||||
|
|
||||||
lines.push(`[See workflow logs for details](https://github.com/${process.env.GITHUB_REPOSITORY}/actions/runs/${process.env.GITHUB_RUN_ID})`);
|
|
||||||
|
|
||||||
await writeFile(outputFile, `${lines.join('\n')}\n`);
|
|
||||||
276
.github/scripts/frontend-bundle-visualizer-report.mjs
vendored
Normal file
276
.github/scripts/frontend-bundle-visualizer-report.mjs
vendored
Normal file
|
|
@ -0,0 +1,276 @@
|
||||||
|
import { readFile, writeFile } from 'node:fs/promises';
|
||||||
|
|
||||||
|
const [beforeFile, afterFile, outputFile] = process.argv.slice(2);
|
||||||
|
|
||||||
|
if (beforeFile == null || afterFile == null || outputFile == null) {
|
||||||
|
console.error('Usage: node .github/scripts/frontend-bundle-visualizer-report.mjs <before-stats.json> <after-stats.json> <report.md>');
|
||||||
|
process.exit(1);
|
||||||
|
}
|
||||||
|
|
||||||
|
const byteFormatter = new Intl.NumberFormat('en-US');
|
||||||
|
const numberFormatter = new Intl.NumberFormat('en-US');
|
||||||
|
|
||||||
|
function formatBytes(value) {
|
||||||
|
if (!Number.isFinite(value) || value <= 0) return '0 B';
|
||||||
|
|
||||||
|
const units = ['B', 'KiB', 'MiB', 'GiB'];
|
||||||
|
let unitIndex = 0;
|
||||||
|
let size = value;
|
||||||
|
while (size >= 1024 && unitIndex < units.length - 1) {
|
||||||
|
size /= 1024;
|
||||||
|
unitIndex += 1;
|
||||||
|
}
|
||||||
|
|
||||||
|
const maximumFractionDigits = size >= 10 || unitIndex === 0 ? 0 : 1;
|
||||||
|
return `${byteFormatter.format(Number(size.toFixed(maximumFractionDigits)))} ${units[unitIndex]}`;
|
||||||
|
}
|
||||||
|
|
||||||
|
function formatNumber(value) {
|
||||||
|
return numberFormatter.format(value);
|
||||||
|
}
|
||||||
|
|
||||||
|
function formatPercent(value) {
|
||||||
|
return `${Math.round(value)}%`;
|
||||||
|
}
|
||||||
|
|
||||||
|
function sharePercent(value, total) {
|
||||||
|
if (total === 0) return '0%';
|
||||||
|
return formatPercent((value / total) * 100);
|
||||||
|
}
|
||||||
|
|
||||||
|
function formatMathText(text) {
|
||||||
|
return text
|
||||||
|
.replaceAll('\\', '\\\\')
|
||||||
|
.replaceAll('{', '\\{')
|
||||||
|
.replaceAll('}', '\\}')
|
||||||
|
.replaceAll('%', '\\%');
|
||||||
|
}
|
||||||
|
|
||||||
|
function formatColoredDiff(text, diff) {
|
||||||
|
const color = diff > 0 ? 'orange' : 'green';
|
||||||
|
return `$\\color{${color}}{\\text{${formatMathText(text)}}}$`;
|
||||||
|
}
|
||||||
|
|
||||||
|
function formatDiff(before, after, formatter) {
|
||||||
|
const diff = after - before;
|
||||||
|
if (diff === 0) return formatter(0);
|
||||||
|
|
||||||
|
const sign = diff > 0 ? '+' : '-';
|
||||||
|
return formatColoredDiff(`${sign}${formatter(Math.abs(diff))}`, diff);
|
||||||
|
}
|
||||||
|
|
||||||
|
function formatDiffPercent(before, after) {
|
||||||
|
if (before === 0 && after === 0) return '0%';
|
||||||
|
if (before === 0) return '-';
|
||||||
|
|
||||||
|
const diff = after - before;
|
||||||
|
if (diff === 0) return '0%';
|
||||||
|
|
||||||
|
const sign = diff > 0 ? '+' : '-';
|
||||||
|
return formatColoredDiff(`${sign}${formatPercent(Math.abs(diff / before) * 100)}`, diff);
|
||||||
|
}
|
||||||
|
|
||||||
|
function tableCell(value) {
|
||||||
|
return String(value).replaceAll('|', '\\|').replaceAll('\r', ' ').replaceAll('\n', ' ');
|
||||||
|
}
|
||||||
|
|
||||||
|
function code(value) {
|
||||||
|
const sanitized = String(value).replaceAll('\r', ' ').replaceAll('\n', ' ');
|
||||||
|
const backtickRuns = sanitized.match(/`+/g) ?? [];
|
||||||
|
const fenceLength = Math.max(1, ...backtickRuns.map((run) => run.length + 1));
|
||||||
|
const fence = '`'.repeat(fenceLength);
|
||||||
|
const padding = sanitized.startsWith('`') || sanitized.endsWith('`') ? ' ' : '';
|
||||||
|
|
||||||
|
return `${fence}${padding}${sanitized}${padding}${fence}`;
|
||||||
|
}
|
||||||
|
|
||||||
|
function tableCode(value) {
|
||||||
|
return tableCell(code(value));
|
||||||
|
}
|
||||||
|
|
||||||
|
function collectReport(data) {
|
||||||
|
const nodeParts = data.nodeParts ?? {};
|
||||||
|
const nodeMetas = Object.values(data.nodeMetas ?? {});
|
||||||
|
const moduleRows = [];
|
||||||
|
const bundleMap = new Map();
|
||||||
|
|
||||||
|
for (const meta of nodeMetas) {
|
||||||
|
const row = {
|
||||||
|
id: meta.id,
|
||||||
|
bundles: 0,
|
||||||
|
renderedLength: 0,
|
||||||
|
gzipLength: 0,
|
||||||
|
brotliLength: 0,
|
||||||
|
importedByCount: meta.importedBy?.length ?? 0,
|
||||||
|
importedCount: meta.imported?.length ?? 0,
|
||||||
|
};
|
||||||
|
|
||||||
|
for (const [bundleId, partUid] of Object.entries(meta.moduleParts ?? {})) {
|
||||||
|
const part = nodeParts[partUid];
|
||||||
|
if (part == null) continue;
|
||||||
|
|
||||||
|
row.bundles += 1;
|
||||||
|
row.renderedLength += part.renderedLength;
|
||||||
|
row.gzipLength += part.gzipLength;
|
||||||
|
row.brotliLength += part.brotliLength;
|
||||||
|
|
||||||
|
const bundle = bundleMap.get(bundleId) ?? {
|
||||||
|
id: bundleId,
|
||||||
|
modules: 0,
|
||||||
|
renderedLength: 0,
|
||||||
|
gzipLength: 0,
|
||||||
|
brotliLength: 0,
|
||||||
|
};
|
||||||
|
bundle.modules += 1;
|
||||||
|
bundle.renderedLength += part.renderedLength;
|
||||||
|
bundle.gzipLength += part.gzipLength;
|
||||||
|
bundle.brotliLength += part.brotliLength;
|
||||||
|
bundleMap.set(bundleId, bundle);
|
||||||
|
}
|
||||||
|
|
||||||
|
if (row.bundles > 0) {
|
||||||
|
moduleRows.push(row);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
let staticImports = 0;
|
||||||
|
let dynamicImports = 0;
|
||||||
|
for (const meta of nodeMetas) {
|
||||||
|
for (const imported of meta.imported ?? []) {
|
||||||
|
if (imported.dynamic) {
|
||||||
|
dynamicImports += 1;
|
||||||
|
} else {
|
||||||
|
staticImports += 1;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
const bundleRows = [...bundleMap.values()].sort((a, b) => b.renderedLength - a.renderedLength);
|
||||||
|
const hotModules = [...moduleRows].sort((a, b) => b.renderedLength - a.renderedLength);
|
||||||
|
const totalRendered = moduleRows.reduce((sum, row) => sum + row.renderedLength, 0);
|
||||||
|
const totalGzip = moduleRows.reduce((sum, row) => sum + row.gzipLength, 0);
|
||||||
|
const totalBrotli = moduleRows.reduce((sum, row) => sum + row.brotliLength, 0);
|
||||||
|
|
||||||
|
return {
|
||||||
|
options: data.options ?? {},
|
||||||
|
summary: {
|
||||||
|
bundles: bundleRows.length,
|
||||||
|
modules: moduleRows.length,
|
||||||
|
entries: nodeMetas.filter((meta) => meta.isEntry).length,
|
||||||
|
externals: nodeMetas.filter((meta) => meta.isExternal).length,
|
||||||
|
staticImports,
|
||||||
|
dynamicImports,
|
||||||
|
},
|
||||||
|
metrics: {
|
||||||
|
renderedLength: totalRendered,
|
||||||
|
gzipLength: totalGzip,
|
||||||
|
brotliLength: totalBrotli,
|
||||||
|
},
|
||||||
|
hotModules,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
function renderSummaryTable(before, after) {
|
||||||
|
const summary = [
|
||||||
|
'bundles',
|
||||||
|
'modules',
|
||||||
|
'entries',
|
||||||
|
//'externals',
|
||||||
|
'staticImports',
|
||||||
|
'dynamicImports',
|
||||||
|
];
|
||||||
|
|
||||||
|
const metrics = [
|
||||||
|
'renderedLength',
|
||||||
|
'gzipLength',
|
||||||
|
'brotliLength',
|
||||||
|
];
|
||||||
|
|
||||||
|
return [
|
||||||
|
`<table>`,
|
||||||
|
`<thead>`,
|
||||||
|
`<tr>`,
|
||||||
|
`<th rowspan="2"></th>`,
|
||||||
|
`<th rowspan="2">Bundles</th>`,
|
||||||
|
`<th rowspan="2">Modules</th>`,
|
||||||
|
`<th rowspan="2">Entries</th>`,
|
||||||
|
`<th colspan="2">Imports</th>`,
|
||||||
|
`<th colspan="3">Size</th>`,
|
||||||
|
`</tr>`,
|
||||||
|
`<tr>`,
|
||||||
|
`<th>Static</th>`,
|
||||||
|
`<th>Dynamic</th>`,
|
||||||
|
`<th>Rendered</th>`,
|
||||||
|
`<th>Gzip</th>`,
|
||||||
|
`<th>Brotli</th>`,
|
||||||
|
`</tr>`,
|
||||||
|
`</thead>`,
|
||||||
|
`<tbody>`,
|
||||||
|
`<tr>`,
|
||||||
|
`<th><b>Before</b></th>`,
|
||||||
|
...summary.map((key) => `<td>${formatNumber(before.summary[key])}</td>`),
|
||||||
|
...metrics.map((key) => `<td>${formatBytes(before.metrics[key])}</td>`),
|
||||||
|
`</tr>`,
|
||||||
|
`<tr>`,
|
||||||
|
`<th><b>After</b></th>`,
|
||||||
|
...summary.map((key) => `<td>${formatNumber(after.summary[key])}</td>`),
|
||||||
|
...metrics.map((key) => `<td>${formatBytes(after.metrics[key])}</td>`),
|
||||||
|
`</tr>`,
|
||||||
|
`<tr><td></td><td></td><td></td><td></td><td></td><td></td><td></td><td></td><td></td></tr>`,
|
||||||
|
`<tr>`,
|
||||||
|
`<th><b>Δ</b></th>`,
|
||||||
|
...summary.map((key) => `<td>${formatDiff(before.summary[key], after.summary[key], formatNumber)}</td>`),
|
||||||
|
...metrics.map((key) => `<td>${formatDiff(before.metrics[key], after.metrics[key], formatBytes)}</td>`),
|
||||||
|
`</tr>`,
|
||||||
|
`<tr>`,
|
||||||
|
`<th><b>Δ (%)</b></th>`,
|
||||||
|
...summary.map((key) => `<td>${formatDiffPercent(before.summary[key], after.summary[key])}</td>`),
|
||||||
|
...metrics.map((key) => `<td>${formatDiffPercent(before.metrics[key], after.metrics[key])}</td>`),
|
||||||
|
`</tr>`,
|
||||||
|
`</tbody>`,
|
||||||
|
`</table>`,
|
||||||
|
];
|
||||||
|
}
|
||||||
|
|
||||||
|
const beforeData = JSON.parse(await readFile(beforeFile, 'utf8'));
|
||||||
|
const afterData = JSON.parse(await readFile(afterFile, 'utf8'));
|
||||||
|
const before = collectReport(beforeData);
|
||||||
|
const after = collectReport(afterData);
|
||||||
|
const lines = [
|
||||||
|
'## Frontend Bundle Report',
|
||||||
|
'',
|
||||||
|
...renderSummaryTable(before, after),
|
||||||
|
'',
|
||||||
|
'<details>',
|
||||||
|
'<summary>Top 10</summary>',
|
||||||
|
'',
|
||||||
|
];
|
||||||
|
|
||||||
|
for (const row of after.hotModules.slice(0, 10)) {
|
||||||
|
lines.push(`- ${code(row.id)}: ${sharePercent(row.renderedLength, after.metrics.renderedLength)} (${formatBytes(row.renderedLength)})`);
|
||||||
|
}
|
||||||
|
|
||||||
|
lines.push(
|
||||||
|
'',
|
||||||
|
'</details>',
|
||||||
|
);
|
||||||
|
|
||||||
|
lines.push(
|
||||||
|
'',
|
||||||
|
'<details>',
|
||||||
|
'<summary>Hot Modules (Self Size)</summary>',
|
||||||
|
'',
|
||||||
|
'| Module | Bundles | Rendered | Share | Gzip | Brotli | Imports | Imported By |',
|
||||||
|
'|---|---:|---:|---:|---:|---:|---:|---:|',
|
||||||
|
);
|
||||||
|
|
||||||
|
for (const row of after.hotModules.slice(0, 15)) {
|
||||||
|
lines.push(`| ${tableCode(row.id)} | ${row.bundles} | ${formatBytes(row.renderedLength)} | ${sharePercent(row.renderedLength, after.metrics.renderedLength)} | ${formatBytes(row.gzipLength)} | ${formatBytes(row.brotliLength)} | ${row.importedCount} | ${row.importedByCount} |`);
|
||||||
|
}
|
||||||
|
|
||||||
|
lines.push(
|
||||||
|
'',
|
||||||
|
'</details>',
|
||||||
|
);
|
||||||
|
|
||||||
|
await writeFile(outputFile, `${lines.join('\n')}\n`);
|
||||||
344
.github/scripts/frontend-js-size.mjs
vendored
Normal file
344
.github/scripts/frontend-js-size.mjs
vendored
Normal file
|
|
@ -0,0 +1,344 @@
|
||||||
|
import { promises as fs } from 'node:fs';
|
||||||
|
import path from 'node:path';
|
||||||
|
|
||||||
|
const marker = '<!-- misskey-frontend-js-size -->';
|
||||||
|
const locale = process.env.FRONTEND_JS_SIZE_LOCALE || 'ja-JP';
|
||||||
|
|
||||||
|
function normalizePath(filePath) {
|
||||||
|
return filePath.split(path.sep).join('/');
|
||||||
|
}
|
||||||
|
|
||||||
|
async function exists(filePath) {
|
||||||
|
try {
|
||||||
|
await fs.access(filePath);
|
||||||
|
return true;
|
||||||
|
} catch {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async function fileSize(filePath) {
|
||||||
|
const stat = await fs.stat(filePath);
|
||||||
|
return stat.size;
|
||||||
|
}
|
||||||
|
|
||||||
|
async function* walk(dir) {
|
||||||
|
for (const entry of await fs.readdir(dir, { withFileTypes: true })) {
|
||||||
|
const fullPath = path.join(dir, entry.name);
|
||||||
|
if (entry.isDirectory()) {
|
||||||
|
yield* walk(fullPath);
|
||||||
|
} else if (entry.isFile()) {
|
||||||
|
yield fullPath;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
function formatBytes(size) {
|
||||||
|
if (size == null) return '-';
|
||||||
|
if (size < 1024) return `${size} B`;
|
||||||
|
if (size < 1024 * 1024) return `${stripTrailingZeros((size / 1024).toFixed(1))} KiB`;
|
||||||
|
return `${stripTrailingZeros((size / 1024 / 1024).toFixed(2))} MiB`;
|
||||||
|
}
|
||||||
|
|
||||||
|
function stripTrailingZeros(value) {
|
||||||
|
return value.replace(/\.0+$/, '').replace(/(\.\d*?)0+$/, '$1');
|
||||||
|
}
|
||||||
|
|
||||||
|
function formatMathText(text) {
|
||||||
|
return text
|
||||||
|
.replaceAll('\\', '\\\\')
|
||||||
|
.replaceAll('{', '\\{')
|
||||||
|
.replaceAll('}', '\\}')
|
||||||
|
.replaceAll('%', '\\\\%');
|
||||||
|
}
|
||||||
|
|
||||||
|
function formatDiff(diff) {
|
||||||
|
if (diff == null) return '-';
|
||||||
|
if (diff === 0) return '0 B';
|
||||||
|
const sign = diff > 0 ? '+' : '-';
|
||||||
|
const text = `${sign}${formatBytes(Math.abs(diff))}`;
|
||||||
|
const color = diff > 0 ? 'orange' : 'green';
|
||||||
|
return `$\\color{${color}}{\\text{${formatMathText(text)}}}$`;
|
||||||
|
}
|
||||||
|
|
||||||
|
function formatDiffPercent(beforeSize, afterSize) {
|
||||||
|
if (beforeSize == null || beforeSize === 0 || afterSize == null || afterSize === 0) return '-';
|
||||||
|
const diff = afterSize - beforeSize;
|
||||||
|
if (diff === 0) return `0%`;
|
||||||
|
const percent = Math.round(diff / beforeSize * 100);
|
||||||
|
const color = diff > 0 ? 'orange' : 'green';
|
||||||
|
return `$\\color{${color}}{\\text{${formatMathText(percent.toString() + '%')}}}$`;
|
||||||
|
}
|
||||||
|
|
||||||
|
function escapeCell(value) {
|
||||||
|
return String(value).replaceAll('|', '\\|').replaceAll('\n', '<br>');
|
||||||
|
}
|
||||||
|
|
||||||
|
function entryDisplayName(entry) {
|
||||||
|
if (!entry) return '';
|
||||||
|
return entry.displayName || entry.file;
|
||||||
|
}
|
||||||
|
|
||||||
|
function findEntryKey(manifest) {
|
||||||
|
const entries = Object.entries(manifest);
|
||||||
|
return entries.find(([key, chunk]) => key === 'src/_boot_.ts' || chunk.src === 'src/_boot_.ts')?.[0]
|
||||||
|
?? entries.find(([, chunk]) => chunk.name === 'entry' && chunk.isEntry)?.[0]
|
||||||
|
?? entries.find(([, chunk]) => chunk.isEntry)?.[0]
|
||||||
|
?? null;
|
||||||
|
}
|
||||||
|
|
||||||
|
function stableChunkKey(manifestKey, chunk) {
|
||||||
|
return chunk.src ?? (chunk.name ? `chunk:${chunk.name}` : manifestKey);
|
||||||
|
}
|
||||||
|
|
||||||
|
function collectStartupKeys(manifest) {
|
||||||
|
const entryKey = findEntryKey(manifest);
|
||||||
|
const keys = new Set();
|
||||||
|
if (entryKey == null) return keys;
|
||||||
|
|
||||||
|
function visit(key) {
|
||||||
|
if (keys.has(key)) return;
|
||||||
|
const chunk = manifest[key];
|
||||||
|
if (!chunk || !chunk.file?.endsWith('.js')) return;
|
||||||
|
keys.add(stableChunkKey(key, chunk));
|
||||||
|
for (const importKey of chunk.imports ?? []) {
|
||||||
|
visit(importKey);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
visit(entryKey);
|
||||||
|
return keys;
|
||||||
|
}
|
||||||
|
|
||||||
|
async function resolveBuiltFile(outDir, file) {
|
||||||
|
if (file.startsWith('scripts/')) {
|
||||||
|
const localizedFile = file.slice('scripts/'.length);
|
||||||
|
const localizedPath = path.join(outDir, locale, localizedFile);
|
||||||
|
if (await exists(localizedPath)) {
|
||||||
|
return {
|
||||||
|
absolutePath: localizedPath,
|
||||||
|
relativePath: `${locale}/${localizedFile}`,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
throw new Error(`Expected ${locale} localized chunk for ${file}, but ${localizedPath} was not found.`);
|
||||||
|
}
|
||||||
|
return {
|
||||||
|
absolutePath: path.join(outDir, file),
|
||||||
|
relativePath: file,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
async function collectReport(repoDir) {
|
||||||
|
const outDir = path.join(repoDir, 'built/_frontend_vite_');
|
||||||
|
const manifestPath = path.join(outDir, 'manifest.json');
|
||||||
|
const manifest = JSON.parse(await fs.readFile(manifestPath, 'utf8'));
|
||||||
|
const byKey = new Map();
|
||||||
|
const byFile = new Set();
|
||||||
|
|
||||||
|
for (const [key, chunk] of Object.entries(manifest)) {
|
||||||
|
if (!chunk.file?.endsWith('.js')) continue;
|
||||||
|
const builtFile = await resolveBuiltFile(outDir, chunk.file);
|
||||||
|
const size = await fileSize(builtFile.absolutePath);
|
||||||
|
const stableKey = stableChunkKey(key, chunk);
|
||||||
|
const displayName = chunk.src ?? chunk.name ?? key;
|
||||||
|
byKey.set(stableKey, {
|
||||||
|
key: stableKey,
|
||||||
|
displayName,
|
||||||
|
file: builtFile.relativePath,
|
||||||
|
size,
|
||||||
|
});
|
||||||
|
byFile.add(builtFile.relativePath);
|
||||||
|
}
|
||||||
|
|
||||||
|
const localeDir = path.join(outDir, locale);
|
||||||
|
if (await exists(localeDir)) {
|
||||||
|
for await (const fullPath of walk(localeDir)) {
|
||||||
|
if (!fullPath.endsWith('.js')) continue;
|
||||||
|
const relativePath = normalizePath(path.relative(outDir, fullPath));
|
||||||
|
if (byFile.has(relativePath)) continue;
|
||||||
|
const size = await fileSize(fullPath);
|
||||||
|
byKey.set(relativePath, {
|
||||||
|
key: relativePath,
|
||||||
|
displayName: relativePath,
|
||||||
|
file: relativePath,
|
||||||
|
size,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return {
|
||||||
|
manifest,
|
||||||
|
chunks: Object.fromEntries(byKey),
|
||||||
|
startupKeys: [...collectStartupKeys(manifest)],
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
function commonKeys(before, after) {
|
||||||
|
return Object.keys(before.chunks)
|
||||||
|
.filter((key) => after.chunks[key] != null);
|
||||||
|
}
|
||||||
|
|
||||||
|
function addedKeys(before, after) {
|
||||||
|
return Object.keys(after.chunks)
|
||||||
|
.filter((key) => before.chunks[key] == null);
|
||||||
|
}
|
||||||
|
|
||||||
|
function removedKeys(before, after) {
|
||||||
|
return Object.keys(before.chunks)
|
||||||
|
.filter((key) => after.chunks[key] == null);
|
||||||
|
}
|
||||||
|
|
||||||
|
function getChunkComparisonRows(keys, before, after) {
|
||||||
|
return keys.map((key) => {
|
||||||
|
const beforeEntry = before.chunks[key];
|
||||||
|
const afterEntry = after.chunks[key];
|
||||||
|
const beforeSize = beforeEntry?.size ?? 0;
|
||||||
|
const afterSize = afterEntry?.size ?? 0;
|
||||||
|
return {
|
||||||
|
key,
|
||||||
|
name: entryDisplayName(beforeEntry ?? afterEntry),
|
||||||
|
chunkFile: beforeEntry?.file ?? afterEntry?.file,
|
||||||
|
beforeSize,
|
||||||
|
afterSize,
|
||||||
|
sortSize: Math.max(beforeSize, afterSize),
|
||||||
|
};
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
function markdownTable(rows, total) {
|
||||||
|
if (rows.length === 0) return '_No data_';
|
||||||
|
|
||||||
|
const lines = [
|
||||||
|
'| Chunk | Before | After | Δ | Δ (%) |',
|
||||||
|
'| --- | ---: | ---: | ---: | ---: |',
|
||||||
|
];
|
||||||
|
if (total != null) {
|
||||||
|
lines.push(`| (total) | ${formatBytes(total.beforeSize)} | ${formatBytes(total.afterSize)} | ${formatDiff(total.afterSize - total.beforeSize)} | ${formatDiffPercent(total.beforeSize, total.afterSize)} |`);
|
||||||
|
lines.push('| | | | | |');
|
||||||
|
}
|
||||||
|
for (const row of rows) {
|
||||||
|
lines.push(`| <details><summary>\`${escapeCell(row.name)}\`</summary> \`${escapeCell(row.chunkFile)}\` </details> | ${formatBytes(row.beforeSize)} | ${formatBytes(row.afterSize)} | ${formatDiff(row.afterSize - row.beforeSize)} | ${formatDiffPercent(row.beforeSize, row.afterSize)} |`);
|
||||||
|
}
|
||||||
|
return lines.join('\n');
|
||||||
|
}
|
||||||
|
|
||||||
|
function chunkRows(keys, report) {
|
||||||
|
return keys.map((key) => {
|
||||||
|
const entry = report.chunks[key];
|
||||||
|
return {
|
||||||
|
key,
|
||||||
|
name: entryDisplayName(entry),
|
||||||
|
chunkFile: entry.file,
|
||||||
|
size: entry.size,
|
||||||
|
};
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
function markdownChunkTable(rows) {
|
||||||
|
if (rows.length === 0) return '_No data_';
|
||||||
|
|
||||||
|
const lines = [
|
||||||
|
'| Chunk | Size |',
|
||||||
|
'| --- | ---: |',
|
||||||
|
];
|
||||||
|
for (const row of rows) {
|
||||||
|
lines.push(`| <details><summary>\`${escapeCell(row.name)}\`</summary> \`${escapeCell(row.chunkFile)}\` </details> | ${formatBytes(row.size)} |`);
|
||||||
|
}
|
||||||
|
return lines.join('\n');
|
||||||
|
}
|
||||||
|
|
||||||
|
const beforeDir = process.argv[2];
|
||||||
|
const afterDir = process.argv[3];
|
||||||
|
const outFile = process.argv[4];
|
||||||
|
const beforeSha = process.env.BASE_SHA;
|
||||||
|
const afterSha = process.env.HEAD_SHA;
|
||||||
|
|
||||||
|
const before = await collectReport(beforeDir);
|
||||||
|
const after = await collectReport(afterDir);
|
||||||
|
|
||||||
|
const commonChunkKeys = commonKeys(before, after);
|
||||||
|
const comparisonRows = getChunkComparisonRows(commonChunkKeys, before, after);
|
||||||
|
|
||||||
|
const diffRows = comparisonRows
|
||||||
|
.filter((row) => row.beforeSize !== row.afterSize)
|
||||||
|
.sort((a, b) => Math.abs(b.afterSize - b.beforeSize) - Math.abs(a.afterSize - a.beforeSize)
|
||||||
|
|| (b.afterSize - b.beforeSize) - (a.afterSize - a.beforeSize)
|
||||||
|
|| b.sortSize - a.sortSize
|
||||||
|
|| a.name.localeCompare(b.name))
|
||||||
|
.slice(0, 30);
|
||||||
|
|
||||||
|
const diffTotal = {
|
||||||
|
beforeSize: comparisonRows.reduce((sum, row) => sum + row.beforeSize, 0),
|
||||||
|
afterSize: comparisonRows.reduce((sum, row) => sum + row.afterSize, 0),
|
||||||
|
};
|
||||||
|
|
||||||
|
const addedRows = chunkRows(addedKeys(before, after), after)
|
||||||
|
.sort((a, b) => b.size - a.size || a.name.localeCompare(b.name));
|
||||||
|
|
||||||
|
const removedRows = chunkRows(removedKeys(before, after), before)
|
||||||
|
.sort((a, b) => b.size - a.size || a.name.localeCompare(b.name));
|
||||||
|
|
||||||
|
const startupKeys = new Set([
|
||||||
|
...before.startupKeys,
|
||||||
|
...after.startupKeys,
|
||||||
|
]);
|
||||||
|
const startupComparisonRows = getChunkComparisonRows([...startupKeys], before, after);
|
||||||
|
const startupRows = startupComparisonRows
|
||||||
|
.sort((a, b) => Math.abs(b.afterSize - b.beforeSize) - Math.abs(a.afterSize - a.beforeSize)
|
||||||
|
|| (b.afterSize - b.beforeSize) - (a.afterSize - a.beforeSize)
|
||||||
|
|| b.sortSize - a.sortSize
|
||||||
|
|| a.name.localeCompare(b.name));
|
||||||
|
const startupTotal = {
|
||||||
|
beforeSize: startupComparisonRows.reduce((sum, row) => sum + row.beforeSize, 0),
|
||||||
|
afterSize: startupComparisonRows.reduce((sum, row) => sum + row.afterSize, 0),
|
||||||
|
};
|
||||||
|
|
||||||
|
const largeRows = comparisonRows
|
||||||
|
.sort((a, b) => b.sortSize - a.sortSize || a.name.localeCompare(b.name))
|
||||||
|
.slice(0, 30);
|
||||||
|
|
||||||
|
const body = [
|
||||||
|
marker,
|
||||||
|
`## Frontend chunk report (${locale})`,
|
||||||
|
'',
|
||||||
|
'<details open>',
|
||||||
|
`<summary>Diffs</summary>`,
|
||||||
|
'',
|
||||||
|
markdownTable(diffRows, diffTotal),
|
||||||
|
'',
|
||||||
|
'</details>',
|
||||||
|
'',
|
||||||
|
'<details>',
|
||||||
|
`<summary>Added (${addedRows.length})</summary>`,
|
||||||
|
'',
|
||||||
|
markdownChunkTable(addedRows),
|
||||||
|
'',
|
||||||
|
'</details>',
|
||||||
|
'',
|
||||||
|
'<details>',
|
||||||
|
`<summary>Removed (${removedRows.length})</summary>`,
|
||||||
|
'',
|
||||||
|
markdownChunkTable(removedRows),
|
||||||
|
'',
|
||||||
|
'</details>',
|
||||||
|
'',
|
||||||
|
'<details>',
|
||||||
|
`<summary>Startup</summary>`,
|
||||||
|
'',
|
||||||
|
markdownTable(startupRows, startupTotal),
|
||||||
|
'',
|
||||||
|
`_Startup chunks are the Vite entry for \`src/_boot_.ts\` and its static imports._`,
|
||||||
|
'',
|
||||||
|
'</details>',
|
||||||
|
'',
|
||||||
|
'<details>',
|
||||||
|
`<summary>Largest</summary>`,
|
||||||
|
'',
|
||||||
|
markdownTable(largeRows),
|
||||||
|
'',
|
||||||
|
'</details>',
|
||||||
|
'',
|
||||||
|
].join('\n');
|
||||||
|
|
||||||
|
await fs.writeFile(outFile, body);
|
||||||
568
.github/scripts/frontend-js-size.mts
vendored
568
.github/scripts/frontend-js-size.mts
vendored
|
|
@ -1,568 +0,0 @@
|
||||||
/*
|
|
||||||
* SPDX-FileCopyrightText: syuilo and misskey-project
|
|
||||||
* SPDX-License-Identifier: AGPL-3.0-only
|
|
||||||
*/
|
|
||||||
|
|
||||||
import { promises as fs } from 'node:fs';
|
|
||||||
import path from 'node:path';
|
|
||||||
import * as util from './utility.mts';
|
|
||||||
|
|
||||||
const marker = '<!-- misskey-frontend-js-size -->';
|
|
||||||
|
|
||||||
const locale = process.env.FRONTEND_JS_SIZE_LOCALE ?? 'ja-JP';
|
|
||||||
|
|
||||||
//function sharePercent(value, total) {
|
|
||||||
// if (total === 0) return '0%';
|
|
||||||
// return Math.round((value / total) * 100) + '%';
|
|
||||||
//}
|
|
||||||
|
|
||||||
function escapeCell(value: string) {
|
|
||||||
return String(value).replaceAll('|', '\\|').replaceAll('\n', '<br>');
|
|
||||||
}
|
|
||||||
|
|
||||||
//function tableCell(value) {
|
|
||||||
// return String(value).replaceAll('|', '\\|').replaceAll('\r', ' ').replaceAll('\n', ' ');
|
|
||||||
//}
|
|
||||||
|
|
||||||
//function code(value) {
|
|
||||||
// const sanitized = String(value).replaceAll('\r', ' ').replaceAll('\n', ' ');
|
|
||||||
// const backtickRuns = sanitized.match(/`+/g) ?? [];
|
|
||||||
// const fenceLength = Math.max(1, ...backtickRuns.map((run) => run.length + 1));
|
|
||||||
// const fence = '`'.repeat(fenceLength);
|
|
||||||
// const padding = sanitized.startsWith('`') || sanitized.endsWith('`') ? ' ' : '';
|
|
||||||
//
|
|
||||||
// return `${fence}${padding}${sanitized}${padding}${fence}`;
|
|
||||||
//}
|
|
||||||
|
|
||||||
//function tableCode(value) {
|
|
||||||
// return tableCell(code(value));
|
|
||||||
//}
|
|
||||||
|
|
||||||
type Manifest = Record<string, { file?: string; src?: string; name?: string; isEntry?: boolean; imports?: string[] }>;
|
|
||||||
|
|
||||||
type FileEntry = {
|
|
||||||
key: string;
|
|
||||||
displayName: string;
|
|
||||||
file: string;
|
|
||||||
size: number;
|
|
||||||
};
|
|
||||||
|
|
||||||
function entryDisplayName(entry: FileEntry) {
|
|
||||||
if (!entry) return '';
|
|
||||||
return entry.displayName || entry.file;
|
|
||||||
}
|
|
||||||
|
|
||||||
function findEntryKey(manifest: Manifest) {
|
|
||||||
const entries = Object.entries(manifest);
|
|
||||||
return entries.find(([key, chunk]) => key === 'src/_boot_.ts' || chunk.src === 'src/_boot_.ts')?.[0]
|
|
||||||
?? entries.find(([, chunk]) => chunk.name === 'entry' && chunk.isEntry)?.[0]
|
|
||||||
?? entries.find(([, chunk]) => chunk.isEntry)?.[0]
|
|
||||||
?? null;
|
|
||||||
}
|
|
||||||
|
|
||||||
function stableChunkKey(manifestKey: string, chunk: Manifest[string]) {
|
|
||||||
return chunk.src ?? (chunk.name ? `chunk:${chunk.name}` : manifestKey);
|
|
||||||
}
|
|
||||||
|
|
||||||
function collectStartupKeys(manifest: Manifest) {
|
|
||||||
const entryKey = findEntryKey(manifest);
|
|
||||||
const keys = new Set<string>();
|
|
||||||
if (entryKey == null) return keys;
|
|
||||||
|
|
||||||
function visit(key: string) {
|
|
||||||
if (keys.has(key)) return;
|
|
||||||
const chunk = manifest[key];
|
|
||||||
if (!chunk || !chunk.file?.endsWith('.js')) return;
|
|
||||||
keys.add(stableChunkKey(key, chunk));
|
|
||||||
for (const importKey of chunk.imports ?? []) {
|
|
||||||
visit(importKey);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
visit(entryKey);
|
|
||||||
return keys;
|
|
||||||
}
|
|
||||||
|
|
||||||
async function resolveBuiltFile(outDir: string, file: string) {
|
|
||||||
if (file.startsWith('scripts/')) {
|
|
||||||
const localizedFile = file.slice('scripts/'.length);
|
|
||||||
const localizedPath = path.join(outDir, locale, localizedFile);
|
|
||||||
if (await util.fileExists(localizedPath)) {
|
|
||||||
return {
|
|
||||||
absolutePath: localizedPath,
|
|
||||||
relativePath: `${locale}/${localizedFile}`,
|
|
||||||
};
|
|
||||||
}
|
|
||||||
|
|
||||||
throw new Error(`Expected ${locale} localized chunk for ${file}, but ${localizedPath} was not found.`);
|
|
||||||
}
|
|
||||||
return {
|
|
||||||
absolutePath: path.join(outDir, file),
|
|
||||||
relativePath: file,
|
|
||||||
};
|
|
||||||
}
|
|
||||||
|
|
||||||
async function collectReport(repoDir: string) {
|
|
||||||
const outDir = path.join(repoDir, 'built/_frontend_vite_');
|
|
||||||
const manifestPath = path.join(outDir, 'manifest.json');
|
|
||||||
const manifest = JSON.parse(await fs.readFile(manifestPath, 'utf8')) as Manifest;
|
|
||||||
const byKey = new Map<string, FileEntry>();
|
|
||||||
const byFile = new Set<string>();
|
|
||||||
|
|
||||||
for (const [key, chunk] of Object.entries(manifest)) {
|
|
||||||
if (!chunk.file?.endsWith('.js')) continue;
|
|
||||||
const builtFile = await resolveBuiltFile(outDir, chunk.file);
|
|
||||||
const size = await util.fileSize(builtFile.absolutePath);
|
|
||||||
const stableKey = stableChunkKey(key, chunk);
|
|
||||||
const displayName = chunk.src ?? chunk.name ?? key;
|
|
||||||
byKey.set(stableKey, {
|
|
||||||
key: stableKey,
|
|
||||||
displayName,
|
|
||||||
file: builtFile.relativePath,
|
|
||||||
size,
|
|
||||||
});
|
|
||||||
byFile.add(builtFile.relativePath);
|
|
||||||
}
|
|
||||||
|
|
||||||
const localeDir = path.join(outDir, locale);
|
|
||||||
if (await util.fileExists(localeDir)) {
|
|
||||||
for await (const fullPath of util.traverseDirectory(localeDir)) {
|
|
||||||
if (!fullPath.endsWith('.js')) continue;
|
|
||||||
const relativePath = util.normalizePath(path.relative(outDir, fullPath));
|
|
||||||
if (byFile.has(relativePath)) continue;
|
|
||||||
const size = await util.fileSize(fullPath);
|
|
||||||
byKey.set(relativePath, {
|
|
||||||
key: relativePath,
|
|
||||||
displayName: relativePath,
|
|
||||||
file: relativePath,
|
|
||||||
size,
|
|
||||||
});
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return {
|
|
||||||
manifest,
|
|
||||||
chunks: Object.fromEntries(byKey),
|
|
||||||
startupKeys: [...collectStartupKeys(manifest)],
|
|
||||||
};
|
|
||||||
}
|
|
||||||
|
|
||||||
type VisualizerReport = {
|
|
||||||
nodeParts?: Record<string, {
|
|
||||||
renderedLength: number;
|
|
||||||
gzipLength: number;
|
|
||||||
brotliLength: number;
|
|
||||||
}>;
|
|
||||||
nodeMetas?: Record<string, {
|
|
||||||
id: string;
|
|
||||||
isEntry?: boolean;
|
|
||||||
isExternal?: boolean;
|
|
||||||
importedBy?: string[];
|
|
||||||
imported?: { id: string; dynamic?: boolean }[];
|
|
||||||
moduleParts?: Record<string, string>;
|
|
||||||
renderedLength: number;
|
|
||||||
gzipLength: number;
|
|
||||||
brotliLength: number;
|
|
||||||
}>;
|
|
||||||
options?: Record<string, unknown>;
|
|
||||||
};
|
|
||||||
|
|
||||||
|
|
||||||
function collectVisualizerReport(data: VisualizerReport) {
|
|
||||||
const nodeParts = data.nodeParts ?? {};
|
|
||||||
const nodeMetas = Object.values(data.nodeMetas ?? {});
|
|
||||||
const moduleRows = [];
|
|
||||||
const bundleMap = new Map();
|
|
||||||
|
|
||||||
for (const meta of nodeMetas) {
|
|
||||||
const row = {
|
|
||||||
id: meta.id,
|
|
||||||
bundles: 0,
|
|
||||||
renderedLength: 0,
|
|
||||||
gzipLength: 0,
|
|
||||||
brotliLength: 0,
|
|
||||||
importedByCount: meta.importedBy?.length ?? 0,
|
|
||||||
importedCount: meta.imported?.length ?? 0,
|
|
||||||
};
|
|
||||||
|
|
||||||
for (const [bundleId, partUid] of Object.entries(meta.moduleParts ?? {})) {
|
|
||||||
const part = nodeParts[partUid];
|
|
||||||
if (part == null) continue;
|
|
||||||
|
|
||||||
row.bundles += 1;
|
|
||||||
row.renderedLength += part.renderedLength;
|
|
||||||
row.gzipLength += part.gzipLength;
|
|
||||||
row.brotliLength += part.brotliLength;
|
|
||||||
|
|
||||||
const bundle = bundleMap.get(bundleId) ?? {
|
|
||||||
id: bundleId,
|
|
||||||
modules: 0,
|
|
||||||
renderedLength: 0,
|
|
||||||
gzipLength: 0,
|
|
||||||
brotliLength: 0,
|
|
||||||
};
|
|
||||||
bundle.modules += 1;
|
|
||||||
bundle.renderedLength += part.renderedLength;
|
|
||||||
bundle.gzipLength += part.gzipLength;
|
|
||||||
bundle.brotliLength += part.brotliLength;
|
|
||||||
bundleMap.set(bundleId, bundle);
|
|
||||||
}
|
|
||||||
|
|
||||||
if (row.bundles > 0) {
|
|
||||||
moduleRows.push(row);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
let staticImports = 0;
|
|
||||||
let dynamicImports = 0;
|
|
||||||
for (const meta of nodeMetas) {
|
|
||||||
for (const imported of meta.imported ?? []) {
|
|
||||||
if (imported.dynamic) {
|
|
||||||
dynamicImports += 1;
|
|
||||||
} else {
|
|
||||||
staticImports += 1;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
const bundleRows = [...bundleMap.values()].sort((a, b) => b.renderedLength - a.renderedLength);
|
|
||||||
const hotModules = [...moduleRows].sort((a, b) => b.renderedLength - a.renderedLength);
|
|
||||||
const totalRendered = moduleRows.reduce((sum, row) => sum + row.renderedLength, 0);
|
|
||||||
const totalGzip = moduleRows.reduce((sum, row) => sum + row.gzipLength, 0);
|
|
||||||
const totalBrotli = moduleRows.reduce((sum, row) => sum + row.brotliLength, 0);
|
|
||||||
|
|
||||||
return {
|
|
||||||
options: data.options ?? {},
|
|
||||||
summary: {
|
|
||||||
bundles: bundleRows.length,
|
|
||||||
modules: moduleRows.length,
|
|
||||||
entries: nodeMetas.filter((meta) => meta.isEntry).length,
|
|
||||||
externals: nodeMetas.filter((meta) => meta.isExternal).length,
|
|
||||||
staticImports,
|
|
||||||
dynamicImports,
|
|
||||||
},
|
|
||||||
metrics: {
|
|
||||||
renderedLength: totalRendered,
|
|
||||||
gzipLength: totalGzip,
|
|
||||||
brotliLength: totalBrotli,
|
|
||||||
},
|
|
||||||
hotModules,
|
|
||||||
};
|
|
||||||
}
|
|
||||||
|
|
||||||
function renderVisualizerSummaryTable(before: ReturnType<typeof collectVisualizerReport>, after: ReturnType<typeof collectVisualizerReport>) {
|
|
||||||
const summary = [
|
|
||||||
'bundles',
|
|
||||||
'modules',
|
|
||||||
'entries',
|
|
||||||
//'externals',
|
|
||||||
'staticImports',
|
|
||||||
'dynamicImports',
|
|
||||||
] as const;
|
|
||||||
|
|
||||||
const metrics = [
|
|
||||||
'renderedLength',
|
|
||||||
'gzipLength',
|
|
||||||
'brotliLength',
|
|
||||||
] as const;
|
|
||||||
|
|
||||||
return [
|
|
||||||
`<table>`,
|
|
||||||
`<thead>`,
|
|
||||||
`<tr>`,
|
|
||||||
`<th rowspan="2"></th>`,
|
|
||||||
`<th rowspan="2">Bundles</th>`,
|
|
||||||
`<th rowspan="2">Modules</th>`,
|
|
||||||
`<th rowspan="2">Entries</th>`,
|
|
||||||
`<th colspan="2">Imports</th>`,
|
|
||||||
`<th colspan="3">Size</th>`,
|
|
||||||
`</tr>`,
|
|
||||||
`<tr>`,
|
|
||||||
`<th>Static</th>`,
|
|
||||||
`<th>Dynamic</th>`,
|
|
||||||
`<th>Rendered</th>`,
|
|
||||||
`<th>Gzip</th>`,
|
|
||||||
`<th>Brotli</th>`,
|
|
||||||
`</tr>`,
|
|
||||||
`</thead>`,
|
|
||||||
`<tbody>`,
|
|
||||||
`<tr>`,
|
|
||||||
`<th><b>Before</b></th>`,
|
|
||||||
...summary.map((key) => `<td>${util.formatNumber(before.summary[key])}</td>`),
|
|
||||||
...metrics.map((key) => `<td>${util.formatBytes(before.metrics[key])}</td>`),
|
|
||||||
`</tr>`,
|
|
||||||
`<tr>`,
|
|
||||||
`<th><b>After</b></th>`,
|
|
||||||
...summary.map((key) => `<td>${util.formatNumber(after.summary[key])}</td>`),
|
|
||||||
...metrics.map((key) => `<td>${util.formatBytes(after.metrics[key])}</td>`),
|
|
||||||
`</tr>`,
|
|
||||||
`<tr><td></td><td></td><td></td><td></td><td></td><td></td><td></td><td></td><td></td></tr>`,
|
|
||||||
`<tr>`,
|
|
||||||
`<th><b>Δ</b></th>`,
|
|
||||||
...summary.map((key) => `<td>${util.calcAndFormatDeltaNumber(before.summary[key], after.summary[key])}</td>`),
|
|
||||||
...metrics.map((key) => `<td>${util.calcAndFormatDeltaBytes(before.metrics[key], after.metrics[key])}</td>`),
|
|
||||||
`</tr>`,
|
|
||||||
`<tr>`,
|
|
||||||
`<th><b>Δ (%)</b></th>`,
|
|
||||||
...summary.map((key) => `<td>${util.calcAndFormatDeltaPercent(before.summary[key], after.summary[key])}</td>`),
|
|
||||||
...metrics.map((key) => `<td>${util.calcAndFormatDeltaPercent(before.metrics[key], after.metrics[key])}</td>`),
|
|
||||||
`</tr>`,
|
|
||||||
`</tbody>`,
|
|
||||||
`</table>`,
|
|
||||||
];
|
|
||||||
}
|
|
||||||
|
|
||||||
function getChunkComparisonRows(keys: string[], before: Awaited<ReturnType<typeof collectReport>>, after: Awaited<ReturnType<typeof collectReport>>) {
|
|
||||||
return keys.map((key) => {
|
|
||||||
const beforeEntry = before.chunks[key];
|
|
||||||
const afterEntry = after.chunks[key];
|
|
||||||
const beforeSize = beforeEntry?.size ?? 0;
|
|
||||||
const afterSize = afterEntry?.size ?? 0;
|
|
||||||
return {
|
|
||||||
key,
|
|
||||||
name: entryDisplayName(beforeEntry ?? afterEntry),
|
|
||||||
chunkFile: beforeEntry?.file ?? afterEntry?.file,
|
|
||||||
beforeSize,
|
|
||||||
afterSize,
|
|
||||||
changeType: beforeEntry == null ? 'added' : afterEntry == null ? 'removed' : beforeSize !== afterSize ? 'updated' : 'unchanged',
|
|
||||||
sortSize: Math.max(beforeSize, afterSize),
|
|
||||||
};
|
|
||||||
});
|
|
||||||
}
|
|
||||||
|
|
||||||
function summarizeChunkChanges(rows: ReturnType<typeof getChunkComparisonRows>) {
|
|
||||||
return {
|
|
||||||
updated: rows.filter((row) => row.changeType === 'updated').length,
|
|
||||||
added: rows.filter((row) => row.changeType === 'added').length,
|
|
||||||
removed: rows.filter((row) => row.changeType === 'removed').length,
|
|
||||||
};
|
|
||||||
}
|
|
||||||
|
|
||||||
function formatChunkChangeSummary(label: string, summary: ReturnType<typeof summarizeChunkChanges>) {
|
|
||||||
return `${label} (${summary.updated} updated, ${summary.added} added, ${summary.removed} removed)`;
|
|
||||||
}
|
|
||||||
|
|
||||||
function compareChunkComparisonRows(a: ReturnType<typeof getChunkComparisonRows>[number], b: ReturnType<typeof getChunkComparisonRows>[number]) {
|
|
||||||
return Math.abs(b.afterSize - b.beforeSize) - Math.abs(a.afterSize - a.beforeSize)
|
|
||||||
|| (b.afterSize - b.beforeSize) - (a.afterSize - a.beforeSize)
|
|
||||||
|| b.sortSize - a.sortSize
|
|
||||||
|| a.name.localeCompare(b.name);
|
|
||||||
}
|
|
||||||
|
|
||||||
function chunkMarkdownTable(rows: ReturnType<typeof getChunkComparisonRows>, total?: { beforeSize: number; afterSize: number }) {
|
|
||||||
if (rows.length === 0) return '_No data_';
|
|
||||||
|
|
||||||
const lines = [
|
|
||||||
'| Chunk | Before | After | Δ | Δ (%) |',
|
|
||||||
'| --- | ---: | ---: | ---: | ---: |',
|
|
||||||
];
|
|
||||||
if (total != null) {
|
|
||||||
lines.push(`| (total) | ${util.formatBytes(total.beforeSize)} | ${util.formatBytes(total.afterSize)} | ${util.calcAndFormatDeltaBytes(total.beforeSize, total.afterSize)} | ${util.calcAndFormatDeltaPercent(total.beforeSize, total.afterSize).replaceAll('\\%', '\\\\%')} |`);
|
|
||||||
lines.push('| | | | | |');
|
|
||||||
}
|
|
||||||
for (const row of rows) {
|
|
||||||
if (row.changeType === 'added') {
|
|
||||||
lines.push(`| <details><summary>\`${escapeCell(row.name)}\`</summary> \`${escapeCell(row.chunkFile)}\` </details> | ${util.formatBytes(row.beforeSize)} | ${util.formatBytes(row.afterSize)} | ${util.calcAndFormatDeltaBytes(row.beforeSize, row.afterSize)} | $\\color{orange}{\\text{(+)}}$ |`);
|
|
||||||
} else if (row.changeType === 'removed') {
|
|
||||||
lines.push(`| <details><summary>\`${escapeCell(row.name)}\`</summary> \`${escapeCell(row.chunkFile)}\` </details> | ${util.formatBytes(row.beforeSize)} | ${util.formatBytes(row.afterSize)} | ${util.calcAndFormatDeltaBytes(row.beforeSize, row.afterSize)} | $\\color{green}{\\text{(-)}}$ |`);
|
|
||||||
} else {
|
|
||||||
lines.push(`| <details><summary>\`${escapeCell(row.name)}\`</summary> \`${escapeCell(row.chunkFile)}\` </details> | ${util.formatBytes(row.beforeSize)} | ${util.formatBytes(row.afterSize)} | ${util.calcAndFormatDeltaBytes(row.beforeSize, row.afterSize)} | ${util.calcAndFormatDeltaPercent(row.beforeSize, row.afterSize).replaceAll('\\%', '\\\\%')} |`);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return lines.join('\n');
|
|
||||||
}
|
|
||||||
|
|
||||||
function renderFrontendChunkReport(before: Awaited<ReturnType<typeof collectReport>>, after: Awaited<ReturnType<typeof collectReport>>) {
|
|
||||||
const commonChunkKeys = Object.keys(before.chunks).filter((key) => after.chunks[key] != null);
|
|
||||||
const addedChunkKeys = Object.keys(after.chunks).filter((key) => before.chunks[key] == null);
|
|
||||||
const removedChunkKeys = Object.keys(before.chunks).filter((key) => after.chunks[key] == null);
|
|
||||||
const allChunkKeys = [
|
|
||||||
...commonChunkKeys,
|
|
||||||
...addedChunkKeys,
|
|
||||||
...removedChunkKeys,
|
|
||||||
];
|
|
||||||
//const comparisonRows = getChunkComparisonRows(commonChunkKeys, before, after);
|
|
||||||
const allComparisonRows = getChunkComparisonRows(allChunkKeys, before, after);
|
|
||||||
|
|
||||||
const changedRows = allComparisonRows.filter((row) => row.changeType !== 'unchanged');
|
|
||||||
const diffSummary = summarizeChunkChanges(changedRows);
|
|
||||||
const diffTotal = {
|
|
||||||
beforeSize: allComparisonRows.reduce((sum, row) => sum + row.beforeSize, 0),
|
|
||||||
afterSize: allComparisonRows.reduce((sum, row) => sum + row.afterSize, 0),
|
|
||||||
};
|
|
||||||
const diffRows = changedRows.sort(compareChunkComparisonRows).slice(0, 30); // TODO: 実際に30を超えて切り捨てられたrowがあった場合はその旨をmarkdown内に表示するようにする
|
|
||||||
|
|
||||||
const startupKeys = new Set([
|
|
||||||
...before.startupKeys,
|
|
||||||
...after.startupKeys,
|
|
||||||
]);
|
|
||||||
const startupComparisonRows = getChunkComparisonRows([...startupKeys], before, after);
|
|
||||||
const startupRows = startupComparisonRows.sort(compareChunkComparisonRows);
|
|
||||||
const startupSummary = summarizeChunkChanges(startupComparisonRows);
|
|
||||||
const startupTotal = {
|
|
||||||
beforeSize: startupComparisonRows.reduce((sum, row) => sum + row.beforeSize, 0),
|
|
||||||
afterSize: startupComparisonRows.reduce((sum, row) => sum + row.afterSize, 0),
|
|
||||||
};
|
|
||||||
|
|
||||||
//const largeRows = comparisonRows
|
|
||||||
// .sort((a, b) => b.sortSize - a.sortSize || a.name.localeCompare(b.name))
|
|
||||||
// .slice(0, 30);
|
|
||||||
|
|
||||||
return [
|
|
||||||
'<details open>',
|
|
||||||
`<summary>${formatChunkChangeSummary('Chunk size diff', diffSummary)}</summary>`,
|
|
||||||
'',
|
|
||||||
chunkMarkdownTable(diffRows, diffTotal),
|
|
||||||
'',
|
|
||||||
'</details>',
|
|
||||||
'',
|
|
||||||
'<details>',
|
|
||||||
`<summary>${formatChunkChangeSummary('Startup chunk size', startupSummary)}</summary>`,
|
|
||||||
'',
|
|
||||||
chunkMarkdownTable(startupRows, startupTotal),
|
|
||||||
'',
|
|
||||||
`_Startup chunks are the Vite entry for \`src/_boot_.ts\` and its static imports._`,
|
|
||||||
'',
|
|
||||||
'</details>',
|
|
||||||
'',
|
|
||||||
//'<details>',
|
|
||||||
//`<summary>Largest</summary>`,
|
|
||||||
//'',
|
|
||||||
//markdownTable(largeRows),
|
|
||||||
//'',
|
|
||||||
//'</details>',
|
|
||||||
//'',
|
|
||||||
].join('\n');
|
|
||||||
}
|
|
||||||
|
|
||||||
function renderFrontendBundleReport(before: ReturnType<typeof collectVisualizerReport>, after: ReturnType<typeof collectVisualizerReport>) {
|
|
||||||
const lines = [
|
|
||||||
...renderVisualizerSummaryTable(before, after),
|
|
||||||
'',
|
|
||||||
//'<details>',
|
|
||||||
//'<summary>Top 10</summary>',
|
|
||||||
//'',
|
|
||||||
];
|
|
||||||
|
|
||||||
/*
|
|
||||||
for (const row of after.hotModules.slice(0, 10)) {
|
|
||||||
lines.push(`- ${code(row.id)}: ${sharePercent(row.renderedLength, after.metrics.renderedLength)} (${formatBytes(row.renderedLength)})`);
|
|
||||||
}
|
|
||||||
|
|
||||||
lines.push(
|
|
||||||
'',
|
|
||||||
'</details>',
|
|
||||||
);
|
|
||||||
|
|
||||||
lines.push(
|
|
||||||
'',
|
|
||||||
'<details>',
|
|
||||||
'<summary>Hot Modules (Self Size)</summary>',
|
|
||||||
'',
|
|
||||||
'| Module | Bundles | Rendered | Share | Gzip | Brotli | Imports | Imported By |',
|
|
||||||
'|---|---:|---:|---:|---:|---:|---:|---:|',
|
|
||||||
);
|
|
||||||
|
|
||||||
for (const row of after.hotModules.slice(0, 15)) {
|
|
||||||
lines.push(`| ${tableCode(row.id)} | ${row.bundles} | ${formatBytes(row.renderedLength)} | ${sharePercent(row.renderedLength, after.metrics.renderedLength)} | ${formatBytes(row.gzipLength)} | ${formatBytes(row.brotliLength)} | ${row.importedCount} | ${row.importedByCount} |`);
|
|
||||||
}
|
|
||||||
|
|
||||||
lines.push(
|
|
||||||
'',
|
|
||||||
'</details>',
|
|
||||||
);
|
|
||||||
*/
|
|
||||||
|
|
||||||
return lines.join('\n');
|
|
||||||
}
|
|
||||||
|
|
||||||
const visualizerTreemapLimit = 50;
|
|
||||||
|
|
||||||
function mermaidTreemapLabel(value: string) {
|
|
||||||
const label = String(value)
|
|
||||||
.replaceAll('\\', '/')
|
|
||||||
.replaceAll('"', "'")
|
|
||||||
.replaceAll('`', "'")
|
|
||||||
.replaceAll('\r', ' ')
|
|
||||||
.replaceAll('\n', ' ')
|
|
||||||
.trim();
|
|
||||||
return label === '' ? '(unknown)' : label;
|
|
||||||
}
|
|
||||||
|
|
||||||
function mermaidTreemapModuleLabel(id: string) {
|
|
||||||
const normalizedId = String(id).replaceAll('\\', '/');
|
|
||||||
const filePath = normalizedId.split(/[?#]/, 1)[0];
|
|
||||||
const fileName = path.posix.basename(filePath);
|
|
||||||
return mermaidTreemapLabel(fileName || normalizedId);
|
|
||||||
}
|
|
||||||
|
|
||||||
function renderVisualizerTreemap(label: string, report: ReturnType<typeof collectVisualizerReport>) {
|
|
||||||
const rows = report.hotModules
|
|
||||||
.filter((row) => row.renderedLength > 0)
|
|
||||||
.slice(0, visualizerTreemapLimit);
|
|
||||||
const topRendered = rows.reduce((sum, row) => sum + row.renderedLength, 0);
|
|
||||||
const otherRendered = Math.max(0, report.metrics.renderedLength - topRendered);
|
|
||||||
const lines = [
|
|
||||||
'```mermaid',
|
|
||||||
`%%{init: ${JSON.stringify({
|
|
||||||
treemap: {
|
|
||||||
diagramPadding: 0,
|
|
||||||
padding: 0,
|
|
||||||
nodeHeight: 70,
|
|
||||||
},
|
|
||||||
})}}%%`,
|
|
||||||
'treemap-beta',
|
|
||||||
`"${mermaidTreemapLabel(label)}"`,
|
|
||||||
];
|
|
||||||
|
|
||||||
for (const row of rows) {
|
|
||||||
lines.push(` "${mermaidTreemapModuleLabel(row.id)}": ${Math.round(row.renderedLength)}`);
|
|
||||||
}
|
|
||||||
if (otherRendered > 0) {
|
|
||||||
lines.push(` "Other": ${Math.round(otherRendered)}`);
|
|
||||||
}
|
|
||||||
|
|
||||||
lines.push('```');
|
|
||||||
return lines.join('\n');
|
|
||||||
}
|
|
||||||
|
|
||||||
function renderVisualizerTreemapDetails(label: string, report: ReturnType<typeof collectVisualizerReport>, open = false) {
|
|
||||||
return [
|
|
||||||
`<details${open ? ' open' : ''}>`,
|
|
||||||
`<summary>${label} rendered size treemap (top ${visualizerTreemapLimit} + Other)</summary>`,
|
|
||||||
'',
|
|
||||||
renderVisualizerTreemap(label, report),
|
|
||||||
'',
|
|
||||||
'</details>',
|
|
||||||
].join('\n');
|
|
||||||
}
|
|
||||||
|
|
||||||
const args = process.argv.slice(2);
|
|
||||||
const [beforeDir, afterDir, beforeStatsFile, afterStatsFile, outFile] = args;
|
|
||||||
const before = await collectReport(beforeDir);
|
|
||||||
const after = await collectReport(afterDir);
|
|
||||||
const beforeStats = JSON.parse(await fs.readFile(beforeStatsFile, 'utf8')) as VisualizerReport;
|
|
||||||
const afterStats = JSON.parse(await fs.readFile(afterStatsFile, 'utf8')) as VisualizerReport;
|
|
||||||
const beforeVisualizerReport = collectVisualizerReport(beforeStats);
|
|
||||||
const afterVisualizerReport = collectVisualizerReport(afterStats);
|
|
||||||
const visualizerArtifactLink = `[Open detailed HTML](${process.env.FRONTEND_BUNDLE_REPORT_ARTIFACT_URL})`;
|
|
||||||
|
|
||||||
const body = [
|
|
||||||
marker,
|
|
||||||
'',
|
|
||||||
`## 📦 Frontend Bundle Report`,
|
|
||||||
'',
|
|
||||||
renderFrontendChunkReport(before, after),
|
|
||||||
'',
|
|
||||||
'## Bundle Stats',
|
|
||||||
'',
|
|
||||||
renderFrontendBundleReport(beforeVisualizerReport, afterVisualizerReport),
|
|
||||||
'',
|
|
||||||
renderVisualizerTreemapDetails('Before', beforeVisualizerReport),
|
|
||||||
'',
|
|
||||||
renderVisualizerTreemapDetails('After', afterVisualizerReport),
|
|
||||||
'',
|
|
||||||
visualizerArtifactLink,
|
|
||||||
].join('\n');
|
|
||||||
|
|
||||||
await fs.writeFile(outFile, body);
|
|
||||||
|
|
@ -1,262 +0,0 @@
|
||||||
/*
|
|
||||||
* SPDX-FileCopyrightText: syuilo and misskey-project
|
|
||||||
* SPDX-License-Identifier: AGPL-3.0-only
|
|
||||||
*/
|
|
||||||
|
|
||||||
import { createRequire } from 'node:module';
|
|
||||||
import { writeFile } from 'node:fs/promises';
|
|
||||||
import { join, resolve } from 'node:path';
|
|
||||||
import * as util from './utility.mts';
|
|
||||||
import type { MemoryReportRaw } from '../../packages/backend/scripts/measure-memory.mts';
|
|
||||||
|
|
||||||
const phases = ['afterGc'] as const;
|
|
||||||
|
|
||||||
export type MemoryReport = {
|
|
||||||
timestamp: string;
|
|
||||||
sampleCount: any;
|
|
||||||
aggregation: string;
|
|
||||||
measurement: {
|
|
||||||
startupTimeoutMs: any;
|
|
||||||
memorySettleTimeMs: any;
|
|
||||||
ipcTimeoutMs: any;
|
|
||||||
requestCount: any;
|
|
||||||
heapSnapshot: {
|
|
||||||
enabled: any;
|
|
||||||
timeoutMs: any;
|
|
||||||
breakdownTopN: any;
|
|
||||||
};
|
|
||||||
};
|
|
||||||
summary: Record<typeof phases[number], {
|
|
||||||
memoryUsage: Record<string, number>;
|
|
||||||
heapSnapshot?: {
|
|
||||||
categories: Record<typeof util.heapSnapshotCategories[number], number>;
|
|
||||||
nodeCounts: Record<typeof util.heapSnapshotCategories[number], number>;
|
|
||||||
breakdowns?: Record<typeof util.heapSnapshotCategories[number], Record<string, number>>;
|
|
||||||
};
|
|
||||||
}>;
|
|
||||||
samples: (MemoryReportRaw['samples'][number] & {
|
|
||||||
round: number;
|
|
||||||
})[];
|
|
||||||
};
|
|
||||||
|
|
||||||
const [baseDirArg, headDirArg, baseOutputArg, headOutputArg] = process.argv.slice(2);
|
|
||||||
|
|
||||||
const HEAP_SNAPSHOT_BREAKDOWN_TOP_N = util.readIntegerEnv('MK_MEMORY_HEAP_SNAPSHOT_BREAKDOWN_TOP_N', 6, 1);
|
|
||||||
|
|
||||||
async function resetState(repoDir: string) {
|
|
||||||
const require = createRequire(join(repoDir, 'packages/backend/package.json'));
|
|
||||||
const pg = require('pg');
|
|
||||||
const Redis = require('ioredis');
|
|
||||||
|
|
||||||
const postgres = new pg.Client({
|
|
||||||
host: '127.0.0.1',
|
|
||||||
port: 54312,
|
|
||||||
database: 'postgres',
|
|
||||||
user: 'postgres',
|
|
||||||
});
|
|
||||||
|
|
||||||
await postgres.connect();
|
|
||||||
try {
|
|
||||||
await postgres.query('DROP DATABASE IF EXISTS "test-misskey" WITH (FORCE)');
|
|
||||||
await postgres.query('CREATE DATABASE "test-misskey"');
|
|
||||||
} finally {
|
|
||||||
await postgres.end();
|
|
||||||
}
|
|
||||||
|
|
||||||
const redis = new Redis({ host: '127.0.0.1', port: 56312 });
|
|
||||||
try {
|
|
||||||
await redis.flushall();
|
|
||||||
} finally {
|
|
||||||
redis.disconnect();
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
function summarizeHeapSnapshotBreakdowns(samples: MemoryReport['samples'], phase: typeof phases[number]) {
|
|
||||||
const breakdowns = {} as Record<typeof util.heapSnapshotCategories[number], Record<string, number>>;
|
|
||||||
|
|
||||||
for (const category of util.heapSnapshotCategories) {
|
|
||||||
if (category === 'Total') continue;
|
|
||||||
|
|
||||||
const childKeys = new Set<string>();
|
|
||||||
for (const sample of samples) {
|
|
||||||
for (const childKey of Object.keys(sample.phases[phase].heapSnapshot?.breakdowns?.[category] ?? {})) {
|
|
||||||
childKeys.add(childKey);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
const categoryBreakdown = {} as Record<string, number>;
|
|
||||||
for (const childKey of childKeys) {
|
|
||||||
const values = samples
|
|
||||||
.map(sample => sample.phases[phase].heapSnapshot?.breakdowns?.[category]?.[childKey])
|
|
||||||
.filter(value => Number.isFinite(value));
|
|
||||||
|
|
||||||
if (values.length > 0) categoryBreakdown[childKey] = util.median(values);
|
|
||||||
}
|
|
||||||
|
|
||||||
if (Object.keys(categoryBreakdown).length > 0) {
|
|
||||||
breakdowns[category] = collapseHeapSnapshotBreakdown(categoryBreakdown);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return breakdowns;
|
|
||||||
}
|
|
||||||
|
|
||||||
function collapseHeapSnapshotBreakdown(breakdown: Record<string, number>) {
|
|
||||||
const entries = Object.entries(breakdown)
|
|
||||||
.filter(([, value]) => value > 0)
|
|
||||||
.toSorted((a, b) => b[1] - a[1]);
|
|
||||||
|
|
||||||
const topEntries = entries.slice(0, HEAP_SNAPSHOT_BREAKDOWN_TOP_N);
|
|
||||||
const otherValue = entries
|
|
||||||
.slice(HEAP_SNAPSHOT_BREAKDOWN_TOP_N)
|
|
||||||
.reduce((sum, [, value]) => sum + value, 0);
|
|
||||||
|
|
||||||
const collapsed = Object.fromEntries(topEntries);
|
|
||||||
if (otherValue > 0) collapsed.Other = otherValue;
|
|
||||||
return collapsed;
|
|
||||||
}
|
|
||||||
|
|
||||||
function summarizeSamples(samples: MemoryReport['samples']) {
|
|
||||||
const summary = {} as MemoryReport['summary'];
|
|
||||||
|
|
||||||
for (const phase of phases) {
|
|
||||||
summary[phase] = {
|
|
||||||
memoryUsage: {},
|
|
||||||
};
|
|
||||||
|
|
||||||
const metricKeys = new Set<string>();
|
|
||||||
for (const sample of samples) {
|
|
||||||
for (const key of Object.keys(sample.phases[phase].memoryUsage)) {
|
|
||||||
metricKeys.add(key);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
for (const key of metricKeys) {
|
|
||||||
const values = samples.map(sample => sample.phases[phase].memoryUsage[key]);
|
|
||||||
summary[phase].memoryUsage[key] = util.median(values);
|
|
||||||
}
|
|
||||||
|
|
||||||
const heapSnapshotCategoryValues = {} as Record<typeof util.heapSnapshotCategories[number], number>;
|
|
||||||
for (const category of util.heapSnapshotCategories) {
|
|
||||||
const values = samples
|
|
||||||
.map(sample => sample.phases[phase].heapSnapshot?.categories?.[category])
|
|
||||||
.filter(value => Number.isFinite(value)) as number[];
|
|
||||||
|
|
||||||
if (values.length > 0) heapSnapshotCategoryValues[category] = util.median(values);
|
|
||||||
}
|
|
||||||
|
|
||||||
const heapSnapshotNodeCountValues = {} as Record<typeof util.heapSnapshotCategories[number], number>;
|
|
||||||
for (const category of util.heapSnapshotCategories) {
|
|
||||||
const values = samples
|
|
||||||
.map(sample => sample.phases[phase].heapSnapshot?.nodeCounts?.[category])
|
|
||||||
.filter(value => Number.isFinite(value)) as number[];
|
|
||||||
|
|
||||||
if (values.length > 0) heapSnapshotNodeCountValues[category] = util.median(values);
|
|
||||||
}
|
|
||||||
|
|
||||||
if (Object.keys(heapSnapshotCategoryValues).length > 0) {
|
|
||||||
const heapSnapshotBreakdowns = summarizeHeapSnapshotBreakdowns(samples, phase);
|
|
||||||
|
|
||||||
summary[phase].heapSnapshot = {
|
|
||||||
categories: heapSnapshotCategoryValues,
|
|
||||||
nodeCounts: heapSnapshotNodeCountValues,
|
|
||||||
...(Object.keys(heapSnapshotBreakdowns).length > 0 ? { breakdowns: heapSnapshotBreakdowns } : {}),
|
|
||||||
};
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return summary;
|
|
||||||
}
|
|
||||||
|
|
||||||
async function measureRepo(label: string, repoDir: string, round: number) {
|
|
||||||
process.stderr.write(`[${label}] Resetting database and Redis\n`);
|
|
||||||
await resetState(repoDir);
|
|
||||||
|
|
||||||
process.stderr.write(`[${label}] Running migrations\n`);
|
|
||||||
await util.run('pnpm', ['--filter', 'backend', 'migrate'], {
|
|
||||||
cwd: repoDir,
|
|
||||||
env: process.env,
|
|
||||||
logStdout: true,
|
|
||||||
});
|
|
||||||
|
|
||||||
process.stderr.write(`[${label}] Measuring memory\n`);
|
|
||||||
const measureEnv = {
|
|
||||||
...process.env,
|
|
||||||
MK_MEMORY_SAMPLE_COUNT: '1',
|
|
||||||
} as NodeJS.ProcessEnv;
|
|
||||||
if (round <= 0) measureEnv.MK_MEMORY_HEAP_SNAPSHOT = '0';
|
|
||||||
|
|
||||||
const stdout = await util.run('node', ['packages/backend/scripts/measure-memory.mts'], {
|
|
||||||
cwd: repoDir,
|
|
||||||
env: measureEnv,
|
|
||||||
});
|
|
||||||
|
|
||||||
const report = JSON.parse(stdout) as MemoryReportRaw;
|
|
||||||
const sample = report.samples[0];
|
|
||||||
|
|
||||||
return sample;
|
|
||||||
}
|
|
||||||
|
|
||||||
async function main() {
|
|
||||||
const baseDir = resolve(baseDirArg);
|
|
||||||
const headDir = resolve(headDirArg);
|
|
||||||
const baseOutput = resolve(baseOutputArg);
|
|
||||||
const headOutput = resolve(headOutputArg);
|
|
||||||
const rounds = util.readIntegerEnv('MK_MEMORY_COMPARE_ROUNDS', 5, 1);
|
|
||||||
const warmupRounds = util.readIntegerEnv('MK_MEMORY_COMPARE_WARMUP_ROUNDS', 1, 0);
|
|
||||||
const startedAt = new Date().toISOString();
|
|
||||||
|
|
||||||
const reports = {
|
|
||||||
base: {
|
|
||||||
dir: baseDir,
|
|
||||||
samples: [] as MemoryReport['samples'],
|
|
||||||
},
|
|
||||||
head: {
|
|
||||||
dir: headDir,
|
|
||||||
samples: [] as MemoryReport['samples'],
|
|
||||||
},
|
|
||||||
};
|
|
||||||
|
|
||||||
for (let round = 1; round <= warmupRounds; round++) {
|
|
||||||
process.stderr.write(`Starting warmup round ${round}/${warmupRounds}\n`);
|
|
||||||
for (const label of ['base', 'head'] as const) {
|
|
||||||
await measureRepo(label, reports[label].dir, -round);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
for (let round = 1; round <= rounds; round++) {
|
|
||||||
const order = round % 2 === 1 ? ['base', 'head'] as const : ['head', 'base'] as const;
|
|
||||||
process.stderr.write(`Starting measurement round ${round}/${rounds}: ${order.join(' -> ')}\n`);
|
|
||||||
|
|
||||||
for (const [orderIndex, label] of order.entries()) {
|
|
||||||
const sample = await measureRepo(label, reports[label].dir, round);
|
|
||||||
reports[label].samples.push({
|
|
||||||
...sample,
|
|
||||||
round,
|
|
||||||
});
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
for (const label of ['base', 'head'] as const) {
|
|
||||||
const report = {
|
|
||||||
timestamp: new Date().toISOString(),
|
|
||||||
sampleCount: reports[label].samples.length,
|
|
||||||
aggregation: 'median',
|
|
||||||
comparison: {
|
|
||||||
strategy: 'interleaved-pairs',
|
|
||||||
rounds,
|
|
||||||
warmupRounds,
|
|
||||||
startedAt,
|
|
||||||
},
|
|
||||||
summary: summarizeSamples(reports[label].samples),
|
|
||||||
samples: reports[label].samples,
|
|
||||||
};
|
|
||||||
|
|
||||||
await writeFile(label === 'base' ? baseOutput : headOutput, `${JSON.stringify(report, null, 2)}\n`);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
main().catch(err => {
|
|
||||||
console.error(err);
|
|
||||||
process.exit(1);
|
|
||||||
});
|
|
||||||
179
.github/scripts/utility.mts
vendored
179
.github/scripts/utility.mts
vendored
|
|
@ -1,179 +0,0 @@
|
||||||
/*
|
|
||||||
* SPDX-FileCopyrightText: syuilo and misskey-project
|
|
||||||
* SPDX-License-Identifier: AGPL-3.0-only
|
|
||||||
*/
|
|
||||||
|
|
||||||
import { spawn } from 'node:child_process';
|
|
||||||
import { promises as fs } from 'node:fs';
|
|
||||||
import path from 'node:path';
|
|
||||||
|
|
||||||
export const heapSnapshotCategories = [
|
|
||||||
'Total',
|
|
||||||
'Code',
|
|
||||||
'Strings',
|
|
||||||
'JS arrays',
|
|
||||||
'Typed arrays',
|
|
||||||
'System objects',
|
|
||||||
'Other JS objects',
|
|
||||||
'Other non-JS objects',
|
|
||||||
] as const;
|
|
||||||
|
|
||||||
export function median(values: number[]) {
|
|
||||||
const sorted = values.toSorted((a, b) => a - b);
|
|
||||||
const center = Math.floor(sorted.length / 2);
|
|
||||||
if (sorted.length % 2 === 1) return sorted[center];
|
|
||||||
return Math.round((sorted[center - 1] + sorted[center]) / 2);
|
|
||||||
}
|
|
||||||
|
|
||||||
export function mad(values: number[]) {
|
|
||||||
if (values.length < 2) return null;
|
|
||||||
|
|
||||||
const center = median(values);
|
|
||||||
return median(values.map(value => Math.abs(value - center)));
|
|
||||||
}
|
|
||||||
|
|
||||||
export function normalizePath(filePath: string) {
|
|
||||||
return filePath.split(path.sep).join('/');
|
|
||||||
}
|
|
||||||
|
|
||||||
export async function fileExists(filePath: string) {
|
|
||||||
try {
|
|
||||||
await fs.access(filePath);
|
|
||||||
return true;
|
|
||||||
} catch {
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
export async function fileSize(filePath: string) {
|
|
||||||
const stat = await fs.stat(filePath);
|
|
||||||
return stat.size;
|
|
||||||
}
|
|
||||||
|
|
||||||
export async function* traverseDirectory(dir: string): AsyncGenerator<string> {
|
|
||||||
for (const entry of await fs.readdir(dir, { withFileTypes: true })) {
|
|
||||||
const fullPath = path.join(dir, entry.name);
|
|
||||||
if (entry.isDirectory()) {
|
|
||||||
yield* traverseDirectory(fullPath);
|
|
||||||
} else if (entry.isFile()) {
|
|
||||||
yield fullPath;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
export function escapeLatex(text: string) {
|
|
||||||
return text
|
|
||||||
.replaceAll('\\', '\\\\')
|
|
||||||
.replaceAll('{', '\\{')
|
|
||||||
.replaceAll('}', '\\}')
|
|
||||||
.replaceAll('%', '\\%');
|
|
||||||
}
|
|
||||||
|
|
||||||
export function formatColoredDelta(text: string, delta: number) {
|
|
||||||
if (delta === 0) return text;
|
|
||||||
const color = delta > 0 ? 'orange' : 'green';
|
|
||||||
const sign = delta > 0 ? '+' : '-';
|
|
||||||
return `$\\color{${color}}{\\text{${sign}${escapeLatex(text)}}}$`;
|
|
||||||
}
|
|
||||||
|
|
||||||
const numberFormatter = new Intl.NumberFormat('en-US', {
|
|
||||||
maximumFractionDigits: 1,
|
|
||||||
});
|
|
||||||
|
|
||||||
export function formatNumber(value: number) {
|
|
||||||
return numberFormatter.format(value);
|
|
||||||
}
|
|
||||||
|
|
||||||
export function formatBytes(value: number) {
|
|
||||||
if (value === 0) return '0 B';
|
|
||||||
const units = ['B', 'KiB', 'MiB', 'GiB'];
|
|
||||||
let unitIndex = 0;
|
|
||||||
let size = value;
|
|
||||||
while (size >= 1024 && unitIndex < units.length - 1) {
|
|
||||||
size /= 1024;
|
|
||||||
unitIndex += 1;
|
|
||||||
}
|
|
||||||
|
|
||||||
const maximumFractionDigits = size >= 10 || unitIndex === 0 ? 0 : 1;
|
|
||||||
return `${numberFormatter.format(Number(size.toFixed(maximumFractionDigits)))} ${units[unitIndex]}`;
|
|
||||||
}
|
|
||||||
|
|
||||||
export function calcAndFormatDeltaNumber(before: number, after: number) {
|
|
||||||
if (before == null || after == null) return '-';
|
|
||||||
const delta = after - before;
|
|
||||||
return formatColoredDelta(formatNumber(Math.abs(delta)), delta);
|
|
||||||
}
|
|
||||||
|
|
||||||
export function formatDeltaBytes(deltaBytes: number) {
|
|
||||||
return formatColoredDelta(formatBytes(Math.abs(deltaBytes)), deltaBytes);
|
|
||||||
}
|
|
||||||
|
|
||||||
export function calcAndFormatDeltaBytes(before: number, after: number) {
|
|
||||||
if (before == null || after == null) return '-';
|
|
||||||
const delta = after - before;
|
|
||||||
return formatDeltaBytes(delta);
|
|
||||||
}
|
|
||||||
|
|
||||||
export function formatPercent(value: number) {
|
|
||||||
return `${formatNumber(value)}%`;
|
|
||||||
}
|
|
||||||
|
|
||||||
export function formatDeltaPercent(deltaPercent: number) {
|
|
||||||
if (deltaPercent === 0) return '0%';
|
|
||||||
return formatColoredDelta(formatPercent(Math.abs(deltaPercent)), deltaPercent);
|
|
||||||
}
|
|
||||||
|
|
||||||
export function calcAndFormatDeltaPercent(before: number, after: number) {
|
|
||||||
if (before == null || before === 0 || after == null || after === 0) return '-';
|
|
||||||
const delta = after - before;
|
|
||||||
return formatDeltaPercent(delta / before * 100);
|
|
||||||
}
|
|
||||||
|
|
||||||
export function commandName(command: string) {
|
|
||||||
if (process.platform !== 'win32') return command;
|
|
||||||
if (command === 'pnpm') return 'pnpm.cmd';
|
|
||||||
return command;
|
|
||||||
}
|
|
||||||
|
|
||||||
export function readIntegerEnv(name: string, defaultValue: number, min: number) {
|
|
||||||
const rawValue = process.env[name];
|
|
||||||
if (rawValue == null || rawValue === '') return defaultValue;
|
|
||||||
if (!/^\d+$/.test(rawValue)) throw new Error(`${name} must be an integer`);
|
|
||||||
|
|
||||||
const value = Number(rawValue);
|
|
||||||
if (!Number.isSafeInteger(value) || value < min) throw new Error(`${name} must be >= ${min}`);
|
|
||||||
return value;
|
|
||||||
}
|
|
||||||
|
|
||||||
export function run(command: string, args: string[], options: { cwd?: string; env?: NodeJS.ProcessEnv; logStdout?: boolean } = {}) {
|
|
||||||
return new Promise<string>((resolvePromise, reject) => {
|
|
||||||
const child = spawn(commandName(command), args, {
|
|
||||||
cwd: options.cwd,
|
|
||||||
env: options.env,
|
|
||||||
stdio: ['ignore', 'pipe', 'pipe'],
|
|
||||||
});
|
|
||||||
|
|
||||||
let stdout = '';
|
|
||||||
let stderr = '';
|
|
||||||
|
|
||||||
child.stdout.on('data', data => {
|
|
||||||
stdout += data;
|
|
||||||
if (options.logStdout) process.stderr.write(data);
|
|
||||||
});
|
|
||||||
|
|
||||||
child.stderr.on('data', data => {
|
|
||||||
stderr += data;
|
|
||||||
process.stderr.write(data);
|
|
||||||
});
|
|
||||||
|
|
||||||
child.on('error', reject);
|
|
||||||
|
|
||||||
child.on('close', code => {
|
|
||||||
if (code === 0) {
|
|
||||||
resolvePromise(stdout);
|
|
||||||
} else {
|
|
||||||
reject(new Error(`${command} ${args.join(' ')} failed with exit code ${code}\n${stderr}`));
|
|
||||||
}
|
|
||||||
});
|
|
||||||
});
|
|
||||||
}
|
|
||||||
|
|
@ -25,8 +25,8 @@ on:
|
||||||
- pnpm-lock.yaml
|
- pnpm-lock.yaml
|
||||||
- pnpm-workspace.yaml
|
- pnpm-workspace.yaml
|
||||||
- .node-version
|
- .node-version
|
||||||
- .github/scripts/utility.mts
|
- .github/scripts/frontend-js-size.mjs
|
||||||
- .github/scripts/frontend-js-size.mts
|
- .github/scripts/frontend-bundle-visualizer-report.mjs
|
||||||
- .github/workflows/frontend-bundle-report.yml
|
- .github/workflows/frontend-bundle-report.yml
|
||||||
- .github/workflows/frontend-bundle-report-comment.yml
|
- .github/workflows/frontend-bundle-report-comment.yml
|
||||||
|
|
||||||
|
|
@ -104,9 +104,6 @@ jobs:
|
||||||
});
|
});
|
||||||
const report = artifacts.find((artifact) => artifact.name === artifactName && !artifact.expired);
|
const report = artifacts.find((artifact) => artifact.name === artifactName && !artifact.expired);
|
||||||
if (report) return { done: true, run };
|
if (report) return { done: true, run };
|
||||||
|
|
||||||
core.info(`Frontend bundle report run ${run.id} did not produce ${artifactName}.`);
|
|
||||||
return { done: true, run: null };
|
|
||||||
}
|
}
|
||||||
|
|
||||||
return { done: false, run: null };
|
return { done: false, run: null };
|
||||||
|
|
@ -129,30 +126,8 @@ jobs:
|
||||||
|
|
||||||
core.warning(`Timed out waiting for ${artifactName} from ${workflow_id} for ${headSha}.`);
|
core.warning(`Timed out waiting for ${artifactName} from ${workflow_id} for ${headSha}.`);
|
||||||
|
|
||||||
- name: Find bundle report artifact
|
|
||||||
if: github.event_name == 'workflow_run'
|
|
||||||
id: find-report-artifact
|
|
||||||
uses: actions/github-script@v9
|
|
||||||
with:
|
|
||||||
script: |
|
|
||||||
const artifactName = 'frontend-bundle-report';
|
|
||||||
const { owner, repo } = context.repo;
|
|
||||||
const artifacts = await github.paginate(github.rest.actions.listWorkflowRunArtifacts, {
|
|
||||||
owner,
|
|
||||||
repo,
|
|
||||||
run_id: context.payload.workflow_run.id,
|
|
||||||
per_page: 100,
|
|
||||||
});
|
|
||||||
const report = artifacts.find((artifact) => artifact.name === artifactName && !artifact.expired);
|
|
||||||
if (report) {
|
|
||||||
core.setOutput('exists', 'true');
|
|
||||||
} else {
|
|
||||||
core.info(`Workflow run ${context.payload.workflow_run.id} did not produce ${artifactName}.`);
|
|
||||||
core.setOutput('exists', 'false');
|
|
||||||
}
|
|
||||||
|
|
||||||
- name: Download bundle report from workflow_run
|
- name: Download bundle report from workflow_run
|
||||||
if: github.event_name == 'workflow_run' && steps.find-report-artifact.outputs.exists == 'true'
|
if: github.event_name == 'workflow_run'
|
||||||
uses: actions/download-artifact@v8
|
uses: actions/download-artifact@v8
|
||||||
with:
|
with:
|
||||||
name: frontend-bundle-report
|
name: frontend-bundle-report
|
||||||
|
|
@ -172,7 +147,7 @@ jobs:
|
||||||
run-id: ${{ steps.find-report-run.outputs.run-id }}
|
run-id: ${{ steps.find-report-run.outputs.run-id }}
|
||||||
|
|
||||||
- name: Comment on pull request
|
- name: Comment on pull request
|
||||||
if: (github.event_name == 'workflow_run' && steps.find-report-artifact.outputs.exists == 'true') || steps.find-report-run.outputs.run-id != ''
|
if: github.event_name == 'workflow_run' || steps.find-report-run.outputs.run-id != ''
|
||||||
uses: actions/github-script@v9
|
uses: actions/github-script@v9
|
||||||
with:
|
with:
|
||||||
github-token: ${{ secrets.FRONTEND_BUNDLE_REPORT_COMMENT_TOKEN || secrets.FRONTEND_JS_SIZE_COMMENT_TOKEN || secrets.FRONTEND_BUNDLE_VISUALIZER_COMMENT_TOKEN || github.token }}
|
github-token: ${{ secrets.FRONTEND_BUNDLE_REPORT_COMMENT_TOKEN || secrets.FRONTEND_JS_SIZE_COMMENT_TOKEN || secrets.FRONTEND_BUNDLE_VISUALIZER_COMMENT_TOKEN || github.token }}
|
||||||
|
|
@ -185,25 +160,30 @@ jobs:
|
||||||
const reportMarkers = [jsSizeMarker, visualizerMarker];
|
const reportMarkers = [jsSizeMarker, visualizerMarker];
|
||||||
const reportDir = path.join(process.env.RUNNER_TEMP, 'frontend-bundle-report');
|
const reportDir = path.join(process.env.RUNNER_TEMP, 'frontend-bundle-report');
|
||||||
const jsSizeReportPath = path.join(reportDir, 'frontend-js-size-report.md');
|
const jsSizeReportPath = path.join(reportDir, 'frontend-js-size-report.md');
|
||||||
|
const visualizerReportPath = path.join(reportDir, 'frontend-bundle-visualizer-report.md');
|
||||||
const prNumberPath = path.join(reportDir, 'pr-number.txt');
|
const prNumberPath = path.join(reportDir, 'pr-number.txt');
|
||||||
const headShaPath = path.join(reportDir, 'head-sha.txt');
|
const headShaPath = path.join(reportDir, 'head-sha.txt');
|
||||||
const workflowRun = context.payload.workflow_run;
|
const workflowRun = context.payload.workflow_run;
|
||||||
const pullRequest = context.payload.pull_request;
|
const pullRequest = context.payload.pull_request;
|
||||||
const eventHeadSha = workflowRun?.head_sha ?? pullRequest?.head?.sha ?? null;
|
const headSha = workflowRun?.head_sha ?? pullRequest?.head?.sha ?? null;
|
||||||
const { owner, repo } = context.repo;
|
const { owner, repo } = context.repo;
|
||||||
|
|
||||||
if (!fs.existsSync(jsSizeReportPath)) {
|
if (!fs.existsSync(jsSizeReportPath)) {
|
||||||
core.setFailed('The frontend bundle report artifact does not contain frontend-js-size-report.md.');
|
core.setFailed('The frontend bundle report artifact does not contain frontend-js-size-report.md.');
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
if (!fs.existsSync(visualizerReportPath)) {
|
||||||
|
core.setFailed('The frontend bundle report artifact does not contain frontend-bundle-visualizer-report.md.');
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
const artifactHeadSha = fs.existsSync(headShaPath)
|
const artifactHeadSha = fs.existsSync(headShaPath)
|
||||||
? fs.readFileSync(headShaPath, 'utf8').trim()
|
? fs.readFileSync(headShaPath, 'utf8').trim()
|
||||||
: null;
|
: null;
|
||||||
if (eventHeadSha != null && artifactHeadSha != null && artifactHeadSha !== eventHeadSha) {
|
if (headSha != null && artifactHeadSha != null && artifactHeadSha !== headSha) {
|
||||||
core.info(`The artifact head SHA (${artifactHeadSha}) differs from the event head SHA (${eventHeadSha}). Using artifact metadata for PR validation.`);
|
core.setFailed(`The artifact head SHA (${artifactHeadSha}) does not match the workflow head SHA (${headSha}).`);
|
||||||
|
return;
|
||||||
}
|
}
|
||||||
const reportHeadSha = artifactHeadSha ?? eventHeadSha;
|
|
||||||
|
|
||||||
const artifactPrNumber = fs.existsSync(prNumberPath)
|
const artifactPrNumber = fs.existsSync(prNumberPath)
|
||||||
? Number(fs.readFileSync(prNumberPath, 'utf8').trim())
|
? Number(fs.readFileSync(prNumberPath, 'utf8').trim())
|
||||||
|
|
@ -223,29 +203,25 @@ jobs:
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
if (reportHeadSha != null) {
|
const pullRequestsForCommit = await github.paginate(github.rest.repos.listPullRequestsAssociatedWithCommit, {
|
||||||
const pullRequestsForCommit = await github.paginate(github.rest.repos.listPullRequestsAssociatedWithCommit, {
|
owner,
|
||||||
owner,
|
repo,
|
||||||
repo,
|
commit_sha: headSha,
|
||||||
commit_sha: reportHeadSha,
|
per_page: 100,
|
||||||
per_page: 100,
|
});
|
||||||
});
|
for (const pullRequest of pullRequestsForCommit) {
|
||||||
for (const pullRequest of pullRequestsForCommit) {
|
associatedPullRequests.set(pullRequest.number, pullRequest);
|
||||||
associatedPullRequests.set(pullRequest.number, pullRequest);
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
if (Number.isInteger(artifactPrNumber) && associatedPullRequests.has(artifactPrNumber)) {
|
if (Number.isInteger(artifactPrNumber) && associatedPullRequests.has(artifactPrNumber)) {
|
||||||
issue_number = artifactPrNumber;
|
issue_number = artifactPrNumber;
|
||||||
} else if (Number.isInteger(artifactPrNumber) && associatedPullRequests.size === 0) {
|
|
||||||
issue_number = artifactPrNumber;
|
|
||||||
} else if (!Number.isInteger(artifactPrNumber) && associatedPullRequests.size === 1) {
|
} else if (!Number.isInteger(artifactPrNumber) && associatedPullRequests.size === 1) {
|
||||||
issue_number = [...associatedPullRequests.keys()][0];
|
issue_number = [...associatedPullRequests.keys()][0];
|
||||||
} else if (Number.isInteger(artifactPrNumber)) {
|
} else if (Number.isInteger(artifactPrNumber)) {
|
||||||
core.setFailed(`The artifact pull request number (${artifactPrNumber}) is not associated with ${reportHeadSha}.`);
|
core.setFailed(`The artifact pull request number (${artifactPrNumber}) is not associated with ${headSha}.`);
|
||||||
return;
|
return;
|
||||||
} else {
|
} else {
|
||||||
core.setFailed(`Could not determine the pull request associated with ${reportHeadSha}.`);
|
core.setFailed(`Could not determine the pull request associated with ${headSha}.`);
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
|
|
@ -253,23 +229,16 @@ jobs:
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
const currentPullRequest = await github.rest.pulls.get({
|
|
||||||
owner,
|
|
||||||
repo,
|
|
||||||
pull_number: issue_number,
|
|
||||||
});
|
|
||||||
const currentHeadSha = currentPullRequest.data.head?.sha;
|
|
||||||
if (reportHeadSha != null && currentHeadSha != null && reportHeadSha !== currentHeadSha) {
|
|
||||||
core.info(`The report head SHA (${reportHeadSha}) is not the current pull request head SHA (${currentHeadSha}). Skipping stale frontend bundle report.`);
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
const jsSizeReport = fs.readFileSync(jsSizeReportPath, 'utf8').trim();
|
const jsSizeReport = fs.readFileSync(jsSizeReportPath, 'utf8').trim();
|
||||||
if (!jsSizeReport.includes(jsSizeMarker)) {
|
if (!jsSizeReport.includes(jsSizeMarker)) {
|
||||||
core.setFailed('The frontend JS size report is missing the expected marker.');
|
core.setFailed('The frontend JS size report is missing the expected marker.');
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
let body = `${jsSizeReport}\n`;
|
const visualizerReport = fs.readFileSync(visualizerReportPath, 'utf8').trim();
|
||||||
|
let body = [
|
||||||
|
jsSizeReport,
|
||||||
|
visualizerReport,
|
||||||
|
].join('\n\n') + '\n';
|
||||||
|
|
||||||
const maxCommentLength = 65_000;
|
const maxCommentLength = 65_000;
|
||||||
if (body.length > maxCommentLength) {
|
if (body.length > maxCommentLength) {
|
||||||
|
|
|
||||||
49
.github/workflows/frontend-bundle-report.yml
vendored
49
.github/workflows/frontend-bundle-report.yml
vendored
|
|
@ -20,8 +20,8 @@ on:
|
||||||
- pnpm-lock.yaml
|
- pnpm-lock.yaml
|
||||||
- pnpm-workspace.yaml
|
- pnpm-workspace.yaml
|
||||||
- .node-version
|
- .node-version
|
||||||
- .github/scripts/utility.mts
|
- .github/scripts/frontend-js-size.mjs
|
||||||
- .github/scripts/frontend-js-size.mts
|
- .github/scripts/frontend-bundle-visualizer-report.mjs
|
||||||
- .github/workflows/frontend-bundle-report.yml
|
- .github/workflows/frontend-bundle-report.yml
|
||||||
- .github/workflows/frontend-bundle-report-comment.yml
|
- .github/workflows/frontend-bundle-report-comment.yml
|
||||||
|
|
||||||
|
|
@ -56,25 +56,21 @@ jobs:
|
||||||
path: after
|
path: after
|
||||||
submodules: true
|
submodules: true
|
||||||
|
|
||||||
- name: Check base visualizer support
|
- name: Backport visualizer tooling to base if needed
|
||||||
id: check-base-visualizer
|
|
||||||
shell: bash
|
shell: bash
|
||||||
run: |
|
run: |
|
||||||
if grep -q 'FRONTEND_BUNDLE_VISUALIZER' before/packages/frontend/vite.config.ts; then
|
if ! grep -q 'FRONTEND_BUNDLE_VISUALIZER' before/packages/frontend/vite.config.ts; then
|
||||||
echo 'supported=true' >> "$GITHUB_OUTPUT"
|
cp after/packages/frontend/package.json before/packages/frontend/package.json
|
||||||
else
|
cp after/packages/frontend/vite.config.ts before/packages/frontend/vite.config.ts
|
||||||
echo 'supported=false' >> "$GITHUB_OUTPUT"
|
cp after/pnpm-lock.yaml before/pnpm-lock.yaml
|
||||||
echo 'Base commit does not support frontend bundle visualizer. Skipping frontend bundle report.' >> "$GITHUB_STEP_SUMMARY"
|
|
||||||
fi
|
fi
|
||||||
|
|
||||||
- name: Setup pnpm
|
- name: Setup pnpm
|
||||||
if: steps.check-base-visualizer.outputs.supported == 'true'
|
|
||||||
uses: pnpm/action-setup@v6.0.3
|
uses: pnpm/action-setup@v6.0.3
|
||||||
with:
|
with:
|
||||||
package_json_file: after/package.json
|
package_json_file: after/package.json
|
||||||
|
|
||||||
- name: Setup Node.js
|
- name: Setup Node.js
|
||||||
if: steps.check-base-visualizer.outputs.supported == 'true'
|
|
||||||
uses: actions/setup-node@v6.4.0
|
uses: actions/setup-node@v6.4.0
|
||||||
with:
|
with:
|
||||||
node-version-file: after/.node-version
|
node-version-file: after/.node-version
|
||||||
|
|
@ -84,84 +80,67 @@ jobs:
|
||||||
after/pnpm-lock.yaml
|
after/pnpm-lock.yaml
|
||||||
|
|
||||||
- name: Install dependencies for base
|
- name: Install dependencies for base
|
||||||
if: steps.check-base-visualizer.outputs.supported == 'true'
|
|
||||||
working-directory: before
|
working-directory: before
|
||||||
run: pnpm i --frozen-lockfile
|
run: pnpm i --frozen-lockfile
|
||||||
|
|
||||||
- name: Build frontend dependencies for base
|
- name: Build frontend dependencies for base
|
||||||
if: steps.check-base-visualizer.outputs.supported == 'true'
|
|
||||||
working-directory: before
|
working-directory: before
|
||||||
run: pnpm --filter "frontend^..." run build
|
run: pnpm --filter "frontend^..." run build
|
||||||
|
|
||||||
- name: Prepare report output
|
- name: Prepare report output
|
||||||
if: steps.check-base-visualizer.outputs.supported == 'true'
|
|
||||||
run: mkdir -p "$RUNNER_TEMP/frontend-bundle-report"
|
run: mkdir -p "$RUNNER_TEMP/frontend-bundle-report"
|
||||||
|
|
||||||
- name: Build frontend report for base
|
- name: Build frontend report for base
|
||||||
if: steps.check-base-visualizer.outputs.supported == 'true'
|
|
||||||
working-directory: before
|
working-directory: before
|
||||||
env:
|
env:
|
||||||
FRONTEND_BUNDLE_VISUALIZER: 'true'
|
FRONTEND_BUNDLE_VISUALIZER: 'true'
|
||||||
|
FRONTEND_BUNDLE_VISUALIZER_TEMPLATE: raw-data
|
||||||
FRONTEND_BUNDLE_VISUALIZER_FILE: ${{ runner.temp }}/frontend-bundle-report/before-stats.json
|
FRONTEND_BUNDLE_VISUALIZER_FILE: ${{ runner.temp }}/frontend-bundle-report/before-stats.json
|
||||||
run: pnpm --filter frontend run build
|
run: pnpm --filter frontend run build
|
||||||
|
|
||||||
- name: Install dependencies for pull request
|
- name: Install dependencies for pull request
|
||||||
if: steps.check-base-visualizer.outputs.supported == 'true'
|
|
||||||
working-directory: after
|
working-directory: after
|
||||||
run: pnpm i --frozen-lockfile
|
run: pnpm i --frozen-lockfile
|
||||||
|
|
||||||
- name: Build frontend dependencies for pull request
|
- name: Build frontend dependencies for pull request
|
||||||
if: steps.check-base-visualizer.outputs.supported == 'true'
|
|
||||||
working-directory: after
|
working-directory: after
|
||||||
run: pnpm --filter "frontend^..." run build
|
run: pnpm --filter "frontend^..." run build
|
||||||
|
|
||||||
- name: Build frontend report for pull request
|
- name: Build frontend report for pull request
|
||||||
if: steps.check-base-visualizer.outputs.supported == 'true'
|
|
||||||
working-directory: after
|
working-directory: after
|
||||||
env:
|
env:
|
||||||
FRONTEND_BUNDLE_VISUALIZER: 'true'
|
FRONTEND_BUNDLE_VISUALIZER: 'true'
|
||||||
|
FRONTEND_BUNDLE_VISUALIZER_TEMPLATE: raw-data
|
||||||
FRONTEND_BUNDLE_VISUALIZER_FILE: ${{ runner.temp }}/frontend-bundle-report/after-stats.json
|
FRONTEND_BUNDLE_VISUALIZER_FILE: ${{ runner.temp }}/frontend-bundle-report/after-stats.json
|
||||||
FRONTEND_BUNDLE_VISUALIZER_HTML_FILE: ${{ runner.temp }}/frontend-bundle-report/frontend-bundle-visualizer.html
|
|
||||||
run: pnpm --filter frontend run build
|
run: pnpm --filter frontend run build
|
||||||
|
|
||||||
- name: Upload bundle visualizer
|
|
||||||
if: steps.check-base-visualizer.outputs.supported == 'true'
|
|
||||||
id: upload-bundle-visualizer
|
|
||||||
uses: actions/upload-artifact@v7
|
|
||||||
with:
|
|
||||||
name: frontend-bundle-visualizer
|
|
||||||
path: ${{ runner.temp }}/frontend-bundle-report/frontend-bundle-visualizer.html
|
|
||||||
if-no-files-found: error
|
|
||||||
archive: false
|
|
||||||
retention-days: 7
|
|
||||||
|
|
||||||
- name: Generate report markdown
|
- name: Generate report markdown
|
||||||
if: steps.check-base-visualizer.outputs.supported == 'true'
|
|
||||||
shell: bash
|
shell: bash
|
||||||
env:
|
env:
|
||||||
BASE_SHA: ${{ github.event.pull_request.base.sha }}
|
BASE_SHA: ${{ github.event.pull_request.base.sha }}
|
||||||
HEAD_SHA: ${{ github.event.pull_request.head.sha }}
|
HEAD_SHA: ${{ github.event.pull_request.head.sha }}
|
||||||
PR_NUMBER: ${{ github.event.pull_request.number }}
|
PR_NUMBER: ${{ github.event.pull_request.number }}
|
||||||
FRONTEND_BUNDLE_REPORT_ARTIFACT_URL: ${{ steps.upload-bundle-visualizer.outputs.artifact-url }}
|
|
||||||
run: |
|
run: |
|
||||||
REPORT_DIR="$RUNNER_TEMP/frontend-bundle-report"
|
REPORT_DIR="$RUNNER_TEMP/frontend-bundle-report"
|
||||||
node after/.github/scripts/frontend-js-size.mts before after "$REPORT_DIR/before-stats.json" "$REPORT_DIR/after-stats.json" "$REPORT_DIR/frontend-js-size-report.md"
|
node after/.github/scripts/frontend-js-size.mjs before after "$REPORT_DIR/frontend-js-size-report.md"
|
||||||
|
node after/.github/scripts/frontend-bundle-visualizer-report.mjs "$REPORT_DIR/before-stats.json" "$REPORT_DIR/after-stats.json" "$REPORT_DIR/frontend-bundle-visualizer-report.md"
|
||||||
printf '%s\n' "$PR_NUMBER" > "$REPORT_DIR/pr-number.txt"
|
printf '%s\n' "$PR_NUMBER" > "$REPORT_DIR/pr-number.txt"
|
||||||
printf '%s\n' "$BASE_SHA" > "$REPORT_DIR/base-sha.txt"
|
printf '%s\n' "$BASE_SHA" > "$REPORT_DIR/base-sha.txt"
|
||||||
printf '%s\n' "$HEAD_SHA" > "$REPORT_DIR/head-sha.txt"
|
printf '%s\n' "$HEAD_SHA" > "$REPORT_DIR/head-sha.txt"
|
||||||
printf '%s\n' "${{ github.event.pull_request.html_url }}" > "$REPORT_DIR/pr-url.txt"
|
printf '%s\n' "${{ github.event.pull_request.html_url }}" > "$REPORT_DIR/pr-url.txt"
|
||||||
|
|
||||||
- name: Check report
|
- name: Check report
|
||||||
if: steps.check-base-visualizer.outputs.supported == 'true'
|
|
||||||
run: |
|
run: |
|
||||||
REPORT_DIR="$RUNNER_TEMP/frontend-bundle-report"
|
REPORT_DIR="$RUNNER_TEMP/frontend-bundle-report"
|
||||||
test -s "$REPORT_DIR/before-stats.json"
|
test -s "$REPORT_DIR/before-stats.json"
|
||||||
test -s "$REPORT_DIR/after-stats.json"
|
test -s "$REPORT_DIR/after-stats.json"
|
||||||
test -s "$REPORT_DIR/frontend-js-size-report.md"
|
test -s "$REPORT_DIR/frontend-js-size-report.md"
|
||||||
|
test -s "$REPORT_DIR/frontend-bundle-visualizer-report.md"
|
||||||
cat "$REPORT_DIR/frontend-js-size-report.md" >> "$GITHUB_STEP_SUMMARY"
|
cat "$REPORT_DIR/frontend-js-size-report.md" >> "$GITHUB_STEP_SUMMARY"
|
||||||
|
printf '\n\n' >> "$GITHUB_STEP_SUMMARY"
|
||||||
|
cat "$REPORT_DIR/frontend-bundle-visualizer-report.md" >> "$GITHUB_STEP_SUMMARY"
|
||||||
|
|
||||||
- name: Upload bundle report
|
- name: Upload bundle report
|
||||||
if: steps.check-base-visualizer.outputs.supported == 'true'
|
|
||||||
uses: actions/upload-artifact@v7
|
uses: actions/upload-artifact@v7
|
||||||
with:
|
with:
|
||||||
name: frontend-bundle-report
|
name: frontend-bundle-report
|
||||||
|
|
|
||||||
90
.github/workflows/get-backend-memory.yml
vendored
90
.github/workflows/get-backend-memory.yml
vendored
|
|
@ -9,14 +9,7 @@ on:
|
||||||
paths:
|
paths:
|
||||||
- packages/backend/**
|
- packages/backend/**
|
||||||
- packages/misskey-js/**
|
- packages/misskey-js/**
|
||||||
- .github/scripts/utility.mts
|
|
||||||
- .github/scripts/backend-memory-report.mts
|
|
||||||
- .github/scripts/measure-backend-memory-comparison.mts
|
|
||||||
- .github/scripts/backend-js-footprint.mjs
|
|
||||||
- .github/scripts/backend-js-footprint-loader.mjs
|
|
||||||
- .github/scripts/backend-js-footprint-require.cjs
|
|
||||||
- .github/workflows/get-backend-memory.yml
|
- .github/workflows/get-backend-memory.yml
|
||||||
- .github/workflows/report-backend-memory.yml
|
|
||||||
|
|
||||||
jobs:
|
jobs:
|
||||||
get-memory-usage:
|
get-memory-usage:
|
||||||
|
|
@ -24,6 +17,15 @@ jobs:
|
||||||
permissions:
|
permissions:
|
||||||
contents: read
|
contents: read
|
||||||
|
|
||||||
|
strategy:
|
||||||
|
matrix:
|
||||||
|
memory-json-name: [memory-base.json, memory-head.json]
|
||||||
|
include:
|
||||||
|
- memory-json-name: memory-base.json
|
||||||
|
ref: ${{ github.base_ref }}
|
||||||
|
- memory-json-name: memory-head.json
|
||||||
|
ref: refs/pull/${{ github.event.number }}/merge
|
||||||
|
|
||||||
services:
|
services:
|
||||||
postgres:
|
postgres:
|
||||||
image: postgres:18
|
image: postgres:18
|
||||||
|
|
@ -38,77 +40,37 @@ jobs:
|
||||||
- 56312:6379
|
- 56312:6379
|
||||||
|
|
||||||
steps:
|
steps:
|
||||||
- name: Checkout base
|
- uses: actions/checkout@v6.0.2
|
||||||
uses: actions/checkout@v6.0.2
|
|
||||||
with:
|
with:
|
||||||
ref: ${{ github.base_ref }}
|
ref: ${{ matrix.ref }}
|
||||||
path: base
|
|
||||||
submodules: true
|
|
||||||
- name: Checkout head
|
|
||||||
uses: actions/checkout@v6.0.2
|
|
||||||
with:
|
|
||||||
ref: refs/pull/${{ github.event.number }}/merge
|
|
||||||
path: head
|
|
||||||
submodules: true
|
submodules: true
|
||||||
- name: Setup pnpm
|
- name: Setup pnpm
|
||||||
uses: pnpm/action-setup@v6.0.3
|
uses: pnpm/action-setup@v6.0.3
|
||||||
with:
|
|
||||||
package_json_file: head/package.json
|
|
||||||
- name: Use Node.js
|
- name: Use Node.js
|
||||||
uses: actions/setup-node@v6.4.0
|
uses: actions/setup-node@v6.4.0
|
||||||
with:
|
with:
|
||||||
node-version-file: 'head/.node-version'
|
node-version-file: '.node-version'
|
||||||
cache: 'pnpm'
|
cache: 'pnpm'
|
||||||
cache-dependency-path: |
|
- run: pnpm i --frozen-lockfile
|
||||||
base/pnpm-lock.yaml
|
- name: Check pnpm-lock.yaml
|
||||||
head/pnpm-lock.yaml
|
|
||||||
- name: Install base dependencies
|
|
||||||
working-directory: base
|
|
||||||
run: pnpm i --frozen-lockfile
|
|
||||||
- name: Check base pnpm-lock.yaml
|
|
||||||
working-directory: base
|
|
||||||
run: git diff --exit-code pnpm-lock.yaml
|
run: git diff --exit-code pnpm-lock.yaml
|
||||||
- name: Configure base
|
- name: Copy Configure
|
||||||
working-directory: base
|
run: cp .github/misskey/test.yml .config/default.yml
|
||||||
run: |
|
- name: Compile Configure
|
||||||
cp .github/misskey/test.yml .config/default.yml
|
run: pnpm compile-config
|
||||||
pnpm compile-config
|
- name: Build
|
||||||
- name: Build base
|
|
||||||
working-directory: base
|
|
||||||
run: pnpm build
|
run: pnpm build
|
||||||
- name: Install head dependencies
|
- name: Run migrations
|
||||||
working-directory: head
|
run: pnpm --filter backend migrate
|
||||||
run: pnpm i --frozen-lockfile
|
- name: Measure memory usage
|
||||||
- name: Check head pnpm-lock.yaml
|
|
||||||
working-directory: head
|
|
||||||
run: git diff --exit-code pnpm-lock.yaml
|
|
||||||
- name: Configure head
|
|
||||||
working-directory: head
|
|
||||||
run: |
|
run: |
|
||||||
cp .github/misskey/test.yml .config/default.yml
|
# Start the server and measure memory usage
|
||||||
pnpm compile-config
|
node packages/backend/scripts/measure-memory.mjs > ${{ matrix.memory-json-name }}
|
||||||
- name: Build head
|
|
||||||
working-directory: head
|
|
||||||
run: pnpm build
|
|
||||||
- name: Measure backend memory usage
|
|
||||||
env:
|
|
||||||
MK_MEMORY_COMPARE_ROUNDS: 5
|
|
||||||
MK_MEMORY_COMPARE_WARMUP_ROUNDS: 1
|
|
||||||
MK_MEMORY_HEAP_SNAPSHOT: 1
|
|
||||||
run: node head/.github/scripts/measure-backend-memory-comparison.mts base head memory-base.json memory-head.json
|
|
||||||
- name: Measure backend loaded JS footprint
|
|
||||||
run: |
|
|
||||||
node head/.github/scripts/backend-js-footprint.mjs base js-footprint-base.json
|
|
||||||
node head/.github/scripts/backend-js-footprint.mjs head js-footprint-head.json
|
|
||||||
- name: Upload Artifact
|
- name: Upload Artifact
|
||||||
uses: actions/upload-artifact@v7
|
uses: actions/upload-artifact@v7
|
||||||
with:
|
with:
|
||||||
name: memory-artifact-results
|
name: memory-artifact-${{ matrix.memory-json-name }}
|
||||||
path: |
|
path: ${{ matrix.memory-json-name }}
|
||||||
memory-base.json
|
|
||||||
memory-head.json
|
|
||||||
js-footprint-base.json
|
|
||||||
js-footprint-head.json
|
|
||||||
|
|
||||||
save-pr-number:
|
save-pr-number:
|
||||||
runs-on: ubuntu-latest
|
runs-on: ubuntu-latest
|
||||||
|
|
|
||||||
122
.github/workflows/report-backend-memory.yml
vendored
122
.github/workflows/report-backend-memory.yml
vendored
|
|
@ -11,14 +11,9 @@ jobs:
|
||||||
runs-on: ubuntu-latest
|
runs-on: ubuntu-latest
|
||||||
if: ${{ github.event.workflow_run.conclusion == 'success' }}
|
if: ${{ github.event.workflow_run.conclusion == 'success' }}
|
||||||
permissions:
|
permissions:
|
||||||
actions: read
|
|
||||||
contents: read
|
|
||||||
pull-requests: write
|
pull-requests: write
|
||||||
|
|
||||||
steps:
|
steps:
|
||||||
- name: Checkout
|
|
||||||
uses: actions/checkout@v6.0.2
|
|
||||||
|
|
||||||
- name: Download artifact
|
- name: Download artifact
|
||||||
uses: actions/github-script@v9
|
uses: actions/github-script@v9
|
||||||
with:
|
with:
|
||||||
|
|
@ -53,13 +48,120 @@ jobs:
|
||||||
run: cat ./artifacts/memory-base.json
|
run: cat ./artifacts/memory-base.json
|
||||||
- name: Output head
|
- name: Output head
|
||||||
run: cat ./artifacts/memory-head.json
|
run: cat ./artifacts/memory-head.json
|
||||||
- name: Output base JS footprint
|
- name: Compare memory usage
|
||||||
run: cat ./artifacts/js-footprint-base.json
|
id: compare
|
||||||
- name: Output head JS footprint
|
run: |
|
||||||
run: cat ./artifacts/js-footprint-head.json
|
BASE_MEMORY=$(cat ./artifacts/memory-base.json)
|
||||||
|
HEAD_MEMORY=$(cat ./artifacts/memory-head.json)
|
||||||
|
|
||||||
|
variation() {
|
||||||
|
calc() {
|
||||||
|
BASE=$(echo "$BASE_MEMORY" | jq -r ".${1}.${2} // 0")
|
||||||
|
HEAD=$(echo "$HEAD_MEMORY" | jq -r ".${1}.${2} // 0")
|
||||||
|
|
||||||
|
DIFF=$((HEAD - BASE))
|
||||||
|
if [ "$BASE" -gt 0 ]; then
|
||||||
|
DIFF_PERCENT=$(echo "scale=2; ($DIFF * 100) / $BASE" | bc)
|
||||||
|
else
|
||||||
|
DIFF_PERCENT=0
|
||||||
|
fi
|
||||||
|
|
||||||
|
# Convert KB to MB for readability
|
||||||
|
BASE_MB=$(echo "scale=2; $BASE / 1024" | bc)
|
||||||
|
HEAD_MB=$(echo "scale=2; $HEAD / 1024" | bc)
|
||||||
|
DIFF_MB=$(echo "scale=2; $DIFF / 1024" | bc)
|
||||||
|
|
||||||
|
JSON=$(jq -c -n \
|
||||||
|
--argjson base "$BASE_MB" \
|
||||||
|
--argjson head "$HEAD_MB" \
|
||||||
|
--argjson diff "$DIFF_MB" \
|
||||||
|
--argjson diff_percent "$DIFF_PERCENT" \
|
||||||
|
'{base: $base, head: $head, diff: $diff, diff_percent: $diff_percent}')
|
||||||
|
|
||||||
|
echo "$JSON"
|
||||||
|
}
|
||||||
|
|
||||||
|
JSON=$(jq -c -n \
|
||||||
|
--argjson VmRSS "$(calc $1 VmRSS)" \
|
||||||
|
--argjson VmHWM "$(calc $1 VmHWM)" \
|
||||||
|
--argjson VmSize "$(calc $1 VmSize)" \
|
||||||
|
--argjson VmData "$(calc $1 VmData)" \
|
||||||
|
'{VmRSS: $VmRSS, VmHWM: $VmHWM, VmSize: $VmSize, VmData: $VmData}')
|
||||||
|
|
||||||
|
echo "$JSON"
|
||||||
|
}
|
||||||
|
|
||||||
|
JSON=$(jq -c -n \
|
||||||
|
--argjson beforeGc "$(variation beforeGc)" \
|
||||||
|
--argjson afterGc "$(variation afterGc)" \
|
||||||
|
--argjson afterRequest "$(variation afterRequest)" \
|
||||||
|
'{beforeGc: $beforeGc, afterGc: $afterGc, afterRequest: $afterRequest}')
|
||||||
|
|
||||||
|
echo "res=$JSON" >> "$GITHUB_OUTPUT"
|
||||||
- id: build-comment
|
- id: build-comment
|
||||||
name: Build memory comment
|
name: Build memory comment
|
||||||
run: node .github/scripts/backend-memory-report.mts ./artifacts/memory-base.json ./artifacts/memory-head.json ./output.md ./artifacts/js-footprint-base.json ./artifacts/js-footprint-head.json
|
env:
|
||||||
|
RES: ${{ steps.compare.outputs.res }}
|
||||||
|
run: |
|
||||||
|
HEADER="## Backend memory usage comparison"
|
||||||
|
FOOTER="[See workflow logs for details](https://github.com/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID})"
|
||||||
|
|
||||||
|
echo "$HEADER" > ./output.md
|
||||||
|
echo >> ./output.md
|
||||||
|
|
||||||
|
table() {
|
||||||
|
echo "| Metric | base (MB) | head (MB) | Diff (MB) | Diff (%) |" >> ./output.md
|
||||||
|
echo "|--------|------:|------:|------:|------:|" >> ./output.md
|
||||||
|
|
||||||
|
line() {
|
||||||
|
METRIC=$2
|
||||||
|
BASE=$(echo "$RES" | jq -r ".${1}.${2}.base")
|
||||||
|
HEAD=$(echo "$RES" | jq -r ".${1}.${2}.head")
|
||||||
|
DIFF=$(echo "$RES" | jq -r ".${1}.${2}.diff")
|
||||||
|
DIFF_PERCENT=$(echo "$RES" | jq -r ".${1}.${2}.diff_percent")
|
||||||
|
|
||||||
|
if (( $(echo "$DIFF_PERCENT > 0" | bc -l) )); then
|
||||||
|
DIFF="+$DIFF"
|
||||||
|
DIFF_PERCENT="+$DIFF_PERCENT"
|
||||||
|
fi
|
||||||
|
|
||||||
|
# highlight VmRSS
|
||||||
|
if [ "$2" = "VmRSS" ]; then
|
||||||
|
METRIC="**${METRIC}**"
|
||||||
|
BASE="**${BASE}**"
|
||||||
|
HEAD="**${HEAD}**"
|
||||||
|
DIFF="**${DIFF}**"
|
||||||
|
DIFF_PERCENT="**${DIFF_PERCENT}**"
|
||||||
|
fi
|
||||||
|
|
||||||
|
echo "| ${METRIC} | ${BASE} MB | ${HEAD} MB | ${DIFF} MB | ${DIFF_PERCENT}% |" >> ./output.md
|
||||||
|
}
|
||||||
|
|
||||||
|
line $1 VmRSS
|
||||||
|
line $1 VmHWM
|
||||||
|
line $1 VmSize
|
||||||
|
line $1 VmData
|
||||||
|
}
|
||||||
|
|
||||||
|
echo "### Before GC" >> ./output.md
|
||||||
|
table beforeGc
|
||||||
|
echo >> ./output.md
|
||||||
|
|
||||||
|
echo "### After GC" >> ./output.md
|
||||||
|
table afterGc
|
||||||
|
echo >> ./output.md
|
||||||
|
|
||||||
|
echo "### After Request" >> ./output.md
|
||||||
|
table afterRequest
|
||||||
|
echo >> ./output.md
|
||||||
|
|
||||||
|
# Determine if this is a significant change (more than 5% increase)
|
||||||
|
if [ "$(echo "$RES" | jq -r '.afterGc.VmRSS.diff_percent | tonumber > 5')" = "true" ]; then
|
||||||
|
echo "⚠️ **Warning**: Memory usage has increased by more than 5%. Please verify this is not an unintended change." >> ./output.md
|
||||||
|
echo >> ./output.md
|
||||||
|
fi
|
||||||
|
|
||||||
|
echo "$FOOTER" >> ./output.md
|
||||||
- uses: thollander/actions-comment-pull-request@v3
|
- uses: thollander/actions-comment-pull-request@v3
|
||||||
with:
|
with:
|
||||||
pr-number: ${{ steps.load-pr-num.outputs.pr-number }}
|
pr-number: ${{ steps.load-pr-num.outputs.pr-number }}
|
||||||
|
|
|
||||||
35
.github/workflows/test-backend.yml
vendored
35
.github/workflows/test-backend.yml
vendored
|
|
@ -19,6 +19,12 @@ on:
|
||||||
- .github/workflows/test-backend.yml
|
- .github/workflows/test-backend.yml
|
||||||
- .github/misskey/test.yml
|
- .github/misskey/test.yml
|
||||||
workflow_dispatch:
|
workflow_dispatch:
|
||||||
|
inputs:
|
||||||
|
force_ffmpeg_cache_update:
|
||||||
|
description: 'Force update ffmpeg cache'
|
||||||
|
required: false
|
||||||
|
default: false
|
||||||
|
type: boolean
|
||||||
|
|
||||||
jobs:
|
jobs:
|
||||||
unit:
|
unit:
|
||||||
|
|
@ -56,9 +62,36 @@ jobs:
|
||||||
submodules: true
|
submodules: true
|
||||||
- name: Setup pnpm
|
- name: Setup pnpm
|
||||||
uses: pnpm/action-setup@v6.0.3
|
uses: pnpm/action-setup@v6.0.3
|
||||||
|
- name: Get current date
|
||||||
|
id: current-date
|
||||||
|
run: echo "today=$(date +'%Y-%m-%d')" >> $GITHUB_OUTPUT
|
||||||
|
- name: Setup and Restore ffmpeg/ffprobe Cache
|
||||||
|
id: cache-ffmpeg
|
||||||
|
uses: actions/cache@v5
|
||||||
|
with:
|
||||||
|
path: |
|
||||||
|
/usr/local/bin/ffmpeg
|
||||||
|
/usr/local/bin/ffprobe
|
||||||
|
# daily cache
|
||||||
|
key: ${{ runner.os }}-ffmpeg-${{ steps.current-date.outputs.today }}
|
||||||
|
restore-keys: |
|
||||||
|
${{ runner.os }}-ffmpeg-${{ steps.current-date.outputs.today }}
|
||||||
- name: Install FFmpeg
|
- name: Install FFmpeg
|
||||||
|
if: steps.cache-ffmpeg.outputs.cache-hit != 'true' || github.event.inputs.force_ffmpeg_cache_update == true
|
||||||
run: |
|
run: |
|
||||||
sudo apt install -y ffmpeg
|
for i in {1..3}; do
|
||||||
|
echo "Attempt $i: Installing FFmpeg..."
|
||||||
|
curl -s -L https://johnvansickle.com/ffmpeg/releases/ffmpeg-release-amd64-static.tar.xz -o ffmpeg.tar.xz && \
|
||||||
|
tar -xf ffmpeg.tar.xz && \
|
||||||
|
mv ffmpeg-*-static/ffmpeg /usr/local/bin/ && \
|
||||||
|
mv ffmpeg-*-static/ffprobe /usr/local/bin/ && \
|
||||||
|
rm -rf ffmpeg.tar.xz ffmpeg-*-static/ && \
|
||||||
|
break || sleep 10
|
||||||
|
if [ $i -eq 3 ]; then
|
||||||
|
echo "Failed to install FFmpeg after 3 attempts"
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
done
|
||||||
- name: Use Node.js
|
- name: Use Node.js
|
||||||
uses: actions/setup-node@v6.4.0
|
uses: actions/setup-node@v6.4.0
|
||||||
with:
|
with:
|
||||||
|
|
|
||||||
35
.github/workflows/test-federation.yml
vendored
35
.github/workflows/test-federation.yml
vendored
|
|
@ -15,6 +15,12 @@ on:
|
||||||
- packages/misskey-js/**
|
- packages/misskey-js/**
|
||||||
- .github/workflows/test-federation.yml
|
- .github/workflows/test-federation.yml
|
||||||
workflow_dispatch:
|
workflow_dispatch:
|
||||||
|
inputs:
|
||||||
|
force_ffmpeg_cache_update:
|
||||||
|
description: 'Force update ffmpeg cache'
|
||||||
|
required: false
|
||||||
|
default: false
|
||||||
|
type: boolean
|
||||||
|
|
||||||
jobs:
|
jobs:
|
||||||
test:
|
test:
|
||||||
|
|
@ -31,9 +37,36 @@ jobs:
|
||||||
submodules: true
|
submodules: true
|
||||||
- name: Setup pnpm
|
- name: Setup pnpm
|
||||||
uses: pnpm/action-setup@v6.0.3
|
uses: pnpm/action-setup@v6.0.3
|
||||||
|
- name: Get current date
|
||||||
|
id: current-date
|
||||||
|
run: echo "today=$(date +'%Y-%m-%d')" >> $GITHUB_OUTPUT
|
||||||
|
- name: Setup and Restore ffmpeg/ffprobe Cache
|
||||||
|
id: cache-ffmpeg
|
||||||
|
uses: actions/cache@v5
|
||||||
|
with:
|
||||||
|
path: |
|
||||||
|
/usr/local/bin/ffmpeg
|
||||||
|
/usr/local/bin/ffprobe
|
||||||
|
# daily cache
|
||||||
|
key: ${{ runner.os }}-ffmpeg-${{ steps.current-date.outputs.today }}
|
||||||
|
restore-keys: |
|
||||||
|
${{ runner.os }}-ffmpeg-${{ steps.current-date.outputs.today }}
|
||||||
- name: Install FFmpeg
|
- name: Install FFmpeg
|
||||||
|
if: steps.cache-ffmpeg.outputs.cache-hit != 'true' || github.event.inputs.force_ffmpeg_cache_update == true
|
||||||
run: |
|
run: |
|
||||||
sudo apt install -y ffmpeg
|
for i in {1..3}; do
|
||||||
|
echo "Attempt $i: Installing FFmpeg..."
|
||||||
|
curl -s -L https://johnvansickle.com/ffmpeg/releases/ffmpeg-release-amd64-static.tar.xz -o ffmpeg.tar.xz && \
|
||||||
|
tar -xf ffmpeg.tar.xz && \
|
||||||
|
mv ffmpeg-*-static/ffmpeg /usr/local/bin/ && \
|
||||||
|
mv ffmpeg-*-static/ffprobe /usr/local/bin/ && \
|
||||||
|
rm -rf ffmpeg.tar.xz ffmpeg-*-static/ && \
|
||||||
|
break || sleep 10
|
||||||
|
if [ $i -eq 3 ]; then
|
||||||
|
echo "Failed to install FFmpeg after 3 attempts"
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
done
|
||||||
- name: Use Node.js
|
- name: Use Node.js
|
||||||
uses: actions/setup-node@v6.4.0
|
uses: actions/setup-node@v6.4.0
|
||||||
with:
|
with:
|
||||||
|
|
|
||||||
|
|
@ -1 +1 @@
|
||||||
22.18.0
|
22.15.0
|
||||||
|
|
|
||||||
12
CHANGELOG.md
12
CHANGELOG.md
|
|
@ -1,15 +1,3 @@
|
||||||
## Unreleased
|
|
||||||
|
|
||||||
### General
|
|
||||||
-
|
|
||||||
|
|
||||||
### Client
|
|
||||||
-
|
|
||||||
|
|
||||||
### Server
|
|
||||||
-
|
|
||||||
|
|
||||||
|
|
||||||
## 2026.6.0
|
## 2026.6.0
|
||||||
|
|
||||||
### General
|
### General
|
||||||
|
|
|
||||||
231
packages/backend/scripts/measure-memory.mjs
Normal file
231
packages/backend/scripts/measure-memory.mjs
Normal file
|
|
@ -0,0 +1,231 @@
|
||||||
|
/*
|
||||||
|
* SPDX-FileCopyrightText: syuilo and misskey-project
|
||||||
|
* SPDX-License-Identifier: AGPL-3.0-only
|
||||||
|
*/
|
||||||
|
|
||||||
|
/**
|
||||||
|
* This script starts the Misskey backend server, waits for it to be ready,
|
||||||
|
* measures memory usage, and outputs the result as JSON.
|
||||||
|
*
|
||||||
|
* Usage: node scripts/measure-memory.mjs
|
||||||
|
*/
|
||||||
|
|
||||||
|
import { fork } from 'node:child_process';
|
||||||
|
import { setTimeout } from 'node:timers/promises';
|
||||||
|
import { fileURLToPath } from 'node:url';
|
||||||
|
import { dirname, join } from 'node:path';
|
||||||
|
import * as http from 'node:http';
|
||||||
|
import * as fs from 'node:fs/promises';
|
||||||
|
|
||||||
|
const __filename = fileURLToPath(import.meta.url);
|
||||||
|
const __dirname = dirname(__filename);
|
||||||
|
|
||||||
|
const SAMPLE_COUNT = 3; // Number of samples to measure
|
||||||
|
const STARTUP_TIMEOUT = 120000; // 120 seconds timeout for server startup
|
||||||
|
const MEMORY_SETTLE_TIME = 10000; // Wait 10 seconds after startup for memory to settle
|
||||||
|
|
||||||
|
const keys = {
|
||||||
|
VmPeak: 0,
|
||||||
|
VmSize: 0,
|
||||||
|
VmHWM: 0,
|
||||||
|
VmRSS: 0,
|
||||||
|
VmData: 0,
|
||||||
|
VmStk: 0,
|
||||||
|
VmExe: 0,
|
||||||
|
VmLib: 0,
|
||||||
|
VmPTE: 0,
|
||||||
|
VmSwap: 0,
|
||||||
|
};
|
||||||
|
|
||||||
|
async function getMemoryUsage(pid) {
|
||||||
|
const status = await fs.readFile(`/proc/${pid}/status`, 'utf-8');
|
||||||
|
|
||||||
|
const result = {};
|
||||||
|
for (const key of Object.keys(keys)) {
|
||||||
|
const match = status.match(new RegExp(`${key}:\\s+(\\d+)\\s+kB`));
|
||||||
|
if (match) {
|
||||||
|
result[key] = parseInt(match[1], 10);
|
||||||
|
} else {
|
||||||
|
throw new Error(`Failed to parse ${key} from /proc/${pid}/status`);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return result;
|
||||||
|
}
|
||||||
|
|
||||||
|
async function measureMemory() {
|
||||||
|
// Start the Misskey backend server using fork to enable IPC
|
||||||
|
const serverProcess = fork(join(__dirname, '../built/entry.js'), ['expose-gc'], {
|
||||||
|
cwd: join(__dirname, '..'),
|
||||||
|
env: {
|
||||||
|
...process.env,
|
||||||
|
NODE_ENV: 'production',
|
||||||
|
MK_DISABLE_CLUSTERING: '1',
|
||||||
|
},
|
||||||
|
stdio: ['pipe', 'pipe', 'pipe', 'ipc'],
|
||||||
|
execArgv: [...process.execArgv, '--expose-gc'],
|
||||||
|
});
|
||||||
|
|
||||||
|
let serverReady = false;
|
||||||
|
|
||||||
|
// Listen for the 'ok' message from the server indicating it's ready
|
||||||
|
serverProcess.on('message', (message) => {
|
||||||
|
if (message === 'ok') {
|
||||||
|
serverReady = true;
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
// Handle server output
|
||||||
|
serverProcess.stdout?.on('data', (data) => {
|
||||||
|
process.stderr.write(`[server stdout] ${data}`);
|
||||||
|
});
|
||||||
|
|
||||||
|
serverProcess.stderr?.on('data', (data) => {
|
||||||
|
process.stderr.write(`[server stderr] ${data}`);
|
||||||
|
});
|
||||||
|
|
||||||
|
// Handle server error
|
||||||
|
serverProcess.on('error', (err) => {
|
||||||
|
process.stderr.write(`[server error] ${err}\n`);
|
||||||
|
});
|
||||||
|
|
||||||
|
async function triggerGc() {
|
||||||
|
const ok = new Promise((resolve) => {
|
||||||
|
serverProcess.once('message', (message) => {
|
||||||
|
if (message === 'gc ok') resolve();
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
serverProcess.send('gc');
|
||||||
|
|
||||||
|
await ok;
|
||||||
|
|
||||||
|
await setTimeout(1000);
|
||||||
|
}
|
||||||
|
|
||||||
|
function createRequest() {
|
||||||
|
return new Promise((resolve, reject) => {
|
||||||
|
const req = http.request({
|
||||||
|
host: 'localhost',
|
||||||
|
port: 61812,
|
||||||
|
path: '/api/meta',
|
||||||
|
method: 'POST',
|
||||||
|
}, (res) => {
|
||||||
|
res.on('data', () => { });
|
||||||
|
res.on('end', () => {
|
||||||
|
resolve();
|
||||||
|
});
|
||||||
|
});
|
||||||
|
req.on('error', (err) => {
|
||||||
|
reject(err);
|
||||||
|
});
|
||||||
|
req.end();
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
// Wait for server to be ready or timeout
|
||||||
|
const startupStartTime = Date.now();
|
||||||
|
while (!serverReady) {
|
||||||
|
if (Date.now() - startupStartTime > STARTUP_TIMEOUT) {
|
||||||
|
serverProcess.kill('SIGTERM');
|
||||||
|
throw new Error('Server startup timeout');
|
||||||
|
}
|
||||||
|
await setTimeout(100);
|
||||||
|
}
|
||||||
|
|
||||||
|
const startupTime = Date.now() - startupStartTime;
|
||||||
|
process.stderr.write(`Server started in ${startupTime}ms\n`);
|
||||||
|
|
||||||
|
// Wait for memory to settle
|
||||||
|
await setTimeout(MEMORY_SETTLE_TIME);
|
||||||
|
|
||||||
|
const pid = serverProcess.pid;
|
||||||
|
|
||||||
|
const beforeGc = await getMemoryUsage(pid);
|
||||||
|
|
||||||
|
await triggerGc();
|
||||||
|
|
||||||
|
const afterGc = await getMemoryUsage(pid);
|
||||||
|
|
||||||
|
// create some http requests to simulate load
|
||||||
|
const REQUEST_COUNT = 10;
|
||||||
|
await Promise.all(
|
||||||
|
Array.from({ length: REQUEST_COUNT }).map(() => createRequest()),
|
||||||
|
);
|
||||||
|
|
||||||
|
await triggerGc();
|
||||||
|
|
||||||
|
const afterRequest = await getMemoryUsage(pid);
|
||||||
|
|
||||||
|
// Stop the server
|
||||||
|
serverProcess.kill('SIGTERM');
|
||||||
|
|
||||||
|
// Wait for process to exit
|
||||||
|
let exited = false;
|
||||||
|
await new Promise((resolve) => {
|
||||||
|
serverProcess.on('exit', () => {
|
||||||
|
exited = true;
|
||||||
|
resolve(undefined);
|
||||||
|
});
|
||||||
|
// Force kill after 10 seconds if not exited
|
||||||
|
setTimeout(10000).then(() => {
|
||||||
|
if (!exited) {
|
||||||
|
serverProcess.kill('SIGKILL');
|
||||||
|
}
|
||||||
|
resolve(undefined);
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
const result = {
|
||||||
|
timestamp: new Date().toISOString(),
|
||||||
|
beforeGc,
|
||||||
|
afterGc,
|
||||||
|
afterRequest,
|
||||||
|
};
|
||||||
|
|
||||||
|
return result;
|
||||||
|
}
|
||||||
|
|
||||||
|
async function main() {
|
||||||
|
// 直列の方が時間的に分散されて正確そうだから直列でやる
|
||||||
|
const results = [];
|
||||||
|
for (let i = 0; i < SAMPLE_COUNT; i++) {
|
||||||
|
const res = await measureMemory();
|
||||||
|
results.push(res);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Calculate averages
|
||||||
|
const beforeGc = structuredClone(keys);
|
||||||
|
const afterGc = structuredClone(keys);
|
||||||
|
const afterRequest = structuredClone(keys);
|
||||||
|
for (const res of results) {
|
||||||
|
for (const key of Object.keys(keys)) {
|
||||||
|
beforeGc[key] += res.beforeGc[key];
|
||||||
|
afterGc[key] += res.afterGc[key];
|
||||||
|
afterRequest[key] += res.afterRequest[key];
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for (const key of Object.keys(keys)) {
|
||||||
|
beforeGc[key] = Math.round(beforeGc[key] / SAMPLE_COUNT);
|
||||||
|
afterGc[key] = Math.round(afterGc[key] / SAMPLE_COUNT);
|
||||||
|
afterRequest[key] = Math.round(afterRequest[key] / SAMPLE_COUNT);
|
||||||
|
}
|
||||||
|
|
||||||
|
const result = {
|
||||||
|
timestamp: new Date().toISOString(),
|
||||||
|
beforeGc,
|
||||||
|
afterGc,
|
||||||
|
afterRequest,
|
||||||
|
};
|
||||||
|
|
||||||
|
// Output as JSON to stdout
|
||||||
|
console.log(JSON.stringify(result, null, 2));
|
||||||
|
}
|
||||||
|
|
||||||
|
main().catch((err) => {
|
||||||
|
console.error(JSON.stringify({
|
||||||
|
error: err.message,
|
||||||
|
timestamp: new Date().toISOString(),
|
||||||
|
}));
|
||||||
|
process.exit(1);
|
||||||
|
});
|
||||||
|
|
@ -1,545 +0,0 @@
|
||||||
/*
|
|
||||||
* SPDX-FileCopyrightText: syuilo and misskey-project
|
|
||||||
* SPDX-License-Identifier: AGPL-3.0-only
|
|
||||||
*/
|
|
||||||
|
|
||||||
import { ChildProcess, fork } from 'node:child_process';
|
|
||||||
import { setTimeout } from 'node:timers/promises';
|
|
||||||
import { fileURLToPath } from 'node:url';
|
|
||||||
import { dirname, join } from 'node:path';
|
|
||||||
import { tmpdir } from 'node:os';
|
|
||||||
//import * as http from 'node:http';
|
|
||||||
import * as fs from 'node:fs/promises';
|
|
||||||
|
|
||||||
const __filename = fileURLToPath(import.meta.url);
|
|
||||||
const __dirname = dirname(__filename);
|
|
||||||
|
|
||||||
function readIntegerEnv(name, defaultValue, min) {
|
|
||||||
const rawValue = process.env[name];
|
|
||||||
if (rawValue == null || rawValue === '') return defaultValue;
|
|
||||||
if (!/^\d+$/.test(rawValue)) throw new Error(`${name} must be an integer`);
|
|
||||||
|
|
||||||
const value = Number(rawValue);
|
|
||||||
if (!Number.isSafeInteger(value) || value < min) throw new Error(`${name} must be >= ${min}`);
|
|
||||||
return value;
|
|
||||||
}
|
|
||||||
|
|
||||||
function readBooleanEnv(name, defaultValue) {
|
|
||||||
const rawValue = process.env[name];
|
|
||||||
if (rawValue == null || rawValue === '') return defaultValue;
|
|
||||||
if (rawValue === '1' || rawValue === 'true') return true;
|
|
||||||
if (rawValue === '0' || rawValue === 'false') return false;
|
|
||||||
throw new Error(`${name} must be one of: 1, 0, true, false`);
|
|
||||||
}
|
|
||||||
|
|
||||||
const SAMPLE_COUNT = readIntegerEnv('MK_MEMORY_SAMPLE_COUNT', 3, 1); // Number of samples to measure
|
|
||||||
const STARTUP_TIMEOUT = readIntegerEnv('MK_MEMORY_STARTUP_TIMEOUT_MS', 120000, 1); // Timeout for server startup
|
|
||||||
const MEMORY_SETTLE_TIME = readIntegerEnv('MK_MEMORY_SETTLE_TIME_MS', 10000, 0); // Wait after startup for memory to settle
|
|
||||||
const IPC_TIMEOUT = readIntegerEnv('MK_MEMORY_IPC_TIMEOUT_MS', 30000, 1); // Timeout for IPC responses
|
|
||||||
const REQUEST_COUNT = readIntegerEnv('MK_MEMORY_REQUEST_COUNT', 10, 0);
|
|
||||||
const HEAP_SNAPSHOT = readBooleanEnv('MK_MEMORY_HEAP_SNAPSHOT', false);
|
|
||||||
const HEAP_SNAPSHOT_TIMEOUT = readIntegerEnv('MK_MEMORY_HEAP_SNAPSHOT_TIMEOUT_MS', 120000, 1);
|
|
||||||
const HEAP_SNAPSHOT_BREAKDOWN_TOP_N = readIntegerEnv('MK_MEMORY_HEAP_SNAPSHOT_BREAKDOWN_TOP_N', 6, 1);
|
|
||||||
|
|
||||||
const procStatusKeys = ['VmPeak', 'VmSize', 'VmHWM', 'VmRSS', 'VmData', 'VmStk', 'VmExe', 'VmLib', 'VmPTE', 'VmSwap'] as const;
|
|
||||||
const smapsRollupKeys = ['Pss', 'Shared_Clean', 'Shared_Dirty', 'Private_Clean', 'Private_Dirty', 'Swap', 'SwapPss'] as const;
|
|
||||||
|
|
||||||
const heapSnapshotCategories = [
|
|
||||||
'Code',
|
|
||||||
'Strings',
|
|
||||||
'JS arrays',
|
|
||||||
'Typed arrays',
|
|
||||||
'System objects',
|
|
||||||
'Other JS objects',
|
|
||||||
'Other non-JS objects',
|
|
||||||
'Total',
|
|
||||||
];
|
|
||||||
|
|
||||||
const typedArrayNames = new Set([
|
|
||||||
'ArrayBuffer',
|
|
||||||
'SharedArrayBuffer',
|
|
||||||
'DataView',
|
|
||||||
'Int8Array',
|
|
||||||
'Uint8Array',
|
|
||||||
'Uint8ClampedArray',
|
|
||||||
'Int16Array',
|
|
||||||
'Uint16Array',
|
|
||||||
'Int32Array',
|
|
||||||
'Uint32Array',
|
|
||||||
'Float16Array',
|
|
||||||
'Float32Array',
|
|
||||||
'Float64Array',
|
|
||||||
'BigInt64Array',
|
|
||||||
'BigUint64Array',
|
|
||||||
'system / JSArrayBufferData',
|
|
||||||
]);
|
|
||||||
|
|
||||||
const otherJsNodeTypes = new Set([
|
|
||||||
'object',
|
|
||||||
'closure',
|
|
||||||
'regexp',
|
|
||||||
'number',
|
|
||||||
'symbol',
|
|
||||||
'bigint',
|
|
||||||
]);
|
|
||||||
|
|
||||||
function parseMemoryFile<KS extends readonly string[]>(content: string, keys: KS, path: string, required: boolean): Record<KS[number], number> {
|
|
||||||
const result = {} as Record<KS[number], number>;
|
|
||||||
for (const _key of keys) {
|
|
||||||
const key = _key as KS[number];
|
|
||||||
const match = content.match(new RegExp(`${key}:\\s+(\\d+)\\s+kB`));
|
|
||||||
if (match) {
|
|
||||||
result[key] = parseInt(match[1], 10);
|
|
||||||
} else if (required) {
|
|
||||||
throw new Error(`Failed to parse ${key} from ${path}`);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return result;
|
|
||||||
}
|
|
||||||
|
|
||||||
function bytesToKiB(value: number) {
|
|
||||||
return Math.round(value / 1024);
|
|
||||||
}
|
|
||||||
|
|
||||||
function createEmptyHeapSnapshotCategoryMap() {
|
|
||||||
return Object.fromEntries(heapSnapshotCategories.map(category => [category, 0]));
|
|
||||||
}
|
|
||||||
|
|
||||||
function isTypedArrayNode(type, name) {
|
|
||||||
return typedArrayNames.has(name) ||
|
|
||||||
(type === 'native' && (name.includes('ArrayBuffer') || name.includes('TypedArray')));
|
|
||||||
}
|
|
||||||
|
|
||||||
function isSystemNode(type, name) {
|
|
||||||
return type === 'hidden' ||
|
|
||||||
type === 'synthetic' ||
|
|
||||||
type === 'object shape' ||
|
|
||||||
name.startsWith('system /') ||
|
|
||||||
name.startsWith('(system ');
|
|
||||||
}
|
|
||||||
|
|
||||||
function classifyHeapSnapshotNode(type, name) {
|
|
||||||
if (type === 'code') return 'Code';
|
|
||||||
if (type === 'string' || type === 'concatenated string' || type === 'sliced string') return 'Strings';
|
|
||||||
if (isTypedArrayNode(type, name)) return 'Typed arrays';
|
|
||||||
if (type === 'array' || (type === 'object' && name === 'Array')) return 'JS arrays';
|
|
||||||
if (isSystemNode(type, name)) return 'System objects';
|
|
||||||
if (otherJsNodeTypes.has(type)) return 'Other JS objects';
|
|
||||||
return 'Other non-JS objects';
|
|
||||||
}
|
|
||||||
|
|
||||||
function sanitizeHeapSnapshotBreakdownLabel(value, fallback = 'unknown') {
|
|
||||||
const label = String(value ?? '').replace(/\s+/g, ' ').trim();
|
|
||||||
if (label === '') return fallback;
|
|
||||||
if (label.length <= 80) return label;
|
|
||||||
return `${label.slice(0, 77)}...`;
|
|
||||||
}
|
|
||||||
|
|
||||||
function classifyHeapSnapshotBreakdown(category, type, name) {
|
|
||||||
if (category === 'Strings') return type;
|
|
||||||
|
|
||||||
if (category === 'JS arrays') {
|
|
||||||
if (type === 'array') return 'array nodes';
|
|
||||||
if (type === 'object' && name === 'Array') return 'Array objects';
|
|
||||||
return sanitizeHeapSnapshotBreakdownLabel(`${type}: ${name}`);
|
|
||||||
}
|
|
||||||
|
|
||||||
if (category === 'Typed arrays') {
|
|
||||||
if (name === 'system / JSArrayBufferData') return 'ArrayBuffer data';
|
|
||||||
if (name === 'Uint8Array') return 'Uint8Array / Buffer';
|
|
||||||
if (typedArrayNames.has(name)) return name;
|
|
||||||
if (type === 'native' && name.includes('ArrayBuffer')) return 'native ArrayBuffer';
|
|
||||||
if (type === 'native' && name.includes('TypedArray')) return 'native TypedArray';
|
|
||||||
return sanitizeHeapSnapshotBreakdownLabel(`${type}: ${name}`);
|
|
||||||
}
|
|
||||||
|
|
||||||
if (category === 'System objects') {
|
|
||||||
if (name.startsWith('system /')) return sanitizeHeapSnapshotBreakdownLabel(name);
|
|
||||||
if (name.startsWith('(system ')) return sanitizeHeapSnapshotBreakdownLabel(name);
|
|
||||||
return sanitizeHeapSnapshotBreakdownLabel(`${type}: ${name}`, type);
|
|
||||||
}
|
|
||||||
|
|
||||||
if (category === 'Other JS objects') {
|
|
||||||
if (type === 'object') return sanitizeHeapSnapshotBreakdownLabel(`object: ${name}`, 'object: unknown');
|
|
||||||
return type;
|
|
||||||
}
|
|
||||||
|
|
||||||
if (category === 'Other non-JS objects') {
|
|
||||||
if (type === 'native') return sanitizeHeapSnapshotBreakdownLabel(`native: ${name}`, 'native: unknown');
|
|
||||||
return sanitizeHeapSnapshotBreakdownLabel(`${type}: ${name}`, type);
|
|
||||||
}
|
|
||||||
|
|
||||||
if (category === 'Code') {
|
|
||||||
const lowerName = name.toLowerCase();
|
|
||||||
if (lowerName.includes('bytecode')) return 'bytecode';
|
|
||||||
if (lowerName.includes('builtin')) return 'builtins';
|
|
||||||
if (lowerName.includes('regexp')) return 'regexp code';
|
|
||||||
if (lowerName.includes('stub')) return 'stubs';
|
|
||||||
return sanitizeHeapSnapshotBreakdownLabel(`code: ${name}`, 'code: unknown');
|
|
||||||
}
|
|
||||||
|
|
||||||
return sanitizeHeapSnapshotBreakdownLabel(`${type}: ${name}`, type);
|
|
||||||
}
|
|
||||||
|
|
||||||
function collapseHeapSnapshotBreakdown(breakdowns) {
|
|
||||||
const collapsed = {};
|
|
||||||
|
|
||||||
for (const [category, children] of Object.entries(breakdowns)) {
|
|
||||||
const entries = Object.entries(children)
|
|
||||||
.filter(([, value]) => value > 0)
|
|
||||||
.toSorted((a, b) => b[1] - a[1]);
|
|
||||||
|
|
||||||
const topEntries = entries.slice(0, HEAP_SNAPSHOT_BREAKDOWN_TOP_N);
|
|
||||||
const otherValue = entries
|
|
||||||
.slice(HEAP_SNAPSHOT_BREAKDOWN_TOP_N)
|
|
||||||
.reduce((sum, [, value]) => sum + value, 0);
|
|
||||||
|
|
||||||
const categoryBreakdown = Object.fromEntries(topEntries);
|
|
||||||
if (otherValue > 0) categoryBreakdown.Other = otherValue;
|
|
||||||
if (Object.keys(categoryBreakdown).length > 0) collapsed[category] = categoryBreakdown;
|
|
||||||
}
|
|
||||||
|
|
||||||
return collapsed;
|
|
||||||
}
|
|
||||||
|
|
||||||
function analyzeHeapSnapshot(snapshot) {
|
|
||||||
const meta = snapshot?.snapshot?.meta;
|
|
||||||
const nodes = snapshot?.nodes;
|
|
||||||
const strings = snapshot?.strings;
|
|
||||||
if (meta == null || !Array.isArray(nodes) || !Array.isArray(strings)) {
|
|
||||||
throw new Error('Invalid heap snapshot format');
|
|
||||||
}
|
|
||||||
|
|
||||||
const nodeFields = meta.node_fields;
|
|
||||||
if (!Array.isArray(nodeFields)) throw new Error('Invalid heap snapshot node fields');
|
|
||||||
|
|
||||||
const typeOffset = nodeFields.indexOf('type');
|
|
||||||
const nameOffset = nodeFields.indexOf('name');
|
|
||||||
const selfSizeOffset = nodeFields.indexOf('self_size');
|
|
||||||
if (typeOffset < 0 || nameOffset < 0 || selfSizeOffset < 0) {
|
|
||||||
throw new Error('Heap snapshot is missing required node fields');
|
|
||||||
}
|
|
||||||
|
|
||||||
const nodeTypeNames = meta.node_types?.[typeOffset];
|
|
||||||
if (!Array.isArray(nodeTypeNames)) throw new Error('Invalid heap snapshot node types');
|
|
||||||
|
|
||||||
const fieldCount = nodeFields.length;
|
|
||||||
const categories = createEmptyHeapSnapshotCategoryMap();
|
|
||||||
const nodeCounts = createEmptyHeapSnapshotCategoryMap();
|
|
||||||
const breakdowns = Object.fromEntries(
|
|
||||||
heapSnapshotCategories
|
|
||||||
.filter(category => category !== 'Total')
|
|
||||||
.map(category => [category, {}]),
|
|
||||||
);
|
|
||||||
|
|
||||||
function addValue(map: Record<string, number>, key: string, value: number) {
|
|
||||||
map[key] = (map[key] ?? 0) + value;
|
|
||||||
}
|
|
||||||
|
|
||||||
for (let offset = 0; offset < nodes.length; offset += fieldCount) {
|
|
||||||
const type = nodeTypeNames[nodes[offset + typeOffset]] ?? 'unknown';
|
|
||||||
const name = strings[nodes[offset + nameOffset]] ?? '';
|
|
||||||
const selfSize = nodes[offset + selfSizeOffset] ?? 0;
|
|
||||||
const category = classifyHeapSnapshotNode(type, name);
|
|
||||||
|
|
||||||
categories[category] += selfSize;
|
|
||||||
categories.Total += selfSize;
|
|
||||||
nodeCounts[category]++;
|
|
||||||
nodeCounts.Total++;
|
|
||||||
addValue(breakdowns[category], classifyHeapSnapshotBreakdown(category, type, name), selfSize);
|
|
||||||
}
|
|
||||||
|
|
||||||
return {
|
|
||||||
categories,
|
|
||||||
nodeCounts,
|
|
||||||
breakdowns: collapseHeapSnapshotBreakdown(breakdowns),
|
|
||||||
};
|
|
||||||
}
|
|
||||||
|
|
||||||
async function getMemoryUsage(pid: number) {
|
|
||||||
const path = `/proc/${pid}/status`;
|
|
||||||
const status = await fs.readFile(path, 'utf-8');
|
|
||||||
return parseMemoryFile(status, procStatusKeys, path, true);
|
|
||||||
}
|
|
||||||
|
|
||||||
async function getSmapsRollupMemoryUsage(pid: number) {
|
|
||||||
const path = `/proc/${pid}/smaps_rollup`;
|
|
||||||
const smapsRollup = await fs.readFile(path, 'utf-8');
|
|
||||||
return parseMemoryFile(smapsRollup, smapsRollupKeys, path, false);
|
|
||||||
}
|
|
||||||
|
|
||||||
function waitForMessage(serverProcess, predicate, description, timeout = IPC_TIMEOUT) {
|
|
||||||
return new Promise((resolve, reject) => {
|
|
||||||
const timer = globalThis.setTimeout(() => {
|
|
||||||
serverProcess.off('message', onMessage);
|
|
||||||
reject(new Error(`Timed out waiting for ${description}`));
|
|
||||||
}, timeout);
|
|
||||||
|
|
||||||
const onMessage = (message) => {
|
|
||||||
if (!predicate(message)) return;
|
|
||||||
globalThis.clearTimeout(timer);
|
|
||||||
serverProcess.off('message', onMessage);
|
|
||||||
resolve(message);
|
|
||||||
};
|
|
||||||
|
|
||||||
serverProcess.on('message', onMessage);
|
|
||||||
});
|
|
||||||
}
|
|
||||||
|
|
||||||
async function getRuntimeMemoryUsage(serverProcess: ChildProcess) {
|
|
||||||
const response = waitForMessage(
|
|
||||||
serverProcess,
|
|
||||||
message => message != null && typeof message === 'object' && message.type === 'memory usage',
|
|
||||||
'memory usage',
|
|
||||||
);
|
|
||||||
|
|
||||||
serverProcess.send('memory usage');
|
|
||||||
|
|
||||||
const message = await response;
|
|
||||||
const memoryUsage = message.value;
|
|
||||||
|
|
||||||
return {
|
|
||||||
HeapTotal: bytesToKiB(memoryUsage.heapTotal),
|
|
||||||
HeapUsed: bytesToKiB(memoryUsage.heapUsed),
|
|
||||||
External: bytesToKiB(memoryUsage.external),
|
|
||||||
ArrayBuffers: bytesToKiB(memoryUsage.arrayBuffers),
|
|
||||||
};
|
|
||||||
}
|
|
||||||
|
|
||||||
async function getHeapSnapshotStatistics(serverProcess: ChildProcess) {
|
|
||||||
if (!HEAP_SNAPSHOT) return null;
|
|
||||||
|
|
||||||
const snapshotPath = join(tmpdir(), `misskey-backend-heap-${process.pid}-${serverProcess.pid}-${Date.now()}.heapsnapshot`);
|
|
||||||
const response = waitForMessage(
|
|
||||||
serverProcess,
|
|
||||||
message => message != null && typeof message === 'object' && (message.type === 'heap snapshot' || message.type === 'heap snapshot error'),
|
|
||||||
'heap snapshot',
|
|
||||||
HEAP_SNAPSHOT_TIMEOUT,
|
|
||||||
);
|
|
||||||
|
|
||||||
serverProcess.send({
|
|
||||||
type: 'heap snapshot',
|
|
||||||
path: snapshotPath,
|
|
||||||
});
|
|
||||||
|
|
||||||
const message = await response;
|
|
||||||
if (message.type === 'heap snapshot error') {
|
|
||||||
throw new Error(`Failed to write heap snapshot: ${message.message}`);
|
|
||||||
}
|
|
||||||
|
|
||||||
const writtenPath = typeof message.path === 'string' ? message.path : snapshotPath;
|
|
||||||
|
|
||||||
try {
|
|
||||||
const snapshot = JSON.parse(await fs.readFile(writtenPath, 'utf-8'));
|
|
||||||
return analyzeHeapSnapshot(snapshot);
|
|
||||||
} finally {
|
|
||||||
await fs.unlink(writtenPath).catch(err => {
|
|
||||||
process.stderr.write(`Failed to delete heap snapshot ${writtenPath}: ${err.message}\n`);
|
|
||||||
});
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
async function getAllMemoryUsage(serverProcess: ChildProcess) {
|
|
||||||
const pid = serverProcess.pid!;
|
|
||||||
return {
|
|
||||||
...await getMemoryUsage(pid),
|
|
||||||
...await getSmapsRollupMemoryUsage(pid),
|
|
||||||
...await getRuntimeMemoryUsage(serverProcess),
|
|
||||||
};
|
|
||||||
}
|
|
||||||
|
|
||||||
async function measureMemory() {
|
|
||||||
// Start the Misskey backend server using fork to enable IPC
|
|
||||||
const serverProcess = fork(join(__dirname, '../built/entry.js'), [], {
|
|
||||||
cwd: join(__dirname, '..'),
|
|
||||||
env: {
|
|
||||||
...process.env,
|
|
||||||
NODE_ENV: 'production',
|
|
||||||
MK_DISABLE_CLUSTERING: '1',
|
|
||||||
MK_ONLY_SERVER: '1',
|
|
||||||
MK_NO_DAEMONS: '1',
|
|
||||||
},
|
|
||||||
stdio: ['pipe', 'pipe', 'pipe', 'ipc'],
|
|
||||||
execArgv: [...process.execArgv, '--expose-gc'],
|
|
||||||
});
|
|
||||||
|
|
||||||
let serverReady = false;
|
|
||||||
|
|
||||||
// Listen for the 'ok' message from the server indicating it's ready
|
|
||||||
serverProcess.on('message', (message) => {
|
|
||||||
if (message === 'ok') {
|
|
||||||
serverReady = true;
|
|
||||||
}
|
|
||||||
});
|
|
||||||
|
|
||||||
// Handle server output
|
|
||||||
serverProcess.stdout?.on('data', (data) => {
|
|
||||||
process.stderr.write(`[server stdout] ${data}`);
|
|
||||||
});
|
|
||||||
|
|
||||||
serverProcess.stderr?.on('data', (data) => {
|
|
||||||
process.stderr.write(`[server stderr] ${data}`);
|
|
||||||
});
|
|
||||||
|
|
||||||
// Handle server error
|
|
||||||
serverProcess.on('error', (err) => {
|
|
||||||
process.stderr.write(`[server error] ${err}\n`);
|
|
||||||
});
|
|
||||||
|
|
||||||
async function triggerGc() {
|
|
||||||
const ok = waitForMessage(
|
|
||||||
serverProcess,
|
|
||||||
message => message === 'gc ok' || message === 'gc unavailable',
|
|
||||||
'GC completion',
|
|
||||||
);
|
|
||||||
|
|
||||||
serverProcess.send('gc');
|
|
||||||
|
|
||||||
const message = await ok;
|
|
||||||
if (message === 'gc unavailable') {
|
|
||||||
throw new Error('GC is unavailable. Start the process with --expose-gc to enable this feature.');
|
|
||||||
}
|
|
||||||
|
|
||||||
await setTimeout(1000);
|
|
||||||
}
|
|
||||||
|
|
||||||
//function createRequest() {
|
|
||||||
// return new Promise((resolve, reject) => {
|
|
||||||
// const req = http.request({
|
|
||||||
// host: 'localhost',
|
|
||||||
// port: 61812,
|
|
||||||
// path: '/api/meta',
|
|
||||||
// method: 'POST',
|
|
||||||
// }, (res) => {
|
|
||||||
// res.on('data', () => { });
|
|
||||||
// res.on('end', () => {
|
|
||||||
// resolve();
|
|
||||||
// });
|
|
||||||
// });
|
|
||||||
// req.on('error', (err) => {
|
|
||||||
// reject(err);
|
|
||||||
// });
|
|
||||||
// req.end();
|
|
||||||
// });
|
|
||||||
//}
|
|
||||||
|
|
||||||
// Wait for server to be ready or timeout
|
|
||||||
const startupStartTime = Date.now();
|
|
||||||
while (!serverReady) {
|
|
||||||
if (Date.now() - startupStartTime > STARTUP_TIMEOUT) {
|
|
||||||
serverProcess.kill('SIGTERM');
|
|
||||||
throw new Error('Server startup timeout');
|
|
||||||
}
|
|
||||||
await setTimeout(100);
|
|
||||||
}
|
|
||||||
|
|
||||||
const startupTime = Date.now() - startupStartTime;
|
|
||||||
process.stderr.write(`Server started in ${startupTime}ms\n`);
|
|
||||||
|
|
||||||
// Wait for memory to settle
|
|
||||||
await setTimeout(MEMORY_SETTLE_TIME);
|
|
||||||
|
|
||||||
//const beforeGc = await getAllMemoryUsage(serverProcess);
|
|
||||||
|
|
||||||
await triggerGc();
|
|
||||||
|
|
||||||
const memoryUsageAfterGC = await getAllMemoryUsage(serverProcess);
|
|
||||||
|
|
||||||
//// create some http requests to simulate load
|
|
||||||
//await Promise.all(
|
|
||||||
// Array.from({ length: REQUEST_COUNT }).map(() => createRequest()),
|
|
||||||
//);
|
|
||||||
|
|
||||||
//await triggerGc();
|
|
||||||
|
|
||||||
//const afterRequest = await getAllMemoryUsage(serverProcess);
|
|
||||||
|
|
||||||
const heapSnapshotAfterGc = await getHeapSnapshotStatistics(serverProcess);
|
|
||||||
|
|
||||||
// Stop the server
|
|
||||||
serverProcess.kill('SIGTERM');
|
|
||||||
|
|
||||||
// Wait for process to exit
|
|
||||||
let exited = false;
|
|
||||||
await new Promise((resolve) => {
|
|
||||||
serverProcess.on('exit', () => {
|
|
||||||
exited = true;
|
|
||||||
resolve(undefined);
|
|
||||||
});
|
|
||||||
// Force kill after 10 seconds if not exited
|
|
||||||
setTimeout(10000).then(() => {
|
|
||||||
if (!exited) {
|
|
||||||
serverProcess.kill('SIGKILL');
|
|
||||||
}
|
|
||||||
resolve(undefined);
|
|
||||||
});
|
|
||||||
});
|
|
||||||
|
|
||||||
const result = {
|
|
||||||
timestamp: new Date().toISOString(),
|
|
||||||
phases: {
|
|
||||||
//beforeGc,
|
|
||||||
afterGc: {
|
|
||||||
memoryUsage: memoryUsageAfterGC,
|
|
||||||
heapSnapshot: heapSnapshotAfterGc,
|
|
||||||
},
|
|
||||||
//afterRequest,
|
|
||||||
},
|
|
||||||
};
|
|
||||||
|
|
||||||
return result;
|
|
||||||
}
|
|
||||||
|
|
||||||
export type MemoryReportRaw = {
|
|
||||||
timestamp: string;
|
|
||||||
sampleCount: number;
|
|
||||||
measurement: {
|
|
||||||
startupTimeoutMs: number;
|
|
||||||
memorySettleTimeMs: number;
|
|
||||||
ipcTimeoutMs: number;
|
|
||||||
requestCount: number;
|
|
||||||
heapSnapshot: {
|
|
||||||
enabled: boolean;
|
|
||||||
timeoutMs: number;
|
|
||||||
breakdownTopN: number;
|
|
||||||
};
|
|
||||||
};
|
|
||||||
samples: Awaited<ReturnType<typeof measureMemory>>[];
|
|
||||||
};
|
|
||||||
|
|
||||||
async function main() {
|
|
||||||
const results = [];
|
|
||||||
for (let i = 0; i < SAMPLE_COUNT; i++) {
|
|
||||||
process.stderr.write(`Starting sample ${i + 1}/${SAMPLE_COUNT}\n`);
|
|
||||||
const res = await measureMemory();
|
|
||||||
results.push(res);
|
|
||||||
}
|
|
||||||
|
|
||||||
const result: MemoryReportRaw = {
|
|
||||||
timestamp: new Date().toISOString(),
|
|
||||||
sampleCount: SAMPLE_COUNT,
|
|
||||||
measurement: {
|
|
||||||
startupTimeoutMs: STARTUP_TIMEOUT,
|
|
||||||
memorySettleTimeMs: MEMORY_SETTLE_TIME,
|
|
||||||
ipcTimeoutMs: IPC_TIMEOUT,
|
|
||||||
requestCount: REQUEST_COUNT,
|
|
||||||
heapSnapshot: {
|
|
||||||
enabled: HEAP_SNAPSHOT,
|
|
||||||
timeoutMs: HEAP_SNAPSHOT_TIMEOUT,
|
|
||||||
breakdownTopN: HEAP_SNAPSHOT_BREAKDOWN_TOP_N,
|
|
||||||
},
|
|
||||||
},
|
|
||||||
samples: results,
|
|
||||||
};
|
|
||||||
|
|
||||||
// Output as JSON to stdout
|
|
||||||
console.log(JSON.stringify(result, null, 2));
|
|
||||||
}
|
|
||||||
|
|
||||||
main().catch((err) => {
|
|
||||||
console.error(JSON.stringify({
|
|
||||||
error: err.message,
|
|
||||||
timestamp: new Date().toISOString(),
|
|
||||||
}));
|
|
||||||
process.exit(1);
|
|
||||||
});
|
|
||||||
|
|
@ -6,7 +6,6 @@
|
||||||
import { NestFactory } from '@nestjs/core';
|
import { NestFactory } from '@nestjs/core';
|
||||||
import { init } from 'slacc';
|
import { init } from 'slacc';
|
||||||
import { NestLogger } from '@/NestLogger.js';
|
import { NestLogger } from '@/NestLogger.js';
|
||||||
import { envOption } from '@/env.js';
|
|
||||||
import type { Config } from '@/config.js';
|
import type { Config } from '@/config.js';
|
||||||
|
|
||||||
let slaccInitialized = false;
|
let slaccInitialized = false;
|
||||||
|
|
@ -32,7 +31,7 @@ export async function server() {
|
||||||
const serverService = app.get(ServerService);
|
const serverService = app.get(ServerService);
|
||||||
await serverService.launch();
|
await serverService.launch();
|
||||||
|
|
||||||
if (process.env.NODE_ENV !== 'test' && !envOption.noDaemons) {
|
if (process.env.NODE_ENV !== 'test') {
|
||||||
const { ChartManagementService } = await import('../core/chart/ChartManagementService.js');
|
const { ChartManagementService } = await import('../core/chart/ChartManagementService.js');
|
||||||
const { QueueStatsService } = await import('../daemons/QueueStatsService.js');
|
const { QueueStatsService } = await import('../daemons/QueueStatsService.js');
|
||||||
const { ServerStatsService } = await import('../daemons/ServerStatsService.js');
|
const { ServerStatsService } = await import('../daemons/ServerStatsService.js');
|
||||||
|
|
@ -55,9 +54,7 @@ export async function jobQueue() {
|
||||||
});
|
});
|
||||||
|
|
||||||
jobQueue.get(QueueProcessorService).start();
|
jobQueue.get(QueueProcessorService).start();
|
||||||
if (!envOption.noDaemons) {
|
jobQueue.get(ChartManagementService).start();
|
||||||
jobQueue.get(ChartManagementService).start();
|
|
||||||
}
|
|
||||||
|
|
||||||
return jobQueue;
|
return jobQueue;
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -9,7 +9,6 @@
|
||||||
|
|
||||||
import cluster from 'node:cluster';
|
import cluster from 'node:cluster';
|
||||||
import { EventEmitter } from 'node:events';
|
import { EventEmitter } from 'node:events';
|
||||||
import { writeHeapSnapshot } from 'node:v8';
|
|
||||||
import chalk from 'chalk';
|
import chalk from 'chalk';
|
||||||
import Xev from 'xev';
|
import Xev from 'xev';
|
||||||
import Logger from '@/logger.js';
|
import Logger from '@/logger.js';
|
||||||
|
|
@ -92,35 +91,10 @@ process.on('message', msg => {
|
||||||
if (msg === 'gc') {
|
if (msg === 'gc') {
|
||||||
if (global.gc != null) {
|
if (global.gc != null) {
|
||||||
logger.info('Manual GC triggered');
|
logger.info('Manual GC triggered');
|
||||||
for (let i = 0; i < 3; i++) {
|
global.gc();
|
||||||
global.gc();
|
|
||||||
}
|
|
||||||
if (process.send != null) process.send('gc ok');
|
if (process.send != null) process.send('gc ok');
|
||||||
} else {
|
} else {
|
||||||
logger.warn('Manual GC requested but gc is not available. Start the process with --expose-gc to enable this feature.');
|
logger.warn('Manual GC requested but gc is not available. Start the process with --expose-gc to enable this feature.');
|
||||||
if (process.send != null) process.send('gc unavailable');
|
|
||||||
}
|
|
||||||
} else if (msg === 'memory usage') {
|
|
||||||
if (process.send != null) {
|
|
||||||
process.send({
|
|
||||||
type: 'memory usage',
|
|
||||||
value: process.memoryUsage(),
|
|
||||||
});
|
|
||||||
}
|
|
||||||
} else if (msg != null && typeof msg === 'object' && 'type' in msg && msg.type === 'heap snapshot' && 'path' in msg && typeof msg.path === 'string') {
|
|
||||||
if (process.send != null) {
|
|
||||||
try {
|
|
||||||
const path = writeHeapSnapshot(msg.path);
|
|
||||||
process.send({
|
|
||||||
type: 'heap snapshot',
|
|
||||||
path,
|
|
||||||
});
|
|
||||||
} catch (err) {
|
|
||||||
process.send({
|
|
||||||
type: 'heap snapshot error',
|
|
||||||
message: err instanceof Error ? err.message : String(err),
|
|
||||||
});
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
|
|
|
||||||
|
|
@ -27,29 +27,27 @@ const extensions = ['.ts', '.tsx', '.js', '.jsx', '.mjs', '.json', '.json5', '.s
|
||||||
function getBundleVisualizerPlugin(): PluginOption[] {
|
function getBundleVisualizerPlugin(): PluginOption[] {
|
||||||
if (process.env.FRONTEND_BUNDLE_VISUALIZER !== 'true') return [];
|
if (process.env.FRONTEND_BUNDLE_VISUALIZER !== 'true') return [];
|
||||||
|
|
||||||
const visualizerOptions = {
|
const template = process.env.FRONTEND_BUNDLE_VISUALIZER_TEMPLATE === 'markdown'
|
||||||
title: 'Misskey frontend bundle visualizer',
|
? 'markdown'
|
||||||
gzipSize: true,
|
: process.env.FRONTEND_BUNDLE_VISUALIZER_TEMPLATE === 'raw-data'
|
||||||
brotliSize: true,
|
? 'raw-data'
|
||||||
projectRoot: path.resolve(__dirname, '../..'),
|
: 'treemap';
|
||||||
};
|
const defaultFilename = template === 'markdown'
|
||||||
const plugins = [
|
? path.resolve(__dirname, '../../built/_frontend_bundle_visualizer_/report.md')
|
||||||
|
: template === 'raw-data'
|
||||||
|
? path.resolve(__dirname, '../../built/_frontend_bundle_visualizer_/stats.json')
|
||||||
|
: path.resolve(__dirname, '../../built/_frontend_bundle_visualizer_/stats.html');
|
||||||
|
|
||||||
|
return [
|
||||||
visualizer({
|
visualizer({
|
||||||
...visualizerOptions,
|
filename: process.env.FRONTEND_BUNDLE_VISUALIZER_FILE ?? defaultFilename,
|
||||||
filename: process.env.FRONTEND_BUNDLE_VISUALIZER_FILE,
|
title: 'Misskey frontend bundle visualizer',
|
||||||
template: 'raw-data',
|
template,
|
||||||
|
gzipSize: true,
|
||||||
|
brotliSize: true,
|
||||||
|
projectRoot: path.resolve(__dirname, '../..'),
|
||||||
}) as PluginOption,
|
}) as PluginOption,
|
||||||
];
|
];
|
||||||
|
|
||||||
if (process.env.FRONTEND_BUNDLE_VISUALIZER_HTML_FILE != null) {
|
|
||||||
plugins.push(visualizer({
|
|
||||||
...visualizerOptions,
|
|
||||||
filename: process.env.FRONTEND_BUNDLE_VISUALIZER_HTML_FILE,
|
|
||||||
template: 'treemap',
|
|
||||||
}) as PluginOption);
|
|
||||||
}
|
|
||||||
|
|
||||||
return plugins;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
|
|
||||||
Loading…
Add table
Add a link
Reference in a new issue