mirror of
https://github.com/horsicq/Detect-It-Easy.git
synced 2026-06-24 01:54:08 +00:00
Skip unchanged files and remove obsolete outputs
Avoid unnecessary file rewrites and prune stale outputs. Add writeIfChanged to worker.js to skip writing identical content (new 'skipped' / '*-skip' result types), update task.js to count skipped/deleted files and report stats, and introduce syncDeleteOldFiles/getAllFilesInDir/deleteEmptyDirs to remove obsolete files from dbs_min before processing. Also restore MAX_PARALLEL to 16 and simplify worker resolution/exit handling. Finally, comment out the unconditional rd command in dbs_min_generate.cmd so the output folder is preserved.
This commit is contained in:
DosX
parent
0d3810e494
commit
455967700f
3 changed files with 139 additions and 36 deletions
|
|
@ -4,13 +4,15 @@ const { Worker } = require("worker_threads");
|
|||
|
||||
const inputDirs = ["db", "db_custom", "db_extra"];
|
||||
const outputDir = "dbs_min";
|
||||
const MAX_PARALLEL = 6; // Reduced from 16 to prevent resource exhaustion
|
||||
const MAX_PARALLEL = 16;
|
||||
|
||||
var stats = {
|
||||
const stats = {
|
||||
total: 0,
|
||||
minified: 0,
|
||||
copied: 0,
|
||||
failed: 0,
|
||||
skipped: 0,
|
||||
deleted: 0,
|
||||
};
|
||||
|
||||
const failedFiles = [];
|
||||
|
|
@ -26,53 +28,55 @@ function processFile(srcFile, dstFile) {
|
|||
}
|
||||
});
|
||||
|
||||
let isResolved = false;
|
||||
|
||||
const cleanup = () => {
|
||||
if (!isResolved) {
|
||||
isResolved = true;
|
||||
worker.terminate().catch(() => { });
|
||||
resolve();
|
||||
}
|
||||
};
|
||||
|
||||
worker.on('message', (result) => {
|
||||
stats.total++;
|
||||
|
||||
if (result.type === 'minified') {
|
||||
stats.minified++;
|
||||
console.log("[MINIFIED] " + result.srcFile);
|
||||
} else if (result.type === 'skipped') {
|
||||
stats.minified++;
|
||||
stats.skipped++;
|
||||
console.log("[SKIP] " + result.srcFile);
|
||||
} else if (result.type === 'copied') {
|
||||
stats.copied++;
|
||||
copiedFiles.push(result.srcFile);
|
||||
console.log("[COPIED] " + result.srcFile);
|
||||
} else if (result.type === 'copied-skip') {
|
||||
stats.copied++;
|
||||
stats.skipped++;
|
||||
console.log("[SKIP] " + result.srcFile);
|
||||
} else if (result.type === 'failed') {
|
||||
stats.failed++;
|
||||
failedFiles.push({ file: result.srcFile, reason: result.error });
|
||||
console.warn("[FAILED] " + result.srcFile + " — " + result.error);
|
||||
} else if (result.type === 'failed-skip') {
|
||||
stats.failed++;
|
||||
stats.skipped++;
|
||||
console.log("[SKIP/FAIL] " + result.srcFile);
|
||||
} else {
|
||||
stats.failed++;
|
||||
failedFiles.push({ file: result.srcFile, reason: "Read error: " + result.error });
|
||||
console.warn("[ERROR/READ] " + result.srcFile + " — " + result.error);
|
||||
}
|
||||
|
||||
cleanup();
|
||||
resolve();
|
||||
});
|
||||
|
||||
worker.on('error', (err) => {
|
||||
stats.failed++;
|
||||
failedFiles.push({ file: srcFile, reason: err.message });
|
||||
console.warn("[ERROR] " + srcFile + " — " + err.message);
|
||||
cleanup();
|
||||
resolve();
|
||||
});
|
||||
|
||||
worker.on('exit', (code) => {
|
||||
if (code !== 0 && !isResolved) {
|
||||
if (code !== 0) {
|
||||
stats.failed++;
|
||||
failedFiles.push({ file: srcFile, reason: `Worker stopped with exit code ${code}` });
|
||||
console.warn("[ERROR] " + srcFile + " — Worker stopped with exit code " + code);
|
||||
resolve();
|
||||
}
|
||||
cleanup();
|
||||
});
|
||||
});
|
||||
}
|
||||
|
|
@ -103,8 +107,8 @@ function collectFiles(srcDir, relBase, dstBase, fileList = []) {
|
|||
const items = fs.readdirSync(srcDir);
|
||||
for (const item of items) {
|
||||
const srcPath = path.join(srcDir, item);
|
||||
|
||||
const stat = fs.statSync(srcPath);
|
||||
|
||||
if (stat.isDirectory()) {
|
||||
collectFiles(srcPath, relBase, dstBase, fileList);
|
||||
} else {
|
||||
|
|
@ -117,6 +121,63 @@ function collectFiles(srcDir, relBase, dstBase, fileList = []) {
|
|||
return fileList;
|
||||
}
|
||||
|
||||
function getAllFilesInDir(dir, fileList = []) {
|
||||
if (!fs.existsSync(dir)) return fileList;
|
||||
|
||||
const items = fs.readdirSync(dir);
|
||||
for (const item of items) {
|
||||
const fullPath = path.join(dir, item);
|
||||
const stat = fs.statSync(fullPath);
|
||||
|
||||
if (stat.isDirectory()) {
|
||||
getAllFilesInDir(fullPath, fileList);
|
||||
} else {
|
||||
fileList.push(fullPath);
|
||||
}
|
||||
}
|
||||
return fileList;
|
||||
}
|
||||
|
||||
function syncDeleteOldFiles(expectedFiles) {
|
||||
const expectedSet = new Set(expectedFiles.map(f => path.normalize(f.dst)));
|
||||
const existingFiles = getAllFilesInDir(outputDir);
|
||||
|
||||
let deletedCount = 0;
|
||||
for (const existingFile of existingFiles) {
|
||||
const normalized = path.normalize(existingFile);
|
||||
if (!expectedSet.has(normalized)) {
|
||||
try {
|
||||
fs.unlinkSync(existingFile);
|
||||
console.log("[DELETED] " + path.relative(process.cwd(), existingFile));
|
||||
deletedCount++;
|
||||
} catch (e) {
|
||||
console.warn("[DELETE FAILED] " + existingFile + " — " + e.message);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
deleteEmptyDirs(outputDir);
|
||||
|
||||
return deletedCount;
|
||||
}
|
||||
|
||||
|
||||
function deleteEmptyDirs(dir) {
|
||||
if (!fs.existsSync(dir)) return;
|
||||
|
||||
const items = fs.readdirSync(dir);
|
||||
for (const item of items) {
|
||||
const fullPath = path.join(dir, item);
|
||||
if (fs.statSync(fullPath).isDirectory()) {
|
||||
deleteEmptyDirs(fullPath);
|
||||
}
|
||||
}
|
||||
|
||||
if (fs.readdirSync(dir).length === 0 && dir !== outputDir) {
|
||||
fs.rmdirSync(dir);
|
||||
}
|
||||
}
|
||||
|
||||
(async () => {
|
||||
console.log(`[i] Processing with ${MAX_PARALLEL} parallel workers...\n`);
|
||||
|
||||
|
|
@ -133,13 +194,20 @@ function collectFiles(srcDir, relBase, dstBase, fileList = []) {
|
|||
|
||||
console.log(`[i] Found ${allFiles.length} files to process\n`);
|
||||
|
||||
stats.deleted = syncDeleteOldFiles(allFiles);
|
||||
if (stats.deleted > 0) {
|
||||
console.log(`\n[i] Deleted ${stats.deleted} obsolete files\n`);
|
||||
}
|
||||
|
||||
await processFilesInParallel(allFiles);
|
||||
|
||||
let report = "\n[V] Done!\n" +
|
||||
`— Total: ${stats.total}\n` +
|
||||
`— Minified: ${stats.minified}\n` +
|
||||
`— Copied: ${stats.copied}\n` +
|
||||
`— Failed: ${stats.failed}\n`;
|
||||
`— Failed: ${stats.failed}\n` +
|
||||
`— Skipped: ${stats.skipped}\n` +
|
||||
`— Deleted: ${stats.deleted}\n`;
|
||||
|
||||
if (copiedFiles.length > 0) {
|
||||
report += "\n[I] Copied (unsupported extension):\n" + copiedFiles.map((f) => " • " + f).join("\n") + "\n";
|
||||
|
|
|
|||
|
|
@ -3,6 +3,19 @@ const fs = require("fs");
|
|||
const path = require("path");
|
||||
const UglifyJS = require("uglify-js");
|
||||
|
||||
function writeIfChanged(filePath, newContent) {
|
||||
if (fs.existsSync(filePath)) {
|
||||
try {
|
||||
const existingContent = fs.readFileSync(filePath, "utf8");
|
||||
if (existingContent === newContent) {
|
||||
return false;
|
||||
}
|
||||
} catch (e) { }
|
||||
}
|
||||
fs.writeFileSync(filePath, newContent, "utf8");
|
||||
return true;
|
||||
}
|
||||
|
||||
function shouldMinify(filePath) {
|
||||
const ext = path.extname(filePath).toLowerCase();
|
||||
return ext === ".sg" || ext === "";
|
||||
|
|
@ -156,28 +169,24 @@ function replaceLetWithVarSafe(text) {
|
|||
function replaceArrowFunctions(text) {
|
||||
// Simple direct replacement without complex parsing
|
||||
// UglifyJS output doesn't have regex/string issues with arrow functions
|
||||
|
||||
|
||||
// 1. ()=>{...} -> function(){...}
|
||||
text = text.replace(/\(\)\s*=>\s*\{/g, 'function(){');
|
||||
|
||||
|
||||
// 2. (args)=>{...} -> function(args){...}
|
||||
// Match balanced parentheses
|
||||
text = text.replace(/\(([^()]*)\)\s*=>\s*\{/g, 'function($1){');
|
||||
|
||||
|
||||
// 3. Single arg with block: arg=>{...} -> function(arg){...}
|
||||
text = text.replace(/\b([a-zA-Z_$][\w$]*)\s*=>\s*\{/g, 'function($1){');
|
||||
|
||||
|
||||
// 4. Concise forms (no braces) - need to find expression end
|
||||
// ()=>expr -> function(){return expr}
|
||||
// This is complex, skip for now as UglifyJS typically uses braces
|
||||
|
||||
|
||||
return text;
|
||||
}
|
||||
|
||||
/**
|
||||
* Fix delete statements for strict mode
|
||||
* Applied BEFORE minification
|
||||
*/
|
||||
function fixDeleteStatements(text) {
|
||||
return parseJSCodeSafe(text, (fragment) => {
|
||||
const match = fragment.match(/^delete\s+([a-zA-Z_$][\w$]*)(\s*;?)/);
|
||||
|
|
@ -208,7 +217,19 @@ function fixDeleteStatements(text) {
|
|||
}
|
||||
|
||||
/**
|
||||
* Replace bDetected=!0 and bDetected=!1
|
||||
* Safely replaces the value of bDetected variable by toggling its boolean state.
|
||||
*
|
||||
* @param {string} text - The JavaScript code text to parse and process
|
||||
* @returns {string} The text with bDetected values toggled (0 becomes 1, 1 becomes 0)
|
||||
*
|
||||
* @description
|
||||
* Parses the provided text using parseJSCodeSafe and searches for patterns matching
|
||||
* "bDetected = !0" or "bDetected = !1". When found, toggles the numeric value and
|
||||
* returns the modified text with the replacement applied.
|
||||
*
|
||||
* @example
|
||||
* replaceBDetectedSafe('bDetected = !0'); // Returns: 'bDetected=1'
|
||||
* replaceBDetectedSafe('bDetected = !1'); // Returns: 'bDetected=0'
|
||||
*/
|
||||
function replaceBDetectedSafe(text) {
|
||||
return parseJSCodeSafe(text, (fragment) => {
|
||||
|
|
@ -227,7 +248,18 @@ function replaceBDetectedSafe(text) {
|
|||
}
|
||||
|
||||
/**
|
||||
* Replace empty constructors with literals
|
||||
* Replaces common constructor calls with their simplified equivalents in JavaScript code.
|
||||
*
|
||||
* Safely transforms:
|
||||
* - `String()` → `""`
|
||||
* - `Boolean()` → `!1`
|
||||
* - `Number()` → `0`
|
||||
*
|
||||
* Only replaces constructors that are not preceded by a dot (.) or identifier character,
|
||||
* ensuring that property accesses and method calls are not affected.
|
||||
*
|
||||
* @param {string} text - The JavaScript code text to process
|
||||
* @returns {string} The text with constructor calls replaced by their simplified forms
|
||||
*/
|
||||
function replaceConstructorsSafe(text) {
|
||||
return parseJSCodeSafe(text, (fragment, index, fullText) => {
|
||||
|
|
@ -314,21 +346,24 @@ try {
|
|||
);
|
||||
|
||||
fs.mkdirSync(path.dirname(dstFile), { recursive: true });
|
||||
fs.writeFileSync(dstFile, legacyCompatibleCode, "utf8");
|
||||
const wasWritten = writeIfChanged(dstFile, legacyCompatibleCode);
|
||||
|
||||
result.success = true;
|
||||
result.type = 'minified';
|
||||
result.type = wasWritten ? 'minified' : 'skipped';
|
||||
} catch (e) {
|
||||
fs.mkdirSync(path.dirname(dstFile), { recursive: true });
|
||||
fs.writeFileSync(dstFile, text, "utf8");
|
||||
const wasWritten = writeIfChanged(dstFile, text);
|
||||
|
||||
result.success = false;
|
||||
result.type = 'failed';
|
||||
result.type = wasWritten ? 'failed' : 'failed-skip';
|
||||
result.error = e.message;
|
||||
}
|
||||
} else {
|
||||
fs.mkdirSync(path.dirname(dstFile), { recursive: true });
|
||||
fs.writeFileSync(dstFile, text, "utf8");
|
||||
const wasWritten = writeIfChanged(dstFile, text);
|
||||
|
||||
result.success = true;
|
||||
result.type = 'copied';
|
||||
result.type = wasWritten ? 'copied' : 'copied-skip';
|
||||
}
|
||||
} catch (e) {
|
||||
result.success = false;
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
@echo off
|
||||
rd dbs_min /q /s
|
||||
:: rd dbs_min /q /s
|
||||
node autotools\dbcompiler\task.js
|
||||
echo Generated: %DATE%>dbs_min\timestamp.log
|
||||
call db_compress
|
||||
Loading…
Add table
Add a link
Reference in a new issue