mirror of
https://github.com/vrc-get/vrc-get.git
synced 2026-06-21 09:58:08 +00:00
Merge pull request #2746 from siloneco/perf/parallel-zip-compression
perf: improve backup project performance by parallelizing zip compression
This commit is contained in:
commit
8babbb0e04
15 changed files with 434 additions and 77 deletions
|
|
@ -17,6 +17,8 @@ The format is based on [Keep a Changelog].
|
|||
- Completely changed how do we build ALCOM and how do we self-update ALCOM `#2759`
|
||||
- This fixes few problems relates to auto update
|
||||
- Please read README for new build instruction.
|
||||
- Improved backup speed by parallelizing the process [`#2746`](https://github.com/vrc-get/vrc-get/pull/2746)
|
||||
- Along with this change, the default compression level has been changed to `zip-fast`
|
||||
|
||||
### Deprecated
|
||||
|
||||
|
|
|
|||
44
Cargo.lock
generated
44
Cargo.lock
generated
|
|
@ -2825,6 +2825,15 @@ dependencies = [
|
|||
"minimal-lexical",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "ntapi"
|
||||
version = "0.4.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "c3b335231dfd352ffb0f8017f3b6027a4917f7df785ea2143d8af2adc66980ae"
|
||||
dependencies = [
|
||||
"winapi",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "nom"
|
||||
version = "8.0.0"
|
||||
|
|
@ -3086,6 +3095,16 @@ dependencies = [
|
|||
"objc2-core-foundation",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "objc2-io-kit"
|
||||
version = "0.3.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "33fafba39597d6dc1fb709123dfa8289d39406734be322956a69f0931c73bb15"
|
||||
dependencies = [
|
||||
"libc",
|
||||
"objc2-core-foundation",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "objc2-io-surface"
|
||||
version = "0.3.2"
|
||||
|
|
@ -4121,6 +4140,15 @@ version = "0.16.0"
|
|||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "57b0b88a509053cbfd535726dcaaceee631313cef981266119527a1d110f6d2b"
|
||||
|
||||
[[package]]
|
||||
name = "rlimit"
|
||||
version = "0.11.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f35ee2729c56bb610f6dba436bf78135f728b7373bdffae2ec815b2d3eb98cc3"
|
||||
dependencies = [
|
||||
"libc",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rpassword"
|
||||
version = "7.4.0"
|
||||
|
|
@ -4900,6 +4928,20 @@ dependencies = [
|
|||
"libc",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "sysinfo"
|
||||
version = "0.38.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "92ab6a2f8bfe508deb3c6406578252e491d299cbbf3bc0529ecc3313aee4a52f"
|
||||
dependencies = [
|
||||
"libc",
|
||||
"memchr",
|
||||
"ntapi",
|
||||
"objc2-core-foundation",
|
||||
"objc2-io-kit",
|
||||
"windows 0.62.2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "system-configuration"
|
||||
version = "0.7.0"
|
||||
|
|
@ -6004,6 +6046,7 @@ dependencies = [
|
|||
"plist",
|
||||
"reqwest 0.12.28",
|
||||
"ringbuffer",
|
||||
"rlimit",
|
||||
"semver",
|
||||
"serde",
|
||||
"serde_json",
|
||||
|
|
@ -6013,6 +6056,7 @@ dependencies = [
|
|||
"specta-typescript",
|
||||
"stable_deref_trait",
|
||||
"sys-locale",
|
||||
"sysinfo",
|
||||
"tar",
|
||||
"tauri",
|
||||
"tauri-build",
|
||||
|
|
|
|||
|
|
@ -42,7 +42,7 @@ tar = "0.4"
|
|||
flate2 = "1"
|
||||
uuid = { version = "1", features = ["v4"] }
|
||||
trash = "5"
|
||||
async_zip = { version = "0.0.18", features = ["deflate", "tokio"] }
|
||||
async_zip = { version = "0.0.18", features = ["tokio", "deflate"] }
|
||||
async-compression = { version = "0.4", features = ["gzip", "futures-io"] }
|
||||
async-stream = "0.3"
|
||||
tauri-plugin-single-instance = "2"
|
||||
|
|
@ -61,6 +61,7 @@ yoke = { version = "0.8", features = ["derive"] }
|
|||
atomicbox = "0.4"
|
||||
stable_deref_trait = "1"
|
||||
itertools = "0.14"
|
||||
sysinfo = "0.38.4"
|
||||
|
||||
[target.'cfg(windows)'.dependencies]
|
||||
windows = { version = "0.62", features = ["Win32_Storage_FileSystem", "Win32_System_IO", "Win32_NetworkManagement_IpHelper", "Wdk_System_SystemServices", "Win32_System_SystemInformation", "Win32_UI_Shell", "Win32_UI_WindowsAndMessaging"] }
|
||||
|
|
@ -74,6 +75,7 @@ objc2-foundation = "0.3.0"
|
|||
block2 = "0.6.0"
|
||||
objc2 = "0.6.0"
|
||||
dispatch2 = "0.3.0"
|
||||
rlimit = "0.11.0"
|
||||
|
||||
[target.'cfg(unix)'.dependencies]
|
||||
nix = { version = "0.31", features = ["fs"] }
|
||||
|
|
|
|||
|
|
@ -476,7 +476,7 @@
|
|||
|
||||
"settings:backup:format": "Backup-Format",
|
||||
"settings:backup:format description": "Backups werden als Archiv komprimiert, das Format kann hier angepasst werden.<br/>Wir empfehlen dies bei der Standard Einstellung zu belassen.<br/>Kompression sorgt für eine längere Backupdauer, reduziert dafür allerdings die Größe des Backups.",
|
||||
"settings:backup:format:default": "Standard (Unkomprimierte zip)",
|
||||
"settings:backup:format:default": "Standard (Leicht komprimierte zip)",
|
||||
"settings:backup:format:zip-store": "Unkomprimierte zip (Schnell)",
|
||||
"settings:backup:format:zip-fast": "Leicht komprimierte zip (Moderat)",
|
||||
"settings:backup:format:zip-best": "Stark komprimierte zip (Langsam)",
|
||||
|
|
|
|||
|
|
@ -476,7 +476,7 @@
|
|||
|
||||
"settings:backup:format": "Backup Archive Format",
|
||||
"settings:backup:format description": "Backups are saved as an archive file. <br/>You can select the format of the backup archive. <br/>Default setting is the best format ALCOM thinks at the moment and current default behavior is represented in braces.<br/>Compressing the archive will make backup took longer but will save disk space.",
|
||||
"settings:backup:format:default": "Default (Uncompressed zip)",
|
||||
"settings:backup:format:default": "Default (Low Compression zip)",
|
||||
"settings:backup:format:zip-store": "Uncompressed zip (Fast)",
|
||||
"settings:backup:format:zip-fast": "Low Compression zip (Slow)",
|
||||
"settings:backup:format:zip-best": "High Compression zip (Slowest)",
|
||||
|
|
|
|||
|
|
@ -261,7 +261,7 @@
|
|||
'settings:backup:exclude vpm packages from backup description': 'Cela réduira la taille du backup, mais si le créateur de packet ne suit pas la recommendation et supprime le packet du dépot, vous devez alors utiliser une autre version du packet afin de le restaurer au sein du projet.',
|
||||
'settings:backup:format': "Format de l'archive de backup",
|
||||
'settings:backup:format description': "Les backups sont stockés sous forme d'achives.<br/>Vous pouvez sélectionner le format d'archive qui vous conviens. <br/>Le réglage par défaut est celui qui est recommandé par ALCOM. Le comportement des autres options est décrite entre crochets.<br/>La compression d'une archive ralentira le processus d'archivage mais réduira l'espace disque pris.",
|
||||
'settings:backup:format:default': 'Défaut (Zip non compressé)',
|
||||
'settings:backup:format:default': 'Défaut (Zip Basse Compression)',
|
||||
'settings:backup:format:zip-best': 'Zip Haute compression (Le plus lent)',
|
||||
'settings:backup:format:zip-fast': 'Zip Basse Compression (Lent)',
|
||||
'settings:backup:format:zip-store': 'Zip non compressé (Rapide)',
|
||||
|
|
|
|||
|
|
@ -465,10 +465,10 @@
|
|||
|
||||
"settings:backup:format": "バックアップの保存形式",
|
||||
"settings:backup:format description": "「デフォルト」設定では、ALCOM開発チームが最も適切だと考えている保存形式が使用されます。(括弧内に詳細が表示されます)<br>圧縮率の高い保存形式ほどバックアップの作成に長い時間を要しますが、バックアップファイルの容量は小さくなります。",
|
||||
"settings:backup:format:default": "デフォルト (無圧縮zip)",
|
||||
"settings:backup:format:default": "デフォルト (低圧縮zip)",
|
||||
"settings:backup:format:zip-store": "無圧縮zip (高速)",
|
||||
"settings:backup:format:zip-fast": "低圧縮zip (低速)",
|
||||
"settings:backup:format:zip-best": "高圧縮zip (超低速)",
|
||||
"settings:backup:format:zip-best": "高圧縮zip (最も低速)",
|
||||
"settings:backup:exclude vpm packages from backup": "バックアップにVPMパッケージの本体を含まないようにする",
|
||||
"settings:backup:exclude vpm packages from backup description": "導入されているパッケージの一覧のみを保持することにより、バックアップファイルの容量を小さくすることができます。<br>ただし、パッケージの作者が(VRChatの勧告に従わずに)VPMリポジトリからパッケージ情報を削除してしまった場合、復元する際に別のバージョンのパッケージを手動で入れ直す必要が生じます。",
|
||||
|
||||
|
|
|
|||
|
|
@ -469,7 +469,7 @@
|
|||
|
||||
"settings:backup:format": "백업 형식",
|
||||
"settings:backup:format description": "\"기본값\" 설정에서는 ALCOM이 가장 적합하다고 판단한 형식이 사용되며, 괄호 안에 현재 기본 동작이 표시됩니다.<br>압축률이 높은 형식을 사용할수록 백업 시간이 길어지지만 디스크 공간을 절약할 수 있습니다.",
|
||||
"settings:backup:format:default": "기본값 (압축하지 않은 zip)",
|
||||
"settings:backup:format:default": "기본값 (낮은 압축 zip)",
|
||||
"settings:backup:format:zip-store": "압축하지 않은 zip (빠름)",
|
||||
"settings:backup:format:zip-fast": "낮은 압축 zip (느림)",
|
||||
"settings:backup:format:zip-best": "높은 압축 zip (가장 느림)",
|
||||
|
|
|
|||
|
|
@ -466,7 +466,7 @@
|
|||
|
||||
"settings:backup:format": "备份文件的压缩格式",
|
||||
"settings:backup:format description": "备份以压缩文件的形式保存。 <br/>您可以选择压缩文件的压缩格式。<br/>默认设置是 ALCOM 目前认为最好的格式,当前的默认行为用括号表示。<br/>高压缩率会增加备份耗时,但会节省磁盘空间。",
|
||||
"settings:backup:format:default": "默认 (未压缩的 zip)",
|
||||
"settings:backup:format:default": "默认 (低压缩率的 zip)",
|
||||
"settings:backup:format:zip-store": "未压缩的 zip (快)",
|
||||
"settings:backup:format:zip-fast": "低压缩率的 zip (慢)",
|
||||
"settings:backup:format:zip-best": "高压缩率的 zip (最慢)",
|
||||
|
|
|
|||
|
|
@ -476,7 +476,7 @@
|
|||
|
||||
"settings:backup:format": "備份檔案格式",
|
||||
"settings:backup:format description": "備份會以壓縮檔案形式儲存。<br/>你可以選擇備份壓縮檔的格式。<br/>預設設定是 ALCOM 認為目前最好的格式,並以括號內的方式表示當前的預設行為。<br/>壓縮檔案會使備份時間較長,但可以節省磁碟空間。",
|
||||
"settings:backup:format:default": "預設(未壓縮的 zip)",
|
||||
"settings:backup:format:default": "預設(低壓縮率的 zip)",
|
||||
"settings:backup:format:zip-store": "未壓縮的 zip(快)",
|
||||
"settings:backup:format:zip-fast": "低壓縮率的 zip(慢)",
|
||||
"settings:backup:format:zip-best": "高壓縮率的 zip(最慢)",
|
||||
|
|
|
|||
18
vrc-get-gui/package-lock.json
generated
18
vrc-get-gui/package-lock.json
generated
|
|
@ -85,6 +85,7 @@
|
|||
"integrity": "sha512-CGOfOJqWjg2qW/Mb6zNsDm+u5vFQ8DxXfbM09z69p5Z6+mE1ikP2jUXw+j42Pf1XTYED2Rni5f95npYeuwMDQA==",
|
||||
"dev": true,
|
||||
"license": "MIT",
|
||||
"peer": true,
|
||||
"dependencies": {
|
||||
"@babel/code-frame": "^7.29.0",
|
||||
"@babel/generator": "^7.29.0",
|
||||
|
|
@ -3198,6 +3199,7 @@
|
|||
"resolved": "https://registry.npmjs.org/@tanstack/react-router/-/react-router-1.168.3.tgz",
|
||||
"integrity": "sha512-hMWXhckeaSvjepHT5x9tUYJVXMvT/kUjaVHOUDmCfyOBtjxJNYJKbEWClXoopGwWlHjRTAzhsndhnQQRbIiKmA==",
|
||||
"license": "MIT",
|
||||
"peer": true,
|
||||
"dependencies": {
|
||||
"@tanstack/history": "1.161.6",
|
||||
"@tanstack/react-store": "^0.9.2",
|
||||
|
|
@ -3239,6 +3241,7 @@
|
|||
"resolved": "https://registry.npmjs.org/@tanstack/router-core/-/router-core-1.168.3.tgz",
|
||||
"integrity": "sha512-qcjArls3v12UQQkEpU0+todc0/MCyrEZeXxhtgZZ0e5gxZDG25BUe/HlNcIjzyb7NZaw0TQAUBXbTClmFaHZiw==",
|
||||
"license": "MIT",
|
||||
"peer": true,
|
||||
"dependencies": {
|
||||
"@tanstack/history": "1.161.6",
|
||||
"cookie-es": "^2.0.0",
|
||||
|
|
@ -3710,6 +3713,7 @@
|
|||
"integrity": "sha512-Uarfe6J91b9HAUXxjvSOdiO2UPOKLm07Q1oh0JHxoZ1y8HoqxDAu3gVrsrOHeiio0kSsoVBt4wFrKOm0dKxVPQ==",
|
||||
"dev": true,
|
||||
"license": "MIT",
|
||||
"peer": true,
|
||||
"dependencies": {
|
||||
"undici-types": "~6.21.0"
|
||||
}
|
||||
|
|
@ -3720,6 +3724,7 @@
|
|||
"integrity": "sha512-ilcTH/UniCkMdtexkoCN0bI7pMcJDvmQFPvuPvmEaYA/NSfFTAgdUSLAoVjaRJm7+6PvcM+q1zYOwS4wTYMF9w==",
|
||||
"devOptional": true,
|
||||
"license": "MIT",
|
||||
"peer": true,
|
||||
"dependencies": {
|
||||
"csstype": "^3.2.2"
|
||||
}
|
||||
|
|
@ -3730,6 +3735,7 @@
|
|||
"integrity": "sha512-jp2L/eY6fn+KgVVQAOqYItbF0VY/YApe5Mz2F0aykSO8gx31bYCZyvSeYxCHKvzHG5eZjc+zyaS5BrBWya2+kQ==",
|
||||
"devOptional": true,
|
||||
"license": "MIT",
|
||||
"peer": true,
|
||||
"peerDependencies": {
|
||||
"@types/react": "^19.2.0"
|
||||
}
|
||||
|
|
@ -3898,6 +3904,7 @@
|
|||
}
|
||||
],
|
||||
"license": "MIT",
|
||||
"peer": true,
|
||||
"dependencies": {
|
||||
"baseline-browser-mapping": "^2.9.0",
|
||||
"caniuse-lite": "^1.0.30001759",
|
||||
|
|
@ -4018,7 +4025,8 @@
|
|||
"version": "3.2.3",
|
||||
"resolved": "https://registry.npmjs.org/csstype/-/csstype-3.2.3.tgz",
|
||||
"integrity": "sha512-z1HGKcYy2xA8AGQfwrn0PAy+PB7X/GSj3UVJW9qKyn43xWa+gl5nXmU4qqLMRzWVLFC8KusUX8T/0kCiOYpAIQ==",
|
||||
"license": "MIT"
|
||||
"license": "MIT",
|
||||
"peer": true
|
||||
},
|
||||
"node_modules/debug": {
|
||||
"version": "4.4.3",
|
||||
|
|
@ -4275,6 +4283,7 @@
|
|||
}
|
||||
],
|
||||
"license": "MIT",
|
||||
"peer": true,
|
||||
"dependencies": {
|
||||
"@babel/runtime": "^7.28.4"
|
||||
},
|
||||
|
|
@ -4795,6 +4804,7 @@
|
|||
"resolved": "https://registry.npmjs.org/react/-/react-19.2.4.tgz",
|
||||
"integrity": "sha512-9nfp2hYpCwOjAN+8TZFGhtWEwgvWHXqESH8qT89AT/lWklpLON22Lc8pEtnpsZz7VmawabSU0gCjnj8aC0euHQ==",
|
||||
"license": "MIT",
|
||||
"peer": true,
|
||||
"engines": {
|
||||
"node": ">=0.10.0"
|
||||
}
|
||||
|
|
@ -4804,6 +4814,7 @@
|
|||
"resolved": "https://registry.npmjs.org/react-dom/-/react-dom-19.2.4.tgz",
|
||||
"integrity": "sha512-AXJdLo8kgMbimY95O2aKQqsz2iWi9jMgKJhRBAxECE4IFxfcazB2LmzloIoibJI3C12IlY20+KFaLv+71bUJeQ==",
|
||||
"license": "MIT",
|
||||
"peer": true,
|
||||
"dependencies": {
|
||||
"scheduler": "^0.27.0"
|
||||
},
|
||||
|
|
@ -5036,6 +5047,7 @@
|
|||
"resolved": "https://registry.npmjs.org/seroval/-/seroval-1.5.0.tgz",
|
||||
"integrity": "sha512-OE4cvmJ1uSPrKorFIH9/w/Qwuvi/IMcGbv5RKgcJ/zjA/IohDLU6SVaxFN9FwajbP7nsX0dQqMDes1whk3y+yw==",
|
||||
"license": "MIT",
|
||||
"peer": true,
|
||||
"engines": {
|
||||
"node": ">=10"
|
||||
}
|
||||
|
|
@ -5151,6 +5163,7 @@
|
|||
"integrity": "sha512-5gTmgEY/sqK6gFXLIsQNH19lWb4ebPDLA4SdLP7dsWkIXHWlG66oPuVvXSGFPppYZz8ZDZq0dYYrbHfBCVUb1Q==",
|
||||
"dev": true,
|
||||
"license": "MIT",
|
||||
"peer": true,
|
||||
"engines": {
|
||||
"node": ">=12"
|
||||
},
|
||||
|
|
@ -5213,6 +5226,7 @@
|
|||
"integrity": "sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw==",
|
||||
"devOptional": true,
|
||||
"license": "Apache-2.0",
|
||||
"peer": true,
|
||||
"bin": {
|
||||
"tsc": "bin/tsc",
|
||||
"tsserver": "bin/tsserver"
|
||||
|
|
@ -5346,6 +5360,7 @@
|
|||
"integrity": "sha512-w+N7Hifpc3gRjZ63vYBXA56dvvRlNWRczTdmCBBa+CotUzAPf5b7YMdMR/8CQoeYE5LX3W4wj6RYTgonm1b9DA==",
|
||||
"dev": true,
|
||||
"license": "MIT",
|
||||
"peer": true,
|
||||
"dependencies": {
|
||||
"esbuild": "^0.27.0",
|
||||
"fdir": "^6.5.0",
|
||||
|
|
@ -5453,6 +5468,7 @@
|
|||
"integrity": "sha512-5gTmgEY/sqK6gFXLIsQNH19lWb4ebPDLA4SdLP7dsWkIXHWlG66oPuVvXSGFPppYZz8ZDZq0dYYrbHfBCVUb1Q==",
|
||||
"dev": true,
|
||||
"license": "MIT",
|
||||
"peer": true,
|
||||
"engines": {
|
||||
"node": ">=12"
|
||||
},
|
||||
|
|
|
|||
|
|
@ -37,6 +37,7 @@ macro_rules! localizable_error {
|
|||
}
|
||||
|
||||
mod async_command;
|
||||
pub(crate) use async_command::AsyncCommandContext;
|
||||
mod environment;
|
||||
mod project;
|
||||
mod start;
|
||||
|
|
@ -368,6 +369,17 @@ impl_from_error!(
|
|||
fs_extra::error::Error,
|
||||
);
|
||||
|
||||
impl From<crate::compressor::CompressError> for RustError {
|
||||
fn from(value: crate::compressor::CompressError) -> Self {
|
||||
match value {
|
||||
crate::compressor::CompressError::Io(e) => e.into(),
|
||||
crate::compressor::CompressError::Zip(e) => e.into(),
|
||||
crate::compressor::CompressError::TaskJoin(e) => RustError::unrecoverable(e),
|
||||
crate::compressor::CompressError::Semaphore(e) => RustError::unrecoverable(e),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl From<crate::updater::Error> for RustError {
|
||||
fn from(value: crate::updater::Error) -> Self {
|
||||
log::error!(gui_toast = false; "updater error: {value}");
|
||||
|
|
|
|||
|
|
@ -1,7 +1,10 @@
|
|||
use crate::commands::DEFAULT_UNITY_ARGUMENTS;
|
||||
use crate::commands::async_command::*;
|
||||
use crate::commands::prelude::*;
|
||||
use crate::compressor::TauriCreateBackupProgress;
|
||||
use crate::compressor::parallel_compress_zip;
|
||||
use crate::utils::{collect_notable_project_files_tree, project_backup_path};
|
||||
use async_zip::{Compression, DeflateOption};
|
||||
use log::{error, info, warn};
|
||||
use serde::Serialize;
|
||||
use std::ffi::OsStr;
|
||||
|
|
@ -9,7 +12,7 @@ use std::path::{Path, PathBuf};
|
|||
use std::process::Stdio;
|
||||
use std::str::FromStr;
|
||||
use tauri::{AppHandle, State, Window};
|
||||
use tokio::io::{AsyncBufReadExt, AsyncWriteExt, BufReader};
|
||||
use tokio::io::{AsyncBufReadExt, BufReader};
|
||||
use tokio::process::Command;
|
||||
use vrc_get_vpm::environment::{PackageInstaller, VccDatabaseConnection};
|
||||
use vrc_get_vpm::io::DefaultEnvironmentIo;
|
||||
|
|
@ -518,14 +521,11 @@ pub fn project_is_unity_launching(project_path: String) -> bool {
|
|||
async fn create_backup_zip(
|
||||
backup_path: &Path,
|
||||
project_path: &Path,
|
||||
compression: async_zip::Compression,
|
||||
deflate_option: async_zip::DeflateOption,
|
||||
compression: Compression,
|
||||
deflate_option: DeflateOption,
|
||||
exclude_vpm: bool,
|
||||
ctx: AsyncCommandContext<TauriCreateBackupProgress>,
|
||||
) -> Result<(), RustError> {
|
||||
let mut file = tokio::fs::File::create_new(&backup_path).await?;
|
||||
let mut writer = async_zip::tokio::write::ZipFileWriter::with_tokio(&mut file);
|
||||
|
||||
info!("Collecting files to backup {}...", project_path.display());
|
||||
|
||||
let start = std::time::Instant::now();
|
||||
|
|
@ -539,45 +539,14 @@ async fn create_backup_zip(
|
|||
start.elapsed().as_secs_f64()
|
||||
);
|
||||
|
||||
let _ = ctx.emit(TauriCreateBackupProgress {
|
||||
total: total_files,
|
||||
proceed: 0,
|
||||
last_proceed: "Collecting files".to_string(),
|
||||
});
|
||||
|
||||
for (proceed, entry) in file_tree.recursive().enumerate() {
|
||||
if entry.is_dir() {
|
||||
writer
|
||||
.write_entry_whole(
|
||||
async_zip::ZipEntryBuilder::new(
|
||||
entry.relative_path().into(),
|
||||
async_zip::Compression::Stored,
|
||||
),
|
||||
b"",
|
||||
)
|
||||
.await?;
|
||||
} else {
|
||||
let file = tokio::fs::read(entry.absolute_path()).await?;
|
||||
writer
|
||||
.write_entry_whole(
|
||||
async_zip::ZipEntryBuilder::new(entry.relative_path().into(), compression)
|
||||
.deflate_option(deflate_option),
|
||||
file.as_ref(),
|
||||
)
|
||||
.await?;
|
||||
}
|
||||
|
||||
let _ = ctx.emit(TauriCreateBackupProgress {
|
||||
total: total_files,
|
||||
proceed: proceed + 1,
|
||||
last_proceed: entry.relative_path().to_string(),
|
||||
});
|
||||
}
|
||||
|
||||
writer.close().await?;
|
||||
file.flush().await?;
|
||||
file.sync_data().await?;
|
||||
drop(file);
|
||||
parallel_compress_zip(
|
||||
file_tree,
|
||||
backup_path.to_path_buf(),
|
||||
compression,
|
||||
deflate_option,
|
||||
ctx,
|
||||
)
|
||||
.await?;
|
||||
|
||||
info!(
|
||||
"Creating backup archive for {} finished!",
|
||||
|
|
@ -605,13 +574,6 @@ impl Drop for RemoveOnDrop<'_> {
|
|||
}
|
||||
}
|
||||
|
||||
#[derive(Serialize, specta::Type, Clone)]
|
||||
pub struct TauriCreateBackupProgress {
|
||||
total: usize,
|
||||
proceed: usize,
|
||||
last_proceed: String,
|
||||
}
|
||||
|
||||
#[tauri::command]
|
||||
#[specta::specta]
|
||||
pub async fn project_create_backup(
|
||||
|
|
@ -648,10 +610,10 @@ pub async fn project_create_backup(
|
|||
log::info!("backup project: {project_name} with {backup_format}");
|
||||
let timer = std::time::Instant::now();
|
||||
|
||||
let backup_path;
|
||||
let backup_path: PathBuf;
|
||||
let remove_on_drop: RemoveOnDrop;
|
||||
match backup_format.as_str() {
|
||||
"default" | "zip-store" => {
|
||||
"zip-store" => {
|
||||
backup_path = Path::new(&backup_dir)
|
||||
.join(&backup_name)
|
||||
.with_added_extension("zip");
|
||||
|
|
@ -659,14 +621,14 @@ pub async fn project_create_backup(
|
|||
create_backup_zip(
|
||||
&backup_path,
|
||||
project_path.as_ref(),
|
||||
async_zip::Compression::Stored,
|
||||
async_zip::DeflateOption::Normal,
|
||||
Compression::Stored,
|
||||
DeflateOption::Fast, // unused
|
||||
exclude_vpm,
|
||||
ctx,
|
||||
)
|
||||
.await?;
|
||||
}
|
||||
"zip-fast" => {
|
||||
"default" | "zip-fast" => {
|
||||
backup_path = Path::new(&backup_dir)
|
||||
.join(&backup_name)
|
||||
.with_added_extension("zip");
|
||||
|
|
@ -674,8 +636,8 @@ pub async fn project_create_backup(
|
|||
create_backup_zip(
|
||||
&backup_path,
|
||||
project_path.as_ref(),
|
||||
async_zip::Compression::Deflate,
|
||||
async_zip::DeflateOption::Other(1),
|
||||
Compression::Deflate,
|
||||
DeflateOption::Fast,
|
||||
exclude_vpm,
|
||||
ctx,
|
||||
)
|
||||
|
|
@ -689,32 +651,32 @@ pub async fn project_create_backup(
|
|||
create_backup_zip(
|
||||
&backup_path,
|
||||
project_path.as_ref(),
|
||||
async_zip::Compression::Deflate,
|
||||
async_zip::DeflateOption::Other(9),
|
||||
Compression::Deflate,
|
||||
DeflateOption::Maximum,
|
||||
exclude_vpm,
|
||||
ctx,
|
||||
)
|
||||
.await?;
|
||||
}
|
||||
backup_format => {
|
||||
_ => {
|
||||
warn!("unknown backup format: {backup_format}, using zip-fast");
|
||||
|
||||
backup_path = Path::new(&backup_dir)
|
||||
.join(&backup_name)
|
||||
.with_added_extension("zip");
|
||||
|
||||
remove_on_drop = RemoveOnDrop::new(&backup_path);
|
||||
create_backup_zip(
|
||||
&backup_path,
|
||||
project_path.as_ref(),
|
||||
async_zip::Compression::Deflate,
|
||||
async_zip::DeflateOption::Other(1),
|
||||
Compression::Deflate,
|
||||
DeflateOption::Fast,
|
||||
exclude_vpm,
|
||||
ctx,
|
||||
)
|
||||
.await?;
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
remove_on_drop.forget();
|
||||
|
||||
log::info!("backup finished in {:?}", timer.elapsed());
|
||||
|
|
|
|||
312
vrc-get-gui/src/compressor.rs
Normal file
312
vrc-get-gui/src/compressor.rs
Normal file
|
|
@ -0,0 +1,312 @@
|
|||
use crate::commands::AsyncCommandContext;
|
||||
use crate::utils::FileSystemTree;
|
||||
use async_zip::base::write::ZipFileWriter;
|
||||
use async_zip::{Compression, DeflateOption, ZipEntryBuilder};
|
||||
use serde::Serialize;
|
||||
use std::collections::BTreeMap;
|
||||
use std::path::PathBuf;
|
||||
use std::sync::Arc;
|
||||
use std::sync::atomic::{AtomicUsize, Ordering};
|
||||
use tokio::fs::File;
|
||||
use tokio::sync::{OwnedSemaphorePermit, Semaphore};
|
||||
use tokio_util::compat::Compat;
|
||||
|
||||
#[derive(Debug, Clone, Serialize, specta::Type)]
|
||||
pub struct TauriCreateBackupProgress {
|
||||
total: usize,
|
||||
proceed: usize,
|
||||
last_proceed: String,
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
pub enum CompressError {
|
||||
Io(std::io::Error),
|
||||
Zip(async_zip::error::ZipError),
|
||||
TaskJoin(tokio::task::JoinError),
|
||||
Semaphore(tokio::sync::AcquireError),
|
||||
}
|
||||
|
||||
impl From<std::io::Error> for CompressError {
|
||||
fn from(value: std::io::Error) -> Self {
|
||||
CompressError::Io(value)
|
||||
}
|
||||
}
|
||||
|
||||
impl From<async_zip::error::ZipError> for CompressError {
|
||||
fn from(value: async_zip::error::ZipError) -> Self {
|
||||
CompressError::Zip(value)
|
||||
}
|
||||
}
|
||||
|
||||
impl From<tokio::task::JoinError> for CompressError {
|
||||
fn from(value: tokio::task::JoinError) -> Self {
|
||||
CompressError::TaskJoin(value)
|
||||
}
|
||||
}
|
||||
|
||||
impl From<tokio::sync::AcquireError> for CompressError {
|
||||
fn from(value: tokio::sync::AcquireError) -> Self {
|
||||
CompressError::Semaphore(value)
|
||||
}
|
||||
}
|
||||
|
||||
struct CompressedData {
|
||||
bytes: Vec<u8>,
|
||||
crc32: u32,
|
||||
uncompressed_size: u64,
|
||||
_permit: Option<OwnedSemaphorePermit>,
|
||||
}
|
||||
|
||||
struct WriteMessage {
|
||||
index: usize,
|
||||
relative_path: String,
|
||||
data: Option<CompressedData>,
|
||||
}
|
||||
|
||||
impl WriteMessage {
|
||||
fn new(index: usize, relative_path: String, data: Option<CompressedData>) -> Self {
|
||||
Self {
|
||||
index,
|
||||
relative_path,
|
||||
data,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
struct WriteState {
|
||||
zip: Option<ZipFileWriter<Compat<File>>>,
|
||||
compression: Compression,
|
||||
deflate_option: DeflateOption,
|
||||
next_write_idx: usize,
|
||||
pending: BTreeMap<usize, (String, Option<CompressedData>)>,
|
||||
|
||||
rx: tokio::sync::mpsc::UnboundedReceiver<WriteMessage>,
|
||||
}
|
||||
|
||||
impl WriteState {
|
||||
fn new(
|
||||
zip: ZipFileWriter<Compat<File>>,
|
||||
compression: Compression,
|
||||
deflate_option: DeflateOption,
|
||||
rx: tokio::sync::mpsc::UnboundedReceiver<WriteMessage>,
|
||||
) -> Self {
|
||||
Self {
|
||||
zip: Some(zip),
|
||||
compression,
|
||||
deflate_option,
|
||||
next_write_idx: 0,
|
||||
pending: BTreeMap::new(),
|
||||
rx,
|
||||
}
|
||||
}
|
||||
|
||||
async fn run(mut self) -> Result<(), CompressError> {
|
||||
while let Some(msg) = self.rx.recv().await {
|
||||
self.submit(msg.index, msg.relative_path, msg.data).await?;
|
||||
}
|
||||
self.finish().await
|
||||
}
|
||||
|
||||
async fn submit(
|
||||
&mut self,
|
||||
idx: usize,
|
||||
relative_path: String,
|
||||
data: Option<CompressedData>,
|
||||
) -> Result<(), CompressError> {
|
||||
self.pending.insert(idx, (relative_path, data));
|
||||
|
||||
while let Some((name, entry_data)) = self.pending.remove(&self.next_write_idx) {
|
||||
if let Some(zip) = self.zip.as_mut() {
|
||||
match entry_data {
|
||||
None => {
|
||||
let entry = ZipEntryBuilder::new(name.into(), self.compression)
|
||||
.deflate_option(self.deflate_option);
|
||||
zip.write_entry_whole(entry.build(), b"").await?;
|
||||
}
|
||||
Some(cd) => {
|
||||
let entry = ZipEntryBuilder::new(name.into(), self.compression)
|
||||
.deflate_option(self.deflate_option)
|
||||
.crc32(cd.crc32)
|
||||
.uncompressed_size(cd.uncompressed_size);
|
||||
zip.write_entry_whole_precompressed(entry.build(), &cd.bytes)
|
||||
.await?;
|
||||
}
|
||||
}
|
||||
}
|
||||
self.next_write_idx += 1;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn finish(&mut self) -> Result<(), CompressError> {
|
||||
if let Some(zip) = self.zip.take() {
|
||||
zip.close().await?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) async fn parallel_compress_zip(
|
||||
tree: FileSystemTree,
|
||||
destination: PathBuf,
|
||||
compression: Compression,
|
||||
deflate_option: DeflateOption,
|
||||
ctx: AsyncCommandContext<TauriCreateBackupProgress>,
|
||||
) -> Result<(), CompressError> {
|
||||
let total = tree.count_all();
|
||||
|
||||
let _ = ctx.emit(TauriCreateBackupProgress {
|
||||
total,
|
||||
proceed: 0,
|
||||
last_proceed: "Collecting files".to_string(),
|
||||
});
|
||||
|
||||
let file = File::create_new(&destination).await?;
|
||||
let writer = ZipFileWriter::with_tokio(file);
|
||||
|
||||
let proceed = Arc::new(AtomicUsize::new(0));
|
||||
|
||||
let threads = std::thread::available_parallelism().map_or(1, |n| n.get());
|
||||
let available_ram = {
|
||||
let mut sys = sysinfo::System::new();
|
||||
sys.refresh_memory();
|
||||
|
||||
// Since the maximum capacity of the semaphore is u32::MAX, it can only handle up to 4GB.
|
||||
// To circumvent this, we will use 1 permit for every 10 bytes, allowing for a capacity of up to 40GB.
|
||||
let available_ram: u32 = ((sys.free_memory() as f64 / 10.0 * 0.8) as u32) // 80% of free memory
|
||||
.max(1);
|
||||
|
||||
log::info!(
|
||||
"Using {:.2} GB soft memory limit for compression",
|
||||
(available_ram as f64) * 10.0 / 1024.0 / 1024.0 / 1024.0
|
||||
);
|
||||
|
||||
available_ram
|
||||
};
|
||||
|
||||
let thread_semaphore = Arc::new(Semaphore::new(threads));
|
||||
let ram_semaphore = Arc::new(Semaphore::new(available_ram as usize));
|
||||
|
||||
let (sender, rx) = tokio::sync::mpsc::unbounded_channel();
|
||||
let write_state = WriteState::new(writer, compression, deflate_option, rx);
|
||||
|
||||
let merge_task = tokio::spawn(write_state.run());
|
||||
|
||||
let mut handles = vec![];
|
||||
|
||||
for (idx, entry) in tree.recursive().enumerate() {
|
||||
if entry.is_dir() {
|
||||
let relative_path = entry.relative_path().to_string();
|
||||
let _ = sender.send(WriteMessage::new(idx, relative_path.clone(), None));
|
||||
let p = proceed.fetch_add(1, Ordering::Relaxed) + 1;
|
||||
let _ = ctx.emit(TauriCreateBackupProgress {
|
||||
total,
|
||||
proceed: p,
|
||||
last_proceed: relative_path,
|
||||
});
|
||||
} else {
|
||||
let relative_path = entry.relative_path().to_string();
|
||||
let absolute_path = entry.absolute_path().to_path_buf();
|
||||
let file_size = tokio::fs::metadata(&absolute_path).await?.len();
|
||||
|
||||
// Permit size is calculated as the number of 10-byte chunks, plus 1 for the remainder.
|
||||
// Since memory usage limiting is a soft limit, if the file size exceeds
|
||||
// the maximum capacity of the semaphore, fall back to acquiring that maximum capacity.
|
||||
let ram_permit_size = ((file_size as f64 / 10.0) as u32)
|
||||
.saturating_add(1)
|
||||
.min(available_ram);
|
||||
|
||||
let thread_permit = thread_semaphore.clone().acquire_owned().await?;
|
||||
let mut ram_permit = ram_semaphore
|
||||
.clone()
|
||||
.acquire_many_owned(ram_permit_size)
|
||||
.await?;
|
||||
|
||||
let sender = sender.clone();
|
||||
let ctx = ctx.clone();
|
||||
let proceed = proceed.clone();
|
||||
|
||||
let handle: tokio::task::JoinHandle<Result<(), CompressError>> =
|
||||
tokio::task::spawn(async move {
|
||||
let (compressed_bytes, crc32, uncompressed_size) = {
|
||||
let raw_data = tokio::fs::read(&absolute_path).await?;
|
||||
let crc32 = async_zip::base::write::crc32(&raw_data);
|
||||
let uncompressed_size = raw_data.len() as u64;
|
||||
|
||||
let bytes = match compression {
|
||||
Compression::Stored => raw_data,
|
||||
_ => {
|
||||
async_zip::base::write::compress(
|
||||
&ZipEntryBuilder::new(
|
||||
relative_path.clone().into(),
|
||||
compression,
|
||||
)
|
||||
.deflate_option(deflate_option)
|
||||
.build(),
|
||||
&raw_data,
|
||||
)
|
||||
.await
|
||||
}
|
||||
};
|
||||
|
||||
(bytes, crc32, uncompressed_size)
|
||||
};
|
||||
|
||||
drop(thread_permit);
|
||||
|
||||
// split semaphore and release unused permits
|
||||
let remain_permit =
|
||||
if let Some(new_permits) = ram_permit.split(compressed_bytes.len()) {
|
||||
drop(ram_permit);
|
||||
new_permits
|
||||
} else {
|
||||
// split() returns None if the compressed size exceeds available permits.
|
||||
// This happens when a file is larger than the semaphore's max capacity,
|
||||
// which is allowed as a soft limit at enqueue time. Keep all permits as-is
|
||||
// rather than acquiring new ones, since doing so could deadlock.
|
||||
ram_permit
|
||||
};
|
||||
|
||||
let compressed_data = CompressedData {
|
||||
bytes: compressed_bytes,
|
||||
crc32,
|
||||
uncompressed_size,
|
||||
_permit: Some(remain_permit),
|
||||
};
|
||||
|
||||
let _ = sender.send(WriteMessage::new(
|
||||
idx,
|
||||
relative_path.clone(),
|
||||
Some(compressed_data),
|
||||
));
|
||||
|
||||
let p = proceed.fetch_add(1, Ordering::Relaxed) + 1;
|
||||
let _ = ctx.emit(TauriCreateBackupProgress {
|
||||
total,
|
||||
proceed: p,
|
||||
last_proceed: relative_path,
|
||||
});
|
||||
|
||||
Ok(())
|
||||
});
|
||||
|
||||
handles.push(handle);
|
||||
}
|
||||
}
|
||||
|
||||
drop(sender);
|
||||
|
||||
for handle in handles {
|
||||
handle.await??;
|
||||
}
|
||||
|
||||
let _ = ctx.emit(TauriCreateBackupProgress {
|
||||
total,
|
||||
proceed: total,
|
||||
last_proceed: "finalizing...".to_string(),
|
||||
});
|
||||
|
||||
merge_task.await??;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
|
@ -5,6 +5,7 @@ use std::path::PathBuf;
|
|||
use tauri::{AppHandle, Manager};
|
||||
|
||||
mod commands;
|
||||
mod compressor;
|
||||
mod config;
|
||||
mod deep_link_support;
|
||||
mod logging;
|
||||
|
|
@ -37,6 +38,12 @@ fn main() {
|
|||
// logger is now initialized, we can use log for panics
|
||||
log_panics::init();
|
||||
|
||||
// prevent errors caused by hitting the file descriptor limit during project backup creation
|
||||
#[cfg(target_os = "macos")]
|
||||
if let Err(e) = rlimit::increase_nofile_limit(4096) {
|
||||
log::error!("error while increasing nofile limit: {e}");
|
||||
}
|
||||
|
||||
#[cfg(dev)]
|
||||
commands::export_ts();
|
||||
|
||||
|
|
@ -92,7 +99,7 @@ fn main() {
|
|||
deep_link_support::process_files(app, files);
|
||||
}
|
||||
_ => {}
|
||||
})
|
||||
});
|
||||
}
|
||||
|
||||
fn process_args(app: &AppHandle, args: &[String]) {
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue