mirrors/misskey
2
0
Fork 1
mirror of https://github.com/misskey-dev/misskey.git synced 2026-06-25 17:10:43 +00:00
Code Issues Projects Releases Packages Wiki Activity

wip

Browse source
This commit is contained in:
syuilo 2026-05-28 08:04:50 +09:00
commit 4c80e92522
6 changed files with 413 additions and 16 deletions
Download patch file Download diff file Expand all files Collapse all files

2
packages/frontend-misskey-world-engine/src/PlayerContainer.ts
View file

@ -5,11 +5,13 @@
import * as BABYLON from '@babylonjs/core';
import { WORLD_SCALE } from 'misskey-world/src/utility.js';
import type { WorldAvatar } from 'misskey-world/src/types.js';
export type PlayerProfile = {
name: string;
username: string;
avatarUrl: string;
worldAvatar: WorldAvatar;
};
export type PlayerState = {

257
packages/frontend-misskey-world-engine/src/avatarPreviewEngine.ts Normal file
View file

@ -0,0 +1,257 @@
/*
* SPDX-FileCopyrightText: syuilo and misskey-project
* SPDX-License-Identifier: AGPL-3.0-only
*/
import * as BABYLON from '@babylonjs/core';
import { registerBuiltInLoaders } from '@babylonjs/loaders/dynamic.js';
import { GridMaterial } from '@babylonjs/materials';
import { cm, WORLD_SCALE } from 'misskey-world/src/utility.js';
import { camelToKebab } from 'misskey-world/src/utility.js';
import { getMeshesBoundingBox, ArcRotateCameraManualInput } from '../utility.js';
import { EngineBase } from '../EngineBase.js';
import { deepClone } from '../clone.js';
import { genId } from '../id.js';
import { SYSTEM_MESH_NAMES, GRAPHICS_QUALITY } from './utility.js';
import type { RawOptions } from './object.js';
import type { PlayerContainer } from './PlayerContainer.js';
export class AvatarPreviewEngine extends EngineBase<{
'loadingProgress': (ctx: { progress: number }) => void;
'contextlost': (ctx: { reason: string; message: string; }) => void;
}> {
private sr: BABYLON.SnapshotRenderingHelper;
private shadowGenerator: BABYLON.ShadowGenerator;
private camera: BABYLON.ArcRotateCamera;
private playerContainer: PlayerContainer | null = null;
private objectOptions: RawOptions | null = null;
private envMapIndoor: BABYLON.CubeTexture;
private roomLight: BABYLON.SpotLight;
private pipeline: BABYLON.DefaultRenderingPipeline;
private graphicsQuality: number;
constructor(options: {
engine: BABYLON.WebGPUEngine;
graphicsQuality: number;
fps: number | null;
}) {
super({
engine: options.engine,
fps: options.fps,
});
registerBuiltInLoaders();
this.graphicsQuality = options.graphicsQuality;
this.scene.autoClear = false;
this.scene.skipPointerMovePicking = true;
this.scene.skipFrustumClipping = true; // snapshot renderingでは全てのメッシュがアクティブになっている必要があるため
this.scene.clearColor = new BABYLON.Color4(0.03, 0.03, 0.03, 1);
this.sr = new BABYLON.SnapshotRenderingHelper(this.scene);
this.camera = new BABYLON.ArcRotateCamera('camera', -Math.PI / 2, Math.PI / 2.5, cm(300), new BABYLON.Vector3(0, cm(90), 0), this.scene);
this.envMapIndoor = BABYLON.CubeTexture.CreateFromPrefilteredData('/client-assets/room/indoor.env', this.scene);
this.envMapIndoor.boundingBoxSize = new BABYLON.Vector3(cm(500), cm(500), cm(500));
this.envMapIndoor.level = 0.6;
this.roomLight = new BABYLON.SpotLight('roomLight', new BABYLON.Vector3(cm(50), cm(249), cm(50)), new BABYLON.Vector3(0, -1, 0), 16, 8, this.scene);
this.roomLight.diffuse = new BABYLON.Color3(1.0, 0.9, 0.8);
this.roomLight.shadowMinZ = cm(10);
this.roomLight.shadowMaxZ = cm(500);
this.roomLight.radius = cm(30);
this.roomLight.intensity = 15 * WORLD_SCALE * WORLD_SCALE;
this.shadowGenerator = new BABYLON.ShadowGenerator(2048, this.roomLight);
this.shadowGenerator.forceBackFacesOnly = true;
this.shadowGenerator.bias = 0.0001;
this.shadowGenerator.usePercentageCloserFiltering = true;
this.shadowGenerator.filteringQuality = BABYLON.ShadowGenerator.QUALITY_HIGH;
this.shadowGenerator.getShadowMap().refreshRate = 60;
const gl = new BABYLON.GlowLayer('glow', this.scene, {
blurKernelSize: 64,
});
gl.intensity = 0.5;
this.scene.setRenderingAutoClearDepthStencil(gl.renderingGroupId, false);
this.sr.updateMeshesForEffectLayer(gl);
this.pipeline = new BABYLON.DefaultRenderingPipeline('default', true, this.scene);
this.pipeline.samples = 4;
if (this.graphicsQuality >= GRAPHICS_QUALITY.HIGH) {
this.pipeline.bloomEnabled = true;
this.pipeline.bloomThreshold = 0.95;
this.pipeline.bloomWeight = 0.1;
this.pipeline.bloomKernel = 256;
this.pipeline.bloomScale = 2;
}
this.pipeline.sharpenEnabled = true;
this.pipeline.sharpen.edgeAmount = 0.5;
}
public async init() {
await this.scene.whenReadyAsync();
this.sr.enableSnapshotRendering();
this.inputs.on('wheel', (ev) => {
this.camera.fov += ev.deltaY * 0.0005;
this.camera.fov = Math.max(0.25, Math.min(0.5, this.camera.fov));
});
this.inputs.on('zoom', (ev) => {
this.camera.fov += -ev.delta * 0.0015;
this.camera.fov = Math.max(0.25, Math.min(0.5, this.camera.fov));
});
this.inputs.on('pointer', (ev) => {
(this.camera.inputs.attached.manual as ArcRotateCameraManualInput).setRotationVector({ x: ev.x, y: ev.y });
});
}
public async loadObject(type: string) {
this.sr.disableSnapshotRendering();
this.clearObject();
this.objectContainer = new ObjectContainer({
id: id,
type: type,
position: new BABYLON.Vector3(0, 0, 0),
rotation: new BABYLON.Vector3(0, 0, 0),
options: this.objectOptions,
roomAttachments: { files: [] },
metadata: {},
sr: this.sr,
getIsSrReady: () => true,
lightContainer: null,
graphicsQuality: this.graphicsQuality,
scene: this.scene,
});
this.objectContainer.registerMeshes = (meshes) => {
for (const mesh of meshes) {
// シェイプキー(morph)を考慮してbounding boxを更新するために必要
mesh.refreshBoundingInfo({ applyMorph: true });
if (SYSTEM_MESH_NAMES.some(n => mesh.name.includes(n))) {
mesh.receiveShadows = false;
mesh.isVisible = false;
} else {
if (def.receiveShadows !== false) mesh.receiveShadows = true;
if (def.castShadows !== false) {
this.shadowGenerator.addShadowCaster(mesh);
}
if (mesh.material) {
if (mesh.material instanceof BABYLON.MultiMaterial) {
for (const subMat of mesh.material.subMaterials) {
(subMat as BABYLON.PBRMaterial).reflectionTexture = this.envMapIndoor;
(subMat as BABYLON.PBRMaterial).useGLTFLightFalloff = true; // Clustered Lightingではphysical falloffを持つマテリアルはアーチファクトが発生する https://doc.babylonjs.com/features/featuresDeepDive/lights/clusteredLighting/#materials-with-a-physical-falloff-may-cause-artefacts
(subMat as BABYLON.PBRMaterial).anisotropy.isEnabled = false; // なんかきれいにレンダリングされないため
}
} else {
(mesh.material as BABYLON.PBRMaterial).reflectionTexture = this.envMapIndoor;
(mesh.material as BABYLON.PBRMaterial).useGLTFLightFalloff = true; // Clustered Lightingではphysical falloffを持つマテリアルはアーチファクトが発生する https://doc.babylonjs.com/features/featuresDeepDive/lights/clusteredLighting/#materials-with-a-physical-falloff-may-cause-artefacts
(mesh.material as BABYLON.PBRMaterial).anisotropy.isEnabled = false; // なんかきれいにレンダリングされないため
}
}
}
if (!this.scene.meshes.includes(mesh)) this.scene.addMesh(mesh);
}
};
await this.objectContainer.load();
const boundingInfo = getMeshesBoundingBox(this.objectContainer.root.getChildMeshes().filter(m => m.isEnabled() && m.isVisible), true);
this.pipeline.removeCamera(this.camera);
this.camera.dispose();
this.camera = new BABYLON.ArcRotateCamera('camera', Math.PI / 2, Math.PI / 2.5, cm(300), new BABYLON.Vector3(0, cm(90), 0), this.scene);
this.camera.minZ = cm(1);
this.camera.maxZ = cm(100000);
this.camera.fov = 0.5;
this.camera.lowerRadiusLimit = cm(50);
this.camera.upperRadiusLimit = cm(1000);
this.camera.useAutoRotationBehavior = true;
this.camera.autoRotationBehavior!.idleRotationSpeed = 0.3;
//this.camera.mode = BABYLON.Camera.ORTHOGRAPHIC_CAMERA;
this.camera.setTarget(new BABYLON.Vector3(0, boundingInfo.centerWorld.y, 0));
this.camera.inputs.clear();
this.camera.inputs.add(new ArcRotateCameraManualInput(this.scene, {
rotationSensitivity: 0.0005,
}));
if (def.placement === 'wall' || def.placement === 'side') {
this.camera.lowerBetaLimit = 0;
this.camera.upperBetaLimit = Math.PI;
this.zGridPreviewPlane.rotation = new BABYLON.Vector3(0, Math.PI, 0);
} else if (def.placement === 'ceiling' || def.placement === 'bottom') {
this.camera.lowerBetaLimit = (Math.PI / 2) - 0.1;
this.camera.upperBetaLimit = Math.PI;
this.camera.beta = Math.PI / 1.75;
this.zGridPreviewPlane.rotation = new BABYLON.Vector3(-Math.PI / 2, 0, 0);
} else {
this.camera.lowerBetaLimit = 0;
this.camera.upperBetaLimit = (Math.PI / 2) + 0.1;
this.zGridPreviewPlane.rotation = new BABYLON.Vector3(Math.PI / 2, 0, 0);
}
// zoom to fit
const size = boundingInfo.extendSize;
const distance = Math.max(size.x, size.y, size.z) * 2;
this.camera.radius = distance * 3;
this.pipeline.addCamera(this.camera);
this.sr.enableSnapshotRendering();
return {
id,
options: this.objectOptions,
};
}
public updateObjectOption(key: string, value: any, attachments?: RoomAttachments) {
if (this.objectOptions == null) return;
this.objectOptions[key] = value;
if (this.objectContainer != null) {
this.objectContainer.optionsUpdated(this.objectOptions, key, value, attachments ?? { files: [] });
}
}
public clearObject() {
this.sr.disableSnapshotRendering();
if (this.objectContainer != null) {
this.objectContainer.destroy();
this.objectContainer = null;
this.objectOptions = null;
}
this.sr.enableSnapshotRendering();
}
public cameraRotate(vector: { x: number; y: number; }) {
(this.camera.inputs.attached.manual as ArcRotateCameraManualInput).setRotationVector(vector);
}
public resize() {
// 一旦snapshot renderingを無効にしておかないとエラーが出る(babylonのバグ？)
// ~~...が、一旦無効にしたらしたで複数のマテリアルがそれぞれ入れ替わる(?)という謎の現象が発生するためコメントアウトしとく(エラー出てもレンダリングが止まったりするわけでもないし)~~
// ↑追記: engine.resizeした後に一瞬待つことで回避できることが判明
this.sr.disableSnapshotRendering();
this.engine.resize(true);
// workerで実行される可能性がある
setTimeout(() => {
this.sr.enableSnapshotRendering();
}, 1);
}
public destroy() {
super.destroy();
this.objectContainer?.destroy();
}
}

29
packages/frontend-misskey-world-engine/src/avatarPreviewEngineNonWorker.ts Normal file
View file

@ -0,0 +1,29 @@
/*
* SPDX-FileCopyrightText: syuilo and misskey-project
* SPDX-License-Identifier: AGPL-3.0-only
*/
import * as BABYLON from '@babylonjs/core';
import { AvatarPreviewEngine } from './avatarPreviewEngine.js';
//BABYLON.RegisterStandardEngineExtensions();
//BABYLON.RegisterEnginesExtensionsEngineRawTexture();
//BABYLON.RegisterCollisionCoordinator();
export async function createAvatarPreviewEngine(params: {
canvas: HTMLCanvasElement; options: { graphicsQuality: number; resolution: number; fps: number | null };
}) {
const babylonEngine = new BABYLON.WebGPUEngine(params.canvas, { doNotHandleContextLost: true, powerPreference: 'low-power', antialias: true });
babylonEngine.compatibilityMode = false;
babylonEngine.enableOfflineSupport = false;
await babylonEngine.initAsync();
if (params.options.resolution === 2) babylonEngine.setHardwareScalingLevel(0.5);
if (params.options.resolution === 0.5) babylonEngine.setHardwareScalingLevel(2);
const engine = new AvatarPreviewEngine({
engine: babylonEngine,
...params.options,
});
return engine;
}

109
packages/frontend-misskey-world-engine/src/avatarPreviewEngineWorker.ts Normal file
View file

@ -0,0 +1,109 @@
/*
* SPDX-FileCopyrightText: syuilo and misskey-project
* SPDX-License-Identifier: AGPL-3.0-only
*/
import * as BABYLON from '@babylonjs/core';
import { AvatarPreviewEngine } from './avatarPreviewEngine.js';
let engine: AvatarPreviewEngine | null = null;
let canvas: OffscreenCanvas | null = null;
//BABYLON.RegisterStandardEngineExtensions();
//BABYLON.RegisterEnginesExtensionsEngineRawTexture();
//BABYLON.RegisterCollisionCoordinator();
// TODO: 他のWorkerと実装を共通化
onmessage = async (event) => {
//console.log('Worker received message:', event.data);
switch (event.data?.type) {
case 'init': {
canvas = event.data.canvas as OffscreenCanvas;
const babylonEngine = new BABYLON.WebGPUEngine(canvas, { doNotHandleContextLost: true, powerPreference: 'low-power', antialias: true });
babylonEngine.compatibilityMode = false;
babylonEngine.enableOfflineSupport = false;
await babylonEngine.initAsync();
if (event.data.options.resolution === 2) babylonEngine.setHardwareScalingLevel(0.5);
if (event.data.options.resolution === 0.5) babylonEngine.setHardwareScalingLevel(2);
engine = new RoomObjectPreviewEngine({
engine: babylonEngine,
...event.data.options,
});
engine.on('ev', ({ type, ctx }) => {
self.postMessage({ type: 'ev', ev: { type, ctx } });
});
await engine.init();
self.postMessage({ type: 'inited' });
break;
}
case 'resize': {
canvas.width = event.data.width;
canvas.height = event.data.height;
if (engine != null) engine.resize();
break;
}
case 'input:keydown': {
if (engine == null) break;
engine.inputs.emit('keydown', event.data.ev);
break;
}
case 'input:keyup': {
if (engine == null) break;
engine.inputs.emit('keyup', event.data.ev);
break;
}
case 'input:click': {
if (engine == null) break;
engine.inputs.emit('click', event.data.ev);
break;
}
case 'input:wheel': {
if (engine == null) break;
engine.inputs.emit('wheel', event.data.ev);
break;
}
case 'input:zoom': {
if (engine == null) break;
engine.inputs.emit('zoom', event.data.ev);
break;
}
case 'input:pointer': {
if (engine == null) break;
engine.inputs.emit('pointer', event.data.ev);
break;
}
case 'call': {
if (engine == null) {
console.error('Failed to call: Engine is not initialized yet!!!');
break;
}
const res = engine[event.data.fn](...(event.data.args ?? []));
if (event.data.needReturnValue) {
if (res instanceof Promise) {
res.then((r) => {
self.postMessage({ type: 'return', id: event.data.id, value: r });
});
} else {
self.postMessage({ type: 'return', id: event.data.id, value: res });
}
}
break;
}
case 'set': {
if (engine == null) {
console.error('Failed to set: Engine is not initialized yet!!!');
break;
}
engine[event.data.key] = event.data.value;
break;
}
default: {
console.warn('Unrecognized message type:', event.data?.type);
}
}
};

16
packages/frontend-misskey-world-engine/src/room/utility.ts
View file

@ -7,22 +7,6 @@ import * as BABYLON from '@babylonjs/core';
import { cm } from 'misskey-world/src/utility.js';
import { applyMorphTargetsToMesh, getPlaneUvIndexes, Timer } from '../utility.js';
export const GRAPHICS_QUALITY = {
HIGH: 1,
MEDIUM: 0,
LOW: -1,
} as const;
export function getLightRangeFactorByGraphicsQuality(quality: number) {
if (quality >= GRAPHICS_QUALITY.HIGH) {
return 1;
} else if (quality >= GRAPHICS_QUALITY.MEDIUM) {
return 0.5;
} else {
return 0.25;
}
}
export const SYSTEM_MESH_NAMES = ['__TOP__', '__SIDE__', '__BOTTOM__', '__PICK__', '__COLLISION__'];
export const SYSTEM_HEYA_MESH_NAMES = ['__ROOM_WALL__', '__ROOM_SIDE__', '__ROOM_FLOOR__', '__ROOM_CEILING__', '__ROOM_TOP__', '__ROOM_BOTTOM__', '__COLLISION__'];

16
packages/frontend-misskey-world-engine/src/utility.ts
View file

@ -5,6 +5,22 @@
import * as BABYLON from '@babylonjs/core';
export const GRAPHICS_QUALITY = {
HIGH: 1,
MEDIUM: 0,
LOW: -1,
} as const;
export function getLightRangeFactorByGraphicsQuality(quality: number) {
if (quality >= GRAPHICS_QUALITY.HIGH) {
return 1;
} else if (quality >= GRAPHICS_QUALITY.MEDIUM) {
return 0.5;
} else {
return 0.25;
}
}
export const TIME_MAP = {
0: 2,
1: 2,