Submobject rendering and further abstraction - WIP for stroke early drop

This commit is contained in:
Tristan Schulz 2023-09-12 05:08:11 +02:00
commit 388604b3f7
11 changed files with 225 additions and 140 deletions

View file

@ -5,61 +5,78 @@ from PIL import Image
from pyglet import shapes
from pyglet.gl import Config
from pyglet.window import Window
import numpy as np
import manim.utils.color.manim_colors as col
from manim._config import tempconfig
from manim.camera.camera import OpenGLCamera, OpenGLCameraFrame
from manim.constants import OUT, RIGHT
from manim.constants import OUT, RIGHT, UP
from manim.mobject.geometry.arc import Circle
from manim.mobject.geometry.polygram import Square
from manim.mobject.logo import ManimBanner
from manim.mobject.opengl.opengl_vectorized_mobject import OpenGLVMobject
from manim.renderer.opengl_renderer import OpenGLRenderer
from manim._config import config
if __name__ == "__main__":
with tempconfig({"renderer": "opengl"}):
renderer = OpenGLRenderer(1920, 1080)
renderer = OpenGLRenderer(1920, 1080, background_color=col.GRAY)
# vm = OpenGLVMobject([col.RED, col.GREEN])
vm = Circle(
radius=1, stroke_color=col.YELLOW, fill_opacity=1, fill_color=col.RED
).shift(RIGHT)
vm2 = Square(stroke_color=col.GREEN, fill_opacity=0, stroke_opacity=1)
# vm3 = ManimBanner()
vm2 = Square(stroke_color=col.GREEN, fill_opacity=0, stroke_opacity=1).move_to((0,0,-0.5))
vm3 = ManimBanner()
# vm.set_points_as_corners([[-1920/2, 0, 0], [1920/2, 0, 0], [0, 1080/2, 0]])
# print(vm.color)
# print(vm.fill_color)
# print(vm.stroke_color)
camera = OpenGLCameraFrame()
camera.save_state()
renderer.init_camera(camera)
renderer.render(camera, [vm, vm2])
image = renderer.get_pixels()
print(image.shape)
Image.fromarray(image, "RGBA").show()
exit(0)
# renderer.render(camera, [vm, vm2])
# image = renderer.get_pixels()
# print(image.shape)
# Image.fromarray(image, "RGBA").show()
# exit(0)
win = Window(
width=1920,
height=1080,
vsync=True,
config=Config(double_buffer=True, samples=4),
)
renderer.use_window_fbo()
renderer.use_window()
vm.apply_depth_test()
vm2.apply_depth_test()
vm3.apply_depth_test()
clock = pyglet.clock.get_default()
def update_circle(dt):
vm.move_to((np.sin(dt), np.cos(dt), -1))
clock.schedule(update_circle)
def p2m(x,y,z):
from manim._config import config
return (config.frame_width*(x/config.pixel_width-0.5), config.frame_height*(y/config.pixel_height-0.5),z)
@win.event
def on_close():
win.close()
pass
@win.event
def on_mouse_motion(x, y, dx, dy):
vm.move_to((14.2222 * (x / 1920 - 0.5), 8 * (y / 1080 - 0.5), 0))
# vm.move_to((14.2222 * (x / 1920 - 0.5), 8 * (y / 1080 - 0.5), 0))
#camera.move_to(p2m(x,y,camera.get_center()[2]))
from scipy.spatial.transform import Rotation
camera.set_orientation(Rotation.from_rotvec((-UP*(x/1920-0.5)+RIGHT*(y/1080-0.5))*2*3.1415))
# vm.set_color(col.RED.interpolate(col.GREEN,x/1920))
# print(x,y)
@win.event
def on_draw():
image = renderer.render(camera, [vm, vm2])
renderer.render(camera, [vm, vm2, vm3])
pass
@win.event

View file

@ -1,6 +1,7 @@
from __future__ import annotations
import copy
from dataclasses import dataclass
import itertools as it
import numbers
import os
@ -51,8 +52,8 @@ if TYPE_CHECKING:
from typing_extensions import Self, TypeAlias
TimeBasedUpdater: TypeAlias = Callable[[OpenGLMobject, float], OpenGLMobject | None]
NonTimeUpdater: TypeAlias = Callable[[OpenGLMobject], OpenGLMobject | None]
TimeBasedUpdater: TypeAlias = Callable[["OpenGLMobject", float], "OpenGLMobject" | None]
NonTimeUpdater: TypeAlias = Callable[["OpenGLMobject"], "OpenGLMobject" | None]
Updater: TypeAlias = Union[TimeBasedUpdater, NonTimeUpdater]
PointUpdateFunction: TypeAlias = Callable[[np.ndarray], np.ndarray]
from manim.renderer.renderer import RendererData
@ -90,6 +91,13 @@ def affects_shader_info_id(func):
return wrapper
@dataclass
class MobjectStatus:
color_changed: bool = False
position_changed: bool = False
rotation_changed: bool = False
scale_changed: bool = False
points_changed: bool = False
class OpenGLMobject:
"""Mathematical Object: base class for objects that can be displayed on screen.
@ -123,6 +131,7 @@ class OpenGLMobject:
gloss: float = 0.0,
texture_paths: dict[str, str] | None = None,
is_fixed_in_frame: bool = False,
is_fixed_orientation: bool = False,
depth_test: bool = False,
name: str | None = None,
**kwargs,
@ -134,6 +143,7 @@ class OpenGLMobject:
self.gloss = gloss
self.texture_paths = texture_paths
self.is_fixed_in_frame = is_fixed_in_frame
self.is_fixed_orientation = is_fixed_orientation
self.depth_test = depth_test
self.name = self.__class__.__name__ if name is None else name
@ -151,8 +161,7 @@ class OpenGLMobject:
self.uniforms: dict[str, float | np.ndarray] = {}
self.renderer_data: T | None = None
self.colors_changed: bool = False
self.points_changed: bool = False
self.status = MobjectStatus()
self.init_data()
self.init_uniforms()

View file

@ -456,10 +456,10 @@ class OpenGLVMobject(OpenGLMobject):
def has_stroke(self) -> bool:
# TODO: This currently doesn't make sense needs fixing
return any(self.data["stroke_width"]) and any(self.data["stroke_rgba"][:, 3])
return self.stroke_width>0 and any(self.get_stroke_opacities())
def has_fill(self) -> bool:
return any(self.data["fill_rgba"][:, 3])
return any(self.get_fill_opacities())
def get_opacity(self) -> float:
if self.has_fill():

View file

@ -26,7 +26,6 @@ from manim.mobject.opengl.opengl_vectorized_mobject import OpenGLVMobject
fill_dtype = [
("point", np.float32, (3,)),
# ("orientation", np.float32, (3,)),
("unit_normal", np.float32, (3,)),
("color", np.float32, (4,)),
("vert_index", np.float32, (1,)),
@ -178,6 +177,24 @@ def compute_bounding_box(mob):
return np.array([mins, mids, maxs])
class ProgramManager:
@staticmethod
def get_available_uniforms(prog):
names = []
for name in prog:
member = prog[name]
if isinstance(member, gl.Uniform):
names.append(name)
@staticmethod
def write_uniforms(prog, uniforms):
for name in prog:
member = prog[name]
if isinstance(member, gl.Uniform):
if name in uniforms:
member.value = uniforms[name]
class OpenGLRenderer(Renderer):
pixel_array_dtype = np.uint8
@ -198,24 +215,23 @@ class OpenGLRenderer(Renderer):
self.samples = samples
self.background_color = background_color.to_rgba()
self.background_image = background_image
self.rgb_max_val: float = np.iinfo(self.pixel_array_dtype).max
# Initializing Context
logger.debug("Initializing OpenGL context and framebuffers")
self.ctx = gl.create_context()
self.target_fbo = self.ctx.simple_framebuffer(
(self.pixel_width, self.pixel_height),
samples=self.samples,
dtype="f1",
components=4,
self.stencil_texture = self.ctx.texture((self.pixel_width, self.pixel_height), components=4,dtype='f1')
self.target_fbo = self.ctx.framebuffer(
color_attachments=[self.ctx.renderbuffer((self.pixel_width, self.pixel_height), components=4,samples=4,dtype='f1')]
,depth_attachment=self.ctx.depth_texture((self.pixel_width, self.pixel_height), samples=4)
)
self.output_fbo = self.ctx.framebuffer(
color_attachments=[
self.ctx.renderbuffer(
(self.pixel_width, self.pixel_height), dtype="f1", components=4
)
),
]
)
@ -228,33 +244,25 @@ class OpenGLRenderer(Renderer):
self.ctx, "quadratic_bezier_stroke"
)
def use_window_fbo(self):
def use_window(self):
self.output_fbo.release()
self.output_fbo = self.ctx.detect_framebuffer()
def init_camera(self, camera: OpenGLCameraFrame) -> ImageType:
self.vmobject_fill_program["is_fixed_in_frame"] = 0.0
self.vmobject_fill_program["frame_shape"] = camera.frame_shape
self.vmobject_fill_program["focal_distance"] = float(
camera.get_focal_distance()
)
self.vmobject_fill_program["camera_center"] = tuple(camera.get_center())
self.vmobject_fill_program["camera_rotation"] = tuple(
def init_camera(self, camera: OpenGLCameraFrame):
uniforms = dict()
uniforms["frame_shape"] = camera.frame_shape
uniforms['pixel_shape'] = (self.pixel_width,self.pixel_height)
uniforms["focal_distance"] = camera.get_focal_distance()
uniforms["camera_center"] = tuple(camera.get_center())
uniforms["camera_rotation"] = tuple(
np.array(camera.get_inverse_camera_rotation_matrix()).T.flatten()
)
self.vmobject_fill_program["light_source_position"] = (-10, 10, 10)
uniforms["light_source_position"] = (-10, 10, 10)
uniforms["anti_alias_width"] = 0.01977
# TODO: convert to singular 4x4 matrix after getting *something* to render
# self.vmobject_fill_program['view'].value = camera.get_view()?
self.vmobject_stroke_program["is_fixed_in_frame"] = 0.0
self.vmobject_stroke_program["anti_alias_width"] = 0.01977
self.vmobject_stroke_program["frame_shape"] = camera.frame_shape
# self.vmobject_stroke_program['pixel_shape'].value = camera.frame_shape
self.vmobject_stroke_program["focal_distance"] = camera.get_focal_distance()
self.vmobject_stroke_program["camera_center"] = camera.get_center()
self.vmobject_stroke_program[
"camera_rotation"
] = camera.get_inverse_camera_rotation_matrix().T.flatten()
self.vmobject_stroke_program["light_source_position"] = [-10, 10, 10]
ProgramManager.write_uniforms(self.vmobject_fill_program, uniforms)
ProgramManager.write_uniforms(self.vmobject_stroke_program, uniforms)
def get_stroke_shader_data(self, mob: OpenGLVMobject) -> np.ndarray:
if not isinstance(mob.renderer_data, GLRenderData):
@ -286,96 +294,88 @@ class OpenGLRenderer(Renderer):
fill_data["vert_index"] = np.reshape(range(len(mob.points)), (-1, 1))
return fill_data
def copy_frame_to_stencil(self):
self.stencil_texture = self.target_fbo.depth_attachment
def pre_render(self, camera):
self.init_camera(camera=camera)
self.target_fbo.use()
self.target_fbo.clear(*self.background_color)
self.copy_frame_to_stencil()
def post_render(self):
self.ctx.copy_framebuffer(self.output_fbo, self.target_fbo)
def render_vmobject(self, mob: OpenGLVMobject) -> None:
def render_program(self, prog, data, indices = None):
vbo = self.ctx.buffer(data.tobytes())
ibo = self.ctx.buffer(np.asarray(indices).astype("i4").tobytes()) if indices is not None else None
# print(prog,vbo,data)
vert_format = gl.detect_format(prog, data.dtype.names)
# print(vert_format)
vao = self.ctx.vertex_array(
program=prog,
content=[(vbo, vert_format, *data.dtype.names)],
index_buffer=ibo,
)
vao.render(gl.TRIANGLES)
vbo.release()
if ibo is not None:
ibo.release()
vao.release()
def render_vmobject(self, mob: OpenGLVMobject) -> None: #type: ignore
# Setting camera uniforms
counter = 0
num_mobs = len(mob.family_members_with_points())
for sub in mob.family_members_with_points():
if sub.renderer_data is None:
# Initialize
GLVMobjectManager.init_render_data(sub)
if mob.renderer_data is None:
# Initialize
# TODO: Initialize all the data also for submobjects
logger.debug("Initializing GLRenderData")
mob.renderer_data = GLRenderData()
# Generate Mesh
mob.renderer_data.vert_indices = get_triangulation(mob)
points_length = len(mob.points)
if not isinstance(sub.renderer_data, GLRenderData):
return
# Generate Fill Color
fill_color = np.array([c._internal_value for c in mob.fill_color])
stroke_color = np.array([c._internal_value for c in mob.stroke_color])
mob.renderer_data.fill_rgbas = prepare_array(fill_color, points_length)
mob.renderer_data.stroke_rgbas = prepare_array(stroke_color, points_length)
mob.renderer_data.stroke_widths = prepare_array(
np.asarray(listify(mob.stroke_width)), points_length
)
mob.renderer_data.normals = np.repeat(
[mob.get_unit_normal()], points_length, axis=0
)
mob.renderer_data.bounding_box = compute_bounding_box(mob)
# print(mob.renderer_data)
# if mob.colors_changed:
# if mob.colors_changed:
# mob.renderer_data.fill_rgbas = np.resize(mob.fill_color, (len(mob.renderer_data.mesh),4))
# mob.renderer_data.fill_rgbas = np.resize(mob.fill_color, (len(mob.renderer_data.mesh),4))
# if mob.points_changed:3357
# if(mob.has_fill()):
# mob.renderer_data.mesh = ... # Triangulation todo
# if mob.points_changed:3357
# if(mob.has_fill()):
# mob.renderer_data.mesh = ... # Triangulation todo
# self.vmobject_fill_program['reflectiveness'].value = mob.reflectiveness
self.vmobject_fill_program["gloss"].value = mob.gloss
self.vmobject_fill_program["shadow"].value = mob.shadow
# self.ctx.enable(gl.CULL_FACE)
self.ctx.enable(gl.BLEND) #type: ignore
# TODO: Because the Triangulation is messing up the normals this won't work
# self.ctx.blend_func = ( #type: ignore
# gl.SRC_ALPHA,
# gl.ONE_MINUS_SRC_ALPHA,
# gl.ONE,
# gl.ONE,
# )
if sub.depth_test:
self.ctx.enable(gl.DEPTH_TEST) #type: ignore
else:
self.ctx.disable(gl.DEPTH_TEST) #type: ignore
uniforms = GLVMobjectManager.read_uniforms(sub)
uniforms['z_shift'] = counter/9
if sub.has_fill():
ProgramManager.write_uniforms(self.vmobject_fill_program, uniforms)
self.render_program(
self.vmobject_fill_program, self.get_fill_shader_data(sub), sub.renderer_data.vert_indices
)
uniforms["z_shift"] -= 1/20
# self.vmobject_stroke_program['reflectiveness'].value = mob.reflectiveness
self.vmobject_stroke_program["gloss"].value = mob.gloss
self.vmobject_stroke_program["shadow"].value = mob.shadow
self.vmobject_stroke_program["joint_type"].value = float(
mob.joint_type.value
) # TODO: This maybe breaks
self.vmobject_stroke_program["flat_stroke"].value = mob.flat_stroke
self.copy_frame_to_stencil()
self.stencil_texture.use(1)
self.vmobject_stroke_program['stencil_texture'] = 1
if sub.has_stroke():
ProgramManager.write_uniforms(self.vmobject_stroke_program, uniforms)
self.render_program(
self.vmobject_stroke_program, self.get_stroke_shader_data(sub), np.array(range(len(sub.points)))[::-1]
)
def render_shader(prog, mob, data, use_ibo):
vbo = self.ctx.buffer(data.tobytes())
ibo = (
self.ctx.buffer(mob.renderer_data.vert_indices.astype("i4").tobytes())
if use_ibo
else None
)
# print(prog,vbo,data)
vert_format = gl.detect_format(prog, data.dtype.names)
# print(vert_format)
vao = self.ctx.vertex_array(
program=prog,
content=[(vbo, vert_format, *data.dtype.names)],
index_buffer=ibo,
)
vao.render(gl.TRIANGLES)
vbo.release()
if use_ibo:
ibo.release()
vao.release()
self.ctx.enable(gl.BLEND)
self.ctx.blend_func = (
gl.SRC_ALPHA,
gl.ONE_MINUS_SRC_ALPHA,
gl.ONE,
gl.ONE,
)
# self.ctx.enable(gl.DEPTH_TEST)
# TODO: Handle Submobjects
render_shader(
self.vmobject_fill_program, mob, self.get_fill_shader_data(mob), True
)
render_shader(
self.vmobject_stroke_program, mob, self.get_stroke_shader_data(mob), False
)
counter += 1
def get_pixels(self) -> ImageType:
raw = self.output_fbo.read(components=4, dtype="f1", clamp=True) # RGBA, floats
@ -383,6 +383,43 @@ class OpenGLRenderer(Renderer):
return buf
class GLVMobjectManager:
@staticmethod
def init_render_data(mob:OpenGLVMobject):
logger.debug("Initializing GLRenderData")
mob.renderer_data = GLRenderData()
# Generate Mesh
mob.renderer_data.vert_indices = get_triangulation(mob)
points_length = len(mob.points)
# Generate Fill Color
fill_color = np.array([c._internal_value for c in mob.fill_color])
stroke_color = np.array([c._internal_value for c in mob.stroke_color])
mob.renderer_data.fill_rgbas = prepare_array(fill_color, points_length)
mob.renderer_data.stroke_rgbas = prepare_array(stroke_color, points_length)
mob.renderer_data.stroke_widths = prepare_array(
np.asarray(listify(mob.stroke_width)), points_length
)
mob.renderer_data.normals = np.repeat(
[mob.get_unit_normal()], points_length, axis=0
)
mob.renderer_data.bounding_box = compute_bounding_box(mob)
# print(mob.renderer_data)
@staticmethod
def read_uniforms(mob: OpenGLVMobject):
uniforms = {}
uniforms['reflectiveness'] = mob.reflectiveness
uniforms["is_fixed_in_frame"] = float(mob.is_fixed_in_frame)
uniforms["is_fixed_orientation"] = float(mob.is_fixed_orientation)
uniforms["gloss"] = mob.gloss
uniforms["shadow"] = mob.shadow
uniforms["flat_stroke"] = float(mob.flat_stroke)
uniforms["joint_type"] = float(mob.joint_type.value)
return uniforms
# def init_frame(self, **config) -> None:
# self.frame = OpenGLCameraFrame(**config)

View file

@ -43,6 +43,9 @@ class Renderer(ABC):
def post_render(self):
raise NotImplementedError
def use_window(self):
raise NotImplementedError
@abstractclassmethod
def render_vmobject(self, mob: OpenGLVMobject) -> None:
raise NotImplementedError

View file

@ -15,7 +15,8 @@ vec4 add_light(vec4 color,
vec3 light_coords,
float gloss,
float shadow){
if(gloss == 0.0 && shadow == 0.0) return color;
if (gloss == 0.0 && shadow == 0.0 && reflectiveness == 0.0)
return color;
// TODO, do we actually want this? It effectively treats surfaces as two-sided
if(unit_normal.z < 0){

View file

@ -2,7 +2,7 @@
// uniform vec2 frame_shape;
// uniform float focal_distance;
// uniform float is_fixed_in_frame;
uniform float z_shift;
const vec2 DEFAULT_FRAME_SHAPE = vec2(8.0 * 16.0 / 9.0, 8.0);
float perspective_scale_factor(float z, float focal_distance)
@ -23,7 +23,9 @@ vec4 get_gl_Position(vec3 point)
result.xy *= psf;
// TODO, what's the better way to do this?
// This is to keep vertices too far out of frame from getting cut.
result.z *= 0.01;
// TODO This will be done by the clipping plane in the future with the transformation matrix
result.z -= z_shift;
result.z *= (1.0 / 100.0);
}
}
else

View file

@ -40,4 +40,8 @@ void main() {
#ifndef ANTI_ALIASING
frag_color.a *= float(sdf() > 0); // No anti-aliasing
#endif
if (frag_color.a <= 0.0)
{
discard;
}
}

View file

@ -15,6 +15,7 @@ uniform vec3 fixed_orientation_center;
uniform vec3 light_source_position;
uniform float gloss;
uniform float shadow;
uniform float reflectiveness;
in vec3 bp[3];
in vec3 v_global_unit_normal[3];

View file

@ -1,6 +1,7 @@
#version 330
#include ../include/camera_uniform_declarations.glsl
uniform vec2 pixel_shape;
in vec2 uv_coords;
in vec2 uv_b2;
@ -16,6 +17,8 @@ in float bevel_end;
in float angle_from_prev;
in float angle_to_next;
uniform sampler2D stencil_texture;
in float bezier_degree;
out vec4 frag_color;
@ -83,11 +86,21 @@ float modify_distance_for_endpoints(vec2 p, float dist, float t){
void main() {
if (uv_stroke_width == 0) discard;
if (uv_stroke_width == 0)
discard;
float dist_to_curve = min_dist_to_curve(uv_coords, uv_b2, bezier_degree);
// An sdf for the region around the curve we wish to color.
float signed_dist = abs(dist_to_curve) - 0.5 * uv_stroke_width;
frag_color = color;
frag_color =
// TODO: The incoming texture should be the depth buffer, discard the pixel on any value this needs to be
// rewritten
vec4(texture2D(stencil_texture, vec2(gl_FragCoord.x / pixel_shape.x, gl_FragCoord.y / pixel_shape.y)).a, 0, 0,
1) +
color / 1000;
frag_color.a *= smoothstep(0.5, -0.5, signed_dist / uv_anti_alias_width);
if (frag_color.a <= 0.0)
{
discard;
}
}

View file

@ -6,8 +6,6 @@ layout (triangle_strip, max_vertices = 5) out;
// Needed for get_gl_Position
uniform vec2 frame_shape;
uniform float focal_distance;
uniform float is_fixed_in_frame;
uniform float is_fixed_orientation;
uniform vec3 fixed_orientation_center;
uniform float anti_alias_width;
@ -15,9 +13,12 @@ uniform float flat_stroke;
//Needed for lighting
uniform vec3 light_source_position;
uniform float joint_type;
uniform float gloss;
uniform float shadow;
uniform float joint_type;
uniform float reflectiveness;
uniform float is_fixed_in_frame;
uniform float is_fixed_orientation;
in vec3 bp[3];
in vec3 prev_bp[3];
@ -264,10 +265,7 @@ void main() {
gloss,
shadow
);
gl_Position = vec4(
get_gl_Position(vec3(corners[i], 0.0)).xy,
get_gl_Position(controls[index_map[i]]).zw
);
gl_Position = vec4(get_gl_Position(vec3(corners[i], 0.0)).xy, get_gl_Position(controls[index_map[i]]).zw);
EmitVertex();
}
EndPrimitive();