From 2a6cccf14c6b6952981eb52950982ba262f6e4b4 Mon Sep 17 00:00:00 2001 From: Dominik Date: Mon, 20 Oct 2025 12:11:15 +0200 Subject: [PATCH 01/19] Added preload property to add-on panel --- bseq/panels.py | 2 ++ bseq/properties.py | 5 +++++ 2 files changed, 7 insertions(+) diff --git a/bseq/panels.py b/bseq/panels.py index c70cdd2..4570be2 100644 --- a/bseq/panels.py +++ b/bseq/panels.py @@ -92,6 +92,8 @@ def draw(self, context): col2.prop(sim_loader, "auto_refresh_active", text="") col1.label(text="Auto Refresh All") col2.prop(sim_loader, "auto_refresh_all", text="") + col1.label(text="Preload Frames") + col2.prop(sim_loader, "preload_next_frame", text="") class BSEQ_Advanced_Panel(BSEQ_Panel, bpy.types.Panel): bl_label = "Advanced Settings" diff --git a/bseq/properties.py b/bseq/properties.py index dd93647..4428845 100644 --- a/bseq/properties.py +++ b/bseq/properties.py @@ -79,6 +79,11 @@ class BSEQ_scene_property(bpy.types.PropertyGroup): description="Auto refresh all sequences every frame", default=False, ) + + preload_next_frame: bpy.props.BoolProperty(name='Preload next frame while rendering', + description="Starts loading the next sequence frame into the RAM while rendering the current frame", + default=True, + ) use_custom_transform: bpy.props.BoolProperty(name='Custom Transform', description="Use a custom transformation matrix when importing", From ad69768d4c29357e2012b9cbed32ab969f953dab Mon Sep 17 00:00:00 2001 From: Dominik Date: Wed, 22 Oct 2025 13:30:25 +0200 Subject: [PATCH 02/19] Started preloading module --- bseq/preloader.py | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) create mode 100644 bseq/preloader.py diff --git a/bseq/preloader.py b/bseq/preloader.py new file mode 100644 index 0000000..1e75f9f --- /dev/null +++ b/bseq/preloader.py @@ -0,0 +1,29 @@ +import concurrent.futures +import bpy + +_executor: concurrent.futures.ThreadPoolExecutor +_init = False + +def init() -> None: + global _executor, _init + _executor = concurrent.futures.ThreadPoolExecutor() + if _executor._initializer_failed(): + print("init failed") + _init = True + print("init") + +def terminate() -> None: + global _init + if not _init: + return + _executor.shutdown(wait=False, cancel_futures=True) + _init = False + print("terminated") + +def is_init() -> bool: + return _init + +def queue_load(scene, depsgraph=None) -> None: + global _executor, _init + if not _init: + raise AttributeError(_executor, "_executor") \ No newline at end of file From e20196cb6fda22ac375e5a30b905c578eca3a2a3 Mon Sep 17 00:00:00 2001 From: Dominik Date: Wed, 22 Oct 2025 13:30:49 +0200 Subject: [PATCH 03/19] Added callback to Preload property --- bseq/callback.py | 10 +++++++++- bseq/properties.py | 3 ++- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/bseq/callback.py b/bseq/callback.py index 079b702..f12c115 100644 --- a/bseq/callback.py +++ b/bseq/callback.py @@ -1,6 +1,8 @@ import bpy import fileseq import traceback +from .preloader import init as preloader_init +from .preloader import terminate as preloader_terminate from .utils import show_message_box @@ -60,4 +62,10 @@ def poll_material(self, material): return not material.is_grease_pencil def poll_edit_obj(self, object): - return object.BSEQ.init \ No newline at end of file + return object.BSEQ.init + +def update_preloader(self, context) -> None: + if self.preload_next_frame: + preloader_init() + else: + preloader_terminate() \ No newline at end of file diff --git a/bseq/properties.py b/bseq/properties.py index 4428845..607c0e3 100644 --- a/bseq/properties.py +++ b/bseq/properties.py @@ -82,7 +82,8 @@ class BSEQ_scene_property(bpy.types.PropertyGroup): preload_next_frame: bpy.props.BoolProperty(name='Preload next frame while rendering', description="Starts loading the next sequence frame into the RAM while rendering the current frame", - default=True, + default=False, + update=update_preloader ) use_custom_transform: bpy.props.BoolProperty(name='Custom Transform', From 2ff7eedd7900ffbca9aada7dff47e873e22c13fc Mon Sep 17 00:00:00 2001 From: Dominik Date: Fri, 24 Oct 2025 13:13:51 +0200 Subject: [PATCH 04/19] implemented preloader prototype --- bseq/__init__.py | 6 ++--- bseq/callback.py | 14 +++++++++--- bseq/importer.py | 2 ++ bseq/load_obj.py | 0 bseq/preloader.py | 58 +++++++++++++++++++++++++++++++++++++++-------- 5 files changed, 65 insertions(+), 15 deletions(-) create mode 100644 bseq/load_obj.py diff --git a/bseq/__init__.py b/bseq/__init__.py index 08a8e37..4430bb4 100644 --- a/bseq/__init__.py +++ b/bseq/__init__.py @@ -3,7 +3,7 @@ from .properties import BSEQ_scene_property, BSEQ_obj_property, BSEQ_mesh_property from .panels import BSEQ_UL_Obj_List, BSEQ_List_Panel, BSEQ_Settings, BSEQ_PT_Import, BSEQ_PT_Import_Child1, BSEQ_PT_Import_Child2, BSEQ_Globals_Panel, BSEQ_Advanced_Panel, BSEQ_Templates, BSEQ_UL_Att_List, draw_template from .messenger import subscribe_to_selected, unsubscribe_to_selected -from .importer import update_obj +from .callback import load_obj from .globals import * import bpy @@ -12,10 +12,10 @@ @persistent def BSEQ_initialize(scene): - if update_obj not in bpy.app.handlers.frame_change_post: + if load_obj not in bpy.app.handlers.frame_change_post: # Insert at the beginning, so that it runs before other frame change handlers. # The other handlers don't need to be in the first position. - bpy.app.handlers.frame_change_post.insert(0, update_obj) + bpy.app.handlers.frame_change_post.insert(0, load_obj) if auto_refresh_active not in bpy.app.handlers.frame_change_post: bpy.app.handlers.frame_change_post.append(auto_refresh_active) if auto_refresh_all not in bpy.app.handlers.frame_change_post: diff --git a/bseq/callback.py b/bseq/callback.py index f12c115..25aab0a 100644 --- a/bseq/callback.py +++ b/bseq/callback.py @@ -1,8 +1,9 @@ import bpy import fileseq import traceback -from .preloader import init as preloader_init -from .preloader import terminate as preloader_terminate +import time +from .preloader import init as preloader_init, terminate as preloader_terminate, queue_load as preloader_queue_load, flush_buffer as preloader_flush_buffer +from .importer import update_obj from .utils import show_message_box @@ -68,4 +69,11 @@ def update_preloader(self, context) -> None: if self.preload_next_frame: preloader_init() else: - preloader_terminate() \ No newline at end of file + preloader_terminate() + +def load_obj(scene, depsgraph=None): + if scene.BSEQ.preload_next_frame: + preloader_flush_buffer(scene, depsgraph) + preloader_queue_load(scene, depsgraph) + return None + update_obj(scene, depsgraph) diff --git a/bseq/importer.py b/bseq/importer.py index 523b6f7..694d351 100644 --- a/bseq/importer.py +++ b/bseq/importer.py @@ -283,6 +283,7 @@ def create_obj(fileseq, use_relative, root_path, transform_matrix=Matrix.Identit bpy.ops.object.select_all(action="DESELECT") bpy.context.view_layer.objects.active = object +# update happens here def update_obj(scene, depsgraph=None): for obj in bpy.data.objects: start_time = time.perf_counter() @@ -360,3 +361,4 @@ def update_obj(scene, depsgraph=None): end_time = time.perf_counter() obj.BSEQ.last_benchmark = (end_time - start_time) * 1000 + print("update_obj(): ", obj.BSEQ.last_benchmark) diff --git a/bseq/load_obj.py b/bseq/load_obj.py new file mode 100644 index 0000000..e69de29 diff --git a/bseq/preloader.py b/bseq/preloader.py index 1e75f9f..6bdd62e 100644 --- a/bseq/preloader.py +++ b/bseq/preloader.py @@ -1,14 +1,17 @@ import concurrent.futures +import time import bpy +from .importer import update_obj +from bpy.app.handlers import persistent _executor: concurrent.futures.ThreadPoolExecutor _init = False +@persistent def init() -> None: global _executor, _init - _executor = concurrent.futures.ThreadPoolExecutor() - if _executor._initializer_failed(): - print("init failed") + _executor = concurrent.futures.ThreadPoolExecutor(max_workers=5) + _executor.submit(pow, 2, 2) _init = True print("init") @@ -19,11 +22,48 @@ def terminate() -> None: _executor.shutdown(wait=False, cancel_futures=True) _init = False print("terminated") - -def is_init() -> bool: - return _init + +class Frame(): + _future: concurrent.futures.Future + _buffer: bpy.types.Scene + _frame: int = -1 + + def flush_buffer(self, scene, depsgraph, *, target_frame: int = -1): + print() + if target_frame == -1 or self._frame + 1 != target_frame: + self._frame = -1 + update_obj(scene, depsgraph) + print("invalidate buffer") + return + print("future done flush?: ", self._future.done()) + concurrent.futures.wait([self._future]) + print("future done wait?: ", self._future.done()) + scene = self._buffer + + def queue_load(self, scene, depsgraph): + start = time.perf_counter() + global _executor, _init + if not _init: + init() + copy_start = time.perf_counter() + self._buffer = scene.copy() + copy_end = time.perf_counter() + self._frame = scene.frame_current + start_queue = time.perf_counter() + end_construct = time.perf_counter() + self._future = _executor.submit(update_obj, self._buffer, depsgraph) + print("future done enqueue?: ", self._future.done()) + end_submit = time.perf_counter() + end = time.perf_counter() + print("queue_load():\n\ttotal: ", (end - start) * 1000, "\n\tcopy: ", (copy_end - copy_start) * 1000, "\n\tqueuing:", (end - start_queue) * 1000) + print("\t\tconstructor: ", (end_construct - start_queue) * 1000, "\n\t\tsubmit: ", (end_submit - end_construct) * 1000) + +_frame = Frame() def queue_load(scene, depsgraph=None) -> None: - global _executor, _init - if not _init: - raise AttributeError(_executor, "_executor") \ No newline at end of file + _frame.queue_load(scene, depsgraph) + return + +def flush_buffer(scene, depsgraph) -> None: + _frame.flush_buffer(scene, depsgraph, target_frame=scene.frame_current) + return \ No newline at end of file From a3c7ffdee6c3756ea11308220f19c2f9213dd748 Mon Sep 17 00:00:00 2001 From: Dominik Date: Fri, 24 Oct 2025 13:33:33 +0200 Subject: [PATCH 05/19] Refactored update_obj() into load_into_ram() and update_scene() functions --- bseq/importer.py | 93 +++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 92 insertions(+), 1 deletion(-) diff --git a/bseq/importer.py b/bseq/importer.py index 694d351..cdf2352 100644 --- a/bseq/importer.py +++ b/bseq/importer.py @@ -10,6 +10,7 @@ import time # this import is not useless from .additional_file_formats import * +from typing import Optional def extract_edges(cell: meshio.CellBlock): if cell.type == "line": @@ -283,10 +284,100 @@ def create_obj(fileseq, use_relative, root_path, transform_matrix=Matrix.Identit bpy.ops.object.select_all(action="DESELECT") bpy.context.view_layer.objects.active = object -# update happens here + +def load_into_ram(obj, scene, depsgraph) -> Optional[meshio.Mesh]: + if obj.BSEQ.init == False: + return None + if obj.BSEQ.enabled == False: + return None + if obj.mode != "OBJECT": + return None + + if depsgraph is not None: + current_frame = obj.evaluated_get(depsgraph).BSEQ.frame + else: + show_message_box("Warning: Might not be able load the correct frame because the dependency graph is not available.", "BSEQ Warning") + current_frame = obj.BSEQ.frame + meshio_mesh = None + + # in case the blender file was created on windows system, but opened in linux system + full_path = get_absolute_path(obj, scene) + + fs = fileseq.FileSequence(full_path) + + if obj.BSEQ.use_advance and obj.BSEQ.script_name: + script = bpy.data.texts[obj.BSEQ.script_name] + try: + exec(script.as_string()) + except Exception as e: + show_message_box(traceback.format_exc(), "running script: " + obj.BSEQ.script_name + " failed: " + str(e), + "ERROR") + return meshio_mesh + + if 'process' in locals(): + user_process = locals()['process'] + try: + user_process(fs, current_frame, obj.data) + obj.BSEQ.current_file = "Controlled by user process" + except Exception as e: + show_message_box("Error when calling user process: " + traceback.format_exc(), icon="ERROR") + del locals()['process'] + # this continue means if process exist, all the remaining code will be ignored, whethere or not error occurs + return meshio_mesh + + elif 'preprocess' in locals(): + user_preprocess = locals()['preprocess'] + try: + meshio_mesh = user_preprocess(fs, current_frame) + obj.BSEQ.current_file = "Controlled by user preprocess" + except Exception as e: + show_message_box("Error when calling user preprocess: " + traceback.format_exc(), icon="ERROR") + # this continue means only if error occures, then goes to next bpy.object + return meshio_mesh + finally: + del locals()['preprocess'] + else: + if obj.BSEQ.match_frames: + fs_frames = fs.frameSet() + if current_frame in fs_frames: + filepath = fs[fs_frames.index(current_frame)] + filepath = os.path.normpath(filepath) + meshio_mesh = load_meshio_from_path(fs, filepath, obj) + else: + meshio_mesh = meshio.Mesh([], []) + else: + filepath = fs[current_frame % len(fs)] + filepath = os.path.normpath(filepath) + meshio_mesh = load_meshio_from_path(fs, filepath, obj) + + if not isinstance(meshio_mesh, meshio.Mesh): + show_message_box('function preprocess does not return meshio object', "ERROR") + return None + + return meshio_mesh + +def update_scene(obj, meshio_mesh, scene, depsgraph): + update_mesh(meshio_mesh, obj.data) + + apply_transformation(meshio_mesh, obj, depsgraph) + + end_time = time.perf_counter() + def update_obj(scene, depsgraph=None): for obj in bpy.data.objects: start_time = time.perf_counter() + + mesh = load_into_ram(obj, scene, depsgraph) + if not isinstance(mesh, meshio.Mesh): + continue + update_scene(obj, mesh, scene, depsgraph) + + end_time = time.perf_counter() + obj.BSEQ.last_benchmark = (end_time - start_time) * 1000 + print("update_obj(): ", obj.BSEQ.last_benchmark) + + continue + start_time = time.perf_counter() if obj.BSEQ.init == False: continue From b39455a10e3b5c80cdeb72c4e692b9d2d6394e80 Mon Sep 17 00:00:00 2001 From: Dominik Date: Wed, 29 Oct 2025 11:33:22 +0100 Subject: [PATCH 06/19] Added threaded diskread --- bseq/importer.py | 12 +++++--- bseq/preloader.py | 76 ++++++++++++++++++++++++++++++++++------------- 2 files changed, 64 insertions(+), 24 deletions(-) diff --git a/bseq/importer.py b/bseq/importer.py index cdf2352..2873c0e 100644 --- a/bseq/importer.py +++ b/bseq/importer.py @@ -167,6 +167,7 @@ def update_mesh(meshio_mesh, mesh): mesh.loops.add(n_loop) mesh.polygons.add(n_poly) + start_time = time.perf_counter() mesh.vertices.foreach_set("co", mesh_vertices.ravel()) mesh.edges.foreach_set("vertices", edges) mesh.loops.foreach_set("vertex_index", loops_vert_idx) @@ -174,6 +175,8 @@ def update_mesh(meshio_mesh, mesh): mesh.polygons.foreach_set("loop_total", faces_loop_total) mesh.polygons.foreach_set("use_smooth", [shade_scheme] * len(faces_loop_total)) + end_time = time.perf_counter() + print("update mesh() took ", (end_time - start_time) * 1000, " ms") # newer function but is about 4 times slower # mesh.clear_geometry() # mesh.from_pydata(mesh_vertices, edge_data, face_data) @@ -194,6 +197,7 @@ def update_mesh(meshio_mesh, mesh): k = "bseq_" + k attribute = create_or_retrieve_attribute(mesh, k, v) if attribute is None: + continue name_string = None if attribute.data_type == "FLOAT": @@ -285,7 +289,7 @@ def create_obj(fileseq, use_relative, root_path, transform_matrix=Matrix.Identit bpy.context.view_layer.objects.active = object -def load_into_ram(obj, scene, depsgraph) -> Optional[meshio.Mesh]: +def load_into_ram(obj, scene, depsgraph, *, target_frame = -1) -> Optional[meshio.Mesh]: if obj.BSEQ.init == False: return None if obj.BSEQ.enabled == False: @@ -293,7 +297,9 @@ def load_into_ram(obj, scene, depsgraph) -> Optional[meshio.Mesh]: if obj.mode != "OBJECT": return None - if depsgraph is not None: + if target_frame != -1: + current_frame = target_frame + elif depsgraph is not None: current_frame = obj.evaluated_get(depsgraph).BSEQ.frame else: show_message_box("Warning: Might not be able load the correct frame because the dependency graph is not available.", "BSEQ Warning") @@ -361,8 +367,6 @@ def update_scene(obj, meshio_mesh, scene, depsgraph): apply_transformation(meshio_mesh, obj, depsgraph) - end_time = time.perf_counter() - def update_obj(scene, depsgraph=None): for obj in bpy.data.objects: start_time = time.perf_counter() diff --git a/bseq/preloader.py b/bseq/preloader.py index 6bdd62e..322880f 100644 --- a/bseq/preloader.py +++ b/bseq/preloader.py @@ -1,7 +1,8 @@ import concurrent.futures import time import bpy -from .importer import update_obj +import meshio +from .importer import load_into_ram, update_scene, update_obj from bpy.app.handlers import persistent _executor: concurrent.futures.ThreadPoolExecutor @@ -15,30 +16,59 @@ def init() -> None: _init = True print("init") -def terminate() -> None: - global _init - if not _init: - return - _executor.shutdown(wait=False, cancel_futures=True) - _init = False - print("terminated") - class Frame(): _future: concurrent.futures.Future - _buffer: bpy.types.Scene + _buffer_objs: dict[str, meshio.Mesh] _frame: int = -1 + def _load_objs(self, scene, depsgraph): + start = time.perf_counter() + self._frame = scene.frame_current + scene.frame_step + self._buffer_objs = {} + for obj in bpy.data.objects: + # TODO: Select next frame (currently seems to load the current frame again) + mesh = load_into_ram(obj, scene, depsgraph, target_frame = self._frame) + if isinstance(mesh, meshio.Mesh): + self._buffer_objs[obj.name_full] = mesh + end = time.perf_counter() + print("load_objs() took ", (end - start) * 1000, " ms") + + def _delete_mesh(self, mesh: meshio.Mesh): + mesh.point_data.clear() + mesh.cell_data.clear() + mesh.point_sets.clear() + mesh.cell_sets.clear() + mesh.cells.clear() + mesh.field_data.clear() + if hasattr(mesh, "attributes"): + mesh.attributes.clear() + + def _clear_buffer(self): + if not hasattr(self, "_buffer_objs") or len(self._buffer_objs) == 0: + print("buffer empty") + return + for name, mesh in self._buffer_objs.items(): + self._delete_mesh(mesh) + self._buffer_objs.clear() + def flush_buffer(self, scene, depsgraph, *, target_frame: int = -1): print() - if target_frame == -1 or self._frame + 1 != target_frame: + if target_frame == -1 or self._frame != target_frame: self._frame = -1 - update_obj(scene, depsgraph) + update_obj(scene, depsgraph) print("invalidate buffer") + self._clear_buffer() return print("future done flush?: ", self._future.done()) concurrent.futures.wait([self._future]) print("future done wait?: ", self._future.done()) - scene = self._buffer + start_time = time.perf_counter() + for obj in bpy.data.objects: + if obj.name_full in self._buffer_objs: + update_scene(obj, self._buffer_objs[obj.name_full], scene, depsgraph) + end_time = time.perf_counter() + print("update_scene() took ", (end_time - start_time) * 1000, " ms") + self._clear_buffer() def queue_load(self, scene, depsgraph): start = time.perf_counter() @@ -48,22 +78,28 @@ def queue_load(self, scene, depsgraph): copy_start = time.perf_counter() self._buffer = scene.copy() copy_end = time.perf_counter() - self._frame = scene.frame_current + self._frame = scene.frame_current + scene.frame_step start_queue = time.perf_counter() - end_construct = time.perf_counter() - self._future = _executor.submit(update_obj, self._buffer, depsgraph) - print("future done enqueue?: ", self._future.done()) + self._future = _executor.submit(self._load_objs, scene, depsgraph) end_submit = time.perf_counter() end = time.perf_counter() print("queue_load():\n\ttotal: ", (end - start) * 1000, "\n\tcopy: ", (copy_end - copy_start) * 1000, "\n\tqueuing:", (end - start_queue) * 1000) - print("\t\tconstructor: ", (end_construct - start_queue) * 1000, "\n\t\tsubmit: ", (end_submit - end_construct) * 1000) _frame = Frame() def queue_load(scene, depsgraph=None) -> None: + start_time = time.perf_counter() _frame.queue_load(scene, depsgraph) - return + print("queue_load() took ", (time.perf_counter() - start_time) * 1000, " ms") def flush_buffer(scene, depsgraph) -> None: _frame.flush_buffer(scene, depsgraph, target_frame=scene.frame_current) - return \ No newline at end of file + +def terminate() -> None: + global _init + if not _init: + return + _executor.shutdown(wait=False, cancel_futures=True) + _init = False + _frame._clear_buffer() + print("terminated") \ No newline at end of file From 83d204b9eafba9f07525599e9656812de686934b Mon Sep 17 00:00:00 2001 From: Dominik Date: Wed, 29 Oct 2025 14:29:24 +0100 Subject: [PATCH 07/19] Added data conversion to preload --- bseq/preloader.py | 34 +++++++++++++++++++++++++--------- 1 file changed, 25 insertions(+), 9 deletions(-) diff --git a/bseq/preloader.py b/bseq/preloader.py index 322880f..f97cfae 100644 --- a/bseq/preloader.py +++ b/bseq/preloader.py @@ -2,7 +2,7 @@ import time import bpy import meshio -from .importer import load_into_ram, update_scene, update_obj +from .importer import load_into_ram, update_scene, update_obj, update_mesh, apply_transformation from bpy.app.handlers import persistent _executor: concurrent.futures.ThreadPoolExecutor @@ -16,26 +16,40 @@ def init() -> None: _init = True print("init") +def _load_data_into_buffer(meshio_mesh, buffer, object: bpy.types.Object): + buffer_data = object.data.copy() + update_mesh(meshio_mesh, buffer_data) + buffer[object.name_full] = buffer_data + class Frame(): _future: concurrent.futures.Future - _buffer_objs: dict[str, meshio.Mesh] + _buffer_meshes: dict[str, meshio.Mesh] + _buffer_data: dict[str, bpy.types.Mesh] _frame: int = -1 + def _load_buffer_to_data(self, object: bpy.types.Object, meshio_mesh, depsgraph): + if object.name_full in self._buffer_data: + object.data = self._buffer_data[object.name_full] + apply_transformation(meshio_mesh, object, depsgraph) + def _load_objs(self, scene, depsgraph): start = time.perf_counter() self._frame = scene.frame_current + scene.frame_step - self._buffer_objs = {} + self._buffer_meshes = {} + self._buffer_data = {} for obj in bpy.data.objects: # TODO: Select next frame (currently seems to load the current frame again) mesh = load_into_ram(obj, scene, depsgraph, target_frame = self._frame) if isinstance(mesh, meshio.Mesh): - self._buffer_objs[obj.name_full] = mesh + self._buffer_meshes[obj.name_full] = mesh + _load_data_into_buffer(mesh, self._buffer_data, obj) end = time.perf_counter() print("load_objs() took ", (end - start) * 1000, " ms") def _delete_mesh(self, mesh: meshio.Mesh): mesh.point_data.clear() mesh.cell_data.clear() + mesh.point_sets.clear() mesh.cell_sets.clear() mesh.cells.clear() @@ -44,12 +58,13 @@ def _delete_mesh(self, mesh: meshio.Mesh): mesh.attributes.clear() def _clear_buffer(self): - if not hasattr(self, "_buffer_objs") or len(self._buffer_objs) == 0: + if not hasattr(self, "_buffer_meshes") or len(self._buffer_meshes) == 0: print("buffer empty") return - for name, mesh in self._buffer_objs.items(): + for name, mesh in self._buffer_meshes.items(): self._delete_mesh(mesh) - self._buffer_objs.clear() + self._buffer_meshes.clear() + self._buffer_data.clear() def flush_buffer(self, scene, depsgraph, *, target_frame: int = -1): print() @@ -64,8 +79,9 @@ def flush_buffer(self, scene, depsgraph, *, target_frame: int = -1): print("future done wait?: ", self._future.done()) start_time = time.perf_counter() for obj in bpy.data.objects: - if obj.name_full in self._buffer_objs: - update_scene(obj, self._buffer_objs[obj.name_full], scene, depsgraph) + if obj.name_full in self._buffer_meshes: + # update_scene(obj, self._buffer_meshes[obj.name_full], scene, depsgraph) + self._load_buffer_to_data(obj, self._buffer_meshes[obj.name_full], depsgraph) end_time = time.perf_counter() print("update_scene() took ", (end_time - start_time) * 1000, " ms") self._clear_buffer() From 260f5c3cf3063fe0814fbc8872016825530cfc39 Mon Sep 17 00:00:00 2001 From: Dominik Date: Thu, 30 Oct 2025 11:26:38 +0100 Subject: [PATCH 08/19] Added loading status to UI --- bseq/panels.py | 6 ++++++ bseq/preloader.py | 8 +++++++- bseq/properties.py | 2 ++ 3 files changed, 15 insertions(+), 1 deletion(-) diff --git a/bseq/panels.py b/bseq/panels.py index 4570be2..e0d55af 100644 --- a/bseq/panels.py +++ b/bseq/panels.py @@ -94,6 +94,12 @@ def draw(self, context): col2.prop(sim_loader, "auto_refresh_all", text="") col1.label(text="Preload Frames") col2.prop(sim_loader, "preload_next_frame", text="") + if sim_loader.preload_next_frame: + col1.label(text="Loading Status") + row2 = col2.row() + row2.enabled = False + row2.prop(sim_loader, "loading_status", text="") + class BSEQ_Advanced_Panel(BSEQ_Panel, bpy.types.Panel): bl_label = "Advanced Settings" diff --git a/bseq/preloader.py b/bseq/preloader.py index f97cfae..4d72e4e 100644 --- a/bseq/preloader.py +++ b/bseq/preloader.py @@ -26,6 +26,7 @@ class Frame(): _buffer_meshes: dict[str, meshio.Mesh] _buffer_data: dict[str, bpy.types.Mesh] _frame: int = -1 + loading_complete: bool = False def _load_buffer_to_data(self, object: bpy.types.Object, meshio_mesh, depsgraph): if object.name_full in self._buffer_data: @@ -37,7 +38,10 @@ def _load_objs(self, scene, depsgraph): self._frame = scene.frame_current + scene.frame_step self._buffer_meshes = {} self._buffer_data = {} + n_loaded = 0 for obj in bpy.data.objects: + scene.BSEQ.loading_status = f"{n_loaded}/{len(bpy.data.objects)}" + n_loaded += 1 # TODO: Select next frame (currently seems to load the current frame again) mesh = load_into_ram(obj, scene, depsgraph, target_frame = self._frame) if isinstance(mesh, meshio.Mesh): @@ -45,6 +49,7 @@ def _load_objs(self, scene, depsgraph): _load_data_into_buffer(mesh, self._buffer_data, obj) end = time.perf_counter() print("load_objs() took ", (end - start) * 1000, " ms") + scene.BSEQ.loading_status = "Complete" def _delete_mesh(self, mesh: meshio.Mesh): mesh.point_data.clear() @@ -93,11 +98,12 @@ def queue_load(self, scene, depsgraph): init() copy_start = time.perf_counter() self._buffer = scene.copy() + copy_end = time.perf_counter() self._frame = scene.frame_current + scene.frame_step start_queue = time.perf_counter() self._future = _executor.submit(self._load_objs, scene, depsgraph) - end_submit = time.perf_counter() + scene.BSEQ.loading_status = "Queued" end = time.perf_counter() print("queue_load():\n\ttotal: ", (end - start) * 1000, "\n\tcopy: ", (copy_end - copy_start) * 1000, "\n\tqueuing:", (end - start_queue) * 1000) diff --git a/bseq/properties.py b/bseq/properties.py index 607c0e3..3fd04d5 100644 --- a/bseq/properties.py +++ b/bseq/properties.py @@ -85,6 +85,8 @@ class BSEQ_scene_property(bpy.types.PropertyGroup): default=False, update=update_preloader ) + + loading_status: bpy.props.StringProperty(default="") use_custom_transform: bpy.props.BoolProperty(name='Custom Transform', description="Use a custom transformation matrix when importing", From 0297047e814a2f14e50ab74c65411be01163d6de Mon Sep 17 00:00:00 2001 From: Dominik Date: Thu, 30 Oct 2025 11:57:53 +0100 Subject: [PATCH 09/19] Added multithreading per object --- bseq/importer.py | 8 ++++---- bseq/preloader.py | 34 +++++++++++++++++++++++----------- 2 files changed, 27 insertions(+), 15 deletions(-) diff --git a/bseq/importer.py b/bseq/importer.py index 2873c0e..cb337b4 100644 --- a/bseq/importer.py +++ b/bseq/importer.py @@ -120,6 +120,7 @@ def create_or_retrieve_attribute(mesh, k, v): return mesh.attributes[k] def update_mesh(meshio_mesh, mesh): + start_time = time.perf_counter() # extract information from the meshio mesh mesh_vertices = meshio_mesh.points @@ -167,16 +168,12 @@ def update_mesh(meshio_mesh, mesh): mesh.loops.add(n_loop) mesh.polygons.add(n_poly) - start_time = time.perf_counter() mesh.vertices.foreach_set("co", mesh_vertices.ravel()) mesh.edges.foreach_set("vertices", edges) mesh.loops.foreach_set("vertex_index", loops_vert_idx) mesh.polygons.foreach_set("loop_start", faces_loop_start) mesh.polygons.foreach_set("loop_total", faces_loop_total) mesh.polygons.foreach_set("use_smooth", [shade_scheme] * len(faces_loop_total)) - - end_time = time.perf_counter() - print("update mesh() took ", (end_time - start_time) * 1000, " ms") # newer function but is about 4 times slower # mesh.clear_geometry() # mesh.from_pydata(mesh_vertices, edge_data, face_data) @@ -228,6 +225,9 @@ def update_mesh(meshio_mesh, mesh): indices = [item for sublist in meshio_mesh.cell_data["obj:vn_face_idx"][0] for item in sublist] mesh.normals_split_custom_set([meshio_mesh.field_data["obj:vn"][i - 1] for i in indices]) + end_time = time.perf_counter() + print("update mesh() took ", (end_time - start_time) * 1000, " ms") + # function to create a single meshio object (not a sequence, this just inports some file using meshio) def create_meshio_obj(filepath): meshio_mesh = None diff --git a/bseq/preloader.py b/bseq/preloader.py index 4d72e4e..e8b61d4 100644 --- a/bseq/preloader.py +++ b/bseq/preloader.py @@ -23,6 +23,7 @@ def _load_data_into_buffer(meshio_mesh, buffer, object: bpy.types.Object): class Frame(): _future: concurrent.futures.Future + _loading_threads: list[concurrent.futures.Future] _buffer_meshes: dict[str, meshio.Mesh] _buffer_data: dict[str, bpy.types.Mesh] _frame: int = -1 @@ -33,23 +34,33 @@ def _load_buffer_to_data(self, object: bpy.types.Object, meshio_mesh, depsgraph) object.data = self._buffer_data[object.name_full] apply_transformation(meshio_mesh, object, depsgraph) + def _obj_load(self, obj, scene, depsgraph): + start_time = time.perf_counter() + mesh = load_into_ram(obj, scene, depsgraph, target_frame=self._frame) + if isinstance(mesh, meshio.Mesh): + self._buffer_meshes[obj.name_full] = mesh + _load_data_into_buffer(mesh, self._buffer_data, obj) + end_time = time.perf_counter() + obj.BSEQ.last_benchmark = (end_time - start_time) * 1000 + def _load_objs(self, scene, depsgraph): start = time.perf_counter() self._frame = scene.frame_current + scene.frame_step self._buffer_meshes = {} self._buffer_data = {} + self._loading_threads = [] n_loaded = 0 for obj in bpy.data.objects: - scene.BSEQ.loading_status = f"{n_loaded}/{len(bpy.data.objects)}" + future = _executor.submit(self._obj_load, obj, scene, depsgraph) + self._loading_threads.append(future) + for future in concurrent.futures.as_completed(self._loading_threads): n_loaded += 1 - # TODO: Select next frame (currently seems to load the current frame again) - mesh = load_into_ram(obj, scene, depsgraph, target_frame = self._frame) - if isinstance(mesh, meshio.Mesh): - self._buffer_meshes[obj.name_full] = mesh - _load_data_into_buffer(mesh, self._buffer_data, obj) + scene.BSEQ.loading_status = f"{n_loaded}/{len(bpy.data.objects)}" + concurrent.futures.wait(self._loading_threads) + scene.BSEQ.loading_status = "Complete" + self._loading_threads.clear() end = time.perf_counter() print("load_objs() took ", (end - start) * 1000, " ms") - scene.BSEQ.loading_status = "Complete" def _delete_mesh(self, mesh: meshio.Mesh): mesh.point_data.clear() @@ -59,6 +70,7 @@ def _delete_mesh(self, mesh: meshio.Mesh): mesh.cell_sets.clear() mesh.cells.clear() mesh.field_data.clear() + del mesh.points if hasattr(mesh, "attributes"): mesh.attributes.clear() @@ -68,10 +80,13 @@ def _clear_buffer(self): return for name, mesh in self._buffer_meshes.items(): self._delete_mesh(mesh) + for _, obj in self._buffer_data.items(): + bpy.data.meshes.remove(obj, do_unlink=False) self._buffer_meshes.clear() self._buffer_data.clear() def flush_buffer(self, scene, depsgraph, *, target_frame: int = -1): + start_time = time.perf_counter() print() if target_frame == -1 or self._frame != target_frame: self._frame = -1 @@ -79,17 +94,14 @@ def flush_buffer(self, scene, depsgraph, *, target_frame: int = -1): print("invalidate buffer") self._clear_buffer() return - print("future done flush?: ", self._future.done()) concurrent.futures.wait([self._future]) - print("future done wait?: ", self._future.done()) - start_time = time.perf_counter() for obj in bpy.data.objects: if obj.name_full in self._buffer_meshes: # update_scene(obj, self._buffer_meshes[obj.name_full], scene, depsgraph) self._load_buffer_to_data(obj, self._buffer_meshes[obj.name_full], depsgraph) + self._clear_buffer() end_time = time.perf_counter() print("update_scene() took ", (end_time - start_time) * 1000, " ms") - self._clear_buffer() def queue_load(self, scene, depsgraph): start = time.perf_counter() From ea21beffa94e38bb90bfceeaa589f63fe561dbb1 Mon Sep 17 00:00:00 2001 From: Dominik Date: Wed, 5 Nov 2025 11:05:19 +0100 Subject: [PATCH 10/19] moved _load_data_into_buffer to class method --- bseq/preloader.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/bseq/preloader.py b/bseq/preloader.py index e8b61d4..840b179 100644 --- a/bseq/preloader.py +++ b/bseq/preloader.py @@ -16,11 +16,6 @@ def init() -> None: _init = True print("init") -def _load_data_into_buffer(meshio_mesh, buffer, object: bpy.types.Object): - buffer_data = object.data.copy() - update_mesh(meshio_mesh, buffer_data) - buffer[object.name_full] = buffer_data - class Frame(): _future: concurrent.futures.Future _loading_threads: list[concurrent.futures.Future] @@ -29,6 +24,11 @@ class Frame(): _frame: int = -1 loading_complete: bool = False + def _load_data_into_buffer(self, meshio_mesh, object: bpy.types.Object): + buffer_data = object.data.copy() + update_mesh(meshio_mesh, buffer_data) + self._buffer_data[object.name_full] = buffer_data + def _load_buffer_to_data(self, object: bpy.types.Object, meshio_mesh, depsgraph): if object.name_full in self._buffer_data: object.data = self._buffer_data[object.name_full] @@ -39,7 +39,7 @@ def _obj_load(self, obj, scene, depsgraph): mesh = load_into_ram(obj, scene, depsgraph, target_frame=self._frame) if isinstance(mesh, meshio.Mesh): self._buffer_meshes[obj.name_full] = mesh - _load_data_into_buffer(mesh, self._buffer_data, obj) + self._load_data_into_buffer(mesh, obj) end_time = time.perf_counter() obj.BSEQ.last_benchmark = (end_time - start_time) * 1000 From f81d36b6930653cd6414bfce023efcb9d14b66eb Mon Sep 17 00:00:00 2001 From: Dominik Date: Wed, 5 Nov 2025 11:05:47 +0100 Subject: [PATCH 11/19] Reduced memory leak by explicitly removing old object.data DataBlock --- bseq/preloader.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/bseq/preloader.py b/bseq/preloader.py index 840b179..0c88726 100644 --- a/bseq/preloader.py +++ b/bseq/preloader.py @@ -31,7 +31,9 @@ def _load_data_into_buffer(self, meshio_mesh, object: bpy.types.Object): def _load_buffer_to_data(self, object: bpy.types.Object, meshio_mesh, depsgraph): if object.name_full in self._buffer_data: + old_mesh = object.data object.data = self._buffer_data[object.name_full] + bpy.data.meshes.remove(old_mesh, do_unlink=False) apply_transformation(meshio_mesh, object, depsgraph) def _obj_load(self, obj, scene, depsgraph): From 51e443cfb0f0bf55ef13d973e56592b1120380f4 Mon Sep 17 00:00:00 2001 From: Dominik Date: Thu, 6 Nov 2025 09:51:12 +0100 Subject: [PATCH 12/19] Added documentation --- bseq/preloader.py | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/bseq/preloader.py b/bseq/preloader.py index 0c88726..a50f726 100644 --- a/bseq/preloader.py +++ b/bseq/preloader.py @@ -8,6 +8,7 @@ _executor: concurrent.futures.ThreadPoolExecutor _init = False +# This needs to be persistent to keep the executor @persistent def init() -> None: global _executor, _init @@ -25,18 +26,22 @@ class Frame(): loading_complete: bool = False def _load_data_into_buffer(self, meshio_mesh, object: bpy.types.Object): + """ Applies the meshio data to a copy of the object mesh """ buffer_data = object.data.copy() update_mesh(meshio_mesh, buffer_data) self._buffer_data[object.name_full] = buffer_data def _load_buffer_to_data(self, object: bpy.types.Object, meshio_mesh, depsgraph): + """ Swaps the object mesh with the buffered mesh """ if object.name_full in self._buffer_data: old_mesh = object.data object.data = self._buffer_data[object.name_full] + # We remove the old mesh data to prevent memory leaks bpy.data.meshes.remove(old_mesh, do_unlink=False) apply_transformation(meshio_mesh, object, depsgraph) def _obj_load(self, obj, scene, depsgraph): + """ Buffering Obj Job for the executor """ start_time = time.perf_counter() mesh = load_into_ram(obj, scene, depsgraph, target_frame=self._frame) if isinstance(mesh, meshio.Mesh): @@ -46,6 +51,9 @@ def _obj_load(self, obj, scene, depsgraph): obj.BSEQ.last_benchmark = (end_time - start_time) * 1000 def _load_objs(self, scene, depsgraph): + """ Job which submits all object buffering jobs + Also updates the buffering status in the UI + """ start = time.perf_counter() self._frame = scene.frame_current + scene.frame_step self._buffer_meshes = {} @@ -76,6 +84,8 @@ def _delete_mesh(self, mesh: meshio.Mesh): if hasattr(mesh, "attributes"): mesh.attributes.clear() + # Needs testing if really needed + # Meshio might work with garbage collection def _clear_buffer(self): if not hasattr(self, "_buffer_meshes") or len(self._buffer_meshes) == 0: print("buffer empty") @@ -88,14 +98,26 @@ def _clear_buffer(self): self._buffer_data.clear() def flush_buffer(self, scene, depsgraph, *, target_frame: int = -1): + """ Applies the buffer to the scene and clears the buffer afterwards + + target_frame -- indicates the current frame which is to be loaded, + '-1' indicates do not use buffered frame + + If target_frame does not coincide with the buffered frame, the buffer will be + invalidated and the scene is loaded without pre-buffering + + """ start_time = time.perf_counter() print() + # if no target_frame is specified or if the target_frame does not coincide + # with the buffered frame, invalidate the buffer and update the scene serially if target_frame == -1 or self._frame != target_frame: self._frame = -1 update_obj(scene, depsgraph) print("invalidate buffer") self._clear_buffer() return + # Barrier to wait until loading is actually completed concurrent.futures.wait([self._future]) for obj in bpy.data.objects: if obj.name_full in self._buffer_meshes: @@ -106,6 +128,10 @@ def flush_buffer(self, scene, depsgraph, *, target_frame: int = -1): print("update_scene() took ", (end_time - start_time) * 1000, " ms") def queue_load(self, scene, depsgraph): + """ Queues the next frame which is determined by the current frame and the set frame step + + Also initialises the executor if not initialized already + """ start = time.perf_counter() global _executor, _init if not _init: From 8b44bbb1902ca1b22954e5d72b9263fd03147382 Mon Sep 17 00:00:00 2001 From: Dominik Date: Thu, 6 Nov 2025 09:51:28 +0100 Subject: [PATCH 13/19] Removed unused buffer --- bseq/preloader.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/bseq/preloader.py b/bseq/preloader.py index a50f726..367d9ae 100644 --- a/bseq/preloader.py +++ b/bseq/preloader.py @@ -136,10 +136,7 @@ def queue_load(self, scene, depsgraph): global _executor, _init if not _init: init() - copy_start = time.perf_counter() - self._buffer = scene.copy() - - copy_end = time.perf_counter() + self._frame = scene.frame_current + scene.frame_step start_queue = time.perf_counter() self._future = _executor.submit(self._load_objs, scene, depsgraph) From 56a8fc6da12453ee0e4ca6f4fd1b0b90ab06c9c5 Mon Sep 17 00:00:00 2001 From: Dominik Date: Thu, 6 Nov 2025 15:38:31 +0100 Subject: [PATCH 14/19] Removed deletemesh() since meshio meshes seem to be garbage collected --- bseq/preloader.py | 15 --------------- 1 file changed, 15 deletions(-) diff --git a/bseq/preloader.py b/bseq/preloader.py index 367d9ae..21e1194 100644 --- a/bseq/preloader.py +++ b/bseq/preloader.py @@ -72,17 +72,6 @@ def _load_objs(self, scene, depsgraph): end = time.perf_counter() print("load_objs() took ", (end - start) * 1000, " ms") - def _delete_mesh(self, mesh: meshio.Mesh): - mesh.point_data.clear() - mesh.cell_data.clear() - - mesh.point_sets.clear() - mesh.cell_sets.clear() - mesh.cells.clear() - mesh.field_data.clear() - del mesh.points - if hasattr(mesh, "attributes"): - mesh.attributes.clear() # Needs testing if really needed # Meshio might work with garbage collection @@ -90,10 +79,6 @@ def _clear_buffer(self): if not hasattr(self, "_buffer_meshes") or len(self._buffer_meshes) == 0: print("buffer empty") return - for name, mesh in self._buffer_meshes.items(): - self._delete_mesh(mesh) - for _, obj in self._buffer_data.items(): - bpy.data.meshes.remove(obj, do_unlink=False) self._buffer_meshes.clear() self._buffer_data.clear() From f44038484d13e1d7c65a719341b73a029e411a8a Mon Sep 17 00:00:00 2001 From: Dominik Date: Thu, 6 Nov 2025 15:40:05 +0100 Subject: [PATCH 15/19] Moved update_mesh() profiling code to preloader --- bseq/importer.py | 4 ---- bseq/preloader.py | 5 +++++ 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/bseq/importer.py b/bseq/importer.py index cb337b4..bb5f4e8 100644 --- a/bseq/importer.py +++ b/bseq/importer.py @@ -120,7 +120,6 @@ def create_or_retrieve_attribute(mesh, k, v): return mesh.attributes[k] def update_mesh(meshio_mesh, mesh): - start_time = time.perf_counter() # extract information from the meshio mesh mesh_vertices = meshio_mesh.points @@ -225,9 +224,6 @@ def update_mesh(meshio_mesh, mesh): indices = [item for sublist in meshio_mesh.cell_data["obj:vn_face_idx"][0] for item in sublist] mesh.normals_split_custom_set([meshio_mesh.field_data["obj:vn"][i - 1] for i in indices]) - end_time = time.perf_counter() - print("update mesh() took ", (end_time - start_time) * 1000, " ms") - # function to create a single meshio object (not a sequence, this just inports some file using meshio) def create_meshio_obj(filepath): meshio_mesh = None diff --git a/bseq/preloader.py b/bseq/preloader.py index 21e1194..abec0a5 100644 --- a/bseq/preloader.py +++ b/bseq/preloader.py @@ -27,10 +27,15 @@ class Frame(): def _load_data_into_buffer(self, meshio_mesh, object: bpy.types.Object): """ Applies the meshio data to a copy of the object mesh """ + start_time = time.perf_counter() + buffer_data = object.data.copy() update_mesh(meshio_mesh, buffer_data) self._buffer_data[object.name_full] = buffer_data + end_time = time.perf_counter() + print("update mesh() took ", (end_time - start_time) * 1000, " ms") + def _load_buffer_to_data(self, object: bpy.types.Object, meshio_mesh, depsgraph): """ Swaps the object mesh with the buffered mesh """ if object.name_full in self._buffer_data: From 67dc20262bacbb121f5739612fc48e7c332633bc Mon Sep 17 00:00:00 2001 From: Dominik Date: Thu, 6 Nov 2025 15:43:33 +0100 Subject: [PATCH 16/19] Renamed Preloader to Framebuffer --- bseq/callback.py | 18 +++++++++--------- bseq/{preloader.py => frame_buffer.py} | 18 +++++++----------- bseq/panels.py | 6 +++--- bseq/properties.py | 4 ++-- 4 files changed, 21 insertions(+), 25 deletions(-) rename bseq/{preloader.py => frame_buffer.py} (89%) diff --git a/bseq/callback.py b/bseq/callback.py index 25aab0a..9a7b3d0 100644 --- a/bseq/callback.py +++ b/bseq/callback.py @@ -2,7 +2,7 @@ import fileseq import traceback import time -from .preloader import init as preloader_init, terminate as preloader_terminate, queue_load as preloader_queue_load, flush_buffer as preloader_flush_buffer +from .frame_buffer import init as framebuffer_init, terminate as framebuffer_terminate, queue_load as framebuffer_queue_load, flush_buffer as framebuffer_flush_buffer from .importer import update_obj from .utils import show_message_box @@ -65,15 +65,15 @@ def poll_material(self, material): def poll_edit_obj(self, object): return object.BSEQ.init -def update_preloader(self, context) -> None: - if self.preload_next_frame: - preloader_init() +def update_framebuffer(self, context) -> None: + if self.buffer_next_frame: + framebuffer_init() else: - preloader_terminate() + framebuffer_terminate() def load_obj(scene, depsgraph=None): - if scene.BSEQ.preload_next_frame: - preloader_flush_buffer(scene, depsgraph) - preloader_queue_load(scene, depsgraph) - return None + if scene.BSEQ.buffer_next_frame: + framebuffer_flush_buffer(scene, depsgraph) + framebuffer_queue_load(scene, depsgraph) + return update_obj(scene, depsgraph) diff --git a/bseq/preloader.py b/bseq/frame_buffer.py similarity index 89% rename from bseq/preloader.py rename to bseq/frame_buffer.py index abec0a5..df2fab8 100644 --- a/bseq/preloader.py +++ b/bseq/frame_buffer.py @@ -2,7 +2,7 @@ import time import bpy import meshio -from .importer import load_into_ram, update_scene, update_obj, update_mesh, apply_transformation +from .importer import load_into_ram, update_obj, update_mesh, apply_transformation from bpy.app.handlers import persistent _executor: concurrent.futures.ThreadPoolExecutor @@ -75,11 +75,8 @@ def _load_objs(self, scene, depsgraph): scene.BSEQ.loading_status = "Complete" self._loading_threads.clear() end = time.perf_counter() - print("load_objs() took ", (end - start) * 1000, " ms") + #print("load_objs() took ", (end - start) * 1000, " ms") - - # Needs testing if really needed - # Meshio might work with garbage collection def _clear_buffer(self): if not hasattr(self, "_buffer_meshes") or len(self._buffer_meshes) == 0: print("buffer empty") @@ -98,7 +95,7 @@ def flush_buffer(self, scene, depsgraph, *, target_frame: int = -1): """ start_time = time.perf_counter() - print() + #print() # if no target_frame is specified or if the target_frame does not coincide # with the buffered frame, invalidate the buffer and update the scene serially if target_frame == -1 or self._frame != target_frame: @@ -111,12 +108,11 @@ def flush_buffer(self, scene, depsgraph, *, target_frame: int = -1): concurrent.futures.wait([self._future]) for obj in bpy.data.objects: if obj.name_full in self._buffer_meshes: - # update_scene(obj, self._buffer_meshes[obj.name_full], scene, depsgraph) self._load_buffer_to_data(obj, self._buffer_meshes[obj.name_full], depsgraph) self._clear_buffer() end_time = time.perf_counter() - print("update_scene() took ", (end_time - start_time) * 1000, " ms") - + #print("update_scene() took ", (end_time - start_time) * 1000, " ms") + def queue_load(self, scene, depsgraph): """ Queues the next frame which is determined by the current frame and the set frame step @@ -132,14 +128,14 @@ def queue_load(self, scene, depsgraph): self._future = _executor.submit(self._load_objs, scene, depsgraph) scene.BSEQ.loading_status = "Queued" end = time.perf_counter() - print("queue_load():\n\ttotal: ", (end - start) * 1000, "\n\tcopy: ", (copy_end - copy_start) * 1000, "\n\tqueuing:", (end - start_queue) * 1000) + #print("queue_load():\n\ttotal: ", (end - start) * 1000, "\n\tcopy: ", (copy_end - copy_start) * 1000, "\n\tqueuing:", (end - start_queue) * 1000) _frame = Frame() def queue_load(scene, depsgraph=None) -> None: start_time = time.perf_counter() _frame.queue_load(scene, depsgraph) - print("queue_load() took ", (time.perf_counter() - start_time) * 1000, " ms") + #print("queue_load() took ", (time.perf_counter() - start_time) * 1000, " ms") def flush_buffer(scene, depsgraph) -> None: _frame.flush_buffer(scene, depsgraph, target_frame=scene.frame_current) diff --git a/bseq/panels.py b/bseq/panels.py index e0d55af..5be9133 100644 --- a/bseq/panels.py +++ b/bseq/panels.py @@ -92,9 +92,9 @@ def draw(self, context): col2.prop(sim_loader, "auto_refresh_active", text="") col1.label(text="Auto Refresh All") col2.prop(sim_loader, "auto_refresh_all", text="") - col1.label(text="Preload Frames") - col2.prop(sim_loader, "preload_next_frame", text="") - if sim_loader.preload_next_frame: + col1.label(text="Buffer Next Frame") + col2.prop(sim_loader, "buffer_next_frame", text="") + if sim_loader.buffer_next_frame: col1.label(text="Loading Status") row2 = col2.row() row2.enabled = False diff --git a/bseq/properties.py b/bseq/properties.py index 3fd04d5..82fed5a 100644 --- a/bseq/properties.py +++ b/bseq/properties.py @@ -80,10 +80,10 @@ class BSEQ_scene_property(bpy.types.PropertyGroup): default=False, ) - preload_next_frame: bpy.props.BoolProperty(name='Preload next frame while rendering', + buffer_next_frame: bpy.props.BoolProperty(name='Buffer next frame while rendering', description="Starts loading the next sequence frame into the RAM while rendering the current frame", default=False, - update=update_preloader + update=update_framebuffer ) loading_status: bpy.props.StringProperty(default="") From 19d0a3c6c49feffe270dd2b1982dbcc5dac0f7d5 Mon Sep 17 00:00:00 2001 From: Dominik Date: Thu, 6 Nov 2025 15:44:49 +0100 Subject: [PATCH 17/19] Removed old unused importer code --- bseq/importer.py | 80 +----------------------------------------------- 1 file changed, 1 insertion(+), 79 deletions(-) diff --git a/bseq/importer.py b/bseq/importer.py index bb5f4e8..cfa460a 100644 --- a/bseq/importer.py +++ b/bseq/importer.py @@ -374,82 +374,4 @@ def update_obj(scene, depsgraph=None): end_time = time.perf_counter() obj.BSEQ.last_benchmark = (end_time - start_time) * 1000 - print("update_obj(): ", obj.BSEQ.last_benchmark) - - continue - start_time = time.perf_counter() - - if obj.BSEQ.init == False: - continue - if obj.BSEQ.enabled == False: - continue - if obj.mode != "OBJECT": - continue - - if depsgraph is not None: - current_frame = obj.evaluated_get(depsgraph).BSEQ.frame - else: - show_message_box("Warning: Might not be able load the correct frame because the dependency graph is not available.", "BSEQ Warning") - current_frame = obj.BSEQ.frame - meshio_mesh = None - - # in case the blender file was created on windows system, but opened in linux system - full_path = get_absolute_path(obj, scene) - - fs = fileseq.FileSequence(full_path) - - if obj.BSEQ.use_advance and obj.BSEQ.script_name: - script = bpy.data.texts[obj.BSEQ.script_name] - try: - exec(script.as_string()) - except Exception as e: - show_message_box(traceback.format_exc(), "running script: " + obj.BSEQ.script_name + " failed: " + str(e), - "ERROR") - continue - - if 'process' in locals(): - user_process = locals()['process'] - try: - user_process(fs, current_frame, obj.data) - obj.BSEQ.current_file = "Controlled by user process" - except Exception as e: - show_message_box("Error when calling user process: " + traceback.format_exc(), icon="ERROR") - del locals()['process'] - # this continue means if process exist, all the remaining code will be ignored, whethere or not error occurs - continue - - elif 'preprocess' in locals(): - user_preprocess = locals()['preprocess'] - try: - meshio_mesh = user_preprocess(fs, current_frame) - obj.BSEQ.current_file = "Controlled by user preprocess" - except Exception as e: - show_message_box("Error when calling user preprocess: " + traceback.format_exc(), icon="ERROR") - # this continue means only if error occures, then goes to next bpy.object - continue - finally: - del locals()['preprocess'] - else: - if obj.BSEQ.match_frames: - fs_frames = fs.frameSet() - if current_frame in fs_frames: - filepath = fs[fs_frames.index(current_frame)] - filepath = os.path.normpath(filepath) - meshio_mesh = load_meshio_from_path(fs, filepath, obj) - else: - meshio_mesh = meshio.Mesh([], []) - else: - filepath = fs[current_frame % len(fs)] - filepath = os.path.normpath(filepath) - meshio_mesh = load_meshio_from_path(fs, filepath, obj) - - if not isinstance(meshio_mesh, meshio.Mesh): - show_message_box('function preprocess does not return meshio object', "ERROR") - continue - update_mesh(meshio_mesh, obj.data) - - apply_transformation(meshio_mesh, obj, depsgraph) - - end_time = time.perf_counter() - obj.BSEQ.last_benchmark = (end_time - start_time) * 1000 - print("update_obj(): ", obj.BSEQ.last_benchmark) + # print("update_obj(): ", obj.BSEQ.last_benchmark) \ No newline at end of file From d58d0a3a7982a23b215d627b49f59d1754d35103 Mon Sep 17 00:00:00 2001 From: Dominik Date: Thu, 13 Nov 2025 09:41:48 +0100 Subject: [PATCH 18/19] Fixed exceptions that may occur while rendering due to write block of data fields --- bseq/frame_buffer.py | 22 ++++++++++++++++++++-- bseq/utils.py | 7 ++++++- 2 files changed, 26 insertions(+), 3 deletions(-) diff --git a/bseq/frame_buffer.py b/bseq/frame_buffer.py index df2fab8..31e3ca4 100644 --- a/bseq/frame_buffer.py +++ b/bseq/frame_buffer.py @@ -22,9 +22,16 @@ class Frame(): _loading_threads: list[concurrent.futures.Future] _buffer_meshes: dict[str, meshio.Mesh] _buffer_data: dict[str, bpy.types.Mesh] + _buffer_timings: dict[str, float] _frame: int = -1 loading_complete: bool = False + def __init__(self): + self._buffer_meshes = {} + self._buffer_data = {} + self._buffer_timings = {} + self._loading_threads = [] + def _load_data_into_buffer(self, meshio_mesh, object: bpy.types.Object): """ Applies the meshio data to a copy of the object mesh """ start_time = time.perf_counter() @@ -53,7 +60,8 @@ def _obj_load(self, obj, scene, depsgraph): self._buffer_meshes[obj.name_full] = mesh self._load_data_into_buffer(mesh, obj) end_time = time.perf_counter() - obj.BSEQ.last_benchmark = (end_time - start_time) * 1000 + # move to main threaded + self._buffer_timings[obj.name_full] = (end_time - start_time) * 1000 def _load_objs(self, scene, depsgraph): """ Job which submits all object buffering jobs @@ -63,6 +71,7 @@ def _load_objs(self, scene, depsgraph): self._frame = scene.frame_current + scene.frame_step self._buffer_meshes = {} self._buffer_data = {} + self._buffer_timings = {} self._loading_threads = [] n_loaded = 0 for obj in bpy.data.objects: @@ -70,7 +79,13 @@ def _load_objs(self, scene, depsgraph): self._loading_threads.append(future) for future in concurrent.futures.as_completed(self._loading_threads): n_loaded += 1 - scene.BSEQ.loading_status = f"{n_loaded}/{len(bpy.data.objects)}" + # Due to multithreading, Blender may forbid writing to the loading status while rendering + # In this case, we just skip the update + try: + scene.BSEQ.loading_status = f"{n_loaded}/{len(bpy.data.objects)}" + except Exception as e: + print("Skipped updating loading status") + concurrent.futures.wait(self._loading_threads) scene.BSEQ.loading_status = "Complete" self._loading_threads.clear() @@ -83,6 +98,7 @@ def _clear_buffer(self): return self._buffer_meshes.clear() self._buffer_data.clear() + self._buffer_timings.clear() def flush_buffer(self, scene, depsgraph, *, target_frame: int = -1): """ Applies the buffer to the scene and clears the buffer afterwards @@ -109,6 +125,8 @@ def flush_buffer(self, scene, depsgraph, *, target_frame: int = -1): for obj in bpy.data.objects: if obj.name_full in self._buffer_meshes: self._load_buffer_to_data(obj, self._buffer_meshes[obj.name_full], depsgraph) + obj.BSEQ.last_benchmark = self._buffer_timings[obj.name_full] + self._clear_buffer() end_time = time.perf_counter() #print("update_scene() took ", (end_time - start_time) * 1000, " ms") diff --git a/bseq/utils.py b/bseq/utils.py index 161542c..f4273eb 100644 --- a/bseq/utils.py +++ b/bseq/utils.py @@ -67,7 +67,12 @@ def load_meshio_from_path(fileseq, filepath, obj = None): try: meshio_mesh = meshio.read(filepath) if obj is not None: - obj.BSEQ.current_file = filepath + # While multithreading, this may raise an exception during rendering + # Since current_file seems not to be used elsewhere, we skip it when an exception occurs + try: + obj.BSEQ.current_file = filepath + except Exception as e: + print("Current file not updated") except Exception as e: show_message_box("Error when reading: " + filepath + ",\n" + traceback.format_exc(), "Meshio Loading Error" + str(e), From 42b172a53d1d295f9726219b543b9eb36d733ba7 Mon Sep 17 00:00:00 2001 From: Dominik Date: Thu, 13 Nov 2025 09:42:24 +0100 Subject: [PATCH 19/19] Mesh data now gets removed on buffer invalidation and on buffer termination --- bseq/frame_buffer.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/bseq/frame_buffer.py b/bseq/frame_buffer.py index 31e3ca4..e117437 100644 --- a/bseq/frame_buffer.py +++ b/bseq/frame_buffer.py @@ -118,6 +118,8 @@ def flush_buffer(self, scene, depsgraph, *, target_frame: int = -1): self._frame = -1 update_obj(scene, depsgraph) print("invalidate buffer") + for _, obj in self._buffer_data.items(): + bpy.data.meshes.remove(obj, do_unlink=False) self._clear_buffer() return # Barrier to wait until loading is actually completed @@ -164,5 +166,7 @@ def terminate() -> None: return _executor.shutdown(wait=False, cancel_futures=True) _init = False + for _, obj in _frame._buffer_data.items(): + bpy.data.meshes.remove(obj, do_unlink=False) _frame._clear_buffer() print("terminated") \ No newline at end of file