From f0e5109c4244713f3914d699548e106f6bcdea30 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20R=C3=B6hl?= Date: Sun, 16 Jun 2024 18:39:16 +0200 Subject: [PATCH 01/19] Fix typo in comment --- bottles/frontend/views/bottle_versioning.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bottles/frontend/views/bottle_versioning.py b/bottles/frontend/views/bottle_versioning.py index 4c2539a176..d364dacfd3 100644 --- a/bottles/frontend/views/bottle_versioning.py +++ b/bottles/frontend/views/bottle_versioning.py @@ -135,8 +135,8 @@ def check_entry_state_message(self, *_args): def add_state(self, widget): """ - This function create ask the versioning manager to - create a new bottle state with the given message. + This function asks the versioning manager to create a new bottle state + with the given message. """ if not self.btn_save.get_sensitive(): return From ca37427933279c522d4808fe18104f93cd7f6254 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20R=C3=B6hl?= Date: Sun, 30 Jun 2024 17:47:57 +0200 Subject: [PATCH 02/19] Refactor: simplify if-else Get rid of the else branch by moving common code out of if-else. --- bottles/backend/managers/manager.py | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/bottles/backend/managers/manager.py b/bottles/backend/managers/manager.py index 5f99fb6782..3b9b925274 100644 --- a/bottles/backend/managers/manager.py +++ b/bottles/backend/managers/manager.py @@ -1060,18 +1060,14 @@ def create_bottle_from_config(self, config: BottleConfig) -> bool: # create the bottle path bottle_path = os.path.join(Paths.bottles, config.Name) - if not os.path.exists(bottle_path): - """ - If the bottle does not exist, create it, else - append a random number to the name. - """ - os.makedirs(bottle_path) - else: + # If the bottle exists append a random number to the name. + if os.path.exists(bottle_path): rnd = random.randint(100, 200) bottle_path = f"{bottle_path}__{rnd}" config.Name = f"{config.Name}__{rnd}" config.Path = f"{config.Path}__{rnd}" - os.makedirs(bottle_path) + + os.makedirs(bottle_path) # Pre-create drive_c directory and set the case-fold flag bottle_drive_c = os.path.join(bottle_path, "drive_c") From a1dd2a5428e781c790a8a4ca1f618cb820c78751 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20R=C3=B6hl?= Date: Sun, 30 Jun 2024 18:09:31 +0200 Subject: [PATCH 03/19] Refactor: define 'reg_files' next to use sites --- bottles/backend/managers/manager.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/bottles/backend/managers/manager.py b/bottles/backend/managers/manager.py index 3b9b925274..9d0e674734 100644 --- a/bottles/backend/managers/manager.py +++ b/bottles/backend/managers/manager.py @@ -1228,12 +1228,6 @@ def components_check(): bottle_name_path = f"{bottle_name_path}__{rnd}" bottle_complete_path = f"{bottle_complete_path}__{rnd}" - # define registers that should be awaited - reg_files = [ - os.path.join(bottle_complete_path, "system.reg"), - os.path.join(bottle_complete_path, "user.reg"), - ] - # create the bottle directory try: os.makedirs(bottle_complete_path) @@ -1349,6 +1343,12 @@ def components_check(): os.unlink(link) os.makedirs(link) + # define registers that should be awaited + reg_files = [ + os.path.join(bottle_complete_path, "system.reg"), + os.path.join(bottle_complete_path, "user.reg"), + ] + # wait for registry files to be created FileUtils.wait_for_files(reg_files) From 8a15f9065844dc7837d472d7cfd9bb9f4b38e42d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20R=C3=B6hl?= Date: Mon, 22 Jul 2024 11:24:32 +0200 Subject: [PATCH 04/19] Add btrfs-progs flatpak module --- com.usebottles.bottles.yml | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/com.usebottles.bottles.yml b/com.usebottles.bottles.yml index 413168fe66..e3cf7d583a 100644 --- a/com.usebottles.bottles.yml +++ b/com.usebottles.bottles.yml @@ -175,6 +175,26 @@ modules: type: git tag-pattern: ^xdpyinfo-([\d.]+)$ + # btrfs-progs contains the btrfsutil library with it's Python bindings + - name: btrfs-progs + buildsystem: simple + build-commands: + - ./autogen.sh + - ./configure --prefix=/app --enable-python --disable-documentation --disable-lzo + - make -j + - make install + # Unfortunately with `make install_python` the `btrfsutil` Python-module + # still can not be imported. It needs this ugly hack: + - cp -a libbtrfsutil/python/btrfsutil.cpython-311-x86_64-linux-gnu.so /app/lib/python3.11/site-packages + sources: + - type: git + url: https://github.com/kdave/btrfs-progs + tag: v6.9.2 + commit: b0e5ef4cf7c8b473119e0d487a26b96058e8f80d + x-checker-data: + type: git + tag-pattern: ^v([\d.]+)$ + # Libraries # ---------------------------------------------------------------------------- From 184bab0855fde2a845f8c35d2c76e23d4d0d44d3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20R=C3=B6hl?= Date: Sun, 30 Jun 2024 19:05:39 +0200 Subject: [PATCH 05/19] Empty BtrfsSubvolumeManager --- bottles/backend/managers/btrfssubvolume.py | 14 ++++++++++++++ bottles/backend/managers/manager.py | 2 ++ bottles/backend/managers/meson.build | 1 + 3 files changed, 17 insertions(+) create mode 100644 bottles/backend/managers/btrfssubvolume.py diff --git a/bottles/backend/managers/btrfssubvolume.py b/bottles/backend/managers/btrfssubvolume.py new file mode 100644 index 0000000000..acc59950d5 --- /dev/null +++ b/bottles/backend/managers/btrfssubvolume.py @@ -0,0 +1,14 @@ +import os +import os.path +import btrfsutil + +class BtrfsSubvolumeManager: + """ + Manager to handle bottles created as btrfs subvolume. + """ + + def __init__( + self, + manager, + ): + self.manager = manager diff --git a/bottles/backend/managers/manager.py b/bottles/backend/managers/manager.py index 9d0e674734..32765a0180 100644 --- a/bottles/backend/managers/manager.py +++ b/bottles/backend/managers/manager.py @@ -36,6 +36,7 @@ from bottles.backend.dlls.vkd3d import VKD3DComponent from bottles.backend.globals import Paths from bottles.backend.logger import Logger +from bottles.backend.managers.btrfssubvolume import BtrfsSubvolumeManager from bottles.backend.managers.component import ComponentManager from bottles.backend.managers.data import DataManager, UserDataKeys from bottles.backend.managers.dependency import DependencyManager @@ -145,6 +146,7 @@ def __init__( times["RepositoryManager"] = time.time() self.versioning_manager = VersioningManager(self) times["VersioningManager"] = time.time() + self.btrfs_subvolume_manager = BtrfsSubvolumeManager(self) self.component_manager = ComponentManager(self, _offline) self.installer_manager = InstallerManager(self, _offline) self.dependency_manager = DependencyManager(self, _offline) diff --git a/bottles/backend/managers/meson.build b/bottles/backend/managers/meson.build index e682c0c3b4..25c2cc0c79 100644 --- a/bottles/backend/managers/meson.build +++ b/bottles/backend/managers/meson.build @@ -4,6 +4,7 @@ managersdir = join_paths(pkgdatadir, 'bottles/backend/managers') bottles_sources = [ '__init__.py', 'backup.py', + 'btrfssubvolume.py', 'component.py', 'dependency.py', 'installer.py', From 236985ff96db2cf2a86b14e3e4a67d5972c78e0a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20R=C3=B6hl?= Date: Sun, 30 Jun 2024 20:08:02 +0200 Subject: [PATCH 06/19] Create bottles as btrfs subvolumes --- bottles/backend/managers/btrfssubvolume.py | 29 +++++++++++++++++++++- bottles/backend/managers/manager.py | 10 +++++--- 2 files changed, 34 insertions(+), 5 deletions(-) diff --git a/bottles/backend/managers/btrfssubvolume.py b/bottles/backend/managers/btrfssubvolume.py index acc59950d5..49bd2751ee 100644 --- a/bottles/backend/managers/btrfssubvolume.py +++ b/bottles/backend/managers/btrfssubvolume.py @@ -2,6 +2,11 @@ import os.path import btrfsutil +# Internal subvolumes created at initialization time: +_internal_subvolumes = [ + "cache", + ] + class BtrfsSubvolumeManager: """ Manager to handle bottles created as btrfs subvolume. @@ -11,4 +16,26 @@ def __init__( self, manager, ): - self.manager = manager + self._manager = manager + + @staticmethod + def create_bottle_as_subvolume(bottle_path) -> bool: + """Create bottle as btrfs subvolume. + + Creates the directory 'bottle_path' as btrfs subvolume and internal + subvolumes inside of it. Returns True on success and False on failure. + In particular it fails, if the filesystem is not btrfs. + """ + + os.makedirs(os.path.dirname(bottle_path), exist_ok=True) + try: + btrfsutil.create_subvolume(bottle_path) + for internal_subvolume in _internal_subvolumes: + path = os.path.join(bottle_path, internal_subvolume) + btrfsutil.create_subvolume(path) + except btrfsutil.BtrfsUtilError as error: + if not error.btrfsutilerror == btrfsutil.ERROR_NOT_BTRFS: + raise + return False + else: + return True diff --git a/bottles/backend/managers/manager.py b/bottles/backend/managers/manager.py index 32765a0180..bf56900924 100644 --- a/bottles/backend/managers/manager.py +++ b/bottles/backend/managers/manager.py @@ -1069,7 +1069,8 @@ def create_bottle_from_config(self, config: BottleConfig) -> bool: config.Name = f"{config.Name}__{rnd}" config.Path = f"{config.Path}__{rnd}" - os.makedirs(bottle_path) + if not self.btrfs_subvolume_manager.create_bottle_as_subvolume(bottle_path): + os.makedirs(bottle_path) # Pre-create drive_c directory and set the case-fold flag bottle_drive_c = os.path.join(bottle_path, "drive_c") @@ -1232,14 +1233,15 @@ def components_check(): # create the bottle directory try: - os.makedirs(bottle_complete_path) + if not self.btrfs_subvolume_manager.create_bottle_as_subvolume(bottle_complete_path): + os.makedirs(bottle_complete_path) # Pre-create drive_c directory and set the case-fold flag bottle_drive_c = os.path.join(bottle_complete_path, "drive_c") os.makedirs(bottle_drive_c) FileUtils.chattr_f(bottle_drive_c) - except: + except RuntimeError as e: logging.error( - f"Failed to create bottle directory: {bottle_complete_path}", jn=True + f"Failed to create bottle directory '{bottle_complete_path}' {e}", jn=True ) log_update(_("Failed to create bottle directory.")) return Result(False) From 53fde2dca40c9b62bb3e0b75c6cd815805e1ed03 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20R=C3=B6hl?= Date: Thu, 4 Jul 2024 13:25:22 +0200 Subject: [PATCH 07/19] BottleSnapshotsHandle class --- bottles/backend/managers/btrfssubvolume.py | 99 +++++++++++++++++++++- 1 file changed, 97 insertions(+), 2 deletions(-) diff --git a/bottles/backend/managers/btrfssubvolume.py b/bottles/backend/managers/btrfssubvolume.py index 49bd2751ee..ff39b7ae27 100644 --- a/bottles/backend/managers/btrfssubvolume.py +++ b/bottles/backend/managers/btrfssubvolume.py @@ -1,5 +1,6 @@ import os import os.path +import shutil import btrfsutil # Internal subvolumes created at initialization time: @@ -7,6 +8,86 @@ "cache", ] +def _delete_subvolume(path): + try: + btrfsutil.delete_subvolume(path) + except btrfsutil.BtrfsUtilError as error: + if not error.btrfsutilerror() == btrfsutil.ERROR_SNAP_DESTROY_FAILED or not issubclass(error, PermissionError): + raise + try: + # Try to delete the subvolume as a normal directory tree. This is + # in particular needed, when the btrfs filesystem is not mounted + # with 'user_subvol_rm_allowed' option. + btrfsutil.set_subvolume_read_only(path, False) + shutil.rmtree(path) + except Exception as e: + # Raise the first error with some appended notes + error.add_note(f"Subvolume path: '{path}' ") + error.add_note(f"Fallback to 'shutil.rmtree()' failed with: '{e}'") + raise error + +class BottleSnapshotsHandle: + """Handle the snapshots of a single bottle created as btrfs subvolume. + """ + + # TODO delete the snapshots, when the bottle get's deleted + + def __init__(self, bottle_path): + """Internal should not be called directly. + + Use BtrfsSubvolumeManager.create_bottle_snapshots_handle() to + potentially create an instance. + """ + self._bottle_path = bottle_path + bottles_dir, bottle_name = os.path.split(bottle_path) + self._snapshots_directory = os.path.join(bottles_dir, "BottlesSnapshots", bottle_name) + + self._snapshots = {} + if os.path.exists(self._snapshots_directory): + for snapshot in os.listdir(self._snapshots_directory): + if not btrfsutil.is_subvolume(os.path.join(self._snapshots_directory, snapshot)): + continue + snapshot_id, separator, description = snapshot.partition("_") + if empty(separator): + continue + self._snapshots[int(snapshot_id)] = description + + def snapshots(self) -> dict: + """A dictionary of all available snapshots. + + Returns a dictionary from snapshot ID (int) to description (str). + """ + return self._snapshots.copy() + + def _snapshot_path(self, snapshot_id: int, description: str): + return os.path.join(self._snapshots_directory, f"{snapshot_id}_{description}") + + def _snapshot_path(self, snapshot_id: int): + return self._snapshot_path(snapshot_id, self._snapshots[snapshot_id]) + + def create_snapshot(self, description: str): + snapshot_id = max(self._snapshots.get_keys(), default=-1) + 1 + snapshot_path = self._snapshot_path(snapshot_id, description) + os.makedirs(self._snapshots_directory, exist_ok=True) + btrfsutil.create_snapshot(self._bottle_path, snapshot_path, read_only=True) + self._snapshots[snapshot_id] = description + + def set_state(self, snapshot_id: int): + """Restore the bottle state from a snapshot. + """ + tmp_bottle_path = f"{self._bottle_path}-tmp" + snapshot_path = self._snapshot_path(snapshot_id) + os.rename(self._bottle_path, tmp_bottle_path) + btrfsutil.create_snapshot(snapshot_path, self._bottle_path, read_only=False) + for internal_subvolume in _internal_subvolumes: + source_path = os.path.join(tmp_bottle_path, internal_subvolume) + if not os.path.exists(source_path) or not btrfsutil.is_subvolume(source_path): + continue + destination_path = os.path.join(self._bottle_path, internal_subvolume) + os.rmdir(destination_path) + os.rename(source_path, destination_path) + _delete_subvolume(tmp_bottle_path) + class BtrfsSubvolumeManager: """ Manager to handle bottles created as btrfs subvolume. @@ -23,8 +104,8 @@ def create_bottle_as_subvolume(bottle_path) -> bool: """Create bottle as btrfs subvolume. Creates the directory 'bottle_path' as btrfs subvolume and internal - subvolumes inside of it. Returns True on success and False on failure. - In particular it fails, if the filesystem is not btrfs. + subvolumes inside of it. Returns True on success and False, if the + filesystem is not btrfs. For other failures an exception is raised. """ os.makedirs(os.path.dirname(bottle_path), exist_ok=True) @@ -39,3 +120,17 @@ def create_bottle_as_subvolume(bottle_path) -> bool: return False else: return True + + @staticmethod + def create_bottle_snapshots_handle(bottle_path): + """Try to create a bottle snapshots handle. + + Checks if the bottle states can be stored as btrfs snapshots and if no + states have been stored by FVS versioning system. Returns + BottleSnapshotsHandle, if checks succeed, None otherwise. + """ + if not btrfsutil.is_subvolume(bottle_path): + return None + if os.path.exists(os.path.join(bottle_path, ".fvs")): + return None + return BottleSnapshotsHandle(bottle_path) From d4b5d83c1e059a1cf7abd4bfc2e5443a497e4fb7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20R=C3=B6hl?= Date: Tue, 2 Jul 2024 19:03:39 +0200 Subject: [PATCH 08/19] Initialize BottleSnapshotsHandle._snapshots lazily --- bottles/backend/managers/btrfssubvolume.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/bottles/backend/managers/btrfssubvolume.py b/bottles/backend/managers/btrfssubvolume.py index ff39b7ae27..f27d3448d0 100644 --- a/bottles/backend/managers/btrfssubvolume.py +++ b/bottles/backend/managers/btrfssubvolume.py @@ -1,3 +1,4 @@ +import functools import os import os.path import shutil @@ -42,7 +43,10 @@ def __init__(self, bottle_path): bottles_dir, bottle_name = os.path.split(bottle_path) self._snapshots_directory = os.path.join(bottles_dir, "BottlesSnapshots", bottle_name) - self._snapshots = {} + # Lazily created + @functools.cached_property + def _snapshot(self): + dict_snapshots = {} if os.path.exists(self._snapshots_directory): for snapshot in os.listdir(self._snapshots_directory): if not btrfsutil.is_subvolume(os.path.join(self._snapshots_directory, snapshot)): @@ -50,7 +54,8 @@ def __init__(self, bottle_path): snapshot_id, separator, description = snapshot.partition("_") if empty(separator): continue - self._snapshots[int(snapshot_id)] = description + dict_snapshots[int(snapshot_id)] = description + return dict_snapshots def snapshots(self) -> dict: """A dictionary of all available snapshots. From 0820a307dff0095e9e32ec52ba653a4dace17322 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20R=C3=B6hl?= Date: Tue, 2 Jul 2024 19:22:10 +0200 Subject: [PATCH 09/19] Free method and rename it --- bottles/backend/managers/btrfssubvolume.py | 31 +++++++++++----------- 1 file changed, 15 insertions(+), 16 deletions(-) diff --git a/bottles/backend/managers/btrfssubvolume.py b/bottles/backend/managers/btrfssubvolume.py index f27d3448d0..8af5567212 100644 --- a/bottles/backend/managers/btrfssubvolume.py +++ b/bottles/backend/managers/btrfssubvolume.py @@ -27,6 +27,19 @@ def _delete_subvolume(path): error.add_note(f"Fallback to 'shutil.rmtree()' failed with: '{e}'") raise error +def try_create_bottle_snapshots_handle(bottle_path): + """Try to create a bottle snapshots handle. + + Checks if the bottle states can be stored as btrfs snapshots and if no + states have been stored by FVS versioning system. Returns + BottleSnapshotsHandle, if checks succeed, None otherwise. + """ + if not btrfsutil.is_subvolume(bottle_path): + return None + if os.path.exists(os.path.join(bottle_path, ".fvs")): + return None + return BottleSnapshotsHandle(bottle_path) + class BottleSnapshotsHandle: """Handle the snapshots of a single bottle created as btrfs subvolume. """ @@ -36,8 +49,8 @@ class BottleSnapshotsHandle: def __init__(self, bottle_path): """Internal should not be called directly. - Use BtrfsSubvolumeManager.create_bottle_snapshots_handle() to - potentially create an instance. + Use try_create_bottle_snapshots_handle() to potentially create an + instance. """ self._bottle_path = bottle_path bottles_dir, bottle_name = os.path.split(bottle_path) @@ -125,17 +138,3 @@ def create_bottle_as_subvolume(bottle_path) -> bool: return False else: return True - - @staticmethod - def create_bottle_snapshots_handle(bottle_path): - """Try to create a bottle snapshots handle. - - Checks if the bottle states can be stored as btrfs snapshots and if no - states have been stored by FVS versioning system. Returns - BottleSnapshotsHandle, if checks succeed, None otherwise. - """ - if not btrfsutil.is_subvolume(bottle_path): - return None - if os.path.exists(os.path.join(bottle_path, ".fvs")): - return None - return BottleSnapshotsHandle(bottle_path) From 9208d86ad20e33aadda62227bfff48202da5a4cb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20R=C3=B6hl?= Date: Tue, 2 Jul 2024 20:27:13 +0200 Subject: [PATCH 10/19] First working btrfs snapshot creation ... but notice the many TODOs --- bottles/backend/managers/btrfssubvolume.py | 103 ++++++++++++++++++--- bottles/backend/managers/versioning.py | 56 +++++++++-- 2 files changed, 136 insertions(+), 23 deletions(-) diff --git a/bottles/backend/managers/btrfssubvolume.py b/bottles/backend/managers/btrfssubvolume.py index 8af5567212..b9944efb3f 100644 --- a/bottles/backend/managers/btrfssubvolume.py +++ b/bottles/backend/managers/btrfssubvolume.py @@ -1,9 +1,15 @@ -import functools +import errno +from functools import cached_property import os import os.path import shutil + +# TODO Properly document and update dependency to libbtrfsutil +# https://github.com/kdave/btrfs-progs/tree/master/libbtrfsutil import btrfsutil +from bottles.backend.models.result import Result + # Internal subvolumes created at initialization time: _internal_subvolumes = [ "cache", @@ -13,17 +19,16 @@ def _delete_subvolume(path): try: btrfsutil.delete_subvolume(path) except btrfsutil.BtrfsUtilError as error: - if not error.btrfsutilerror() == btrfsutil.ERROR_SNAP_DESTROY_FAILED or not issubclass(error, PermissionError): + if not error.btrfsutilerror == btrfsutil.ERROR_SNAP_DESTROY_FAILED or not error.errno == errno.EPERM: raise try: # Try to delete the subvolume as a normal directory tree. This is - # in particular needed, when the btrfs filesystem is not mounted - # with 'user_subvol_rm_allowed' option. + # in particular needed, if the btrfs filesystem is not mounted with + # 'user_subvol_rm_allowed' option. btrfsutil.set_subvolume_read_only(path, False) shutil.rmtree(path) except Exception as e: - # Raise the first error with some appended notes - error.add_note(f"Subvolume path: '{path}' ") + # Raise the first error with some appended note error.add_note(f"Fallback to 'shutil.rmtree()' failed with: '{e}'") raise error @@ -57,15 +62,15 @@ def __init__(self, bottle_path): self._snapshots_directory = os.path.join(bottles_dir, "BottlesSnapshots", bottle_name) # Lazily created - @functools.cached_property - def _snapshot(self): + @cached_property + def _snapshots(self): dict_snapshots = {} if os.path.exists(self._snapshots_directory): for snapshot in os.listdir(self._snapshots_directory): if not btrfsutil.is_subvolume(os.path.join(self._snapshots_directory, snapshot)): continue snapshot_id, separator, description = snapshot.partition("_") - if empty(separator): + if len(separator) == 0: continue dict_snapshots[int(snapshot_id)] = description return dict_snapshots @@ -77,18 +82,19 @@ def snapshots(self) -> dict: """ return self._snapshots.copy() - def _snapshot_path(self, snapshot_id: int, description: str): + def _snapshot_path2(self, snapshot_id: int, description: str): return os.path.join(self._snapshots_directory, f"{snapshot_id}_{description}") def _snapshot_path(self, snapshot_id: int): - return self._snapshot_path(snapshot_id, self._snapshots[snapshot_id]) + return self._snapshot_path2(snapshot_id, self._snapshots[snapshot_id]) - def create_snapshot(self, description: str): - snapshot_id = max(self._snapshots.get_keys(), default=-1) + 1 - snapshot_path = self._snapshot_path(snapshot_id, description) + def create_snapshot(self, description: str) -> int: + snapshot_id = max(self._snapshots.keys(), default=-1) + 1 + snapshot_path = self._snapshot_path2(snapshot_id, description) os.makedirs(self._snapshots_directory, exist_ok=True) btrfsutil.create_snapshot(self._bottle_path, snapshot_path, read_only=True) self._snapshots[snapshot_id] = description + return snapshot_id def set_state(self, snapshot_id: int): """Restore the bottle state from a snapshot. @@ -106,11 +112,80 @@ def set_state(self, snapshot_id: int): os.rename(source_path, destination_path) _delete_subvolume(tmp_bottle_path) +def try_create_bottle_snapshots_versioning_wrapper(bottle_path): + handle = try_create_bottle_snapshots_handle(bottle_path) + if not handle: + return None + return BottleSnapshotsVersioningWrapper(handle) + +class BottleSnapshotsVersioningWrapper: + def __init__(self, handle: BottleSnapshotsHandle): + self._handle = handle + + def convert_states(self): + states = {} + for snapshot_id, description in self._handle.snapshots().items(): + # TODO: return meaningful timestamps + states[snapshot_id] = {"message": description, "timestamp": 0} + return states + + def is_initialized(self): + # Nothing to initialize + return true + + def re_initialize(self): + # Nothing to initialize + pass + + def update_system(self): + # Nothing to update + pass + + def create_state(self, message: str) -> Result: + newly_created_snapshot_id = self._handle.create_snapshot(message) + return Result( + status=True, + data={"state_id": newly_created_snapshot_id, "states": self.convert_states()}, + message="Created new BTRFS snapshot", + ) + + def list_states(self) -> Result: + # TODO Save active state id + active_state_id = -1 + return Result( + status=True, + data={"state_id": active_state_id, "states": self.convert_states()}, + message="Retrieved list of states", + ) + + def set_state( + self, state_id: int, after: callable + ) -> Result: + self._handle.set_state(state_id) + if after: + after() + return Result(True) + + def get_state_files( + self, state_id: int, plain: bool = False + ) -> dict: + raise NotImplementedError + + def get_index(self): + raise NotImplementedError + class BtrfsSubvolumeManager: """ Manager to handle bottles created as btrfs subvolume. """ + # TODO ask in the GUI, if a bottle should be created as subvolume. + # TODO duplicate bottles as subvolumes. Nice to have, using lightweight + # subvolume cloning, if the source bottle is a subvolume. + # TODO Add logging + # TODO Better error handling + # TODO Refactoring + def __init__( self, manager, diff --git a/bottles/backend/managers/versioning.py b/bottles/backend/managers/versioning.py index 1d88ecd41c..a04e3dfea5 100644 --- a/bottles/backend/managers/versioning.py +++ b/bottles/backend/managers/versioning.py @@ -30,6 +30,7 @@ from fvs.repo import FVSRepo from bottles.backend.logger import Logger +from bottles.backend.managers.btrfssubvolume import try_create_bottle_snapshots_versioning_wrapper, BottleSnapshotsVersioningWrapper from bottles.backend.models.config import BottleConfig from bottles.backend.models.result import Result from bottles.backend.state import TaskManager, Task @@ -54,9 +55,14 @@ def __get_patterns(config: BottleConfig): @staticmethod def is_initialized(config: BottleConfig): + bottle_path = ManagerUtils.get_bottle_path(config) + bottle_snapshots_wrapper = try_create_bottle_snapshots_versioning_wrapper(bottle_path) + if bottle_snapshots_wrapper: + return bottle_snapshots_wrapper.is_initialized() + try: repo = FVSRepo( - repo_path=ManagerUtils.get_bottle_path(config), + repo_path=bottle_path, use_compression=config.Parameters.versioning_compression, no_init=True, ) @@ -66,20 +72,35 @@ def is_initialized(config: BottleConfig): @staticmethod def re_initialize(config: BottleConfig): - fvs_path = os.path.join(ManagerUtils.get_bottle_path(config), ".fvs") + bottle_path = ManagerUtils.get_bottle_path(config) + bottle_snapshots_wrapper = try_create_bottle_snapshots_versioning_wrapper(bottle_path) + if bottle_snapshots_wrapper: + return bottle_snapshots_wrapper.re_initialize() + + fvs_path = os.path.join(bottle_path, ".fvs") if os.path.exists(fvs_path): shutil.rmtree(fvs_path) def update_system(self, config: BottleConfig): - states_path = os.path.join(ManagerUtils.get_bottle_path(config), "states") + bottle_path = ManagerUtils.get_bottle_path(config) + bottle_snapshots_wrapper = try_create_bottle_snapshots_versioning_wrapper(bottle_path) + if bottle_snapshots_wrapper: + return bottle_snapshots_wrapper.update_system() + + states_path = os.path.join(bottle_path, "states") if os.path.exists(states_path): shutil.rmtree(states_path) return self.manager.update_config(config, "Versioning", False) def create_state(self, config: BottleConfig, message: str = "No message"): + bottle_path = ManagerUtils.get_bottle_path(config) + bottle_snapshots_wrapper = try_create_bottle_snapshots_versioning_wrapper(bottle_path) + if bottle_snapshots_wrapper: + return bottle_snapshots_wrapper.create_state(message) + patterns = self.__get_patterns(config) repo = FVSRepo( - repo_path=ManagerUtils.get_bottle_path(config), + repo_path=bottle_path, use_compression=config.Parameters.versioning_compression, ) task_id = TaskManager.add(Task(title=_("Committing state …"))) @@ -103,10 +124,15 @@ def list_states(self, config: BottleConfig) -> Result: This function take all the states from the states.yml file of the given bottle and return them as a dict. """ + bottle_path = ManagerUtils.get_bottle_path(config) + bottle_snapshots_wrapper = try_create_bottle_snapshots_versioning_wrapper(bottle_path) + if bottle_snapshots_wrapper: + return bottle_snapshots_wrapper.list_states() + if not config.Versioning: try: repo = FVSRepo( - repo_path=ManagerUtils.get_bottle_path(config), + repo_path=bottle_path, use_compression=config.Parameters.versioning_compression, ) except FVSStateNotFound: @@ -124,7 +150,6 @@ def list_states(self, config: BottleConfig) -> Result: data={"state_id": repo.active_state_id, "states": repo.states}, ) - bottle_path = ManagerUtils.get_bottle_path(config) states = {} try: @@ -141,10 +166,15 @@ def list_states(self, config: BottleConfig) -> Result: def set_state( self, config: BottleConfig, state_id: int, after: callable = None ) -> Result: + bottle_path = ManagerUtils.get_bottle_path(config) + bottle_snapshots_wrapper = try_create_bottle_snapshots_versioning_wrapper(bottle_path) + if bottle_snapshots_wrapper: + return bottle_snapshots_wrapper.set_state(state_id, after) + if not config.Versioning: patterns = self.__get_patterns(config) repo = FVSRepo( - repo_path=ManagerUtils.get_bottle_path(config), + repo_path=bottle_path, use_compression=config.Parameters.versioning_compression, ) res = Result( @@ -168,7 +198,6 @@ def set_state( TaskManager.remove(task_id) return res - bottle_path = ManagerUtils.get_bottle_path(config) logging.info(f"Restoring to state: [{state_id}]") # get bottle and state indexes @@ -238,10 +267,15 @@ def get_state_files( Return the files.yml content of the state. Use the plain argument to return the content as plain text. """ + bottle_path = ManagerUtils.get_bottle_path(config) + bottle_snapshots_wrapper = try_create_bottle_snapshots_versioning_wrapper(bottle_path) + if bottle_snapshots_wrapper: + return bottle_snapshots_wrapper.get_state_files(state_id, plain) + try: file = open( "%s/states/%s/files.yml" - % (ManagerUtils.get_bottle_path(config), state_id) + % (bottle_path, state_id) ) files = file.read() if plain else yaml.load(file.read()) file.close() @@ -254,6 +288,10 @@ def get_state_files( def get_index(config: BottleConfig): """List all files in a bottle and return as dict.""" bottle_path = ManagerUtils.get_bottle_path(config) + bottle_snapshots_wrapper = try_create_bottle_snapshots_versioning_wrapper(bottle_path) + if bottle_snapshots_wrapper: + return bottle_snapshots_wrapper.get_index() + cur_index = {"Update_Date": str(datetime.now()), "Files": []} for file in glob("%s/drive_c/**" % bottle_path, recursive=True): if not os.path.isfile(file): From 0c16b900f7d6ad13bdb4aed8bc9080bf78a1c35a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20R=C3=B6hl?= Date: Mon, 8 Jul 2024 12:56:06 +0200 Subject: [PATCH 11/19] Return meaningful timestamps of subvolume snapshots --- bottles/backend/managers/btrfssubvolume.py | 35 ++++++++++++++-------- 1 file changed, 22 insertions(+), 13 deletions(-) diff --git a/bottles/backend/managers/btrfssubvolume.py b/bottles/backend/managers/btrfssubvolume.py index b9944efb3f..b7abaa412a 100644 --- a/bottles/backend/managers/btrfssubvolume.py +++ b/bottles/backend/managers/btrfssubvolume.py @@ -1,3 +1,4 @@ +from dataclasses import dataclass import errno from functools import cached_property import os @@ -45,6 +46,11 @@ def try_create_bottle_snapshots_handle(bottle_path): return None return BottleSnapshotsHandle(bottle_path) +@dataclass(frozen=True) +class SnapshotMetaData: + description: str + timestamp: float = 0.0 + class BottleSnapshotsHandle: """Handle the snapshots of a single bottle created as btrfs subvolume. """ @@ -66,19 +72,22 @@ def __init__(self, bottle_path): def _snapshots(self): dict_snapshots = {} if os.path.exists(self._snapshots_directory): - for snapshot in os.listdir(self._snapshots_directory): - if not btrfsutil.is_subvolume(os.path.join(self._snapshots_directory, snapshot)): - continue - snapshot_id, separator, description = snapshot.partition("_") - if len(separator) == 0: - continue - dict_snapshots[int(snapshot_id)] = description + with os.scandir(self._snapshots_directory) as it: + for snapshot in it: + if not snapshot.is_dir(follow_symlinks=False): + continue + if not btrfsutil.is_subvolume(snapshot.path): + continue + snapshot_id, separator, description = snapshot.name.partition("_") + if len(separator) == 0: + continue + dict_snapshots[int(snapshot_id)] = SnapshotMetaData(description, timestamp=snapshot.stat().st_mtime) return dict_snapshots def snapshots(self) -> dict: """A dictionary of all available snapshots. - Returns a dictionary from snapshot ID (int) to description (str). + Returns a dictionary from snapshot ID (int) to SnapshotMetaData. """ return self._snapshots.copy() @@ -86,14 +95,15 @@ def _snapshot_path2(self, snapshot_id: int, description: str): return os.path.join(self._snapshots_directory, f"{snapshot_id}_{description}") def _snapshot_path(self, snapshot_id: int): - return self._snapshot_path2(snapshot_id, self._snapshots[snapshot_id]) + return self._snapshot_path2(snapshot_id, self._snapshots[snapshot_id].description) def create_snapshot(self, description: str) -> int: snapshot_id = max(self._snapshots.keys(), default=-1) + 1 snapshot_path = self._snapshot_path2(snapshot_id, description) os.makedirs(self._snapshots_directory, exist_ok=True) btrfsutil.create_snapshot(self._bottle_path, snapshot_path, read_only=True) - self._snapshots[snapshot_id] = description + stat = os.stat(snapshot_path) + self._snapshots[snapshot_id] = SnapshotMetaData(description, stat.st_mtime) return snapshot_id def set_state(self, snapshot_id: int): @@ -124,9 +134,8 @@ def __init__(self, handle: BottleSnapshotsHandle): def convert_states(self): states = {} - for snapshot_id, description in self._handle.snapshots().items(): - # TODO: return meaningful timestamps - states[snapshot_id] = {"message": description, "timestamp": 0} + for snapshot_id, metadata in self._handle.snapshots().items(): + states[snapshot_id] = {"message": metadata.description, "timestamp": metadata.timestamp} return states def is_initialized(self): From 6887727f620febe4bd2c8140ee07a0960fc3cfc9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20R=C3=B6hl?= Date: Mon, 8 Jul 2024 14:58:07 +0200 Subject: [PATCH 12/19] Save active btrfs snapshot ID --- bottles/backend/managers/btrfssubvolume.py | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/bottles/backend/managers/btrfssubvolume.py b/bottles/backend/managers/btrfssubvolume.py index b7abaa412a..9178ddd965 100644 --- a/bottles/backend/managers/btrfssubvolume.py +++ b/bottles/backend/managers/btrfssubvolume.py @@ -97,6 +97,20 @@ def _snapshot_path2(self, snapshot_id: int, description: str): def _snapshot_path(self, snapshot_id: int): return self._snapshot_path2(snapshot_id, self._snapshots[snapshot_id].description) + def _active_snapshot_id_path(self): + return os.path.join(self._bottle_path, ".active_state_id") + + def _save_active_snapshot_id(self, active_state_id: int): + with open(self._active_snapshot_id_path(), "w") as file: + file.write(str(active_state_id)) + + def read_active_snapshot_id(self) -> int: + try: + with open(self._active_snapshot_id_path(), "r") as file: + return int(file.read()) + except OSError as error: + return -1 + def create_snapshot(self, description: str) -> int: snapshot_id = max(self._snapshots.keys(), default=-1) + 1 snapshot_path = self._snapshot_path2(snapshot_id, description) @@ -104,6 +118,7 @@ def create_snapshot(self, description: str) -> int: btrfsutil.create_snapshot(self._bottle_path, snapshot_path, read_only=True) stat = os.stat(snapshot_path) self._snapshots[snapshot_id] = SnapshotMetaData(description, stat.st_mtime) + self._save_active_snapshot_id(snapshot_id) return snapshot_id def set_state(self, snapshot_id: int): @@ -121,6 +136,7 @@ def set_state(self, snapshot_id: int): os.rmdir(destination_path) os.rename(source_path, destination_path) _delete_subvolume(tmp_bottle_path) + self._save_active_snapshot_id(snapshot_id) def try_create_bottle_snapshots_versioning_wrapper(bottle_path): handle = try_create_bottle_snapshots_handle(bottle_path) @@ -159,8 +175,7 @@ def create_state(self, message: str) -> Result: ) def list_states(self) -> Result: - # TODO Save active state id - active_state_id = -1 + active_state_id = self._handle.read_active_snapshot_id() return Result( status=True, data={"state_id": active_state_id, "states": self.convert_states()}, From 3ac15cf0ff0ceded4908f9ae7c765160543a94f0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20R=C3=B6hl?= Date: Wed, 10 Jul 2024 11:59:02 +0200 Subject: [PATCH 13/19] Refactor perform FVS versioning work in dedicated class --- bottles/backend/managers/btrfssubvolume.py | 8 - bottles/backend/managers/versioning.py | 292 ++------------------- bottles/backend/models/fvs_versioning.py | 268 +++++++++++++++++++ bottles/backend/models/meson.build | 1 + 4 files changed, 291 insertions(+), 278 deletions(-) create mode 100644 bottles/backend/models/fvs_versioning.py diff --git a/bottles/backend/managers/btrfssubvolume.py b/bottles/backend/managers/btrfssubvolume.py index 9178ddd965..310b7ec4b1 100644 --- a/bottles/backend/managers/btrfssubvolume.py +++ b/bottles/backend/managers/btrfssubvolume.py @@ -190,14 +190,6 @@ def set_state( after() return Result(True) - def get_state_files( - self, state_id: int, plain: bool = False - ) -> dict: - raise NotImplementedError - - def get_index(self): - raise NotImplementedError - class BtrfsSubvolumeManager: """ Manager to handle bottles created as btrfs subvolume. diff --git a/bottles/backend/managers/versioning.py b/bottles/backend/managers/versioning.py index a04e3dfea5..966363f80e 100644 --- a/bottles/backend/managers/versioning.py +++ b/bottles/backend/managers/versioning.py @@ -15,298 +15,50 @@ # along with this program. If not, see . # -import os -import shutil -from datetime import datetime -from gettext import gettext as _ -from glob import glob - -from fvs.exceptions import ( - FVSNothingToCommit, - FVSStateNotFound, - FVSNothingToRestore, - FVSStateZeroNotDeletable, -) -from fvs.repo import FVSRepo - -from bottles.backend.logger import Logger -from bottles.backend.managers.btrfssubvolume import try_create_bottle_snapshots_versioning_wrapper, BottleSnapshotsVersioningWrapper from bottles.backend.models.config import BottleConfig from bottles.backend.models.result import Result -from bottles.backend.state import TaskManager, Task -from bottles.backend.utils import yaml -from bottles.backend.utils.file import FileUtils from bottles.backend.utils.manager import ManagerUtils -logging = Logger() - +# The implementations doing the actual work +from bottles.backend.managers.btrfssubvolume import try_create_bottle_snapshots_versioning_wrapper, BottleSnapshotsVersioningWrapper +from bottles.backend.models.fvs_versioning import BottleFvsVersioning # noinspection PyTypeChecker class VersioningManager: def __init__(self, manager): self.manager = manager - @staticmethod - def __get_patterns(config: BottleConfig): - patterns = ["*dosdevices*", "*cache*"] - if config.Parameters.versioning_exclusion_patterns: - patterns += config.Versioning_Exclusion_Patterns - return patterns - - @staticmethod - def is_initialized(config: BottleConfig): + def _get_bottle_versioning_system(self, config: BottleConfig): bottle_path = ManagerUtils.get_bottle_path(config) bottle_snapshots_wrapper = try_create_bottle_snapshots_versioning_wrapper(bottle_path) if bottle_snapshots_wrapper: - return bottle_snapshots_wrapper.is_initialized() - - try: - repo = FVSRepo( - repo_path=bottle_path, - use_compression=config.Parameters.versioning_compression, - no_init=True, - ) - except FileNotFoundError: - return False - return not repo.has_no_states + return bottle_snapshots_wrapper + def update_config(config: BottleConfig, key: str, value: any): + self.manager.update_config(config, key, value) + return BottleFvsVersioning(config, bottle_path, update_config) - @staticmethod - def re_initialize(config: BottleConfig): - bottle_path = ManagerUtils.get_bottle_path(config) - bottle_snapshots_wrapper = try_create_bottle_snapshots_versioning_wrapper(bottle_path) - if bottle_snapshots_wrapper: - return bottle_snapshots_wrapper.re_initialize() + def is_initialized(self, config: BottleConfig): + bottle_versioning_system = self._get_bottle_versioning_system(config) + return bottle_versioning_system.is_initialized() - fvs_path = os.path.join(bottle_path, ".fvs") - if os.path.exists(fvs_path): - shutil.rmtree(fvs_path) + def re_initialize(self, config: BottleConfig): + bottle_versioning_system = self._get_bottle_versioning_system(config) + return bottle_versioning_system.re_initialize() def update_system(self, config: BottleConfig): - bottle_path = ManagerUtils.get_bottle_path(config) - bottle_snapshots_wrapper = try_create_bottle_snapshots_versioning_wrapper(bottle_path) - if bottle_snapshots_wrapper: - return bottle_snapshots_wrapper.update_system() - - states_path = os.path.join(bottle_path, "states") - if os.path.exists(states_path): - shutil.rmtree(states_path) - return self.manager.update_config(config, "Versioning", False) + bottle_versioning_system = self._get_bottle_versioning_system(config) + return bottle_versioning_system.update_system() def create_state(self, config: BottleConfig, message: str = "No message"): - bottle_path = ManagerUtils.get_bottle_path(config) - bottle_snapshots_wrapper = try_create_bottle_snapshots_versioning_wrapper(bottle_path) - if bottle_snapshots_wrapper: - return bottle_snapshots_wrapper.create_state(message) - - patterns = self.__get_patterns(config) - repo = FVSRepo( - repo_path=bottle_path, - use_compression=config.Parameters.versioning_compression, - ) - task_id = TaskManager.add(Task(title=_("Committing state …"))) - try: - repo.commit(message, ignore=patterns) - except FVSNothingToCommit: - TaskManager.remove(task_id) - return Result(status=False, message=_("Nothing to commit")) - - TaskManager.remove(task_id) - return Result( - status=True, - message=_("New state [{0}] created successfully!").format( - repo.active_state_id - ), - data={"state_id": repo.active_state_id, "states": repo.states}, - ) + bottle_versioning_system = self._get_bottle_versioning_system(config) + return bottle_versioning_system.create_state(message) def list_states(self, config: BottleConfig) -> Result: - """ - This function take all the states from the states.yml file - of the given bottle and return them as a dict. - """ - bottle_path = ManagerUtils.get_bottle_path(config) - bottle_snapshots_wrapper = try_create_bottle_snapshots_versioning_wrapper(bottle_path) - if bottle_snapshots_wrapper: - return bottle_snapshots_wrapper.list_states() - - if not config.Versioning: - try: - repo = FVSRepo( - repo_path=bottle_path, - use_compression=config.Parameters.versioning_compression, - ) - except FVSStateNotFound: - logging.warning( - "The FVS repository may be corrupted, trying to re-initialize it" - ) - self.re_initialize(config) - repo = FVSRepo( - repo_path=ManagerUtils.get_bottle_path(config), - use_compression=config.Parameters.versioning_compression, - ) - return Result( - status=True, - message=_("States list retrieved successfully!"), - data={"state_id": repo.active_state_id, "states": repo.states}, - ) - - states = {} - - try: - states_file = open("%s/states/states.yml" % bottle_path) - states_file_yaml = yaml.load(states_file) - states_file.close() - states = states_file_yaml.get("States") - logging.info(f"Found [{len(states)}] states for bottle: [{config.Name}]") - except (FileNotFoundError, yaml.YAMLError): - logging.info(f"No states found for bottle: [{config.Name}]") - - return states + bottle_versioning_system = self._get_bottle_versioning_system(config) + return bottle_versioning_system.list_states() def set_state( self, config: BottleConfig, state_id: int, after: callable = None ) -> Result: - bottle_path = ManagerUtils.get_bottle_path(config) - bottle_snapshots_wrapper = try_create_bottle_snapshots_versioning_wrapper(bottle_path) - if bottle_snapshots_wrapper: - return bottle_snapshots_wrapper.set_state(state_id, after) - - if not config.Versioning: - patterns = self.__get_patterns(config) - repo = FVSRepo( - repo_path=bottle_path, - use_compression=config.Parameters.versioning_compression, - ) - res = Result( - status=True, - message=_("State {0} restored successfully!").format(state_id), - ) - task_id = TaskManager.add( - Task(title=_("Restoring state {} …".format(state_id))) - ) - try: - repo.restore_state(state_id, ignore=patterns) - except FVSStateNotFound: - logging.error(f"State {state_id} not found.") - res = Result(status=False, message=_("State not found")) - except (FVSNothingToRestore, FVSStateZeroNotDeletable): - logging.error(f"State {state_id} is the active state.") - res = Result( - status=False, - message=_("State {} is already the active state").format(state_id), - ) - TaskManager.remove(task_id) - return res - - logging.info(f"Restoring to state: [{state_id}]") - - # get bottle and state indexes - bottle_index = self.get_index(config) - state_index = self.get_state_files(config, state_id) - - search_sources = list(range(int(state_id) + 1)) - search_sources.reverse() - - # check for removed and changed files - remove_files = [] - edit_files = [] - for file in bottle_index.get("Files"): - if file["file"] not in [f["file"] for f in state_index.get("Files")]: - remove_files.append(file) - elif file["checksum"] not in [ - f["checksum"] for f in state_index.get("Files") - ]: - edit_files.append(file) - logging.info(f"[{len(remove_files)}] files to remove.") - logging.info(f"[{len(edit_files)}] files to replace.") - - # check for new files - add_files = [] - for file in state_index.get("Files"): - if file["file"] not in [f["file"] for f in bottle_index.get("Files")]: - add_files.append(file) - logging.info(f"[{len(add_files)}] files to add.") - - # perform file updates - for file in remove_files: - os.remove("%s/drive_c/%s" % (bottle_path, file["file"])) - - for file in add_files: - source = "%s/states/%s/drive_c/%s" % ( - bottle_path, - str(state_id), - file["file"], - ) - target = "%s/drive_c/%s" % (bottle_path, file["file"]) - shutil.copy2(source, target) - - for file in edit_files: - for i in search_sources: - source = "%s/states/%s/drive_c/%s" % (bottle_path, str(i), file["file"]) - if os.path.isfile(source): - checksum = FileUtils().get_checksum(source) - if file["checksum"] == checksum: - break - target = "%s/drive_c/%s" % (bottle_path, file["file"]) - shutil.copy2(source, target) - - # update State in bottle config - self.manager.update_config(config, "State", state_id) - - # execute caller function after all - if after: - after() - - return Result(True) - - @staticmethod - def get_state_files( - config: BottleConfig, state_id: int, plain: bool = False - ) -> dict: - """ - Return the files.yml content of the state. Use the plain argument - to return the content as plain text. - """ - bottle_path = ManagerUtils.get_bottle_path(config) - bottle_snapshots_wrapper = try_create_bottle_snapshots_versioning_wrapper(bottle_path) - if bottle_snapshots_wrapper: - return bottle_snapshots_wrapper.get_state_files(state_id, plain) - - try: - file = open( - "%s/states/%s/files.yml" - % (bottle_path, state_id) - ) - files = file.read() if plain else yaml.load(file.read()) - file.close() - return files - except (OSError, IOError, yaml.YAMLError): - logging.error(f"Could not read the state files file.") - return {} - - @staticmethod - def get_index(config: BottleConfig): - """List all files in a bottle and return as dict.""" - bottle_path = ManagerUtils.get_bottle_path(config) - bottle_snapshots_wrapper = try_create_bottle_snapshots_versioning_wrapper(bottle_path) - if bottle_snapshots_wrapper: - return bottle_snapshots_wrapper.get_index() - - cur_index = {"Update_Date": str(datetime.now()), "Files": []} - for file in glob("%s/drive_c/**" % bottle_path, recursive=True): - if not os.path.isfile(file): - continue - - if os.path.islink(os.path.dirname(file)): - continue - - if file[len(bottle_path) + 9 :].split("/")[0] in ["users"]: - continue - - cur_index["Files"].append( - { - "file": file[len(bottle_path) + 9 :], - "checksum": FileUtils().get_checksum(file), - } - ) - return cur_index + bottle_versioning_system = self._get_bottle_versioning_system(config) + return bottle_versioning_system.set_state(state_id, after) diff --git a/bottles/backend/models/fvs_versioning.py b/bottles/backend/models/fvs_versioning.py new file mode 100644 index 0000000000..0b39093025 --- /dev/null +++ b/bottles/backend/models/fvs_versioning.py @@ -0,0 +1,268 @@ +# fvs_versioning.py +# +# Copyright 2022 brombinmirko +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, in version 3 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# + +import os +import shutil +from datetime import datetime +from gettext import gettext as _ +from glob import glob + +from fvs.exceptions import ( + FVSNothingToCommit, + FVSStateNotFound, + FVSNothingToRestore, + FVSStateZeroNotDeletable, +) +from fvs.repo import FVSRepo + +from bottles.backend.logger import Logger +from bottles.backend.models.config import BottleConfig +from bottles.backend.models.result import Result +from bottles.backend.state import TaskManager, Task +from bottles.backend.utils import yaml +from bottles.backend.utils.file import FileUtils + +logging = Logger() + +class BottleFvsVersioning: + """Provide versioning based on FVS for a single bottle""" + + def __init__(self, config: BottleConfig, bottle_path: str, update_config: callable): + self._config = config + self._bottle_path = bottle_path + self._update_config = update_config + + @staticmethod + def __get_patterns(config: BottleConfig): + patterns = ["*dosdevices*", "*cache*"] + if config.Parameters.versioning_exclusion_patterns: + patterns += config.Versioning_Exclusion_Patterns + return patterns + + def is_initialized(self): + try: + repo = FVSRepo( + repo_path=self._bottle_path, + use_compression=self._config.Parameters.versioning_compression, + no_init=True, + ) + except FileNotFoundError: + return False + return not repo.has_no_states + + def re_initialize(self): + fvs_path = os.path.join(self._bottle_path, ".fvs") + if os.path.exists(fvs_path): + shutil.rmtree(fvs_path) + + def update_system(self): + states_path = os.path.join(self._bottle_path, "states") + if os.path.exists(states_path): + shutil.rmtree(states_path) + return self._update_config(self._config, "Versioning", False) + + def create_state(self, message: str = "No message"): + patterns = self.__get_patterns(self._config) + repo = FVSRepo( + repo_path=self._bottle_path, + use_compression=self._config.Parameters.versioning_compression, + ) + task_id = TaskManager.add(Task(title=_("Committing state …"))) + try: + repo.commit(message, ignore=patterns) + except FVSNothingToCommit: + TaskManager.remove(task_id) + return Result(status=False, message=_("Nothing to commit")) + + TaskManager.remove(task_id) + return Result( + status=True, + message=_("New state [{0}] created successfully!").format( + repo.active_state_id + ), + data={"state_id": repo.active_state_id, "states": repo.states}, + ) + + def list_states(self) -> Result: + """ + This function take all the states from the states.yml file + of the given bottle and return them as a dict. + """ + if not self._config.Versioning: + try: + repo = FVSRepo( + repo_path=self._bottle_path, + use_compression=self._config.Parameters.versioning_compression, + ) + except FVSStateNotFound: + logging.warning( + "The FVS repository may be corrupted, trying to re-initialize it" + ) + self.re_initialize() + repo = FVSRepo( + repo_path=self._bottle_path, + use_compression=self._config.Parameters.versioning_compression, + ) + return Result( + status=True, + message=_("States list retrieved successfully!"), + data={"state_id": repo.active_state_id, "states": repo.states}, + ) + + states = {} + + try: + states_file = open("%s/states/states.yml" % self._bottle_path) + states_file_yaml = yaml.load(states_file) + states_file.close() + states = states_file_yaml.get("States") + logging.info(f"Found [{len(states)}] states for bottle: [{self._config.Name}]") + except (FileNotFoundError, yaml.YAMLError): + logging.info(f"No states found for bottle: [{self._config.Name}]") + + return states + + def set_state( + self, state_id: int, after: callable = None + ) -> Result: + if not self._config.Versioning: + patterns = self.__get_patterns(self._config) + repo = FVSRepo( + repo_path=self._bottle_path, + use_compression=self._config.Parameters.versioning_compression, + ) + res = Result( + status=True, + message=_("State {0} restored successfully!").format(state_id), + ) + task_id = TaskManager.add( + Task(title=_("Restoring state {} …".format(state_id))) + ) + try: + repo.restore_state(state_id, ignore=patterns) + except FVSStateNotFound: + logging.error(f"State {state_id} not found.") + res = Result(status=False, message=_("State not found")) + except (FVSNothingToRestore, FVSStateZeroNotDeletable): + logging.error(f"State {state_id} is the active state.") + res = Result( + status=False, + message=_("State {} is already the active state").format(state_id), + ) + TaskManager.remove(task_id) + return res + + logging.info(f"Restoring to state: [{state_id}]") + + # get bottle and state indexes + bottle_index = self.get_index() + state_index = self.get_state_files(state_id) + + search_sources = list(range(int(state_id) + 1)) + search_sources.reverse() + + # check for removed and changed files + remove_files = [] + edit_files = [] + for file in bottle_index.get("Files"): + if file["file"] not in [f["file"] for f in state_index.get("Files")]: + remove_files.append(file) + elif file["checksum"] not in [ + f["checksum"] for f in state_index.get("Files") + ]: + edit_files.append(file) + logging.info(f"[{len(remove_files)}] files to remove.") + logging.info(f"[{len(edit_files)}] files to replace.") + + # check for new files + add_files = [] + for file in state_index.get("Files"): + if file["file"] not in [f["file"] for f in bottle_index.get("Files")]: + add_files.append(file) + logging.info(f"[{len(add_files)}] files to add.") + + # perform file updates + for file in remove_files: + os.remove("%s/drive_c/%s" % (self._bottle_path, file["file"])) + + for file in add_files: + source = "%s/states/%s/drive_c/%s" % ( + self._bottle_path, + str(state_id), + file["file"], + ) + target = "%s/drive_c/%s" % (self._bottle_path, file["file"]) + shutil.copy2(source, target) + + for file in edit_files: + for i in search_sources: + source = "%s/states/%s/drive_c/%s" % (self._bottle_path, str(i), file["file"]) + if os.path.isfile(source): + checksum = FileUtils().get_checksum(source) + if file["checksum"] == checksum: + break + target = "%s/drive_c/%s" % (self._bottle_path, file["file"]) + shutil.copy2(source, target) + + # update State in bottle config + self._update_config(self._config, "State", state_id) + + # execute caller function after all + if after: + after() + + return Result(True) + + def get_state_files( + self, state_id: int, plain: bool = False + ) -> dict: + """ + Return the files.yml content of the state. Use the plain argument + to return the content as plain text. + """ + try: + file = open( + "%s/states/%s/files.yml" + % (self._bottle_path, state_id) + ) + files = file.read() if plain else yaml.load(file.read()) + file.close() + return files + except (OSError, IOError, yaml.YAMLError): + logging.error(f"Could not read the state files file.") + return {} + + def get_index(self): + """List all files in a bottle and return as dict.""" + cur_index = {"Update_Date": str(datetime.now()), "Files": []} + for file in glob("%s/drive_c/**" % self._bottle_path, recursive=True): + if not os.path.isfile(file): + continue + + if os.path.islink(os.path.dirname(file)): + continue + + if file[len(self._bottle_path) + 9 :].split("/")[0] in ["users"]: + continue + + cur_index["Files"].append( + { + "file": file[len(self._bottle_path) + 9 :], + "checksum": FileUtils().get_checksum(file), + } + ) + return cur_index diff --git a/bottles/backend/models/meson.build b/bottles/backend/models/meson.build index 3da22eed8b..e3d91aa1ee 100644 --- a/bottles/backend/models/meson.build +++ b/bottles/backend/models/meson.build @@ -3,6 +3,7 @@ modelsdir = join_paths(pkgdatadir, 'bottles/backend/models') bottles_sources = [ '__init__.py', + 'fvs_versioning.py', 'result.py', 'samples.py', 'vdict.py', From 64ee3b0cdedf659c4a9d2c76b68f3fda6ba179f2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20R=C3=B6hl?= Date: Sat, 13 Jul 2024 12:34:30 +0200 Subject: [PATCH 14/19] Refactor move code from manager to model module --- bottles/backend/managers/btrfssubvolume.py | 219 +------------------- bottles/backend/managers/versioning.py | 2 +- bottles/backend/models/btrfssubvolume.py | 220 +++++++++++++++++++++ bottles/backend/models/meson.build | 1 + 4 files changed, 224 insertions(+), 218 deletions(-) create mode 100644 bottles/backend/models/btrfssubvolume.py diff --git a/bottles/backend/managers/btrfssubvolume.py b/bottles/backend/managers/btrfssubvolume.py index 310b7ec4b1..7a53c59f6e 100644 --- a/bottles/backend/managers/btrfssubvolume.py +++ b/bottles/backend/managers/btrfssubvolume.py @@ -1,207 +1,10 @@ -from dataclasses import dataclass -import errno -from functools import cached_property -import os -import os.path -import shutil - -# TODO Properly document and update dependency to libbtrfsutil -# https://github.com/kdave/btrfs-progs/tree/master/libbtrfsutil -import btrfsutil - -from bottles.backend.models.result import Result - -# Internal subvolumes created at initialization time: -_internal_subvolumes = [ - "cache", - ] - -def _delete_subvolume(path): - try: - btrfsutil.delete_subvolume(path) - except btrfsutil.BtrfsUtilError as error: - if not error.btrfsutilerror == btrfsutil.ERROR_SNAP_DESTROY_FAILED or not error.errno == errno.EPERM: - raise - try: - # Try to delete the subvolume as a normal directory tree. This is - # in particular needed, if the btrfs filesystem is not mounted with - # 'user_subvol_rm_allowed' option. - btrfsutil.set_subvolume_read_only(path, False) - shutil.rmtree(path) - except Exception as e: - # Raise the first error with some appended note - error.add_note(f"Fallback to 'shutil.rmtree()' failed with: '{e}'") - raise error - -def try_create_bottle_snapshots_handle(bottle_path): - """Try to create a bottle snapshots handle. - - Checks if the bottle states can be stored as btrfs snapshots and if no - states have been stored by FVS versioning system. Returns - BottleSnapshotsHandle, if checks succeed, None otherwise. - """ - if not btrfsutil.is_subvolume(bottle_path): - return None - if os.path.exists(os.path.join(bottle_path, ".fvs")): - return None - return BottleSnapshotsHandle(bottle_path) - -@dataclass(frozen=True) -class SnapshotMetaData: - description: str - timestamp: float = 0.0 - -class BottleSnapshotsHandle: - """Handle the snapshots of a single bottle created as btrfs subvolume. - """ - - # TODO delete the snapshots, when the bottle get's deleted - - def __init__(self, bottle_path): - """Internal should not be called directly. - - Use try_create_bottle_snapshots_handle() to potentially create an - instance. - """ - self._bottle_path = bottle_path - bottles_dir, bottle_name = os.path.split(bottle_path) - self._snapshots_directory = os.path.join(bottles_dir, "BottlesSnapshots", bottle_name) - - # Lazily created - @cached_property - def _snapshots(self): - dict_snapshots = {} - if os.path.exists(self._snapshots_directory): - with os.scandir(self._snapshots_directory) as it: - for snapshot in it: - if not snapshot.is_dir(follow_symlinks=False): - continue - if not btrfsutil.is_subvolume(snapshot.path): - continue - snapshot_id, separator, description = snapshot.name.partition("_") - if len(separator) == 0: - continue - dict_snapshots[int(snapshot_id)] = SnapshotMetaData(description, timestamp=snapshot.stat().st_mtime) - return dict_snapshots - - def snapshots(self) -> dict: - """A dictionary of all available snapshots. - - Returns a dictionary from snapshot ID (int) to SnapshotMetaData. - """ - return self._snapshots.copy() - - def _snapshot_path2(self, snapshot_id: int, description: str): - return os.path.join(self._snapshots_directory, f"{snapshot_id}_{description}") - - def _snapshot_path(self, snapshot_id: int): - return self._snapshot_path2(snapshot_id, self._snapshots[snapshot_id].description) - - def _active_snapshot_id_path(self): - return os.path.join(self._bottle_path, ".active_state_id") - - def _save_active_snapshot_id(self, active_state_id: int): - with open(self._active_snapshot_id_path(), "w") as file: - file.write(str(active_state_id)) - - def read_active_snapshot_id(self) -> int: - try: - with open(self._active_snapshot_id_path(), "r") as file: - return int(file.read()) - except OSError as error: - return -1 - - def create_snapshot(self, description: str) -> int: - snapshot_id = max(self._snapshots.keys(), default=-1) + 1 - snapshot_path = self._snapshot_path2(snapshot_id, description) - os.makedirs(self._snapshots_directory, exist_ok=True) - btrfsutil.create_snapshot(self._bottle_path, snapshot_path, read_only=True) - stat = os.stat(snapshot_path) - self._snapshots[snapshot_id] = SnapshotMetaData(description, stat.st_mtime) - self._save_active_snapshot_id(snapshot_id) - return snapshot_id - - def set_state(self, snapshot_id: int): - """Restore the bottle state from a snapshot. - """ - tmp_bottle_path = f"{self._bottle_path}-tmp" - snapshot_path = self._snapshot_path(snapshot_id) - os.rename(self._bottle_path, tmp_bottle_path) - btrfsutil.create_snapshot(snapshot_path, self._bottle_path, read_only=False) - for internal_subvolume in _internal_subvolumes: - source_path = os.path.join(tmp_bottle_path, internal_subvolume) - if not os.path.exists(source_path) or not btrfsutil.is_subvolume(source_path): - continue - destination_path = os.path.join(self._bottle_path, internal_subvolume) - os.rmdir(destination_path) - os.rename(source_path, destination_path) - _delete_subvolume(tmp_bottle_path) - self._save_active_snapshot_id(snapshot_id) - -def try_create_bottle_snapshots_versioning_wrapper(bottle_path): - handle = try_create_bottle_snapshots_handle(bottle_path) - if not handle: - return None - return BottleSnapshotsVersioningWrapper(handle) - -class BottleSnapshotsVersioningWrapper: - def __init__(self, handle: BottleSnapshotsHandle): - self._handle = handle - - def convert_states(self): - states = {} - for snapshot_id, metadata in self._handle.snapshots().items(): - states[snapshot_id] = {"message": metadata.description, "timestamp": metadata.timestamp} - return states - - def is_initialized(self): - # Nothing to initialize - return true - - def re_initialize(self): - # Nothing to initialize - pass - - def update_system(self): - # Nothing to update - pass - - def create_state(self, message: str) -> Result: - newly_created_snapshot_id = self._handle.create_snapshot(message) - return Result( - status=True, - data={"state_id": newly_created_snapshot_id, "states": self.convert_states()}, - message="Created new BTRFS snapshot", - ) - - def list_states(self) -> Result: - active_state_id = self._handle.read_active_snapshot_id() - return Result( - status=True, - data={"state_id": active_state_id, "states": self.convert_states()}, - message="Retrieved list of states", - ) - - def set_state( - self, state_id: int, after: callable - ) -> Result: - self._handle.set_state(state_id) - if after: - after() - return Result(True) +import bottles.backend.models.btrfssubvolume as btrfssubvolume class BtrfsSubvolumeManager: """ Manager to handle bottles created as btrfs subvolume. """ - # TODO ask in the GUI, if a bottle should be created as subvolume. - # TODO duplicate bottles as subvolumes. Nice to have, using lightweight - # subvolume cloning, if the source bottle is a subvolume. - # TODO Add logging - # TODO Better error handling - # TODO Refactoring - def __init__( self, manager, @@ -210,22 +13,4 @@ def __init__( @staticmethod def create_bottle_as_subvolume(bottle_path) -> bool: - """Create bottle as btrfs subvolume. - - Creates the directory 'bottle_path' as btrfs subvolume and internal - subvolumes inside of it. Returns True on success and False, if the - filesystem is not btrfs. For other failures an exception is raised. - """ - - os.makedirs(os.path.dirname(bottle_path), exist_ok=True) - try: - btrfsutil.create_subvolume(bottle_path) - for internal_subvolume in _internal_subvolumes: - path = os.path.join(bottle_path, internal_subvolume) - btrfsutil.create_subvolume(path) - except btrfsutil.BtrfsUtilError as error: - if not error.btrfsutilerror == btrfsutil.ERROR_NOT_BTRFS: - raise - return False - else: - return True + return btrfssubvolume.create_bottle_as_subvolume(bottle_path) diff --git a/bottles/backend/managers/versioning.py b/bottles/backend/managers/versioning.py index 966363f80e..7915c2712f 100644 --- a/bottles/backend/managers/versioning.py +++ b/bottles/backend/managers/versioning.py @@ -20,7 +20,7 @@ from bottles.backend.utils.manager import ManagerUtils # The implementations doing the actual work -from bottles.backend.managers.btrfssubvolume import try_create_bottle_snapshots_versioning_wrapper, BottleSnapshotsVersioningWrapper +from bottles.backend.models.btrfssubvolume import try_create_bottle_snapshots_versioning_wrapper, BottleSnapshotsVersioningWrapper from bottles.backend.models.fvs_versioning import BottleFvsVersioning # noinspection PyTypeChecker diff --git a/bottles/backend/models/btrfssubvolume.py b/bottles/backend/models/btrfssubvolume.py new file mode 100644 index 0000000000..88af37f934 --- /dev/null +++ b/bottles/backend/models/btrfssubvolume.py @@ -0,0 +1,220 @@ +from dataclasses import dataclass +import errno +from functools import cached_property +import os +import os.path +import shutil + +# TODO Properly document and update dependency to libbtrfsutil +# https://github.com/kdave/btrfs-progs/tree/master/libbtrfsutil +import btrfsutil + +from bottles.backend.models.result import Result + +# TODO ask in the GUI, if a bottle should be created as subvolume. +# TODO duplicate bottles as subvolumes. Nice to have, using lightweight +# subvolume cloning, if the source bottle is a subvolume. +# TODO Add logging +# TODO Better error handling + +# Internal subvolumes created at initialization time: +_internal_subvolumes = [ + "cache", + ] + +def _delete_subvolume(path): + try: + btrfsutil.delete_subvolume(path) + except btrfsutil.BtrfsUtilError as error: + if not error.btrfsutilerror == btrfsutil.ERROR_SNAP_DESTROY_FAILED or not error.errno == errno.EPERM: + raise + try: + # Try to delete the subvolume as a normal directory tree. This is + # in particular needed, if the btrfs filesystem is not mounted with + # 'user_subvol_rm_allowed' option. + btrfsutil.set_subvolume_read_only(path, False) + shutil.rmtree(path) + except Exception as e: + # Raise the first error with some appended note + error.add_note(f"Fallback to 'shutil.rmtree()' failed with: '{e}'") + raise error + + +def create_bottle_as_subvolume(bottle_path) -> bool: + """Create bottle as btrfs subvolume. + + Creates the directory 'bottle_path' as btrfs subvolume and internal + subvolumes inside of it. Returns True on success and False, if the + filesystem is not btrfs. For other failures an exception is raised. + """ + + os.makedirs(os.path.dirname(bottle_path), exist_ok=True) + try: + btrfsutil.create_subvolume(bottle_path) + for internal_subvolume in _internal_subvolumes: + path = os.path.join(bottle_path, internal_subvolume) + btrfsutil.create_subvolume(path) + except btrfsutil.BtrfsUtilError as error: + if not error.btrfsutilerror == btrfsutil.ERROR_NOT_BTRFS: + raise + return False + else: + return True + +def try_create_bottle_snapshots_handle(bottle_path): + """Try to create a bottle snapshots handle. + + Checks if the bottle states can be stored as btrfs snapshots and if no + states have been stored by FVS versioning system. Returns + BottleSnapshotsHandle, if checks succeed, None otherwise. + """ + if not btrfsutil.is_subvolume(bottle_path): + return None + if os.path.exists(os.path.join(bottle_path, ".fvs")): + return None + return BottleSnapshotsHandle(bottle_path) + +@dataclass(frozen=True) +class SnapshotMetaData: + description: str + timestamp: float = 0.0 + +class BottleSnapshotsHandle: + """Handle the snapshots of a single bottle created as btrfs subvolume. + """ + + # TODO delete the snapshots, when the bottle get's deleted + + def __init__(self, bottle_path): + """Internal should not be called directly. + + Use try_create_bottle_snapshots_handle() to potentially create an + instance. + """ + self._bottle_path = bottle_path + bottles_dir, bottle_name = os.path.split(bottle_path) + self._snapshots_directory = os.path.join(bottles_dir, "BottlesSnapshots", bottle_name) + + # Lazily created + @cached_property + def _snapshots(self): + dict_snapshots = {} + if os.path.exists(self._snapshots_directory): + with os.scandir(self._snapshots_directory) as it: + for snapshot in it: + if not snapshot.is_dir(follow_symlinks=False): + continue + if not btrfsutil.is_subvolume(snapshot.path): + continue + snapshot_id, separator, description = snapshot.name.partition("_") + if len(separator) == 0: + continue + dict_snapshots[int(snapshot_id)] = SnapshotMetaData(description, timestamp=snapshot.stat().st_mtime) + return dict_snapshots + + def snapshots(self) -> dict: + """A dictionary of all available snapshots. + + Returns a dictionary from snapshot ID (int) to SnapshotMetaData. + """ + return self._snapshots.copy() + + def _snapshot_path2(self, snapshot_id: int, description: str): + return os.path.join(self._snapshots_directory, f"{snapshot_id}_{description}") + + def _snapshot_path(self, snapshot_id: int): + return self._snapshot_path2(snapshot_id, self._snapshots[snapshot_id].description) + + def _active_snapshot_id_path(self): + return os.path.join(self._bottle_path, ".active_state_id") + + def _save_active_snapshot_id(self, active_state_id: int): + with open(self._active_snapshot_id_path(), "w") as file: + file.write(str(active_state_id)) + + def read_active_snapshot_id(self) -> int: + try: + with open(self._active_snapshot_id_path(), "r") as file: + return int(file.read()) + except OSError as error: + return -1 + + def create_snapshot(self, description: str) -> int: + snapshot_id = max(self._snapshots.keys(), default=-1) + 1 + snapshot_path = self._snapshot_path2(snapshot_id, description) + os.makedirs(self._snapshots_directory, exist_ok=True) + btrfsutil.create_snapshot(self._bottle_path, snapshot_path, read_only=True) + stat = os.stat(snapshot_path) + self._snapshots[snapshot_id] = SnapshotMetaData(description, stat.st_mtime) + self._save_active_snapshot_id(snapshot_id) + return snapshot_id + + def set_state(self, snapshot_id: int): + """Restore the bottle state from a snapshot. + """ + tmp_bottle_path = f"{self._bottle_path}-tmp" + snapshot_path = self._snapshot_path(snapshot_id) + os.rename(self._bottle_path, tmp_bottle_path) + btrfsutil.create_snapshot(snapshot_path, self._bottle_path, read_only=False) + for internal_subvolume in _internal_subvolumes: + source_path = os.path.join(tmp_bottle_path, internal_subvolume) + if not os.path.exists(source_path) or not btrfsutil.is_subvolume(source_path): + continue + destination_path = os.path.join(self._bottle_path, internal_subvolume) + os.rmdir(destination_path) + os.rename(source_path, destination_path) + _delete_subvolume(tmp_bottle_path) + self._save_active_snapshot_id(snapshot_id) + +def try_create_bottle_snapshots_versioning_wrapper(bottle_path): + handle = try_create_bottle_snapshots_handle(bottle_path) + if not handle: + return None + return BottleSnapshotsVersioningWrapper(handle) + +class BottleSnapshotsVersioningWrapper: + def __init__(self, handle: BottleSnapshotsHandle): + self._handle = handle + + def convert_states(self): + states = {} + for snapshot_id, metadata in self._handle.snapshots().items(): + states[snapshot_id] = {"message": metadata.description, "timestamp": metadata.timestamp} + return states + + def is_initialized(self): + # Nothing to initialize + return true + + def re_initialize(self): + # Nothing to initialize + pass + + def update_system(self): + # Nothing to update + pass + + def create_state(self, message: str) -> Result: + newly_created_snapshot_id = self._handle.create_snapshot(message) + return Result( + status=True, + data={"state_id": newly_created_snapshot_id, "states": self.convert_states()}, + message="Created new BTRFS snapshot", + ) + + def list_states(self) -> Result: + active_state_id = self._handle.read_active_snapshot_id() + return Result( + status=True, + data={"state_id": active_state_id, "states": self.convert_states()}, + message="Retrieved list of states", + ) + + def set_state( + self, state_id: int, after: callable + ) -> Result: + self._handle.set_state(state_id) + if after: + after() + return Result(True) + diff --git a/bottles/backend/models/meson.build b/bottles/backend/models/meson.build index e3d91aa1ee..cbe2870c04 100644 --- a/bottles/backend/models/meson.build +++ b/bottles/backend/models/meson.build @@ -3,6 +3,7 @@ modelsdir = join_paths(pkgdatadir, 'bottles/backend/models') bottles_sources = [ '__init__.py', + 'btrfssubvolume.py', 'fvs_versioning.py', 'result.py', 'samples.py', From 059b1b90e8fa7cf449827ef22db8ad19647cf57d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20R=C3=B6hl?= Date: Sat, 13 Jul 2024 13:07:50 +0200 Subject: [PATCH 15/19] Delete bottles' btrfs snapshots during deletion --- bottles/backend/managers/btrfssubvolume.py | 6 ++++++ bottles/backend/managers/manager.py | 1 + bottles/backend/models/btrfssubvolume.py | 11 +++++++++-- 3 files changed, 16 insertions(+), 2 deletions(-) diff --git a/bottles/backend/managers/btrfssubvolume.py b/bottles/backend/managers/btrfssubvolume.py index 7a53c59f6e..76b2218632 100644 --- a/bottles/backend/managers/btrfssubvolume.py +++ b/bottles/backend/managers/btrfssubvolume.py @@ -14,3 +14,9 @@ def __init__( @staticmethod def create_bottle_as_subvolume(bottle_path) -> bool: return btrfssubvolume.create_bottle_as_subvolume(bottle_path) + + @staticmethod + def delete_all_snapshots(bottle_path): + snapshots_handle = btrfssubvolume.try_create_bottle_snapshots_handle(bottle_path) + if snapshots_handle: + snapshots_handle.delete_all_snapshots() diff --git a/bottles/backend/managers/manager.py b/bottles/backend/managers/manager.py index bf56900924..535279b7f3 100644 --- a/bottles/backend/managers/manager.py +++ b/bottles/backend/managers/manager.py @@ -1557,6 +1557,7 @@ def delete_bottle(self, config: BottleConfig) -> bool: logging.info(f"Removing the bottle…") path = ManagerUtils.get_bottle_path(config) + self.btrfs_subvolume_manager.delete_all_snapshots(path) subprocess.run(["rm", "-rf", path], stdout=subprocess.DEVNULL) self.update_bottles(silent=True) diff --git a/bottles/backend/models/btrfssubvolume.py b/bottles/backend/models/btrfssubvolume.py index 88af37f934..6ccf4ab048 100644 --- a/bottles/backend/models/btrfssubvolume.py +++ b/bottles/backend/models/btrfssubvolume.py @@ -83,8 +83,6 @@ class BottleSnapshotsHandle: """Handle the snapshots of a single bottle created as btrfs subvolume. """ - # TODO delete the snapshots, when the bottle get's deleted - def __init__(self, bottle_path): """Internal should not be called directly. @@ -166,6 +164,15 @@ def set_state(self, snapshot_id: int): _delete_subvolume(tmp_bottle_path) self._save_active_snapshot_id(snapshot_id) + def delete_all_snapshots(self) -> None: + for snapshot_id, metadata in self._snapshots.items(): + snapshot_path = self._snapshot_path2(snapshot_id, metadata.description) + _delete_subvolume(snapshot_path) + try: + os.rmdir(self._snapshots_directory) + except FileNotFoundError: + pass + def try_create_bottle_snapshots_versioning_wrapper(bottle_path): handle = try_create_bottle_snapshots_handle(bottle_path) if not handle: From 2e8fd579f1ebd753148a0298a91726ec57688510 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20R=C3=B6hl?= Date: Sat, 13 Jul 2024 13:59:04 +0200 Subject: [PATCH 16/19] Add btrfsutil to the requirements --- bottles/backend/models/btrfssubvolume.py | 2 -- requirements.txt | 1 + 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/bottles/backend/models/btrfssubvolume.py b/bottles/backend/models/btrfssubvolume.py index 6ccf4ab048..8fa6211739 100644 --- a/bottles/backend/models/btrfssubvolume.py +++ b/bottles/backend/models/btrfssubvolume.py @@ -5,8 +5,6 @@ import os.path import shutil -# TODO Properly document and update dependency to libbtrfsutil -# https://github.com/kdave/btrfs-progs/tree/master/libbtrfsutil import btrfsutil from bottles.backend.models.result import Result diff --git a/requirements.txt b/requirements.txt index 8cf0ebef4a..ae16e9a602 100644 --- a/requirements.txt +++ b/requirements.txt @@ -17,3 +17,4 @@ idna==3.7 urllib3==2.2.1 certifi==2024.2.2 pefile==2023.2.7 +btrfsutil=6.8 From 6908bc5ca63a07eac55cdf67390565b0cf62f605 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20R=C3=B6hl?= Date: Sat, 13 Jul 2024 15:01:17 +0200 Subject: [PATCH 17/19] Restore previous state if the restore from snapshot fails --- bottles/backend/models/btrfssubvolume.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/bottles/backend/models/btrfssubvolume.py b/bottles/backend/models/btrfssubvolume.py index 8fa6211739..5f387162a2 100644 --- a/bottles/backend/models/btrfssubvolume.py +++ b/bottles/backend/models/btrfssubvolume.py @@ -13,7 +13,6 @@ # TODO duplicate bottles as subvolumes. Nice to have, using lightweight # subvolume cloning, if the source bottle is a subvolume. # TODO Add logging -# TODO Better error handling # Internal subvolumes created at initialization time: _internal_subvolumes = [ @@ -151,7 +150,11 @@ def set_state(self, snapshot_id: int): tmp_bottle_path = f"{self._bottle_path}-tmp" snapshot_path = self._snapshot_path(snapshot_id) os.rename(self._bottle_path, tmp_bottle_path) - btrfsutil.create_snapshot(snapshot_path, self._bottle_path, read_only=False) + try: + btrfsutil.create_snapshot(snapshot_path, self._bottle_path, read_only=False) + except btrfsutil.BtrfsUtilError as error: + os.rename(tmp_bottle_path, self._bottle_path) + raise error for internal_subvolume in _internal_subvolumes: source_path = os.path.join(tmp_bottle_path, internal_subvolume) if not os.path.exists(source_path) or not btrfsutil.is_subvolume(source_path): From b5a99c02a7267319488146a68e8f4a1dec58633a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20R=C3=B6hl?= Date: Sat, 13 Jul 2024 15:55:25 +0200 Subject: [PATCH 18/19] Duplicate bottle as subvolumes --- bottles/backend/managers/backup.py | 41 ++++++++++++---------- bottles/backend/models/btrfssubvolume.py | 44 ++++++++++++++++++++++-- 2 files changed, 64 insertions(+), 21 deletions(-) diff --git a/bottles/backend/managers/backup.py b/bottles/backend/managers/backup.py index 7a60311521..4ba32ce428 100644 --- a/bottles/backend/managers/backup.py +++ b/bottles/backend/managers/backup.py @@ -24,6 +24,7 @@ from bottles.backend.globals import Paths from bottles.backend.logger import Logger from bottles.backend.managers.manager import Manager +from bottles.backend.models.btrfssubvolume import duplicate_bottle_as_subvolume, DuplicateResult from bottles.backend.models.config import BottleConfig from bottles.backend.models.result import Result from bottles.backend.state import TaskManager, Task @@ -185,26 +186,28 @@ def _duplicate_bottle_directory( config: BottleConfig, source_path: str, destination_path: str, new_name: str ) -> Result: try: - if not os.path.exists(destination_path): + duplicate_result = duplicate_bottle_as_subvolume(source_path, destination_path) + if not duplicate_result.destination_directories_created(): os.makedirs(destination_path) - for item in [ - "drive_c", - "system.reg", - "user.reg", - "userdef.reg", - "bottle.yml", - ]: - source_item = os.path.join(source_path, item) - destination_item = os.path.join(destination_path, item) - if os.path.isdir(source_item): - shutil.copytree( - source_item, - destination_item, - ignore=shutil.ignore_patterns(".*"), - symlinks=True, - ) - elif os.path.isfile(source_item): - shutil.copy(source_item, destination_item) + if not duplicate_result.bottle_contents_is_duplicated(): + for item in [ + "drive_c", + "system.reg", + "user.reg", + "userdef.reg", + "bottle.yml", + ]: + source_item = os.path.join(source_path, item) + destination_item = os.path.join(destination_path, item) + if os.path.isdir(source_item): + shutil.copytree( + source_item, + destination_item, + ignore=shutil.ignore_patterns(".*"), + symlinks=True, + ) + elif os.path.isfile(source_item): + shutil.copy(source_item, destination_item) # Update the bottle configuration config_path = os.path.join(destination_path, "bottle.yml") diff --git a/bottles/backend/models/btrfssubvolume.py b/bottles/backend/models/btrfssubvolume.py index 5f387162a2..5c2936436b 100644 --- a/bottles/backend/models/btrfssubvolume.py +++ b/bottles/backend/models/btrfssubvolume.py @@ -1,4 +1,5 @@ from dataclasses import dataclass +from enum import Enum import errno from functools import cached_property import os @@ -10,8 +11,6 @@ from bottles.backend.models.result import Result # TODO ask in the GUI, if a bottle should be created as subvolume. -# TODO duplicate bottles as subvolumes. Nice to have, using lightweight -# subvolume cloning, if the source bottle is a subvolume. # TODO Add logging # Internal subvolumes created at initialization time: @@ -58,6 +57,47 @@ def create_bottle_as_subvolume(bottle_path) -> bool: else: return True +class DuplicateResult(Enum): + NOTHING = 1 + EMPTY_SUBVOLUMES = 2 + SNAPSHOTS_FROM_SOURCE = 3 + + def destination_directories_created(self) -> bool: + return not self == DuplicateResult.NOTHING + + def bottle_contents_is_duplicated(self) -> bool: + return self == DuplicateResult.SNAPSHOTS_FROM_SOURCE + +def duplicate_bottle_as_subvolume(source_path, destination_path) -> DuplicateResult: + def create_bare_destination() -> DuplicateResult: + if create_bottle_as_subvolume(destination_path): + return DuplicateResult.EMPTY_SUBVOLUMES + else: + return DuplicateResult.NOTHING + + if not btrfsutil.is_subvolume(source_path): + return create_bare_destination() + else: + try: + btrfsutil.create_snapshot(source_path, destination_path, read_only=False) + except btrfsutil.BtrfsUtilError as error: + match error.btrfsutilerror: + case btrfsutil.ERROR_NOT_BTRFS: + return DuplicateResult.NOTHING + case btrfsutil.ERROR_SNAP_CREATE_FAILED: + return create_bare_destination() + case other: + raise error + for internal_subvolume in _internal_subvolumes: + internal_source_path = os.path.join(source_path, internal_subvolume) + if not btrfsutil.is_subvolume(internal_source_path): + continue + internal_destination_path = os.path.join(destination_path, internal_subvolume) + if os.path.isdir(internal_destination_path): + os.rmdir(internal_destination_path) + btrfsutil.create_snapshot(internal_source_path, internal_destination_path, read_only=False) + return DuplicateResult.SNAPSHOTS_FROM_SOURCE + def try_create_bottle_snapshots_handle(bottle_path): """Try to create a bottle snapshots handle. From 234b633e0346c2a43d6df003d208ec72b62f2e64 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20R=C3=B6hl?= Date: Sat, 13 Jul 2024 16:22:01 +0200 Subject: [PATCH 19/19] Fix some pylint warnings --- bottles/backend/models/btrfssubvolume.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/bottles/backend/models/btrfssubvolume.py b/bottles/backend/models/btrfssubvolume.py index 5c2936436b..8e7614caa4 100644 --- a/bottles/backend/models/btrfssubvolume.py +++ b/bottles/backend/models/btrfssubvolume.py @@ -86,7 +86,7 @@ def create_bare_destination() -> DuplicateResult: return DuplicateResult.NOTHING case btrfsutil.ERROR_SNAP_CREATE_FAILED: return create_bare_destination() - case other: + case _: raise error for internal_subvolume in _internal_subvolumes: internal_source_path = os.path.join(source_path, internal_subvolume) @@ -171,9 +171,9 @@ def read_active_snapshot_id(self) -> int: try: with open(self._active_snapshot_id_path(), "r") as file: return int(file.read()) - except OSError as error: + except OSError: return -1 - + def create_snapshot(self, description: str) -> int: snapshot_id = max(self._snapshots.keys(), default=-1) + 1 snapshot_path = self._snapshot_path2(snapshot_id, description) @@ -232,7 +232,7 @@ def convert_states(self): def is_initialized(self): # Nothing to initialize - return true + return True def re_initialize(self): # Nothing to initialize @@ -257,7 +257,7 @@ def list_states(self) -> Result: data={"state_id": active_state_id, "states": self.convert_states()}, message="Retrieved list of states", ) - + def set_state( self, state_id: int, after: callable ) -> Result: @@ -265,4 +265,3 @@ def set_state( if after: after() return Result(True) -