From 42b902e80c00990f5798b9c150fbd033891cf240 Mon Sep 17 00:00:00 2001 From: Martin Abente Lahaye Date: Tue, 21 May 2024 12:23:28 -0400 Subject: [PATCH 01/46] src: Add basic operations for image management --- .gitignore | 3 + COPYING | 22 +++ README.md | 62 ++++++++ pyproject.toml | 14 ++ src/sysext_utils/__init__.py | 24 +++ src/sysext_utils/commands.py | 141 +++++++++++++++++ src/sysext_utils/config.py | 27 ++++ src/sysext_utils/definitions.py | 50 ++++++ src/sysext_utils/helpers.py | 106 +++++++++++++ src/sysext_utils/main.py | 205 +++++++++++++++++++++++++ src/sysext_utils/utilities/__init__.py | 24 +++ src/sysext_utils/utilities/image.py | 105 +++++++++++++ src/sysext_utils/workflows.py | 79 ++++++++++ 13 files changed, 862 insertions(+) create mode 100644 .gitignore create mode 100644 COPYING create mode 100644 pyproject.toml create mode 100644 src/sysext_utils/__init__.py create mode 100644 src/sysext_utils/commands.py create mode 100644 src/sysext_utils/config.py create mode 100644 src/sysext_utils/definitions.py create mode 100644 src/sysext_utils/helpers.py create mode 100644 src/sysext_utils/main.py create mode 100644 src/sysext_utils/utilities/__init__.py create mode 100644 src/sysext_utils/utilities/image.py create mode 100644 src/sysext_utils/workflows.py diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..3d4f499 --- /dev/null +++ b/.gitignore @@ -0,0 +1,3 @@ +*.pyc +*.egg-info +version.py \ No newline at end of file diff --git a/COPYING b/COPYING new file mode 100644 index 0000000..90366dd --- /dev/null +++ b/COPYING @@ -0,0 +1,22 @@ +MIT License + +Copyright (c) 2024 Codethink Limited +Copyright (c) 2024 GNOME Foundation + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/README.md b/README.md index e69de29..16cdd59 100644 --- a/README.md +++ b/README.md @@ -0,0 +1,62 @@ +# sysext-utils + +A tool that provides a better experience for developing and testing system components on GNOME OS (and eventually other immutable OSes). It facilitates the steps of the workflow described in this [proposal](https://discourse.gnome.org/t/towards-a-better-way-to-hack-and-test-your-system-components/21075). + +## Requirements + +* [Install](https://gitlab.gnome.org/GNOME/gnome-build-meta/-/wikis/gnome_os/Start-GNOME-OS#getting-the-images) the sysupdate variant of GNOME OS. +* Turn that installation in to a development environment by using systemd-sysupdate to [import](https://gitlab.gnome.org/GNOME/gnome-build-meta/-/wikis/gnome_os/Install-Software#install-developer-tools-and-system-extensions-with-systemd-sysext) the development tree sysext. + +## Installation + +```bash +$ git clone https://gitlab.gnome.org/tchx84/sysext-utils.git +$ cd sysext-utils +$ python3 -m venv env +$ source env/bin/activate +$ python -m pip install . +``` + +## Examples + +### Build sysext images out of existing artifacts + +```bash +$ sysext-utils --dry image build sdk ./sdk --ignore-release --format=ddi --private_key=key --certificate=cert +$ sysext-utils --dry image build sdk ./sdk --ignore-release --format=compressed +$ sysext-utils --dry image build sdk ./sdk --format=compressed --integrations schemas +``` + +### Manage sysext images + +```bash +$ sysext-utils --dry image add sdk sdk.sysext.raw +$ sysext-utils --dry image add sdk https://os.gnome.org/sysext/sdk.sysext.raw --runtime +$ sysext-utils --dry image remove sdk +``` + +## License + +MIT License + +Copyright (c) 2024 Codethink Limited + +Copyright (c) 2024 GNOME Foundation + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..9139f5f --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,14 @@ +[build-system] +requires = ["setuptools >= 61.0", "setuptools-scm>=8.0"] +build-backend = "setuptools.build_meta" + +[project] +name = "sysext-utils" +dynamic = ["version"] +requires-python = ">=3.8" + +[project.scripts] +sysext-utils = "sysext_utils:main.main" + +[tool.setuptools_scm] +version_file = "src/sysext_utils/version.py" \ No newline at end of file diff --git a/src/sysext_utils/__init__.py b/src/sysext_utils/__init__.py new file mode 100644 index 0000000..adbf580 --- /dev/null +++ b/src/sysext_utils/__init__.py @@ -0,0 +1,24 @@ +# MIT License +# +# Copyright (c) 2024 Codethink Limited +# Copyright (c) 2024 GNOME Foundation +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# +# SPDX-License-Identifier: MIT diff --git a/src/sysext_utils/commands.py b/src/sysext_utils/commands.py new file mode 100644 index 0000000..0d10a16 --- /dev/null +++ b/src/sysext_utils/commands.py @@ -0,0 +1,141 @@ +# MIT License +# +# Copyright (c) 2024 Codethink Limited +# Copyright (c) 2024 GNOME Foundation +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# +# SPDX-License-Identifier: MIT + +import subprocess + +from . import config + +from .definitions import DryStepType + + +class BaseCommand: + @classmethod + def run(cls, *args) -> None: + raise NotImplementedError() + + @classmethod + def do_run(cls, command: str, *args) -> None: + commands = [command] + list(*args) + + if config.dry_run: + print([str(DryStepType.RUNNING)] + commands) + else: + subprocess.run(commands, capture_output=not config.verbose, check=True) + + +class GLibCompileSchemas(BaseCommand): + @classmethod + def compile(cls, source: str, destination: str) -> None: + cls.run( + [ + "--strict", + source, + "--targetdir", + destination, + ] + ) + + @classmethod + def run(cls, *args) -> None: + cls.do_run("glib-compile-schemas", *args) + + +class Importctl(BaseCommand): + @classmethod + def import_raw(cls, name: str, image: str) -> None: + cls.run( + [ + "import-raw", + "--class=sysext", + image, + name, + ] + ) + + @classmethod + def pull_raw(cls, name: str, url: str) -> None: + cls.run( + [ + "pull-raw", + "--verify=no", + "--class=sysext", + url, + name, + ] + ) + + @classmethod + def run(cls, *args) -> None: + cls.do_run("importctl", *args) + + +class Mksquashfs(BaseCommand): + @classmethod + def build( + cls, + source: str, + path: str, + ) -> None: + cls.run([source, path]) + + @classmethod + def run(cls, *args) -> None: + cls.do_run("mksquashfs", *args) + + +class SystemdRepart(BaseCommand): + @classmethod + def build( + cls, + source: str, + path: str, + private_key: str, + certificate: str, + ) -> None: + cls.run( + [ + "--make-ddi=sysext", + "--offline=true", + "--seed=random", + f"--private-key={private_key}", + f"--certificate={certificate}", + f"--copy-source={source}", + path, + ] + ) + + @classmethod + def run(cls, *args) -> None: + cls.do_run("systemd-repart", *args) + + +class SystemdSysext(BaseCommand): + @classmethod + def refresh(cls) -> None: + cls.run(["refresh"]) + + @classmethod + def run(cls, *args) -> None: + cls.do_run("systemd-sysext", *args) diff --git a/src/sysext_utils/config.py b/src/sysext_utils/config.py new file mode 100644 index 0000000..3ee08b6 --- /dev/null +++ b/src/sysext_utils/config.py @@ -0,0 +1,27 @@ +# MIT License +# +# Copyright (c) 2024 Codethink Limited +# Copyright (c) 2024 GNOME Foundation +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# +# SPDX-License-Identifier: MIT + +dry_run = False +verbose = False diff --git a/src/sysext_utils/definitions.py b/src/sysext_utils/definitions.py new file mode 100644 index 0000000..dc2f8cf --- /dev/null +++ b/src/sysext_utils/definitions.py @@ -0,0 +1,50 @@ +# MIT License +# +# Copyright (c) 2024 Codethink Limited +# Copyright (c) 2024 GNOME Foundation +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# +# SPDX-License-Identifier: MIT + +import os + +from enum import StrEnum + +OS_RELEASE_PATH = os.path.join(os.sep, "etc", "os-release") +RUN_EXTENSIONS_DIR = os.path.join(os.sep, "run", "extensions") +VAR_EXTENSIONS_DIR = os.path.join(os.sep, "var", "lib", "extensions") +SCHEMAS_DIR = os.path.join("usr", "share", "glib-2.0", "schemas") + + +class ImageFormat(StrEnum): + DDI = "ddi" + COMPRESSED = "compressed" + + +class IntegrationType(StrEnum): + SCHEMAS = "schemas" + + +class DryStepType(StrEnum): + RUNNING = "running" + MAKING = "making" + WRITING = "writing" + MOVING = "moving" + REMOVING = "removing" diff --git a/src/sysext_utils/helpers.py b/src/sysext_utils/helpers.py new file mode 100644 index 0000000..c2ffa0c --- /dev/null +++ b/src/sysext_utils/helpers.py @@ -0,0 +1,106 @@ +# MIT License +# +# Copyright (c) 2024 Codethink Limited +# Copyright (c) 2024 GNOME Foundation +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# +# SPDX-License-Identifier: MIT + +import os +import re +import shutil + +from typing import Dict + +from . import config + +from .definitions import ( + RUN_EXTENSIONS_DIR, + VAR_EXTENSIONS_DIR, + OS_RELEASE_PATH, + DryStepType, +) + + +def get_release_data() -> Dict[str, str]: + data = {} + + try: + with open(OS_RELEASE_PATH, encoding="utf-8") as f: + data = dict([l.strip().split("=", 1) for l in f.readlines()]) + except FileNotFoundError: + pass + + return data + + +def get_sysext_metadata(ignore_release: bool) -> str: + if ignore_release: + return "ID=_any" + + data = get_release_data() + return f"ID={data['ID']}\nVERSION_ID={data['VERSION_ID']}" + + +def is_url(string: str) -> bool: + return re.match("http[s]?://", string) is not None + + +def get_path_for_input_sysext_name(name: str, directory: str) -> str: + return os.path.join(directory, f"{name}.sysext.raw") + + +def get_path_for_active_sysext_name(name: str, runtime: bool) -> str: + if runtime: + return os.path.join(RUN_EXTENSIONS_DIR, f"{name}.raw") + + return os.path.join(VAR_EXTENSIONS_DIR, f"{name}.raw") + + +def make_directory(directory: str) -> None: + if config.dry_run: + print([str(DryStepType.MAKING), directory]) + else: + os.makedirs(directory, exist_ok=True) + + +def write_file(content: str, path: str) -> None: + if config.dry_run: + print([str(DryStepType.WRITING), content, path]) + else: + with open(path, "w", encoding="utf-8") as f: + f.write(content) + + +def move_file(source: str, destination: str) -> None: + if config.dry_run: + print([str(DryStepType.MOVING), source, destination]) + else: + shutil.move(source, destination) + + +def remove_file(path: str) -> None: + if config.dry_run: + print([str(DryStepType.REMOVING), path]) + else: + try: + os.remove(path) + except FileNotFoundError: + pass diff --git a/src/sysext_utils/main.py b/src/sysext_utils/main.py new file mode 100644 index 0000000..bfd62ae --- /dev/null +++ b/src/sysext_utils/main.py @@ -0,0 +1,205 @@ +# MIT License +# +# Copyright (c) 2024 Codethink Limited +# Copyright (c) 2024 GNOME Foundation +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# +# SPDX-License-Identifier: MIT + +import os +import argparse + +from . import config +from .definitions import ImageFormat, IntegrationType +from .version import version +from .workflows import ( + run_image_build_workflow, + run_image_add_workflow, + run_image_remove_workflow, +) + + +def _handle_image_build( + parser: argparse.ArgumentParser, + args: argparse.Namespace, +) -> None: + if args.format == ImageFormat.DDI and not (args.private_key and args.certificate): + parser.error("DDI image creation require private_key and certificate") + + run_image_build_workflow( + args.name, + args.artifact, + args.directory, + args.format, + args.private_key, + args.certificate, + args.ignore_release, + args.integrations, + ) + + +def _handle_image_add( + parser: argparse.ArgumentParser, + args: argparse.Namespace, +) -> None: + run_image_add_workflow(args.name, args.image, args.runtime) + + +def _handle_image_remove( + parser: argparse.ArgumentParser, + args: argparse.Namespace, +) -> None: + run_image_remove_workflow(args.name) + + +def main(): + parser = argparse.ArgumentParser( + prog="sysext-utils", + description="A tool to develop and test system components on immutable OSes", + ) + parser.add_argument( + "--version", + help="print the current version of this utility", + action="version", + version=version, + ) + parser.add_argument( + "--dry", + help="prints to stdout the commands involved in the operation", + action="store_true", + ) + parser.add_argument( + "--verbose", + help="captures all system calls stdout and stderr", + action="store_true", + ) + + # Arguments for the different supported utilities + + subparser = parser.add_subparsers( + dest="utility", + help="a supported utility, e.g., image", + required=True, + ) + + # Arguments for image management + + image_parser = subparser.add_parser("image") + image_subparser = image_parser.add_subparsers( + dest="operation", + help="a supported operation, e.g., build", + required=True, + ) + + # Arguments for building an image + + image_build_parser = image_subparser.add_parser( + "build", + help="build a sysext image out of an artifact", + ) + image_build_parser.add_argument( + "name", + help="what to name the sysext e.g., sdk", + ) + image_build_parser.add_argument( + "artifact", + help="path to the input artifact directory e.g., ./sdk/", + ) + image_build_parser.add_argument( + "--format", + help="format to be used for the image creation", + choices=[str(ImageFormat.DDI), str(ImageFormat.COMPRESSED)], + default=ImageFormat.DDI, + ) + image_build_parser.add_argument( + "--private_key", + help="path to the private key file e.g., files/boot-keys/SYSEXT.key", + ) + image_build_parser.add_argument( + "--certificate", + help="path to the certificate file, e.g., files/boot-keys/SYSEXT.crt", + ) + image_build_parser.add_argument( + "--directory", + help="where to place the image to, e.g., ./images/. Defaults to $PWD", + default=os.path.abspath(os.getcwd()), + ) + image_build_parser.add_argument( + "--ignore-release", + help="ignore the host release data and use 'ID=_any' instead", + action="store_true", + ) + image_build_parser.add_argument( + "--integrations", + help="list of integration steps to be run", + nargs="+", + choices=[str(IntegrationType.SCHEMAS)], + default=[], + ) + + # Arguments for adding an image to the host + + image_add_parser = image_subparser.add_parser( + "add", + help="add and activate a sysext image onto host OS", + ) + image_add_parser.add_argument( + "name", + help="name of the sysext e.g., sdk", + ) + image_add_parser.add_argument( + "image", + help="path or URL to the sysext image e.g., sdk.sysext.raw", + ) + image_add_parser.add_argument( + "--runtime", + help="add the sysext image under /run/extensions/ instead", + action="store_true", + ) + + # Arguments for removing a sysext image from the host OS + + image_remove_parser = image_subparser.add_parser( + "remove", + help="deactivate and remove a sysext image from the host OS", + ) + image_remove_parser.add_argument( + "name", + help="name of the the sysext e.g., sdk", + ) + + args = parser.parse_args() + + # Handle global flags + + config.dry_run = args.dry + config.verbose = args.verbose + + # Handle supported operations + + handlers = { + "image": { + "build": _handle_image_build, + "add": _handle_image_add, + "remove": _handle_image_remove, + } + } + + handlers[args.utility][args.operation](parser, args) diff --git a/src/sysext_utils/utilities/__init__.py b/src/sysext_utils/utilities/__init__.py new file mode 100644 index 0000000..adbf580 --- /dev/null +++ b/src/sysext_utils/utilities/__init__.py @@ -0,0 +1,24 @@ +# MIT License +# +# Copyright (c) 2024 Codethink Limited +# Copyright (c) 2024 GNOME Foundation +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# +# SPDX-License-Identifier: MIT diff --git a/src/sysext_utils/utilities/image.py b/src/sysext_utils/utilities/image.py new file mode 100644 index 0000000..e7940fc --- /dev/null +++ b/src/sysext_utils/utilities/image.py @@ -0,0 +1,105 @@ +# MIT License +# +# Copyright (c) 2024 Codethink Limited +# Copyright (c) 2024 GNOME Foundation +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# +# SPDX-License-Identifier: MIT + +import os + +from typing import Optional, List + +from ..definitions import ImageFormat, IntegrationType, RUN_EXTENSIONS_DIR, SCHEMAS_DIR +from ..commands import SystemdRepart +from ..commands import Mksquashfs +from ..commands import Importctl +from ..commands import SystemdSysext +from ..commands import GLibCompileSchemas +from ..helpers import ( + get_sysext_metadata, + is_url, + get_path_for_active_sysext_name, + get_path_for_input_sysext_name, + make_directory, + write_file, + move_file, + remove_file, +) + + +def build( + name: str, + artifact: str, + directory: str, + image_format: str, + private_key: Optional[str], + certificate: Optional[str], + ignore_release: bool, +) -> None: + image_path = get_path_for_input_sysext_name(name, directory) + + remove_file(image_path) + make_directory(directory) + + metadata_directory = os.path.join(artifact, "usr", "lib", "extension-release.d") + metadata_path = os.path.join(metadata_directory, f"extension-release.{name}") + metadata_content = get_sysext_metadata(ignore_release) + + make_directory(metadata_directory) + write_file(metadata_content, metadata_path) + + if image_format == ImageFormat.DDI and private_key and certificate: + SystemdRepart.build(artifact, image_path, private_key, certificate) + else: + Mksquashfs.build(artifact, image_path) + + +def add(name: str, image: str, runtime: bool) -> None: + if is_url(image): + Importctl.pull_raw(name, image) + else: + Importctl.import_raw(name, image) + + # See https://github.com/systemd/systemd/issues/32938 + if runtime is True: + make_directory(RUN_EXTENSIONS_DIR) + move_file( + get_path_for_active_sysext_name(name, runtime=False), + get_path_for_active_sysext_name(name, runtime=True), + ) + + SystemdSysext.refresh() + + +def remove(name: str) -> None: + remove_file(get_path_for_active_sysext_name(name, runtime=True)) + remove_file(get_path_for_active_sysext_name(name, runtime=False)) + + SystemdSysext.refresh() + + +def integrate(artifact: str, integrations: List[str]) -> None: + if IntegrationType.SCHEMAS in integrations: + host_schemas_dir = os.path.join(os.sep, SCHEMAS_DIR) + artifact_schemas_dir = os.path.join(artifact, SCHEMAS_DIR) + + make_directory(artifact_schemas_dir) + GLibCompileSchemas.compile(host_schemas_dir, artifact_schemas_dir) diff --git a/src/sysext_utils/workflows.py b/src/sysext_utils/workflows.py new file mode 100644 index 0000000..2664444 --- /dev/null +++ b/src/sysext_utils/workflows.py @@ -0,0 +1,79 @@ +# MIT License +# +# Copyright (c) 2024 Codethink Limited +# Copyright (c) 2024 GNOME Foundation +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# +# SPDX-License-Identifier: MIT + +from typing import List + +from .helpers import get_path_for_input_sysext_name +from .utilities import image as image_utility + + +def run_image_build_workflow( + name: str, + artifact: str, + directory: str, + image_format: str, + private_key: str, + certificate: str, + ignore_release: bool, + integrations: List[str], +) -> None: + image_utility.build( + name, + artifact, + directory, + image_format, + private_key, + certificate, + ignore_release, + ) + + if not integrations: + return + + image_name = get_path_for_input_sysext_name(name, directory) + + image_utility.remove(name) + image_utility.add(name, image_name, runtime=False) + image_utility.integrate(artifact, integrations) + image_utility.remove(name) + + image_utility.build( + name, + artifact, + directory, + image_format, + private_key, + certificate, + ignore_release, + ) + + +def run_image_add_workflow(name: str, image: str, runtime: bool) -> None: + image_utility.remove(name) + image_utility.add(name, image, runtime) + + +def run_image_remove_workflow(name: str) -> None: + image_utility.remove(name) -- GitLab From 1d985233c543becb086c487af334bbb4737f9df4 Mon Sep 17 00:00:00 2001 From: Martin Abente Lahaye Date: Fri, 24 May 2024 13:20:00 -0400 Subject: [PATCH 02/46] tests: Add initial tests --- .gitlab-ci.yml | 13 +++++ pyproject.toml | 22 +++++++- tests/__init__.py | 24 +++++++++ tests/test_workflows.py | 108 ++++++++++++++++++++++++++++++++++++++++ 4 files changed, 166 insertions(+), 1 deletion(-) create mode 100644 .gitlab-ci.yml create mode 100644 tests/__init__.py create mode 100644 tests/test_workflows.py diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml new file mode 100644 index 0000000..2904877 --- /dev/null +++ b/.gitlab-ci.yml @@ -0,0 +1,13 @@ +image: python:latest + +before_script: + - python3 -m venv env + - source env/bin/activate + +test: + script: + - python -m pip install .[dev,test] + - python -m black src tests + - python -m pyflakes src tests + - python -m mypy src tests + - python -m pytest diff --git a/pyproject.toml b/pyproject.toml index 9139f5f..1eb91d3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -11,4 +11,24 @@ requires-python = ">=3.8" sysext-utils = "sysext_utils:main.main" [tool.setuptools_scm] -version_file = "src/sysext_utils/version.py" \ No newline at end of file +version_file = "src/sysext_utils/version.py" + +[project.optional-dependencies] +dev = [ + "black >=24.4.2", + "pyflakes >=3.2.0", + "mypy >=1.10.0", +] +test = [ + "pytest-cov >=5.0.0", +] + +[tool.pytest.ini_options] +addopts = "--cov --cov-report html --cov-report term-missing --cov-fail-under 90" + +[tool.coverage.run] +source = ["sysext_utils"] +omit = [ + "main.py", + "version.py", +] diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 0000000..adbf580 --- /dev/null +++ b/tests/__init__.py @@ -0,0 +1,24 @@ +# MIT License +# +# Copyright (c) 2024 Codethink Limited +# Copyright (c) 2024 GNOME Foundation +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# +# SPDX-License-Identifier: MIT diff --git a/tests/test_workflows.py b/tests/test_workflows.py new file mode 100644 index 0000000..0ac428e --- /dev/null +++ b/tests/test_workflows.py @@ -0,0 +1,108 @@ +# MIT License +# +# Copyright (c) 2024 Codethink Limited +# Copyright (c) 2024 GNOME Foundation +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# +# SPDX-License-Identifier: MIT + +from sysext_utils import config +from sysext_utils.definitions import ImageFormat, IntegrationType +from sysext_utils.workflows import ( + run_image_build_workflow, + run_image_add_workflow, + run_image_remove_workflow, +) + + +def setup_module(): + config.dry_run = True + + +def test_image_build_workflow(): + run_image_build_workflow( + name="name", + artifact="artifact", + directory="directory", + image_format=ImageFormat.DDI, + private_key="private_key", + certificate="certificate", + ignore_release=False, + integrations=[], + ) + + +def test_image_build_workflow_with_compressed_format(): + run_image_build_workflow( + name="name", + artifact="artifact", + directory="directory", + image_format=ImageFormat.COMPRESSED, + private_key="private_key", + certificate="certificate", + ignore_release=False, + integrations=[], + ) + + +def test_image_build_workflow_with_integrations(): + run_image_build_workflow( + name="name", + artifact="artifact", + directory="directory", + image_format=ImageFormat.COMPRESSED, + private_key="private_key", + certificate="certificate", + ignore_release=False, + integrations=[ + IntegrationType.SCHEMAS, + ], + ) + + +def test_image_build_work_with_ignore_release(): + run_image_build_workflow( + name="name", + artifact="artifact", + directory="directory", + image_format=ImageFormat.COMPRESSED, + private_key="private_key", + certificate="certificate", + ignore_release=True, + integrations=[ + IntegrationType.SCHEMAS, + ], + ) + + +def test_image_add_workflow(): + run_image_add_workflow(name="name", image="image", runtime=False) + + +def test_image_add_workflow_with_remote_image(): + run_image_add_workflow(name="name", image="https://image", runtime=False) + + +def test_image_add_workflow_with_runtime_flag(): + run_image_add_workflow(name="name", image="image", runtime=True) + + +def test_image_remove_workflow(): + run_image_remove_workflow(name="name") -- GitLab From 796ed7d01cd824cab9fb1c396ebbb6e53ec51467 Mon Sep 17 00:00:00 2001 From: Martin Abente Lahaye Date: Mon, 27 May 2024 11:06:57 -0400 Subject: [PATCH 03/46] utilities: Add basic operation to build artifacts --- README.md | 6 ++++ src/sysext_utils/commands.py | 26 ++++++++++++++ src/sysext_utils/definitions.py | 4 +++ src/sysext_utils/helpers.py | 7 ++++ src/sysext_utils/main.py | 50 ++++++++++++++++++++++++-- src/sysext_utils/utilities/artifact.py | 45 +++++++++++++++++++++++ src/sysext_utils/workflows.py | 9 +++++ tests/test_workflows.py | 11 +++++- 8 files changed, 155 insertions(+), 3 deletions(-) create mode 100644 src/sysext_utils/utilities/artifact.py diff --git a/README.md b/README.md index 16cdd59..9fb8f95 100644 --- a/README.md +++ b/README.md @@ -19,6 +19,12 @@ $ python -m pip install . ## Examples +### Build projects artifacts + +```bash +$ sysext-utils --dry artifact build ./sdk bst sdk/gtk.bst sdk/libadwaita.bst +``` + ### Build sysext images out of existing artifacts ```bash diff --git a/src/sysext_utils/commands.py b/src/sysext_utils/commands.py index 0d10a16..80c9360 100644 --- a/src/sysext_utils/commands.py +++ b/src/sysext_utils/commands.py @@ -25,6 +25,8 @@ import subprocess +from typing import List + from . import config from .definitions import DryStepType @@ -139,3 +141,27 @@ class SystemdSysext(BaseCommand): @classmethod def run(cls, *args) -> None: cls.do_run("systemd-sysext", *args) + + +class Bst(BaseCommand): + @classmethod + def build(cls, elements: List[str]) -> None: + cls.run(["build", "--retry-failed"] + elements) + + @classmethod + def artifact_checkout(cls, element: str, directory: str) -> None: + cls.run( + [ + "artifact", + "checkout", + "--force", + "--deps=none", + "--directory", + directory, + element, + ] + ) + + @classmethod + def run(cls, *args) -> None: + cls.do_run("bst", *args) diff --git a/src/sysext_utils/definitions.py b/src/sysext_utils/definitions.py index dc2f8cf..034d4bb 100644 --- a/src/sysext_utils/definitions.py +++ b/src/sysext_utils/definitions.py @@ -48,3 +48,7 @@ class DryStepType(StrEnum): WRITING = "writing" MOVING = "moving" REMOVING = "removing" + + +class BuildSystemType(StrEnum): + BST = "bst" diff --git a/src/sysext_utils/helpers.py b/src/sysext_utils/helpers.py index c2ffa0c..b530cc2 100644 --- a/src/sysext_utils/helpers.py +++ b/src/sysext_utils/helpers.py @@ -81,6 +81,13 @@ def make_directory(directory: str) -> None: os.makedirs(directory, exist_ok=True) +def remove_directory(directory: str) -> None: + if config.dry_run: + print([str(DryStepType.REMOVING), directory]) + else: + shutil.rmtree(directory, ignore_errors=True) + + def write_file(content: str, path: str) -> None: if config.dry_run: print([str(DryStepType.WRITING), content, path]) diff --git a/src/sysext_utils/main.py b/src/sysext_utils/main.py index bfd62ae..8052f47 100644 --- a/src/sysext_utils/main.py +++ b/src/sysext_utils/main.py @@ -27,12 +27,13 @@ import os import argparse from . import config -from .definitions import ImageFormat, IntegrationType +from .definitions import ImageFormat, IntegrationType, BuildSystemType from .version import version from .workflows import ( run_image_build_workflow, run_image_add_workflow, run_image_remove_workflow, + run_artifact_build_workflow, ) @@ -69,6 +70,16 @@ def _handle_image_remove( run_image_remove_workflow(args.name) +def _handle_artifact_build( + parser: argparse.ArgumentParser, + args: argparse.Namespace, +) -> None: + if args.system == BuildSystemType.BST and not args.arguments: + parser.error("One or more elements are required") + + run_artifact_build_workflow(args.directory, args.system, args.arguments) + + def main(): parser = argparse.ArgumentParser( prog="sysext-utils", @@ -185,6 +196,38 @@ def main(): help="name of the the sysext e.g., sdk", ) + # Arguments for artifact management + + artifact_parser = subparser.add_parser("artifact") + artifact_subparser = artifact_parser.add_subparsers( + dest="operation", + help="a supported operation, e.g., build", + required=True, + ) + + # Arguments for building an artifact + + artifact_build_parser = artifact_subparser.add_parser( + "build", + help="build artifact out of a project", + ) + artifact_build_parser.add_argument( + "directory", + help="path to the output artifact directory", + ) + artifact_build_parser.add_argument( + "system", + help="build system to use e.g., bst", + choices=[str(BuildSystemType.BST)], + default=BuildSystemType.BST, + ) + artifact_build_parser.add_argument( + "arguments", + help="list of build-system specific arguments", + nargs=argparse.REMAINDER, + default=[], + ) + args = parser.parse_args() # Handle global flags @@ -199,7 +242,10 @@ def main(): "build": _handle_image_build, "add": _handle_image_add, "remove": _handle_image_remove, - } + }, + "artifact": { + "build": _handle_artifact_build, + }, } handlers[args.utility][args.operation](parser, args) diff --git a/src/sysext_utils/utilities/artifact.py b/src/sysext_utils/utilities/artifact.py new file mode 100644 index 0000000..9d1635b --- /dev/null +++ b/src/sysext_utils/utilities/artifact.py @@ -0,0 +1,45 @@ +# MIT License +# +# Copyright (c) 2024 Codethink Limited +# Copyright (c) 2024 GNOME Foundation +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# +# SPDX-License-Identifier: MIT + +from typing import List + +from ..commands import Bst +from ..definitions import BuildSystemType +from ..helpers import make_directory, remove_directory + + +def build(directory: str, system: str, arguments: List[str]) -> None: + remove_directory(directory) + make_directory(directory) + + if system == BuildSystemType.BST: + build_bst_elements(directory, arguments) + + +def build_bst_elements(directory: str, elements: List[str]) -> None: + Bst.build(elements) + + for element in elements: + Bst.artifact_checkout(element, directory) diff --git a/src/sysext_utils/workflows.py b/src/sysext_utils/workflows.py index 2664444..816dc92 100644 --- a/src/sysext_utils/workflows.py +++ b/src/sysext_utils/workflows.py @@ -27,6 +27,7 @@ from typing import List from .helpers import get_path_for_input_sysext_name from .utilities import image as image_utility +from .utilities import artifact as artifact_utility def run_image_build_workflow( @@ -77,3 +78,11 @@ def run_image_add_workflow(name: str, image: str, runtime: bool) -> None: def run_image_remove_workflow(name: str) -> None: image_utility.remove(name) + + +def run_artifact_build_workflow( + directory: str, + system: str, + arguments: List[str], +) -> None: + artifact_utility.build(directory, system, arguments) diff --git a/tests/test_workflows.py b/tests/test_workflows.py index 0ac428e..c371b0e 100644 --- a/tests/test_workflows.py +++ b/tests/test_workflows.py @@ -24,11 +24,12 @@ # SPDX-License-Identifier: MIT from sysext_utils import config -from sysext_utils.definitions import ImageFormat, IntegrationType +from sysext_utils.definitions import ImageFormat, IntegrationType, BuildSystemType from sysext_utils.workflows import ( run_image_build_workflow, run_image_add_workflow, run_image_remove_workflow, + run_artifact_build_workflow, ) @@ -106,3 +107,11 @@ def test_image_add_workflow_with_runtime_flag(): def test_image_remove_workflow(): run_image_remove_workflow(name="name") + + +def test_artifact_build_workflow(): + run_artifact_build_workflow( + directory="directory", + system=BuildSystemType.BST, + arguments=["element/one.bst"], + ) -- GitLab From 35e6bab30ec12003f40bec31a6dbba9ac73c6bb1 Mon Sep 17 00:00:00 2001 From: Martin Abente Lahaye Date: Mon, 27 May 2024 11:45:36 -0400 Subject: [PATCH 04/46] aritfact: Add meson support --- README.md | 1 + src/sysext_utils/commands.py | 18 ++++++++++++++++++ src/sysext_utils/definitions.py | 1 + src/sysext_utils/main.py | 2 +- src/sysext_utils/utilities/artifact.py | 18 +++++++++++++++++- tests/test_workflows.py | 8 ++++++++ 6 files changed, 46 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 9fb8f95..bec5e2c 100644 --- a/README.md +++ b/README.md @@ -23,6 +23,7 @@ $ python -m pip install . ```bash $ sysext-utils --dry artifact build ./sdk bst sdk/gtk.bst sdk/libadwaita.bst +$ sysext-utils --dry artifact build ./sdk meson -Dsysprof=enabled ``` ### Build sysext images out of existing artifacts diff --git a/src/sysext_utils/commands.py b/src/sysext_utils/commands.py index 80c9360..38df1f4 100644 --- a/src/sysext_utils/commands.py +++ b/src/sysext_utils/commands.py @@ -165,3 +165,21 @@ class Bst(BaseCommand): @classmethod def run(cls, *args) -> None: cls.do_run("bst", *args) + + +class Meson(BaseCommand): + @classmethod + def setup(cls, build_directory: str, arguments: List[str]) -> None: + cls.run(["setup", build_directory, "--prefix", "/usr"] + arguments) + + @classmethod + def compile(cls, build_directory: str) -> None: + cls.run(["compile", "-C", build_directory]) + + @classmethod + def install(cls, build_directory: str, destination: str) -> None: + cls.run(["install", "-C", build_directory, "--destdir", destination]) + + @classmethod + def run(cls, *args) -> None: + cls.do_run("meson", *args) diff --git a/src/sysext_utils/definitions.py b/src/sysext_utils/definitions.py index 034d4bb..eee5955 100644 --- a/src/sysext_utils/definitions.py +++ b/src/sysext_utils/definitions.py @@ -52,3 +52,4 @@ class DryStepType(StrEnum): class BuildSystemType(StrEnum): BST = "bst" + MESON = "meson" diff --git a/src/sysext_utils/main.py b/src/sysext_utils/main.py index 8052f47..f845a22 100644 --- a/src/sysext_utils/main.py +++ b/src/sysext_utils/main.py @@ -218,7 +218,7 @@ def main(): artifact_build_parser.add_argument( "system", help="build system to use e.g., bst", - choices=[str(BuildSystemType.BST)], + choices=[str(BuildSystemType.BST), str(BuildSystemType.MESON)], default=BuildSystemType.BST, ) artifact_build_parser.add_argument( diff --git a/src/sysext_utils/utilities/artifact.py b/src/sysext_utils/utilities/artifact.py index 9d1635b..18290ea 100644 --- a/src/sysext_utils/utilities/artifact.py +++ b/src/sysext_utils/utilities/artifact.py @@ -23,9 +23,11 @@ # # SPDX-License-Identifier: MIT +import os + from typing import List -from ..commands import Bst +from ..commands import Bst, Meson from ..definitions import BuildSystemType from ..helpers import make_directory, remove_directory @@ -36,6 +38,8 @@ def build(directory: str, system: str, arguments: List[str]) -> None: if system == BuildSystemType.BST: build_bst_elements(directory, arguments) + if system == BuildSystemType.MESON: + build_meson_project(directory, arguments) def build_bst_elements(directory: str, elements: List[str]) -> None: @@ -43,3 +47,15 @@ def build_bst_elements(directory: str, elements: List[str]) -> None: for element in elements: Bst.artifact_checkout(element, directory) + + +def build_meson_project(directory: str, arguments: List[str]) -> None: + current_directory = os.getcwd() + build_directory = os.path.join(current_directory, "_build") + dest_directory = os.path.join(current_directory, directory) + + remove_directory(build_directory) + + Meson.setup(build_directory, arguments) + Meson.compile(build_directory) + Meson.install(build_directory, dest_directory) diff --git a/tests/test_workflows.py b/tests/test_workflows.py index c371b0e..780ced7 100644 --- a/tests/test_workflows.py +++ b/tests/test_workflows.py @@ -115,3 +115,11 @@ def test_artifact_build_workflow(): system=BuildSystemType.BST, arguments=["element/one.bst"], ) + + +def test_artifact_build_workflow_with_meson(): + run_artifact_build_workflow( + directory="directory", + system=BuildSystemType.MESON, + arguments=[], + ) -- GitLab From 177137605325dd52910537f8e276184f9258f9bb Mon Sep 17 00:00:00 2001 From: Martin Abente Lahaye Date: Wed, 29 May 2024 14:56:22 -0400 Subject: [PATCH 05/46] utilities: Reorganize code to faciliate following refactoring --- .../{utilities/image.py => utilities.py} | 62 +++++++++++++++---- src/sysext_utils/utilities/__init__.py | 24 ------- src/sysext_utils/utilities/artifact.py | 61 ------------------ src/sysext_utils/workflows.py | 29 +++++---- 4 files changed, 68 insertions(+), 108 deletions(-) rename src/sysext_utils/{utilities/image.py => utilities.py} (68%) delete mode 100644 src/sysext_utils/utilities/__init__.py delete mode 100644 src/sysext_utils/utilities/artifact.py diff --git a/src/sysext_utils/utilities/image.py b/src/sysext_utils/utilities.py similarity index 68% rename from src/sysext_utils/utilities/image.py rename to src/sysext_utils/utilities.py index e7940fc..1732b3d 100644 --- a/src/sysext_utils/utilities/image.py +++ b/src/sysext_utils/utilities.py @@ -27,25 +27,36 @@ import os from typing import Optional, List -from ..definitions import ImageFormat, IntegrationType, RUN_EXTENSIONS_DIR, SCHEMAS_DIR -from ..commands import SystemdRepart -from ..commands import Mksquashfs -from ..commands import Importctl -from ..commands import SystemdSysext -from ..commands import GLibCompileSchemas -from ..helpers import ( +from .definitions import ( + ImageFormat, + IntegrationType, + BuildSystemType, + RUN_EXTENSIONS_DIR, + SCHEMAS_DIR, +) +from .commands import ( + SystemdRepart, + Mksquashfs, + Importctl, + SystemdSysext, + GLibCompileSchemas, + Bst, + Meson, +) +from .helpers import ( get_sysext_metadata, is_url, get_path_for_active_sysext_name, get_path_for_input_sysext_name, make_directory, + remove_directory, write_file, move_file, remove_file, ) -def build( +def build_image( name: str, artifact: str, directory: str, @@ -72,7 +83,7 @@ def build( Mksquashfs.build(artifact, image_path) -def add(name: str, image: str, runtime: bool) -> None: +def add_image(name: str, image: str, runtime: bool) -> None: if is_url(image): Importctl.pull_raw(name, image) else: @@ -89,17 +100,46 @@ def add(name: str, image: str, runtime: bool) -> None: SystemdSysext.refresh() -def remove(name: str) -> None: +def remove_image(name: str) -> None: remove_file(get_path_for_active_sysext_name(name, runtime=True)) remove_file(get_path_for_active_sysext_name(name, runtime=False)) SystemdSysext.refresh() -def integrate(artifact: str, integrations: List[str]) -> None: +def integrate_image(artifact: str, integrations: List[str]) -> None: if IntegrationType.SCHEMAS in integrations: host_schemas_dir = os.path.join(os.sep, SCHEMAS_DIR) artifact_schemas_dir = os.path.join(artifact, SCHEMAS_DIR) make_directory(artifact_schemas_dir) GLibCompileSchemas.compile(host_schemas_dir, artifact_schemas_dir) + + +def build_artifact(directory: str, system: str, arguments: List[str]) -> None: + remove_directory(directory) + make_directory(directory) + + if system == BuildSystemType.BST: + build_artifact_bst_elements(directory, arguments) + if system == BuildSystemType.MESON: + build_artifact_meson_project(directory, arguments) + + +def build_artifact_bst_elements(directory: str, elements: List[str]) -> None: + Bst.build(elements) + + for element in elements: + Bst.artifact_checkout(element, directory) + + +def build_artifact_meson_project(directory: str, arguments: List[str]) -> None: + current_directory = os.getcwd() + build_directory = os.path.join(current_directory, "_build") + dest_directory = os.path.join(current_directory, directory) + + remove_directory(build_directory) + + Meson.setup(build_directory, arguments) + Meson.compile(build_directory) + Meson.install(build_directory, dest_directory) diff --git a/src/sysext_utils/utilities/__init__.py b/src/sysext_utils/utilities/__init__.py deleted file mode 100644 index adbf580..0000000 --- a/src/sysext_utils/utilities/__init__.py +++ /dev/null @@ -1,24 +0,0 @@ -# MIT License -# -# Copyright (c) 2024 Codethink Limited -# Copyright (c) 2024 GNOME Foundation -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in all -# copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. -# -# SPDX-License-Identifier: MIT diff --git a/src/sysext_utils/utilities/artifact.py b/src/sysext_utils/utilities/artifact.py deleted file mode 100644 index 18290ea..0000000 --- a/src/sysext_utils/utilities/artifact.py +++ /dev/null @@ -1,61 +0,0 @@ -# MIT License -# -# Copyright (c) 2024 Codethink Limited -# Copyright (c) 2024 GNOME Foundation -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in all -# copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. -# -# SPDX-License-Identifier: MIT - -import os - -from typing import List - -from ..commands import Bst, Meson -from ..definitions import BuildSystemType -from ..helpers import make_directory, remove_directory - - -def build(directory: str, system: str, arguments: List[str]) -> None: - remove_directory(directory) - make_directory(directory) - - if system == BuildSystemType.BST: - build_bst_elements(directory, arguments) - if system == BuildSystemType.MESON: - build_meson_project(directory, arguments) - - -def build_bst_elements(directory: str, elements: List[str]) -> None: - Bst.build(elements) - - for element in elements: - Bst.artifact_checkout(element, directory) - - -def build_meson_project(directory: str, arguments: List[str]) -> None: - current_directory = os.getcwd() - build_directory = os.path.join(current_directory, "_build") - dest_directory = os.path.join(current_directory, directory) - - remove_directory(build_directory) - - Meson.setup(build_directory, arguments) - Meson.compile(build_directory) - Meson.install(build_directory, dest_directory) diff --git a/src/sysext_utils/workflows.py b/src/sysext_utils/workflows.py index 816dc92..e3ae51e 100644 --- a/src/sysext_utils/workflows.py +++ b/src/sysext_utils/workflows.py @@ -26,8 +26,13 @@ from typing import List from .helpers import get_path_for_input_sysext_name -from .utilities import image as image_utility -from .utilities import artifact as artifact_utility +from .utilities import ( + build_image, + add_image, + remove_image, + integrate_image, + build_artifact, +) def run_image_build_workflow( @@ -40,7 +45,7 @@ def run_image_build_workflow( ignore_release: bool, integrations: List[str], ) -> None: - image_utility.build( + build_image( name, artifact, directory, @@ -55,12 +60,12 @@ def run_image_build_workflow( image_name = get_path_for_input_sysext_name(name, directory) - image_utility.remove(name) - image_utility.add(name, image_name, runtime=False) - image_utility.integrate(artifact, integrations) - image_utility.remove(name) + remove_image(name) + add_image(name, image_name, runtime=False) + integrate_image(artifact, integrations) + remove_image(name) - image_utility.build( + build_image( name, artifact, directory, @@ -72,12 +77,12 @@ def run_image_build_workflow( def run_image_add_workflow(name: str, image: str, runtime: bool) -> None: - image_utility.remove(name) - image_utility.add(name, image, runtime) + remove_image(name) + add_image(name, image, runtime) def run_image_remove_workflow(name: str) -> None: - image_utility.remove(name) + remove_image(name) def run_artifact_build_workflow( @@ -85,4 +90,4 @@ def run_artifact_build_workflow( system: str, arguments: List[str], ) -> None: - artifact_utility.build(directory, system, arguments) + build_artifact(directory, system, arguments) -- GitLab From ddfd075397ae491041f89a4d0a0ba9d5b50c3803 Mon Sep 17 00:00:00 2001 From: Martin Abente Lahaye Date: Wed, 29 May 2024 15:04:18 -0400 Subject: [PATCH 06/46] main: Remove subcommands for simplified verbs --- README.md | 16 +++++++------- src/sysext_utils/main.py | 48 +++++++++++----------------------------- 2 files changed, 21 insertions(+), 43 deletions(-) diff --git a/README.md b/README.md index bec5e2c..6c1929a 100644 --- a/README.md +++ b/README.md @@ -22,24 +22,24 @@ $ python -m pip install . ### Build projects artifacts ```bash -$ sysext-utils --dry artifact build ./sdk bst sdk/gtk.bst sdk/libadwaita.bst -$ sysext-utils --dry artifact build ./sdk meson -Dsysprof=enabled +$ sysext-utils --dry build ./sdk bst sdk/gtk.bst sdk/libadwaita.bst +$ sysext-utils --dry build ./sdk meson -Dsysprof=enabled ``` ### Build sysext images out of existing artifacts ```bash -$ sysext-utils --dry image build sdk ./sdk --ignore-release --format=ddi --private_key=key --certificate=cert -$ sysext-utils --dry image build sdk ./sdk --ignore-release --format=compressed -$ sysext-utils --dry image build sdk ./sdk --format=compressed --integrations schemas +$ sysext-utils --dry create sdk ./sdk --ignore-release --format=ddi --private_key=key --certificate=cert +$ sysext-utils --dry create sdk ./sdk --ignore-release --format=compressed +$ sysext-utils --dry create sdk ./sdk --format=compressed --integrations schemas ``` ### Manage sysext images ```bash -$ sysext-utils --dry image add sdk sdk.sysext.raw -$ sysext-utils --dry image add sdk https://os.gnome.org/sysext/sdk.sysext.raw --runtime -$ sysext-utils --dry image remove sdk +$ sysext-utils --dry add sdk sdk.sysext.raw +$ sysext-utils --dry add sdk https://os.gnome.org/sysext/sdk.sysext.raw --runtime +$ sysext-utils --dry remove sdk ``` ## License diff --git a/src/sysext_utils/main.py b/src/sysext_utils/main.py index f845a22..ff0acd2 100644 --- a/src/sysext_utils/main.py +++ b/src/sysext_utils/main.py @@ -37,7 +37,7 @@ from .workflows import ( ) -def _handle_image_build( +def _handle_image_create( parser: argparse.ArgumentParser, args: argparse.Namespace, ) -> None: @@ -105,25 +105,16 @@ def main(): # Arguments for the different supported utilities subparser = parser.add_subparsers( - dest="utility", - help="a supported utility, e.g., image", - required=True, - ) - - # Arguments for image management - - image_parser = subparser.add_parser("image") - image_subparser = image_parser.add_subparsers( dest="operation", help="a supported operation, e.g., build", required=True, ) - # Arguments for building an image + # Arguments for creating an image - image_build_parser = image_subparser.add_parser( - "build", - help="build a sysext image out of an artifact", + image_build_parser = subparser.add_parser( + "create", + help="create a sysext image out of an artifact", ) image_build_parser.add_argument( "name", @@ -167,7 +158,7 @@ def main(): # Arguments for adding an image to the host - image_add_parser = image_subparser.add_parser( + image_add_parser = subparser.add_parser( "add", help="add and activate a sysext image onto host OS", ) @@ -187,7 +178,7 @@ def main(): # Arguments for removing a sysext image from the host OS - image_remove_parser = image_subparser.add_parser( + image_remove_parser = subparser.add_parser( "remove", help="deactivate and remove a sysext image from the host OS", ) @@ -196,18 +187,9 @@ def main(): help="name of the the sysext e.g., sdk", ) - # Arguments for artifact management - - artifact_parser = subparser.add_parser("artifact") - artifact_subparser = artifact_parser.add_subparsers( - dest="operation", - help="a supported operation, e.g., build", - required=True, - ) - # Arguments for building an artifact - artifact_build_parser = artifact_subparser.add_parser( + artifact_build_parser = subparser.add_parser( "build", help="build artifact out of a project", ) @@ -238,14 +220,10 @@ def main(): # Handle supported operations handlers = { - "image": { - "build": _handle_image_build, - "add": _handle_image_add, - "remove": _handle_image_remove, - }, - "artifact": { - "build": _handle_artifact_build, - }, + "create": _handle_image_create, + "add": _handle_image_add, + "remove": _handle_image_remove, + "build": _handle_artifact_build, } - handlers[args.utility][args.operation](parser, args) + handlers[args.operation](parser, args) -- GitLab From 19d34121ff8ceca9b629b63e3bdece599ade4724 Mon Sep 17 00:00:00 2001 From: Martin Abente Lahaye Date: Wed, 29 May 2024 15:15:18 -0400 Subject: [PATCH 07/46] main: Set compressed as the default image format --- README.md | 7 ++++--- src/sysext_utils/main.py | 2 +- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 6c1929a..d5f1d4a 100644 --- a/README.md +++ b/README.md @@ -29,9 +29,10 @@ $ sysext-utils --dry build ./sdk meson -Dsysprof=enabled ### Build sysext images out of existing artifacts ```bash -$ sysext-utils --dry create sdk ./sdk --ignore-release --format=ddi --private_key=key --certificate=cert -$ sysext-utils --dry create sdk ./sdk --ignore-release --format=compressed -$ sysext-utils --dry create sdk ./sdk --format=compressed --integrations schemas +$ sysext-utils --dry create sdk ./sdk +$ sysext-utils --dry create sdk ./sdk --ignore-release +$ sysext-utils --dry create sdk ./sdk --integrations schemas +$ sysext-utils --dry create sdk ./sdk --format=ddi --private_key=key --certificate=cert ``` ### Manage sysext images diff --git a/src/sysext_utils/main.py b/src/sysext_utils/main.py index ff0acd2..7ba83a8 100644 --- a/src/sysext_utils/main.py +++ b/src/sysext_utils/main.py @@ -128,7 +128,7 @@ def main(): "--format", help="format to be used for the image creation", choices=[str(ImageFormat.DDI), str(ImageFormat.COMPRESSED)], - default=ImageFormat.DDI, + default=ImageFormat.COMPRESSED, ) image_build_parser.add_argument( "--private_key", -- GitLab From 027b9784cd6e22463a9ba1835e527917583fc431 Mon Sep 17 00:00:00 2001 From: Martin Abente Lahaye Date: Wed, 29 May 2024 15:34:22 -0400 Subject: [PATCH 08/46] main: Remove name from add operation If we stick to this convention we can save up an unnecessary command. --- README.md | 4 ++-- src/sysext_utils/definitions.py | 1 + src/sysext_utils/helpers.py | 11 +++++++++++ src/sysext_utils/main.py | 6 +----- src/sysext_utils/workflows.py | 6 ++++-- tests/test_workflows.py | 6 +++--- 6 files changed, 22 insertions(+), 12 deletions(-) diff --git a/README.md b/README.md index d5f1d4a..cf2e9a1 100644 --- a/README.md +++ b/README.md @@ -38,8 +38,8 @@ $ sysext-utils --dry create sdk ./sdk --format=ddi --private_key=key --certifica ### Manage sysext images ```bash -$ sysext-utils --dry add sdk sdk.sysext.raw -$ sysext-utils --dry add sdk https://os.gnome.org/sysext/sdk.sysext.raw --runtime +$ sysext-utils --dry add sdk.sysext.raw +$ sysext-utils --dry add https://os.gnome.org/sysext/sdk.sysext.raw --runtime $ sysext-utils --dry remove sdk ``` diff --git a/src/sysext_utils/definitions.py b/src/sysext_utils/definitions.py index eee5955..f876be6 100644 --- a/src/sysext_utils/definitions.py +++ b/src/sysext_utils/definitions.py @@ -31,6 +31,7 @@ OS_RELEASE_PATH = os.path.join(os.sep, "etc", "os-release") RUN_EXTENSIONS_DIR = os.path.join(os.sep, "run", "extensions") VAR_EXTENSIONS_DIR = os.path.join(os.sep, "var", "lib", "extensions") SCHEMAS_DIR = os.path.join("usr", "share", "glib-2.0", "schemas") +SYSEXT_IMAGE_SUFFIX = ".sysext.raw" class ImageFormat(StrEnum): diff --git a/src/sysext_utils/helpers.py b/src/sysext_utils/helpers.py index b530cc2..59e96bc 100644 --- a/src/sysext_utils/helpers.py +++ b/src/sysext_utils/helpers.py @@ -27,6 +27,7 @@ import os import re import shutil +from urllib.parse import urlparse from typing import Dict from . import config @@ -35,6 +36,7 @@ from .definitions import ( RUN_EXTENSIONS_DIR, VAR_EXTENSIONS_DIR, OS_RELEASE_PATH, + SYSEXT_IMAGE_SUFFIX, DryStepType, ) @@ -67,6 +69,15 @@ def get_path_for_input_sysext_name(name: str, directory: str) -> str: return os.path.join(directory, f"{name}.sysext.raw") +def get_name_from_input_sysext_image(image: str) -> str: + if is_url(image): + filename = os.path.basename(urlparse(image).path) + else: + filename = os.path.basename(image) + + return filename.removesuffix(SYSEXT_IMAGE_SUFFIX) + + def get_path_for_active_sysext_name(name: str, runtime: bool) -> str: if runtime: return os.path.join(RUN_EXTENSIONS_DIR, f"{name}.raw") diff --git a/src/sysext_utils/main.py b/src/sysext_utils/main.py index 7ba83a8..f8aae36 100644 --- a/src/sysext_utils/main.py +++ b/src/sysext_utils/main.py @@ -60,7 +60,7 @@ def _handle_image_add( parser: argparse.ArgumentParser, args: argparse.Namespace, ) -> None: - run_image_add_workflow(args.name, args.image, args.runtime) + run_image_add_workflow(args.image, args.runtime) def _handle_image_remove( @@ -162,10 +162,6 @@ def main(): "add", help="add and activate a sysext image onto host OS", ) - image_add_parser.add_argument( - "name", - help="name of the sysext e.g., sdk", - ) image_add_parser.add_argument( "image", help="path or URL to the sysext image e.g., sdk.sysext.raw", diff --git a/src/sysext_utils/workflows.py b/src/sysext_utils/workflows.py index e3ae51e..50a31fe 100644 --- a/src/sysext_utils/workflows.py +++ b/src/sysext_utils/workflows.py @@ -25,7 +25,7 @@ from typing import List -from .helpers import get_path_for_input_sysext_name +from .helpers import get_path_for_input_sysext_name, get_name_from_input_sysext_image from .utilities import ( build_image, add_image, @@ -76,7 +76,9 @@ def run_image_build_workflow( ) -def run_image_add_workflow(name: str, image: str, runtime: bool) -> None: +def run_image_add_workflow(image: str, runtime: bool) -> None: + name = get_name_from_input_sysext_image(image) + remove_image(name) add_image(name, image, runtime) diff --git a/tests/test_workflows.py b/tests/test_workflows.py index 780ced7..442811a 100644 --- a/tests/test_workflows.py +++ b/tests/test_workflows.py @@ -94,15 +94,15 @@ def test_image_build_work_with_ignore_release(): def test_image_add_workflow(): - run_image_add_workflow(name="name", image="image", runtime=False) + run_image_add_workflow(image="image", runtime=False) def test_image_add_workflow_with_remote_image(): - run_image_add_workflow(name="name", image="https://image", runtime=False) + run_image_add_workflow(image="https://image", runtime=False) def test_image_add_workflow_with_runtime_flag(): - run_image_add_workflow(name="name", image="image", runtime=True) + run_image_add_workflow(image="image", runtime=True) def test_image_remove_workflow(): -- GitLab From f15a8bd61bcff30090e03943a5ccfe0c6738d521 Mon Sep 17 00:00:00 2001 From: Martin Abente Lahaye Date: Thu, 30 May 2024 08:43:30 -0400 Subject: [PATCH 09/46] main: Merge build and create verbs --- README.md | 18 ++---- src/sysext_utils/helpers.py | 4 ++ src/sysext_utils/main.py | 102 +++++++++++++--------------------- src/sysext_utils/workflows.py | 20 +++---- tests/test_workflows.py | 45 ++++++++------- 5 files changed, 83 insertions(+), 106 deletions(-) diff --git a/README.md b/README.md index cf2e9a1..7ab1abf 100644 --- a/README.md +++ b/README.md @@ -19,20 +19,14 @@ $ python -m pip install . ## Examples -### Build projects artifacts +### Build sysext images out of a project ```bash -$ sysext-utils --dry build ./sdk bst sdk/gtk.bst sdk/libadwaita.bst -$ sysext-utils --dry build ./sdk meson -Dsysprof=enabled -``` - -### Build sysext images out of existing artifacts - -```bash -$ sysext-utils --dry create sdk ./sdk -$ sysext-utils --dry create sdk ./sdk --ignore-release -$ sysext-utils --dry create sdk ./sdk --integrations schemas -$ sysext-utils --dry create sdk ./sdk --format=ddi --private_key=key --certificate=cert +$ sysext-utils --dry build sdk meson -Dsysprof=enabled +$ sysext-utils --dry build sdk bst sdk/gtk.bst sdk/libadwaita.bst +$ sysext-utils --dry build sdk --ignore-release meson +$ sysext-utils --dry build sdk --format=ddi --private_key=key --certificate=cert meson +$ sysext-utils --dry build sdk --integrations schemas -- meson ``` ### Manage sysext images diff --git a/src/sysext_utils/helpers.py b/src/sysext_utils/helpers.py index 59e96bc..50fbade 100644 --- a/src/sysext_utils/helpers.py +++ b/src/sysext_utils/helpers.py @@ -85,6 +85,10 @@ def get_path_for_active_sysext_name(name: str, runtime: bool) -> str: return os.path.join(VAR_EXTENSIONS_DIR, f"{name}.raw") +def get_path_for_artifacts() -> str: + return os.path.join(os.getcwd(), "_artifacts") + + def make_directory(directory: str) -> None: if config.dry_run: print([str(DryStepType.MAKING), directory]) diff --git a/src/sysext_utils/main.py b/src/sysext_utils/main.py index f8aae36..0b89d65 100644 --- a/src/sysext_utils/main.py +++ b/src/sysext_utils/main.py @@ -33,26 +33,28 @@ from .workflows import ( run_image_build_workflow, run_image_add_workflow, run_image_remove_workflow, - run_artifact_build_workflow, ) -def _handle_image_create( +def _handle_image_build( parser: argparse.ArgumentParser, args: argparse.Namespace, ) -> None: if args.format == ImageFormat.DDI and not (args.private_key and args.certificate): parser.error("DDI image creation require private_key and certificate") + if args.system == BuildSystemType.BST and not args.arguments: + parser.error("One or more elements are required") run_image_build_workflow( args.name, - args.artifact, args.directory, args.format, args.private_key, args.certificate, args.ignore_release, args.integrations, + args.system, + args.arguments, ) @@ -70,16 +72,6 @@ def _handle_image_remove( run_image_remove_workflow(args.name) -def _handle_artifact_build( - parser: argparse.ArgumentParser, - args: argparse.Namespace, -) -> None: - if args.system == BuildSystemType.BST and not args.arguments: - parser.error("One or more elements are required") - - run_artifact_build_workflow(args.directory, args.system, args.arguments) - - def main(): parser = argparse.ArgumentParser( prog="sysext-utils", @@ -110,20 +102,43 @@ def main(): required=True, ) - # Arguments for creating an image + # Arguments for adding an image to the host + + image_add_parser = subparser.add_parser( + "add", + help="add and activate a sysext image onto host OS", + ) + image_add_parser.add_argument( + "image", + help="path or URL to the sysext image e.g., sdk.sysext.raw", + ) + image_add_parser.add_argument( + "--runtime", + help="add the sysext image under /run/extensions/ instead", + action="store_true", + ) + + # Arguments for removing a sysext image from the host OS + + image_remove_parser = subparser.add_parser( + "remove", + help="deactivate and remove a sysext image from the host OS", + ) + image_remove_parser.add_argument( + "name", + help="name of the the sysext e.g., sdk", + ) + + # Arguments for building an image image_build_parser = subparser.add_parser( - "create", - help="create a sysext image out of an artifact", + "build", + help="build a sysext image out of a project", ) image_build_parser.add_argument( "name", help="what to name the sysext e.g., sdk", ) - image_build_parser.add_argument( - "artifact", - help="path to the input artifact directory e.g., ./sdk/", - ) image_build_parser.add_argument( "--format", help="format to be used for the image creation", @@ -140,7 +155,7 @@ def main(): ) image_build_parser.add_argument( "--directory", - help="where to place the image to, e.g., ./images/. Defaults to $PWD", + help="where to place the image, e.g., ./images/. Defaults to $PWD", default=os.path.abspath(os.getcwd()), ) image_build_parser.add_argument( @@ -155,51 +170,13 @@ def main(): choices=[str(IntegrationType.SCHEMAS)], default=[], ) - - # Arguments for adding an image to the host - - image_add_parser = subparser.add_parser( - "add", - help="add and activate a sysext image onto host OS", - ) - image_add_parser.add_argument( - "image", - help="path or URL to the sysext image e.g., sdk.sysext.raw", - ) - image_add_parser.add_argument( - "--runtime", - help="add the sysext image under /run/extensions/ instead", - action="store_true", - ) - - # Arguments for removing a sysext image from the host OS - - image_remove_parser = subparser.add_parser( - "remove", - help="deactivate and remove a sysext image from the host OS", - ) - image_remove_parser.add_argument( - "name", - help="name of the the sysext e.g., sdk", - ) - - # Arguments for building an artifact - - artifact_build_parser = subparser.add_parser( - "build", - help="build artifact out of a project", - ) - artifact_build_parser.add_argument( - "directory", - help="path to the output artifact directory", - ) - artifact_build_parser.add_argument( + image_build_parser.add_argument( "system", help="build system to use e.g., bst", choices=[str(BuildSystemType.BST), str(BuildSystemType.MESON)], default=BuildSystemType.BST, ) - artifact_build_parser.add_argument( + image_build_parser.add_argument( "arguments", help="list of build-system specific arguments", nargs=argparse.REMAINDER, @@ -216,10 +193,9 @@ def main(): # Handle supported operations handlers = { - "create": _handle_image_create, + "build": _handle_image_build, "add": _handle_image_add, "remove": _handle_image_remove, - "build": _handle_artifact_build, } handlers[args.operation](parser, args) diff --git a/src/sysext_utils/workflows.py b/src/sysext_utils/workflows.py index 50a31fe..df0ef39 100644 --- a/src/sysext_utils/workflows.py +++ b/src/sysext_utils/workflows.py @@ -25,7 +25,11 @@ from typing import List -from .helpers import get_path_for_input_sysext_name, get_name_from_input_sysext_image +from .helpers import ( + get_path_for_input_sysext_name, + get_name_from_input_sysext_image, + get_path_for_artifacts, +) from .utilities import ( build_image, add_image, @@ -37,14 +41,18 @@ from .utilities import ( def run_image_build_workflow( name: str, - artifact: str, directory: str, image_format: str, private_key: str, certificate: str, ignore_release: bool, integrations: List[str], + system: str, + arguments: List[str], ) -> None: + artifact = get_path_for_artifacts() + + build_artifact(artifact, system, arguments) build_image( name, artifact, @@ -85,11 +93,3 @@ def run_image_add_workflow(image: str, runtime: bool) -> None: def run_image_remove_workflow(name: str) -> None: remove_image(name) - - -def run_artifact_build_workflow( - directory: str, - system: str, - arguments: List[str], -) -> None: - build_artifact(directory, system, arguments) diff --git a/tests/test_workflows.py b/tests/test_workflows.py index 442811a..8cbea55 100644 --- a/tests/test_workflows.py +++ b/tests/test_workflows.py @@ -29,7 +29,6 @@ from sysext_utils.workflows import ( run_image_build_workflow, run_image_add_workflow, run_image_remove_workflow, - run_artifact_build_workflow, ) @@ -40,33 +39,50 @@ def setup_module(): def test_image_build_workflow(): run_image_build_workflow( name="name", - artifact="artifact", directory="directory", image_format=ImageFormat.DDI, private_key="private_key", certificate="certificate", ignore_release=False, integrations=[], + system=BuildSystemType.BST, + arguments=[ + "element/one.bst", + ], + ) + + +def test_image_build_workflow_with_meson(): + run_image_build_workflow( + name="name", + directory="directory", + image_format=ImageFormat.DDI, + private_key="private_key", + certificate="certificate", + ignore_release=False, + integrations=[], + system=BuildSystemType.MESON, + arguments=[], ) def test_image_build_workflow_with_compressed_format(): run_image_build_workflow( name="name", - artifact="artifact", directory="directory", image_format=ImageFormat.COMPRESSED, private_key="private_key", certificate="certificate", ignore_release=False, integrations=[], + system=BuildSystemType.MESON, + arguments=[], ) def test_image_build_workflow_with_integrations(): run_image_build_workflow( name="name", - artifact="artifact", directory="directory", image_format=ImageFormat.COMPRESSED, private_key="private_key", @@ -75,13 +91,14 @@ def test_image_build_workflow_with_integrations(): integrations=[ IntegrationType.SCHEMAS, ], + system=BuildSystemType.MESON, + arguments=[], ) def test_image_build_work_with_ignore_release(): run_image_build_workflow( name="name", - artifact="artifact", directory="directory", image_format=ImageFormat.COMPRESSED, private_key="private_key", @@ -90,6 +107,8 @@ def test_image_build_work_with_ignore_release(): integrations=[ IntegrationType.SCHEMAS, ], + system=BuildSystemType.MESON, + arguments=[], ) @@ -107,19 +126,3 @@ def test_image_add_workflow_with_runtime_flag(): def test_image_remove_workflow(): run_image_remove_workflow(name="name") - - -def test_artifact_build_workflow(): - run_artifact_build_workflow( - directory="directory", - system=BuildSystemType.BST, - arguments=["element/one.bst"], - ) - - -def test_artifact_build_workflow_with_meson(): - run_artifact_build_workflow( - directory="directory", - system=BuildSystemType.MESON, - arguments=[], - ) -- GitLab From 6a75d3f105fc793ec5f49cc0d6128927d8ff2703 Mon Sep 17 00:00:00 2001 From: Martin Abente Lahaye Date: Thu, 30 May 2024 09:59:13 -0400 Subject: [PATCH 10/46] main: Support building images out of an OS tree This would cover for unsupported build systems and future unforeseen use cases. --- README.md | 1 + src/sysext_utils/definitions.py | 2 ++ src/sysext_utils/helpers.py | 7 +++++++ src/sysext_utils/main.py | 8 +++++++- src/sysext_utils/utilities.py | 7 +++++++ tests/test_workflows.py | 14 ++++++++++++++ 6 files changed, 38 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 7ab1abf..35fc335 100644 --- a/README.md +++ b/README.md @@ -24,6 +24,7 @@ $ python -m pip install . ```bash $ sysext-utils --dry build sdk meson -Dsysprof=enabled $ sysext-utils --dry build sdk bst sdk/gtk.bst sdk/libadwaita.bst +$ sysext-utils --dry build sdk import ./destdir $ sysext-utils --dry build sdk --ignore-release meson $ sysext-utils --dry build sdk --format=ddi --private_key=key --certificate=cert meson $ sysext-utils --dry build sdk --integrations schemas -- meson diff --git a/src/sysext_utils/definitions.py b/src/sysext_utils/definitions.py index f876be6..e7922a3 100644 --- a/src/sysext_utils/definitions.py +++ b/src/sysext_utils/definitions.py @@ -49,8 +49,10 @@ class DryStepType(StrEnum): WRITING = "writing" MOVING = "moving" REMOVING = "removing" + COPYING = "copying" class BuildSystemType(StrEnum): BST = "bst" MESON = "meson" + IMPORT = "import" diff --git a/src/sysext_utils/helpers.py b/src/sysext_utils/helpers.py index 50fbade..70c36c4 100644 --- a/src/sysext_utils/helpers.py +++ b/src/sysext_utils/helpers.py @@ -96,6 +96,13 @@ def make_directory(directory: str) -> None: os.makedirs(directory, exist_ok=True) +def copy_directory(source: str, destination: str) -> None: + if config.dry_run: + print([str(DryStepType.COPYING), source, destination]) + else: + shutil.copytree(source, destination, dirs_exist_ok=True) + + def remove_directory(directory: str) -> None: if config.dry_run: print([str(DryStepType.REMOVING), directory]) diff --git a/src/sysext_utils/main.py b/src/sysext_utils/main.py index 0b89d65..bb8c17a 100644 --- a/src/sysext_utils/main.py +++ b/src/sysext_utils/main.py @@ -44,6 +44,8 @@ def _handle_image_build( parser.error("DDI image creation require private_key and certificate") if args.system == BuildSystemType.BST and not args.arguments: parser.error("One or more elements are required") + if args.system == BuildSystemType.IMPORT and len(args.arguments) != 1: + parser.error("A single path is required") run_image_build_workflow( args.name, @@ -173,7 +175,11 @@ def main(): image_build_parser.add_argument( "system", help="build system to use e.g., bst", - choices=[str(BuildSystemType.BST), str(BuildSystemType.MESON)], + choices=[ + str(BuildSystemType.BST), + str(BuildSystemType.MESON), + str(BuildSystemType.IMPORT), + ], default=BuildSystemType.BST, ) image_build_parser.add_argument( diff --git a/src/sysext_utils/utilities.py b/src/sysext_utils/utilities.py index 1732b3d..2da6caa 100644 --- a/src/sysext_utils/utilities.py +++ b/src/sysext_utils/utilities.py @@ -49,6 +49,7 @@ from .helpers import ( get_path_for_active_sysext_name, get_path_for_input_sysext_name, make_directory, + copy_directory, remove_directory, write_file, move_file, @@ -124,6 +125,8 @@ def build_artifact(directory: str, system: str, arguments: List[str]) -> None: build_artifact_bst_elements(directory, arguments) if system == BuildSystemType.MESON: build_artifact_meson_project(directory, arguments) + if system == BuildSystemType.IMPORT: + build_artifact_import_directory(directory, arguments) def build_artifact_bst_elements(directory: str, elements: List[str]) -> None: @@ -143,3 +146,7 @@ def build_artifact_meson_project(directory: str, arguments: List[str]) -> None: Meson.setup(build_directory, arguments) Meson.compile(build_directory) Meson.install(build_directory, dest_directory) + + +def build_artifact_import_directory(directory: str, arguments: List[str]) -> None: + copy_directory(arguments[0], directory) diff --git a/tests/test_workflows.py b/tests/test_workflows.py index 8cbea55..fc330e7 100644 --- a/tests/test_workflows.py +++ b/tests/test_workflows.py @@ -52,6 +52,20 @@ def test_image_build_workflow(): ) +def test_image_build_workflow_with_import(): + run_image_build_workflow( + name="name", + directory="directory", + image_format=ImageFormat.DDI, + private_key="private_key", + certificate="certificate", + ignore_release=False, + integrations=[], + system=BuildSystemType.IMPORT, + arguments=["./destdir"], + ) + + def test_image_build_workflow_with_meson(): run_image_build_workflow( name="name", -- GitLab From 4a1fe337ff5d15adf7fc29d3ba896eb671444e25 Mon Sep 17 00:00:00 2001 From: Martin Abente Lahaye Date: Thu, 30 May 2024 11:55:18 -0400 Subject: [PATCH 11/46] main: Add basic pieces for proper error handling --- src/sysext_utils/errors.py | 28 ++++++++++++++++++++++++++++ src/sysext_utils/main.py | 37 ++++++++++++++++++++++++------------- 2 files changed, 52 insertions(+), 13 deletions(-) create mode 100644 src/sysext_utils/errors.py diff --git a/src/sysext_utils/errors.py b/src/sysext_utils/errors.py new file mode 100644 index 0000000..425727b --- /dev/null +++ b/src/sysext_utils/errors.py @@ -0,0 +1,28 @@ +# MIT License +# +# Copyright (c) 2024 Codethink Limited +# Copyright (c) 2024 GNOME Foundation +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# +# SPDX-License-Identifier: MIT + + +class HandledError(BaseException): + pass diff --git a/src/sysext_utils/main.py b/src/sysext_utils/main.py index bb8c17a..73dfff5 100644 --- a/src/sysext_utils/main.py +++ b/src/sysext_utils/main.py @@ -24,9 +24,11 @@ # SPDX-License-Identifier: MIT import os +import sys import argparse from . import config +from .errors import HandledError from .definitions import ImageFormat, IntegrationType, BuildSystemType from .version import version from .workflows import ( @@ -47,31 +49,40 @@ def _handle_image_build( if args.system == BuildSystemType.IMPORT and len(args.arguments) != 1: parser.error("A single path is required") - run_image_build_workflow( - args.name, - args.directory, - args.format, - args.private_key, - args.certificate, - args.ignore_release, - args.integrations, - args.system, - args.arguments, - ) + try: + run_image_build_workflow( + args.name, + args.directory, + args.format, + args.private_key, + args.certificate, + args.ignore_release, + args.integrations, + args.system, + args.arguments, + ) + except HandledError as e: + sys.exit(str(e)) def _handle_image_add( parser: argparse.ArgumentParser, args: argparse.Namespace, ) -> None: - run_image_add_workflow(args.image, args.runtime) + try: + run_image_add_workflow(args.image, args.runtime) + except HandledError as e: + sys.exit(str(e)) def _handle_image_remove( parser: argparse.ArgumentParser, args: argparse.Namespace, ) -> None: - run_image_remove_workflow(args.name) + try: + run_image_remove_workflow(args.name) + except HandledError as e: + sys.exit(str(e)) def main(): -- GitLab From daef28ec6c70dfd8527efc24e4bea330c1c0c0a9 Mon Sep 17 00:00:00 2001 From: Martin Abente Lahaye Date: Thu, 30 May 2024 11:56:22 -0400 Subject: [PATCH 12/46] workflows: Ensure priviledged user when it's needed --- src/sysext_utils/errors.py | 5 +++++ src/sysext_utils/helpers.py | 6 ++++++ src/sysext_utils/workflows.py | 4 ++++ 3 files changed, 15 insertions(+) diff --git a/src/sysext_utils/errors.py b/src/sysext_utils/errors.py index 425727b..0e88fb4 100644 --- a/src/sysext_utils/errors.py +++ b/src/sysext_utils/errors.py @@ -26,3 +26,8 @@ class HandledError(BaseException): pass + + +class NotPriviledgedError(HandledError): + def __init__(self): + super().__init__("Need to be privileged.") diff --git a/src/sysext_utils/helpers.py b/src/sysext_utils/helpers.py index 70c36c4..95775b7 100644 --- a/src/sysext_utils/helpers.py +++ b/src/sysext_utils/helpers.py @@ -32,6 +32,7 @@ from typing import Dict from . import config +from .errors import NotPriviledgedError from .definitions import ( RUN_EXTENSIONS_DIR, VAR_EXTENSIONS_DIR, @@ -133,3 +134,8 @@ def remove_file(path: str) -> None: os.remove(path) except FileNotFoundError: pass + + +def ensure_priviledged() -> None: + if not config.dry_run and os.getuid() != 0: + raise NotPriviledgedError() diff --git a/src/sysext_utils/workflows.py b/src/sysext_utils/workflows.py index df0ef39..f325c7d 100644 --- a/src/sysext_utils/workflows.py +++ b/src/sysext_utils/workflows.py @@ -29,6 +29,7 @@ from .helpers import ( get_path_for_input_sysext_name, get_name_from_input_sysext_image, get_path_for_artifacts, + ensure_priviledged, ) from .utilities import ( build_image, @@ -68,6 +69,7 @@ def run_image_build_workflow( image_name = get_path_for_input_sysext_name(name, directory) + ensure_priviledged() remove_image(name) add_image(name, image_name, runtime=False) integrate_image(artifact, integrations) @@ -87,9 +89,11 @@ def run_image_build_workflow( def run_image_add_workflow(image: str, runtime: bool) -> None: name = get_name_from_input_sysext_image(image) + ensure_priviledged() remove_image(name) add_image(name, image, runtime) def run_image_remove_workflow(name: str) -> None: + ensure_priviledged() remove_image(name) -- GitLab From 8c516fc4e21e8b60998ab81ed3fb1e155ead6bce Mon Sep 17 00:00:00 2001 From: Martin Abente Lahaye Date: Thu, 30 May 2024 13:10:53 -0400 Subject: [PATCH 13/46] commands: Handle missing and failed commands gracefully --- src/sysext_utils/commands.py | 9 ++++++++- src/sysext_utils/errors.py | 17 +++++++++++++++++ 2 files changed, 25 insertions(+), 1 deletion(-) diff --git a/src/sysext_utils/commands.py b/src/sysext_utils/commands.py index 38df1f4..649432d 100644 --- a/src/sysext_utils/commands.py +++ b/src/sysext_utils/commands.py @@ -29,6 +29,7 @@ from typing import List from . import config +from .errors import CommandNotAvailableError, CommandFailedError from .definitions import DryStepType @@ -43,8 +44,14 @@ class BaseCommand: if config.dry_run: print([str(DryStepType.RUNNING)] + commands) - else: + return + + try: subprocess.run(commands, capture_output=not config.verbose, check=True) + except FileNotFoundError: + raise CommandNotAvailableError(command) + except subprocess.CalledProcessError as e: + raise CommandFailedError(command, e.output) class GLibCompileSchemas(BaseCommand): diff --git a/src/sysext_utils/errors.py b/src/sysext_utils/errors.py index 0e88fb4..0bbdfe0 100644 --- a/src/sysext_utils/errors.py +++ b/src/sysext_utils/errors.py @@ -23,6 +23,8 @@ # # SPDX-License-Identifier: MIT +from typing import Optional + class HandledError(BaseException): pass @@ -31,3 +33,18 @@ class HandledError(BaseException): class NotPriviledgedError(HandledError): def __init__(self): super().__init__("Need to be privileged.") + + +class CommandNotAvailableError(HandledError): + def __init__(self, command: str): + super().__init__(f"{command} is not installed.") + + +class CommandFailedError(HandledError): + def __init__(self, command: str, output: Optional[bytes]): + if output is not None: + message = f"{command} failed due to: {output.decode('UTF-8')}" + else: + message = f"{command} failed." + + super().__init__(message) -- GitLab From 3ad10082c50d078e489cfaae00dc4ff27ecc381b Mon Sep 17 00:00:00 2001 From: Martin Abente Lahaye Date: Thu, 30 May 2024 13:39:35 -0400 Subject: [PATCH 14/46] workflows: Ensure there're contents to build an image from --- src/sysext_utils/errors.py | 5 +++++ src/sysext_utils/helpers.py | 7 ++++++- src/sysext_utils/workflows.py | 2 ++ 3 files changed, 13 insertions(+), 1 deletion(-) diff --git a/src/sysext_utils/errors.py b/src/sysext_utils/errors.py index 0bbdfe0..6ce2f3a 100644 --- a/src/sysext_utils/errors.py +++ b/src/sysext_utils/errors.py @@ -48,3 +48,8 @@ class CommandFailedError(HandledError): message = f"{command} failed." super().__init__(message) + + +class EmptyDirectoryError(HandledError): + def __init__(self, directory: str): + super().__init__(f"No contents were found at: {directory}") diff --git a/src/sysext_utils/helpers.py b/src/sysext_utils/helpers.py index 95775b7..ff8b978 100644 --- a/src/sysext_utils/helpers.py +++ b/src/sysext_utils/helpers.py @@ -32,7 +32,7 @@ from typing import Dict from . import config -from .errors import NotPriviledgedError +from .errors import NotPriviledgedError, EmptyDirectoryError from .definitions import ( RUN_EXTENSIONS_DIR, VAR_EXTENSIONS_DIR, @@ -139,3 +139,8 @@ def remove_file(path: str) -> None: def ensure_priviledged() -> None: if not config.dry_run and os.getuid() != 0: raise NotPriviledgedError() + + +def ensure_directory(path: str) -> None: + if not config.dry_run and (not os.path.isdir(path) or not os.listdir(path)): + raise EmptyDirectoryError(path) diff --git a/src/sysext_utils/workflows.py b/src/sysext_utils/workflows.py index f325c7d..cfcf2ef 100644 --- a/src/sysext_utils/workflows.py +++ b/src/sysext_utils/workflows.py @@ -30,6 +30,7 @@ from .helpers import ( get_name_from_input_sysext_image, get_path_for_artifacts, ensure_priviledged, + ensure_directory, ) from .utilities import ( build_image, @@ -54,6 +55,7 @@ def run_image_build_workflow( artifact = get_path_for_artifacts() build_artifact(artifact, system, arguments) + ensure_directory(artifact) build_image( name, artifact, -- GitLab From 0538b52e434e05965833f3ea4842004d41f82eaa Mon Sep 17 00:00:00 2001 From: Martin Abente Lahaye Date: Thu, 30 May 2024 14:05:16 -0400 Subject: [PATCH 15/46] utilities: Ensure there's contents to import from --- src/sysext_utils/utilities.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/sysext_utils/utilities.py b/src/sysext_utils/utilities.py index 2da6caa..f42805e 100644 --- a/src/sysext_utils/utilities.py +++ b/src/sysext_utils/utilities.py @@ -51,6 +51,7 @@ from .helpers import ( make_directory, copy_directory, remove_directory, + ensure_directory, write_file, move_file, remove_file, @@ -149,4 +150,6 @@ def build_artifact_meson_project(directory: str, arguments: List[str]) -> None: def build_artifact_import_directory(directory: str, arguments: List[str]) -> None: - copy_directory(arguments[0], directory) + source = arguments[0] + ensure_directory(source) + copy_directory(source, directory) -- GitLab From 7d462f68d78dee48fa171b623ca7f9d6ea405681 Mon Sep 17 00:00:00 2001 From: Martin Abente Lahaye Date: Thu, 30 May 2024 14:10:39 -0400 Subject: [PATCH 16/46] utilities: Guard against bogus image formats --- src/sysext_utils/errors.py | 5 +++++ src/sysext_utils/utilities.py | 5 ++++- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/src/sysext_utils/errors.py b/src/sysext_utils/errors.py index 6ce2f3a..a7df220 100644 --- a/src/sysext_utils/errors.py +++ b/src/sysext_utils/errors.py @@ -53,3 +53,8 @@ class CommandFailedError(HandledError): class EmptyDirectoryError(HandledError): def __init__(self, directory: str): super().__init__(f"No contents were found at: {directory}") + + +class UnsupportedImageFormat(HandledError): + def __init__(self, image_format: str): + super().__init__(f"Image format not supported: {image_format}") diff --git a/src/sysext_utils/utilities.py b/src/sysext_utils/utilities.py index f42805e..6bb47c4 100644 --- a/src/sysext_utils/utilities.py +++ b/src/sysext_utils/utilities.py @@ -27,6 +27,7 @@ import os from typing import Optional, List +from .errors import UnsupportedImageFormat from .definitions import ( ImageFormat, IntegrationType, @@ -81,8 +82,10 @@ def build_image( if image_format == ImageFormat.DDI and private_key and certificate: SystemdRepart.build(artifact, image_path, private_key, certificate) - else: + elif image_format == ImageFormat.COMPRESSED: Mksquashfs.build(artifact, image_path) + else: + raise UnsupportedImageFormat(image_format) def add_image(name: str, image: str, runtime: bool) -> None: -- GitLab From bdffba203165f665e38682b2b0ba1651d4ce3ce3 Mon Sep 17 00:00:00 2001 From: Martin Abente Lahaye Date: Thu, 30 May 2024 14:16:31 -0400 Subject: [PATCH 17/46] utilities: Guard against bogus build systems --- src/sysext_utils/errors.py | 5 +++++ src/sysext_utils/utilities.py | 8 +++++--- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/src/sysext_utils/errors.py b/src/sysext_utils/errors.py index a7df220..7a8c9e0 100644 --- a/src/sysext_utils/errors.py +++ b/src/sysext_utils/errors.py @@ -58,3 +58,8 @@ class EmptyDirectoryError(HandledError): class UnsupportedImageFormat(HandledError): def __init__(self, image_format: str): super().__init__(f"Image format not supported: {image_format}") + + +class UnsupportedBuildSystem(HandledError): + def __init__(self, system: str): + super().__init__(f"Build system not supported: {system}") diff --git a/src/sysext_utils/utilities.py b/src/sysext_utils/utilities.py index 6bb47c4..f1e5828 100644 --- a/src/sysext_utils/utilities.py +++ b/src/sysext_utils/utilities.py @@ -27,7 +27,7 @@ import os from typing import Optional, List -from .errors import UnsupportedImageFormat +from .errors import UnsupportedImageFormat, UnsupportedBuildSystem from .definitions import ( ImageFormat, IntegrationType, @@ -127,10 +127,12 @@ def build_artifact(directory: str, system: str, arguments: List[str]) -> None: if system == BuildSystemType.BST: build_artifact_bst_elements(directory, arguments) - if system == BuildSystemType.MESON: + elif system == BuildSystemType.MESON: build_artifact_meson_project(directory, arguments) - if system == BuildSystemType.IMPORT: + elif system == BuildSystemType.IMPORT: build_artifact_import_directory(directory, arguments) + else: + raise UnsupportedBuildSystem(system) def build_artifact_bst_elements(directory: str, elements: List[str]) -> None: -- GitLab From bce1c979d0c3dbf8abdbc3c2b8c33ce1d09ae836 Mon Sep 17 00:00:00 2001 From: Martin Abente Lahaye Date: Thu, 30 May 2024 14:39:44 -0400 Subject: [PATCH 18/46] tests: Cover most of the error handling pieces --- tests/test_errors.py | 95 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 95 insertions(+) create mode 100644 tests/test_errors.py diff --git a/tests/test_errors.py b/tests/test_errors.py new file mode 100644 index 0000000..fd175e8 --- /dev/null +++ b/tests/test_errors.py @@ -0,0 +1,95 @@ +# MIT License +# +# Copyright (c) 2024 Codethink Limited +# Copyright (c) 2024 GNOME Foundation +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# +# SPDX-License-Identifier: MIT + +import pytest + +from sysext_utils import config +from sysext_utils.errors import ( + CommandNotAvailableError, + CommandFailedError, + EmptyDirectoryError, + UnsupportedImageFormat, + UnsupportedBuildSystem, +) +from sysext_utils.helpers import ensure_directory +from sysext_utils.commands import BaseCommand +from sysext_utils.utilities import build_image, build_artifact + + +class NoExistentCommand(BaseCommand): + @classmethod + def run(cls, *args) -> None: + cls.do_run("non-existent-command", *args) + + +class AlwaysFailsCommand(BaseCommand): + @classmethod + def run(cls, *args) -> None: + cls.do_run("false", *args) + + +def test_command_not_available_error(): + with pytest.raises(CommandNotAvailableError) as _: + NoExistentCommand.run([]) + + +def test_command_failed_error(): + with pytest.raises(CommandFailedError) as _: + AlwaysFailsCommand.run([]) + + +def test_empty_directory_error(): + with pytest.raises(EmptyDirectoryError) as _: + ensure_directory("does_not_exists") + + +def test_unsupported_image_format_error(): + config.dry_run = True + + with pytest.raises(UnsupportedImageFormat) as _: + build_image( + name="name", + artifact="artifact", + directory="directory", + image_format="BOGUS", + private_key=None, + certificate=None, + ignore_release=True, + ) + + config.dry_run = False + + +def test_unsupported_build_system_error(): + config.dry_run = True + + with pytest.raises(UnsupportedBuildSystem) as _: + build_artifact( + directory="directory", + system="BOGUS", + arguments=[], + ) + + config.dry_run = False -- GitLab From 54b78afe140f4976b20c850ac2137814045a6bb1 Mon Sep 17 00:00:00 2001 From: Martin Abente Lahaye Date: Fri, 31 May 2024 10:40:53 -0400 Subject: [PATCH 19/46] commands: Handle commands like meson that don't report on stderr --- src/sysext_utils/commands.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/src/sysext_utils/commands.py b/src/sysext_utils/commands.py index 649432d..4a2447c 100644 --- a/src/sysext_utils/commands.py +++ b/src/sysext_utils/commands.py @@ -47,11 +47,16 @@ class BaseCommand: return try: - subprocess.run(commands, capture_output=not config.verbose, check=True) + subprocess.run( + commands, + stdout=None if config.verbose else subprocess.PIPE, + stderr=subprocess.STDOUT, + check=True, + ) except FileNotFoundError: raise CommandNotAvailableError(command) except subprocess.CalledProcessError as e: - raise CommandFailedError(command, e.output) + raise CommandFailedError(command, e.stdout) class GLibCompileSchemas(BaseCommand): -- GitLab From f3ec954a14ff67453df304c1b03751047d0cb0d3 Mon Sep 17 00:00:00 2001 From: Martin Abente Lahaye Date: Fri, 31 May 2024 11:38:38 -0400 Subject: [PATCH 20/46] main: Validate sysext names --- src/sysext_utils/main.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/src/sysext_utils/main.py b/src/sysext_utils/main.py index 73dfff5..5e7848c 100644 --- a/src/sysext_utils/main.py +++ b/src/sysext_utils/main.py @@ -24,6 +24,7 @@ # SPDX-License-Identifier: MIT import os +import re import sys import argparse @@ -38,6 +39,12 @@ from .workflows import ( ) +def _name_validator(name: str) -> str: + if not re.match("^[\\w\\-.]+$", name): + raise argparse.ArgumentTypeError(f"'{name}' is not a valid sysext name") + return name + + def _handle_image_build( parser: argparse.ArgumentParser, args: argparse.Namespace, @@ -140,6 +147,7 @@ def main(): image_remove_parser.add_argument( "name", help="name of the the sysext e.g., sdk", + type=_name_validator, ) # Arguments for building an image @@ -151,6 +159,7 @@ def main(): image_build_parser.add_argument( "name", help="what to name the sysext e.g., sdk", + type=_name_validator, ) image_build_parser.add_argument( "--format", -- GitLab From ec54b2e8404910b102765d073aa4635e6c3e1ade Mon Sep 17 00:00:00 2001 From: Martin Abente Lahaye Date: Fri, 31 May 2024 11:48:20 -0400 Subject: [PATCH 21/46] errors: Fix name inconsistency --- src/sysext_utils/errors.py | 4 ++-- src/sysext_utils/utilities.py | 6 +++--- tests/test_errors.py | 8 ++++---- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/src/sysext_utils/errors.py b/src/sysext_utils/errors.py index 7a8c9e0..e5faef9 100644 --- a/src/sysext_utils/errors.py +++ b/src/sysext_utils/errors.py @@ -55,11 +55,11 @@ class EmptyDirectoryError(HandledError): super().__init__(f"No contents were found at: {directory}") -class UnsupportedImageFormat(HandledError): +class UnsupportedImageFormatError(HandledError): def __init__(self, image_format: str): super().__init__(f"Image format not supported: {image_format}") -class UnsupportedBuildSystem(HandledError): +class UnsupportedBuildSystemError(HandledError): def __init__(self, system: str): super().__init__(f"Build system not supported: {system}") diff --git a/src/sysext_utils/utilities.py b/src/sysext_utils/utilities.py index f1e5828..cb45530 100644 --- a/src/sysext_utils/utilities.py +++ b/src/sysext_utils/utilities.py @@ -27,7 +27,7 @@ import os from typing import Optional, List -from .errors import UnsupportedImageFormat, UnsupportedBuildSystem +from .errors import UnsupportedImageFormatError, UnsupportedBuildSystemError from .definitions import ( ImageFormat, IntegrationType, @@ -85,7 +85,7 @@ def build_image( elif image_format == ImageFormat.COMPRESSED: Mksquashfs.build(artifact, image_path) else: - raise UnsupportedImageFormat(image_format) + raise UnsupportedImageFormatError(image_format) def add_image(name: str, image: str, runtime: bool) -> None: @@ -132,7 +132,7 @@ def build_artifact(directory: str, system: str, arguments: List[str]) -> None: elif system == BuildSystemType.IMPORT: build_artifact_import_directory(directory, arguments) else: - raise UnsupportedBuildSystem(system) + raise UnsupportedBuildSystemError(system) def build_artifact_bst_elements(directory: str, elements: List[str]) -> None: diff --git a/tests/test_errors.py b/tests/test_errors.py index fd175e8..b0aa378 100644 --- a/tests/test_errors.py +++ b/tests/test_errors.py @@ -30,8 +30,8 @@ from sysext_utils.errors import ( CommandNotAvailableError, CommandFailedError, EmptyDirectoryError, - UnsupportedImageFormat, - UnsupportedBuildSystem, + UnsupportedImageFormatError, + UnsupportedBuildSystemError, ) from sysext_utils.helpers import ensure_directory from sysext_utils.commands import BaseCommand @@ -68,7 +68,7 @@ def test_empty_directory_error(): def test_unsupported_image_format_error(): config.dry_run = True - with pytest.raises(UnsupportedImageFormat) as _: + with pytest.raises(UnsupportedImageFormatError) as _: build_image( name="name", artifact="artifact", @@ -85,7 +85,7 @@ def test_unsupported_image_format_error(): def test_unsupported_build_system_error(): config.dry_run = True - with pytest.raises(UnsupportedBuildSystem) as _: + with pytest.raises(UnsupportedBuildSystemError) as _: build_artifact( directory="directory", system="BOGUS", -- GitLab From 1985f2deb39a9ed8c7404fdded24a819928c00ec Mon Sep 17 00:00:00 2001 From: Martin Abente Lahaye Date: Fri, 31 May 2024 14:06:57 -0400 Subject: [PATCH 22/46] utilities: Fix redundant and inconsistent check for signed image requirements --- src/sysext_utils/main.py | 2 ++ src/sysext_utils/utilities.py | 8 ++++---- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/src/sysext_utils/main.py b/src/sysext_utils/main.py index 5e7848c..7333d02 100644 --- a/src/sysext_utils/main.py +++ b/src/sysext_utils/main.py @@ -170,10 +170,12 @@ def main(): image_build_parser.add_argument( "--private_key", help="path to the private key file e.g., files/boot-keys/SYSEXT.key", + default="", ) image_build_parser.add_argument( "--certificate", help="path to the certificate file, e.g., files/boot-keys/SYSEXT.crt", + default="", ) image_build_parser.add_argument( "--directory", diff --git a/src/sysext_utils/utilities.py b/src/sysext_utils/utilities.py index cb45530..03980ca 100644 --- a/src/sysext_utils/utilities.py +++ b/src/sysext_utils/utilities.py @@ -25,7 +25,7 @@ import os -from typing import Optional, List +from typing import List from .errors import UnsupportedImageFormatError, UnsupportedBuildSystemError from .definitions import ( @@ -64,8 +64,8 @@ def build_image( artifact: str, directory: str, image_format: str, - private_key: Optional[str], - certificate: Optional[str], + private_key: str, + certificate: str, ignore_release: bool, ) -> None: image_path = get_path_for_input_sysext_name(name, directory) @@ -80,7 +80,7 @@ def build_image( make_directory(metadata_directory) write_file(metadata_content, metadata_path) - if image_format == ImageFormat.DDI and private_key and certificate: + if image_format == ImageFormat.DDI: SystemdRepart.build(artifact, image_path, private_key, certificate) elif image_format == ImageFormat.COMPRESSED: Mksquashfs.build(artifact, image_path) -- GitLab From 27c73ca871dd82506636d32d691ffbdf7d68c96a Mon Sep 17 00:00:00 2001 From: Martin Abente Lahaye Date: Fri, 31 May 2024 15:00:40 -0400 Subject: [PATCH 23/46] workflows: Add basic confirmation messages --- src/sysext_utils/workflows.py | 37 ++++++++++++++++++++++++++++++----- 1 file changed, 32 insertions(+), 5 deletions(-) diff --git a/src/sysext_utils/workflows.py b/src/sysext_utils/workflows.py index cfcf2ef..0815b58 100644 --- a/src/sysext_utils/workflows.py +++ b/src/sysext_utils/workflows.py @@ -66,17 +66,40 @@ def run_image_build_workflow( ignore_release, ) - if not integrations: - return + image = get_path_for_input_sysext_name(name, directory) - image_name = get_path_for_input_sysext_name(name, directory) + if integrations: + run_image_integration_workflow( + name, + image, + directory, + artifact, + image_format, + private_key, + certificate, + ignore_release, + integrations, + ) + print(f"Successfully built {image}") + + +def run_image_integration_workflow( + name: str, + image: str, + directory: str, + artifact: str, + image_format: str, + private_key: str, + certificate: str, + ignore_release: bool, + integrations: List[str], +) -> None: ensure_priviledged() remove_image(name) - add_image(name, image_name, runtime=False) + add_image(name, image, runtime=False) integrate_image(artifact, integrations) remove_image(name) - build_image( name, artifact, @@ -95,7 +118,11 @@ def run_image_add_workflow(image: str, runtime: bool) -> None: remove_image(name) add_image(name, image, runtime) + print(f"Successfully added {image}") + def run_image_remove_workflow(name: str) -> None: ensure_priviledged() remove_image(name) + + print(f"Successfully removed {name}") -- GitLab From 54d4db1f64e9e39fdd3a1bd7e86bd0c546fd6ebe Mon Sep 17 00:00:00 2001 From: Martin Abente Lahaye Date: Mon, 3 Jun 2024 07:43:34 -0400 Subject: [PATCH 24/46] workflow: Move some details out to the utilities themselves --- src/sysext_utils/utilities.py | 31 ++++++++++++++++++++----------- src/sysext_utils/workflows.py | 12 ++---------- tests/test_errors.py | 1 - 3 files changed, 22 insertions(+), 22 deletions(-) diff --git a/src/sysext_utils/utilities.py b/src/sysext_utils/utilities.py index 03980ca..14f424f 100644 --- a/src/sysext_utils/utilities.py +++ b/src/sysext_utils/utilities.py @@ -49,6 +49,7 @@ from .helpers import ( is_url, get_path_for_active_sysext_name, get_path_for_input_sysext_name, + get_path_for_artifacts, make_directory, copy_directory, remove_directory, @@ -67,10 +68,10 @@ def build_image( private_key: str, certificate: str, ignore_release: bool, -) -> None: - image_path = get_path_for_input_sysext_name(name, directory) +) -> str: + image = get_path_for_input_sysext_name(name, directory) - remove_file(image_path) + remove_file(image) make_directory(directory) metadata_directory = os.path.join(artifact, "usr", "lib", "extension-release.d") @@ -81,12 +82,14 @@ def build_image( write_file(metadata_content, metadata_path) if image_format == ImageFormat.DDI: - SystemdRepart.build(artifact, image_path, private_key, certificate) + SystemdRepart.build(artifact, image, private_key, certificate) elif image_format == ImageFormat.COMPRESSED: - Mksquashfs.build(artifact, image_path) + Mksquashfs.build(artifact, image) else: raise UnsupportedImageFormatError(image_format) + return image + def add_image(name: str, image: str, runtime: bool) -> None: if is_url(image): @@ -121,19 +124,25 @@ def integrate_image(artifact: str, integrations: List[str]) -> None: GLibCompileSchemas.compile(host_schemas_dir, artifact_schemas_dir) -def build_artifact(directory: str, system: str, arguments: List[str]) -> None: - remove_directory(directory) - make_directory(directory) +def build_artifact(system: str, arguments: List[str]) -> str: + artifact = get_path_for_artifacts() + + remove_directory(artifact) + make_directory(artifact) if system == BuildSystemType.BST: - build_artifact_bst_elements(directory, arguments) + build_artifact_bst_elements(artifact, arguments) elif system == BuildSystemType.MESON: - build_artifact_meson_project(directory, arguments) + build_artifact_meson_project(artifact, arguments) elif system == BuildSystemType.IMPORT: - build_artifact_import_directory(directory, arguments) + build_artifact_import_directory(artifact, arguments) else: raise UnsupportedBuildSystemError(system) + ensure_directory(artifact) + + return artifact + def build_artifact_bst_elements(directory: str, elements: List[str]) -> None: Bst.build(elements) diff --git a/src/sysext_utils/workflows.py b/src/sysext_utils/workflows.py index 0815b58..904bf45 100644 --- a/src/sysext_utils/workflows.py +++ b/src/sysext_utils/workflows.py @@ -26,11 +26,8 @@ from typing import List from .helpers import ( - get_path_for_input_sysext_name, get_name_from_input_sysext_image, - get_path_for_artifacts, ensure_priviledged, - ensure_directory, ) from .utilities import ( build_image, @@ -52,11 +49,8 @@ def run_image_build_workflow( system: str, arguments: List[str], ) -> None: - artifact = get_path_for_artifacts() - - build_artifact(artifact, system, arguments) - ensure_directory(artifact) - build_image( + artifact = build_artifact(system, arguments) + image = build_image( name, artifact, directory, @@ -66,8 +60,6 @@ def run_image_build_workflow( ignore_release, ) - image = get_path_for_input_sysext_name(name, directory) - if integrations: run_image_integration_workflow( name, diff --git a/tests/test_errors.py b/tests/test_errors.py index b0aa378..a2aad84 100644 --- a/tests/test_errors.py +++ b/tests/test_errors.py @@ -87,7 +87,6 @@ def test_unsupported_build_system_error(): with pytest.raises(UnsupportedBuildSystemError) as _: build_artifact( - directory="directory", system="BOGUS", arguments=[], ) -- GitLab From f6639a885c5890cf6fc5f556727113cc1715ea05 Mon Sep 17 00:00:00 2001 From: Martin Abente Lahaye Date: Mon, 3 Jun 2024 07:48:26 -0400 Subject: [PATCH 25/46] definitions: Move more key paths in --- src/sysext_utils/definitions.py | 2 ++ src/sysext_utils/utilities.py | 6 ++++-- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/src/sysext_utils/definitions.py b/src/sysext_utils/definitions.py index e7922a3..1ffdd91 100644 --- a/src/sysext_utils/definitions.py +++ b/src/sysext_utils/definitions.py @@ -31,6 +31,8 @@ OS_RELEASE_PATH = os.path.join(os.sep, "etc", "os-release") RUN_EXTENSIONS_DIR = os.path.join(os.sep, "run", "extensions") VAR_EXTENSIONS_DIR = os.path.join(os.sep, "var", "lib", "extensions") SCHEMAS_DIR = os.path.join("usr", "share", "glib-2.0", "schemas") +METADATA_DIR = os.path.join("usr", "lib", "extension-release.d") +METADATA_FILENAME = "extension-release.%s" SYSEXT_IMAGE_SUFFIX = ".sysext.raw" diff --git a/src/sysext_utils/utilities.py b/src/sysext_utils/utilities.py index 14f424f..0a37b89 100644 --- a/src/sysext_utils/utilities.py +++ b/src/sysext_utils/utilities.py @@ -34,6 +34,8 @@ from .definitions import ( BuildSystemType, RUN_EXTENSIONS_DIR, SCHEMAS_DIR, + METADATA_DIR, + METADATA_FILENAME, ) from .commands import ( SystemdRepart, @@ -74,8 +76,8 @@ def build_image( remove_file(image) make_directory(directory) - metadata_directory = os.path.join(artifact, "usr", "lib", "extension-release.d") - metadata_path = os.path.join(metadata_directory, f"extension-release.{name}") + metadata_directory = os.path.join(artifact, METADATA_DIR) + metadata_path = os.path.join(metadata_directory, METADATA_FILENAME % name) metadata_content = get_sysext_metadata(ignore_release) make_directory(metadata_directory) -- GitLab From 4bb8af8db03bab6c016f8a2c5a340fb8742034e5 Mon Sep 17 00:00:00 2001 From: Martin Abente Lahaye Date: Mon, 3 Jun 2024 07:57:10 -0400 Subject: [PATCH 26/46] workflows: Change integrations check to an early exit for better readability --- src/sysext_utils/workflows.py | 26 ++++++++++++++------------ 1 file changed, 14 insertions(+), 12 deletions(-) diff --git a/src/sysext_utils/workflows.py b/src/sysext_utils/workflows.py index 904bf45..5eaf93e 100644 --- a/src/sysext_utils/workflows.py +++ b/src/sysext_utils/workflows.py @@ -60,18 +60,17 @@ def run_image_build_workflow( ignore_release, ) - if integrations: - run_image_integration_workflow( - name, - image, - directory, - artifact, - image_format, - private_key, - certificate, - ignore_release, - integrations, - ) + run_image_integration_workflow( + name, + image, + directory, + artifact, + image_format, + private_key, + certificate, + ignore_release, + integrations, + ) print(f"Successfully built {image}") @@ -87,6 +86,9 @@ def run_image_integration_workflow( ignore_release: bool, integrations: List[str], ) -> None: + if not integrations: + return + ensure_priviledged() remove_image(name) add_image(name, image, runtime=False) -- GitLab From 56b52f014f5cba6f4c174c0cf9f54b1945a71f72 Mon Sep 17 00:00:00 2001 From: Martin Abente Lahaye Date: Mon, 3 Jun 2024 08:34:08 -0400 Subject: [PATCH 27/46] utilities: Manage a cleaner workspace --- src/sysext_utils/definitions.py | 1 + src/sysext_utils/helpers.py | 9 +++++++-- src/sysext_utils/utilities.py | 11 +++++------ 3 files changed, 13 insertions(+), 8 deletions(-) diff --git a/src/sysext_utils/definitions.py b/src/sysext_utils/definitions.py index 1ffdd91..d3bd944 100644 --- a/src/sysext_utils/definitions.py +++ b/src/sysext_utils/definitions.py @@ -34,6 +34,7 @@ SCHEMAS_DIR = os.path.join("usr", "share", "glib-2.0", "schemas") METADATA_DIR = os.path.join("usr", "lib", "extension-release.d") METADATA_FILENAME = "extension-release.%s" SYSEXT_IMAGE_SUFFIX = ".sysext.raw" +WORKSPACE_DIR = os.path.join(os.getcwd(), ".sysext-utils") class ImageFormat(StrEnum): diff --git a/src/sysext_utils/helpers.py b/src/sysext_utils/helpers.py index ff8b978..18708cf 100644 --- a/src/sysext_utils/helpers.py +++ b/src/sysext_utils/helpers.py @@ -36,6 +36,7 @@ from .errors import NotPriviledgedError, EmptyDirectoryError from .definitions import ( RUN_EXTENSIONS_DIR, VAR_EXTENSIONS_DIR, + WORKSPACE_DIR, OS_RELEASE_PATH, SYSEXT_IMAGE_SUFFIX, DryStepType, @@ -86,8 +87,12 @@ def get_path_for_active_sysext_name(name: str, runtime: bool) -> str: return os.path.join(VAR_EXTENSIONS_DIR, f"{name}.raw") -def get_path_for_artifacts() -> str: - return os.path.join(os.getcwd(), "_artifacts") +def get_path_for_artifact() -> str: + return os.path.join(WORKSPACE_DIR, "artifact") + + +def get_path_for_build() -> str: + return os.path.join(WORKSPACE_DIR, "build") def make_directory(directory: str) -> None: diff --git a/src/sysext_utils/utilities.py b/src/sysext_utils/utilities.py index 0a37b89..0c05a5b 100644 --- a/src/sysext_utils/utilities.py +++ b/src/sysext_utils/utilities.py @@ -51,7 +51,8 @@ from .helpers import ( is_url, get_path_for_active_sysext_name, get_path_for_input_sysext_name, - get_path_for_artifacts, + get_path_for_artifact, + get_path_for_build, make_directory, copy_directory, remove_directory, @@ -127,7 +128,7 @@ def integrate_image(artifact: str, integrations: List[str]) -> None: def build_artifact(system: str, arguments: List[str]) -> str: - artifact = get_path_for_artifacts() + artifact = get_path_for_artifact() remove_directory(artifact) make_directory(artifact) @@ -154,15 +155,13 @@ def build_artifact_bst_elements(directory: str, elements: List[str]) -> None: def build_artifact_meson_project(directory: str, arguments: List[str]) -> None: - current_directory = os.getcwd() - build_directory = os.path.join(current_directory, "_build") - dest_directory = os.path.join(current_directory, directory) + build_directory = get_path_for_build() remove_directory(build_directory) Meson.setup(build_directory, arguments) Meson.compile(build_directory) - Meson.install(build_directory, dest_directory) + Meson.install(build_directory, directory) def build_artifact_import_directory(directory: str, arguments: List[str]) -> None: -- GitLab From 059dbaf6a5ed0f568c571192bceef1e97a615731 Mon Sep 17 00:00:00 2001 From: Martin Abente Lahaye Date: Mon, 3 Jun 2024 14:48:47 -0400 Subject: [PATCH 28/46] main: Move integration steps to the add verb --- README.md | 2 +- src/sysext_utils/helpers.py | 14 +++++++-- src/sysext_utils/main.py | 17 +++++------ src/sysext_utils/utilities.py | 54 +++++++++++++++++++++++++-------- src/sysext_utils/workflows.py | 57 +++++++++++++---------------------- tests/test_workflows.py | 39 ++++++++---------------- 6 files changed, 96 insertions(+), 87 deletions(-) diff --git a/README.md b/README.md index 35fc335..b3a3fd6 100644 --- a/README.md +++ b/README.md @@ -27,13 +27,13 @@ $ sysext-utils --dry build sdk bst sdk/gtk.bst sdk/libadwaita.bst $ sysext-utils --dry build sdk import ./destdir $ sysext-utils --dry build sdk --ignore-release meson $ sysext-utils --dry build sdk --format=ddi --private_key=key --certificate=cert meson -$ sysext-utils --dry build sdk --integrations schemas -- meson ``` ### Manage sysext images ```bash $ sysext-utils --dry add sdk.sysext.raw +$ sysext-utils --dry add sdk.sysext.raw --integrations schemas $ sysext-utils --dry add https://os.gnome.org/sysext/sdk.sysext.raw --runtime $ sysext-utils --dry remove sdk ``` diff --git a/src/sysext_utils/helpers.py b/src/sysext_utils/helpers.py index 18708cf..19c41e5 100644 --- a/src/sysext_utils/helpers.py +++ b/src/sysext_utils/helpers.py @@ -80,11 +80,19 @@ def get_name_from_input_sysext_image(image: str) -> str: return filename.removesuffix(SYSEXT_IMAGE_SUFFIX) -def get_path_for_active_sysext_name(name: str, runtime: bool) -> str: +def get_name_for_integrations_sysext(name: str) -> str: + return f"{name}.integrations" + + +def get_tree_path_for_active_sysext_name(name: str, runtime: bool) -> str: if runtime: - return os.path.join(RUN_EXTENSIONS_DIR, f"{name}.raw") + return os.path.join(RUN_EXTENSIONS_DIR, name) + + return os.path.join(VAR_EXTENSIONS_DIR, name) + - return os.path.join(VAR_EXTENSIONS_DIR, f"{name}.raw") +def get_image_path_for_active_sysext_name(name: str, runtime: bool) -> str: + return get_tree_path_for_active_sysext_name(name, runtime) + ".raw" def get_path_for_artifact() -> str: diff --git a/src/sysext_utils/main.py b/src/sysext_utils/main.py index 7333d02..0aa27de 100644 --- a/src/sysext_utils/main.py +++ b/src/sysext_utils/main.py @@ -64,7 +64,6 @@ def _handle_image_build( args.private_key, args.certificate, args.ignore_release, - args.integrations, args.system, args.arguments, ) @@ -77,7 +76,7 @@ def _handle_image_add( args: argparse.Namespace, ) -> None: try: - run_image_add_workflow(args.image, args.runtime) + run_image_add_workflow(args.image, args.runtime, args.integrations) except HandledError as e: sys.exit(str(e)) @@ -137,6 +136,13 @@ def main(): help="add the sysext image under /run/extensions/ instead", action="store_true", ) + image_add_parser.add_argument( + "--integrations", + help="list of integration steps to be run", + nargs="+", + choices=[str(IntegrationType.SCHEMAS)], + default=[], + ) # Arguments for removing a sysext image from the host OS @@ -187,13 +193,6 @@ def main(): help="ignore the host release data and use 'ID=_any' instead", action="store_true", ) - image_build_parser.add_argument( - "--integrations", - help="list of integration steps to be run", - nargs="+", - choices=[str(IntegrationType.SCHEMAS)], - default=[], - ) image_build_parser.add_argument( "system", help="build system to use e.g., bst", diff --git a/src/sysext_utils/utilities.py b/src/sysext_utils/utilities.py index 0c05a5b..16c98ca 100644 --- a/src/sysext_utils/utilities.py +++ b/src/sysext_utils/utilities.py @@ -49,7 +49,8 @@ from .commands import ( from .helpers import ( get_sysext_metadata, is_url, - get_path_for_active_sysext_name, + get_image_path_for_active_sysext_name, + get_tree_path_for_active_sysext_name, get_path_for_input_sysext_name, get_path_for_artifact, get_path_for_build, @@ -77,12 +78,7 @@ def build_image( remove_file(image) make_directory(directory) - metadata_directory = os.path.join(artifact, METADATA_DIR) - metadata_path = os.path.join(metadata_directory, METADATA_FILENAME % name) - metadata_content = get_sysext_metadata(ignore_release) - - make_directory(metadata_directory) - write_file(metadata_content, metadata_path) + build_tree(name, artifact, ignore_release) if image_format == ImageFormat.DDI: SystemdRepart.build(artifact, image, private_key, certificate) @@ -94,6 +90,15 @@ def build_image( return image +def build_tree(name: str, artifact: str, ignore_release: bool) -> None: + metadata_directory = os.path.join(artifact, METADATA_DIR) + metadata_path = os.path.join(metadata_directory, METADATA_FILENAME % name) + metadata_content = get_sysext_metadata(ignore_release) + + make_directory(metadata_directory) + write_file(metadata_content, metadata_path) + + def add_image(name: str, image: str, runtime: bool) -> None: if is_url(image): Importctl.pull_raw(name, image) @@ -104,21 +109,42 @@ def add_image(name: str, image: str, runtime: bool) -> None: if runtime is True: make_directory(RUN_EXTENSIONS_DIR) move_file( - get_path_for_active_sysext_name(name, runtime=False), - get_path_for_active_sysext_name(name, runtime=True), + get_image_path_for_active_sysext_name(name, runtime=False), + get_image_path_for_active_sysext_name(name, runtime=True), ) SystemdSysext.refresh() +def add_tree(name: str, artifact: str, runtime: bool) -> None: + copy_directory( + artifact, + get_tree_path_for_active_sysext_name(name, runtime=runtime), + ) + + SystemdSysext.refresh() + + def remove_image(name: str) -> None: - remove_file(get_path_for_active_sysext_name(name, runtime=True)) - remove_file(get_path_for_active_sysext_name(name, runtime=False)) + remove_file(get_image_path_for_active_sysext_name(name, runtime=True)) + remove_file(get_image_path_for_active_sysext_name(name, runtime=False)) + + SystemdSysext.refresh() + + +def remove_tree(name: str) -> None: + remove_directory(get_tree_path_for_active_sysext_name(name, runtime=True)) + remove_directory(get_tree_path_for_active_sysext_name(name, runtime=False)) SystemdSysext.refresh() -def integrate_image(artifact: str, integrations: List[str]) -> None: +def build_integrations_artifact(integrations: List[str]) -> str: + artifact = get_path_for_artifact() + + remove_directory(artifact) + make_directory(artifact) + if IntegrationType.SCHEMAS in integrations: host_schemas_dir = os.path.join(os.sep, SCHEMAS_DIR) artifact_schemas_dir = os.path.join(artifact, SCHEMAS_DIR) @@ -126,6 +152,10 @@ def integrate_image(artifact: str, integrations: List[str]) -> None: make_directory(artifact_schemas_dir) GLibCompileSchemas.compile(host_schemas_dir, artifact_schemas_dir) + ensure_directory(artifact) + + return artifact + def build_artifact(system: str, arguments: List[str]) -> str: artifact = get_path_for_artifact() diff --git a/src/sysext_utils/workflows.py b/src/sysext_utils/workflows.py index 5eaf93e..b656c8f 100644 --- a/src/sysext_utils/workflows.py +++ b/src/sysext_utils/workflows.py @@ -27,14 +27,18 @@ from typing import List from .helpers import ( get_name_from_input_sysext_image, + get_name_for_integrations_sysext, ensure_priviledged, ) from .utilities import ( build_image, add_image, remove_image, - integrate_image, build_artifact, + build_tree, + add_tree, + remove_tree, + build_integrations_artifact, ) @@ -45,7 +49,6 @@ def run_image_build_workflow( private_key: str, certificate: str, ignore_release: bool, - integrations: List[str], system: str, arguments: List[str], ) -> None: @@ -60,57 +63,38 @@ def run_image_build_workflow( ignore_release, ) - run_image_integration_workflow( - name, - image, - directory, - artifact, - image_format, - private_key, - certificate, - ignore_release, - integrations, - ) - print(f"Successfully built {image}") def run_image_integration_workflow( name: str, - image: str, - directory: str, - artifact: str, - image_format: str, - private_key: str, - certificate: str, - ignore_release: bool, + runtime: bool, integrations: List[str], ) -> None: if not integrations: return - ensure_priviledged() - remove_image(name) - add_image(name, image, runtime=False) - integrate_image(artifact, integrations) - remove_image(name) - build_image( - name, - artifact, - directory, - image_format, - private_key, - certificate, - ignore_release, - ) + remove_tree(name) + artifact = build_integrations_artifact(integrations) + build_tree(name, artifact, False) + add_tree(name, artifact, runtime) -def run_image_add_workflow(image: str, runtime: bool) -> None: +def run_image_add_workflow( + image: str, + runtime: bool, + integrations: List[str], +) -> None: name = get_name_from_input_sysext_image(image) ensure_priviledged() remove_image(name) add_image(name, image, runtime) + run_image_integration_workflow( + get_name_for_integrations_sysext(name), + runtime, + integrations, + ) print(f"Successfully added {image}") @@ -118,5 +102,6 @@ def run_image_add_workflow(image: str, runtime: bool) -> None: def run_image_remove_workflow(name: str) -> None: ensure_priviledged() remove_image(name) + remove_tree(get_name_for_integrations_sysext(name)) print(f"Successfully removed {name}") diff --git a/tests/test_workflows.py b/tests/test_workflows.py index fc330e7..1907a28 100644 --- a/tests/test_workflows.py +++ b/tests/test_workflows.py @@ -44,7 +44,6 @@ def test_image_build_workflow(): private_key="private_key", certificate="certificate", ignore_release=False, - integrations=[], system=BuildSystemType.BST, arguments=[ "element/one.bst", @@ -60,7 +59,6 @@ def test_image_build_workflow_with_import(): private_key="private_key", certificate="certificate", ignore_release=False, - integrations=[], system=BuildSystemType.IMPORT, arguments=["./destdir"], ) @@ -74,7 +72,6 @@ def test_image_build_workflow_with_meson(): private_key="private_key", certificate="certificate", ignore_release=False, - integrations=[], system=BuildSystemType.MESON, arguments=[], ) @@ -88,23 +85,6 @@ def test_image_build_workflow_with_compressed_format(): private_key="private_key", certificate="certificate", ignore_release=False, - integrations=[], - system=BuildSystemType.MESON, - arguments=[], - ) - - -def test_image_build_workflow_with_integrations(): - run_image_build_workflow( - name="name", - directory="directory", - image_format=ImageFormat.COMPRESSED, - private_key="private_key", - certificate="certificate", - ignore_release=False, - integrations=[ - IntegrationType.SCHEMAS, - ], system=BuildSystemType.MESON, arguments=[], ) @@ -118,24 +98,31 @@ def test_image_build_work_with_ignore_release(): private_key="private_key", certificate="certificate", ignore_release=True, - integrations=[ - IntegrationType.SCHEMAS, - ], system=BuildSystemType.MESON, arguments=[], ) def test_image_add_workflow(): - run_image_add_workflow(image="image", runtime=False) + run_image_add_workflow(image="image", runtime=False, integrations=[]) def test_image_add_workflow_with_remote_image(): - run_image_add_workflow(image="https://image", runtime=False) + run_image_add_workflow(image="https://image", runtime=False, integrations=[]) def test_image_add_workflow_with_runtime_flag(): - run_image_add_workflow(image="image", runtime=True) + run_image_add_workflow(image="image", runtime=True, integrations=[]) + + +def test_image_add_workflow_with_integrations(): + run_image_add_workflow( + image="image", + runtime=True, + integrations=[ + IntegrationType.SCHEMAS, + ], + ) def test_image_remove_workflow(): -- GitLab From 03aec70b62255d3616a476030355b6f75fd163c5 Mon Sep 17 00:00:00 2001 From: Martin Abente Lahaye Date: Tue, 4 Jun 2024 07:38:16 -0400 Subject: [PATCH 29/46] main: Cleanup workspace every time --- src/sysext_utils/main.py | 5 +++++ src/sysext_utils/workflows.py | 6 ++++++ 2 files changed, 11 insertions(+) diff --git a/src/sysext_utils/main.py b/src/sysext_utils/main.py index 0aa27de..19088ab 100644 --- a/src/sysext_utils/main.py +++ b/src/sysext_utils/main.py @@ -36,6 +36,7 @@ from .workflows import ( run_image_build_workflow, run_image_add_workflow, run_image_remove_workflow, + run_cleanup_workflow, ) @@ -69,6 +70,8 @@ def _handle_image_build( ) except HandledError as e: sys.exit(str(e)) + finally: + run_cleanup_workflow() def _handle_image_add( @@ -79,6 +82,8 @@ def _handle_image_add( run_image_add_workflow(args.image, args.runtime, args.integrations) except HandledError as e: sys.exit(str(e)) + finally: + run_cleanup_workflow() def _handle_image_remove( diff --git a/src/sysext_utils/workflows.py b/src/sysext_utils/workflows.py index b656c8f..e236412 100644 --- a/src/sysext_utils/workflows.py +++ b/src/sysext_utils/workflows.py @@ -25,10 +25,12 @@ from typing import List +from .definitions import WORKSPACE_DIR from .helpers import ( get_name_from_input_sysext_image, get_name_for_integrations_sysext, ensure_priviledged, + remove_directory, ) from .utilities import ( build_image, @@ -105,3 +107,7 @@ def run_image_remove_workflow(name: str) -> None: remove_tree(get_name_for_integrations_sysext(name)) print(f"Successfully removed {name}") + + +def run_cleanup_workflow() -> None: + remove_directory(WORKSPACE_DIR) -- GitLab From 06639798218a9f5d6cbe3be9c20ad4fc27fee0df Mon Sep 17 00:00:00 2001 From: Martin Abente Lahaye Date: Tue, 4 Jun 2024 11:42:28 -0400 Subject: [PATCH 30/46] main: Add support for updating icons cache as an integration step --- src/sysext_utils/commands.py | 17 +++++++++++++++++ src/sysext_utils/definitions.py | 3 +++ src/sysext_utils/main.py | 5 ++++- src/sysext_utils/utilities.py | 18 ++++++++++++++++++ tests/test_workflows.py | 1 + 5 files changed, 43 insertions(+), 1 deletion(-) diff --git a/src/sysext_utils/commands.py b/src/sysext_utils/commands.py index 4a2447c..44885bd 100644 --- a/src/sysext_utils/commands.py +++ b/src/sysext_utils/commands.py @@ -76,6 +76,23 @@ class GLibCompileSchemas(BaseCommand): cls.do_run("glib-compile-schemas", *args) +class Gtk4UpdateIconCache(BaseCommand): + @classmethod + def update(cls, path: str) -> None: + cls.run( + [ + "--quiet", + "--ignore-theme-index", + "--force", + path, + ] + ) + + @classmethod + def run(cls, *args) -> None: + cls.do_run("gtk4-update-icon-cache", *args) + + class Importctl(BaseCommand): @classmethod def import_raw(cls, name: str, image: str) -> None: diff --git a/src/sysext_utils/definitions.py b/src/sysext_utils/definitions.py index d3bd944..1f9457b 100644 --- a/src/sysext_utils/definitions.py +++ b/src/sysext_utils/definitions.py @@ -31,6 +31,8 @@ OS_RELEASE_PATH = os.path.join(os.sep, "etc", "os-release") RUN_EXTENSIONS_DIR = os.path.join(os.sep, "run", "extensions") VAR_EXTENSIONS_DIR = os.path.join(os.sep, "var", "lib", "extensions") SCHEMAS_DIR = os.path.join("usr", "share", "glib-2.0", "schemas") +ICONS_DIR = os.path.join("usr", "share", "icons", "hicolor") +ICONS_CACHE_FILENAME = "icon-theme.cache" METADATA_DIR = os.path.join("usr", "lib", "extension-release.d") METADATA_FILENAME = "extension-release.%s" SYSEXT_IMAGE_SUFFIX = ".sysext.raw" @@ -44,6 +46,7 @@ class ImageFormat(StrEnum): class IntegrationType(StrEnum): SCHEMAS = "schemas" + ICONS = "icons" class DryStepType(StrEnum): diff --git a/src/sysext_utils/main.py b/src/sysext_utils/main.py index 19088ab..eea4fc6 100644 --- a/src/sysext_utils/main.py +++ b/src/sysext_utils/main.py @@ -145,7 +145,10 @@ def main(): "--integrations", help="list of integration steps to be run", nargs="+", - choices=[str(IntegrationType.SCHEMAS)], + choices=[ + str(IntegrationType.SCHEMAS), + str(IntegrationType.ICONS), + ], default=[], ) diff --git a/src/sysext_utils/utilities.py b/src/sysext_utils/utilities.py index 16c98ca..22a907f 100644 --- a/src/sysext_utils/utilities.py +++ b/src/sysext_utils/utilities.py @@ -34,6 +34,8 @@ from .definitions import ( BuildSystemType, RUN_EXTENSIONS_DIR, SCHEMAS_DIR, + ICONS_DIR, + ICONS_CACHE_FILENAME, METADATA_DIR, METADATA_FILENAME, ) @@ -43,6 +45,7 @@ from .commands import ( Importctl, SystemdSysext, GLibCompileSchemas, + Gtk4UpdateIconCache, Bst, Meson, ) @@ -151,6 +154,21 @@ def build_integrations_artifact(integrations: List[str]) -> str: make_directory(artifact_schemas_dir) GLibCompileSchemas.compile(host_schemas_dir, artifact_schemas_dir) + if IntegrationType.ICONS in integrations: + host_icons_dir = os.path.join(os.sep, ICONS_DIR) + tmp_icons_dir = os.path.join(artifact, ICONS_DIR + ".tmp") + artifact_icons_dir = os.path.join(artifact, ICONS_DIR) + + make_directory(tmp_icons_dir) + copy_directory(host_icons_dir, tmp_icons_dir) + Gtk4UpdateIconCache.update(tmp_icons_dir) + + make_directory(artifact_icons_dir) + move_file( + os.path.join(tmp_icons_dir, ICONS_CACHE_FILENAME), + os.path.join(artifact_icons_dir, ICONS_CACHE_FILENAME), + ) + remove_directory(tmp_icons_dir) ensure_directory(artifact) diff --git a/tests/test_workflows.py b/tests/test_workflows.py index 1907a28..7c6bd26 100644 --- a/tests/test_workflows.py +++ b/tests/test_workflows.py @@ -121,6 +121,7 @@ def test_image_add_workflow_with_integrations(): runtime=True, integrations=[ IntegrationType.SCHEMAS, + IntegrationType.ICONS, ], ) -- GitLab From b037e14a7ee8319c9903c71a9d7f29d21a09f48b Mon Sep 17 00:00:00 2001 From: Martin Abente Lahaye Date: Wed, 5 Jun 2024 12:54:26 -0400 Subject: [PATCH 31/46] main: Simplify integrations interface and make it default Instead of specifying each individual integration step we want to run, let's just assume it's all or nothing. Also, assume we always want to run integrations unless explicitly stated. --- README.md | 2 +- src/sysext_utils/definitions.py | 5 ---- src/sysext_utils/main.py | 15 ++++------ src/sysext_utils/utilities.py | 49 ++++++++++++++++++--------------- src/sysext_utils/workflows.py | 12 ++++---- tests/test_workflows.py | 27 ++++++++++++------ 6 files changed, 57 insertions(+), 53 deletions(-) diff --git a/README.md b/README.md index b3a3fd6..04567f1 100644 --- a/README.md +++ b/README.md @@ -33,7 +33,7 @@ $ sysext-utils --dry build sdk --format=ddi --private_key=key --certificate=cert ```bash $ sysext-utils --dry add sdk.sysext.raw -$ sysext-utils --dry add sdk.sysext.raw --integrations schemas +$ sysext-utils --dry add sdk.sysext.raw --skip-integration $ sysext-utils --dry add https://os.gnome.org/sysext/sdk.sysext.raw --runtime $ sysext-utils --dry remove sdk ``` diff --git a/src/sysext_utils/definitions.py b/src/sysext_utils/definitions.py index 1f9457b..8d49f51 100644 --- a/src/sysext_utils/definitions.py +++ b/src/sysext_utils/definitions.py @@ -44,11 +44,6 @@ class ImageFormat(StrEnum): COMPRESSED = "compressed" -class IntegrationType(StrEnum): - SCHEMAS = "schemas" - ICONS = "icons" - - class DryStepType(StrEnum): RUNNING = "running" MAKING = "making" diff --git a/src/sysext_utils/main.py b/src/sysext_utils/main.py index eea4fc6..12654c2 100644 --- a/src/sysext_utils/main.py +++ b/src/sysext_utils/main.py @@ -30,7 +30,7 @@ import argparse from . import config from .errors import HandledError -from .definitions import ImageFormat, IntegrationType, BuildSystemType +from .definitions import ImageFormat, BuildSystemType from .version import version from .workflows import ( run_image_build_workflow, @@ -79,7 +79,7 @@ def _handle_image_add( args: argparse.Namespace, ) -> None: try: - run_image_add_workflow(args.image, args.runtime, args.integrations) + run_image_add_workflow(args.image, args.runtime, args.skip_integration) except HandledError as e: sys.exit(str(e)) finally: @@ -142,14 +142,9 @@ def main(): action="store_true", ) image_add_parser.add_argument( - "--integrations", - help="list of integration steps to be run", - nargs="+", - choices=[ - str(IntegrationType.SCHEMAS), - str(IntegrationType.ICONS), - ], - default=[], + "--skip-integration", + help="skip integration steps", + action="store_true", ) # Arguments for removing a sysext image from the host OS diff --git a/src/sysext_utils/utilities.py b/src/sysext_utils/utilities.py index 22a907f..feb597e 100644 --- a/src/sysext_utils/utilities.py +++ b/src/sysext_utils/utilities.py @@ -30,7 +30,6 @@ from typing import List from .errors import UnsupportedImageFormatError, UnsupportedBuildSystemError from .definitions import ( ImageFormat, - IntegrationType, BuildSystemType, RUN_EXTENSIONS_DIR, SCHEMAS_DIR, @@ -142,37 +141,43 @@ def remove_tree(name: str) -> None: SystemdSysext.refresh() -def build_integrations_artifact(integrations: List[str]) -> str: +def build_integration_artifact() -> str: artifact = get_path_for_artifact() remove_directory(artifact) make_directory(artifact) - if IntegrationType.SCHEMAS in integrations: - host_schemas_dir = os.path.join(os.sep, SCHEMAS_DIR) - artifact_schemas_dir = os.path.join(artifact, SCHEMAS_DIR) + build_integration_schemas(artifact) + build_integration_icons(artifact) - make_directory(artifact_schemas_dir) - GLibCompileSchemas.compile(host_schemas_dir, artifact_schemas_dir) - if IntegrationType.ICONS in integrations: - host_icons_dir = os.path.join(os.sep, ICONS_DIR) - tmp_icons_dir = os.path.join(artifact, ICONS_DIR + ".tmp") - artifact_icons_dir = os.path.join(artifact, ICONS_DIR) + ensure_directory(artifact) - make_directory(tmp_icons_dir) - copy_directory(host_icons_dir, tmp_icons_dir) - Gtk4UpdateIconCache.update(tmp_icons_dir) + return artifact - make_directory(artifact_icons_dir) - move_file( - os.path.join(tmp_icons_dir, ICONS_CACHE_FILENAME), - os.path.join(artifact_icons_dir, ICONS_CACHE_FILENAME), - ) - remove_directory(tmp_icons_dir) - ensure_directory(artifact) +def build_integration_schemas(artifact: str) -> None: + host_schemas_dir = os.path.join(os.sep, SCHEMAS_DIR) + artifact_schemas_dir = os.path.join(artifact, SCHEMAS_DIR) - return artifact + make_directory(artifact_schemas_dir) + GLibCompileSchemas.compile(host_schemas_dir, artifact_schemas_dir) + + +def build_integration_icons(artifact: str) -> None: + host_icons_dir = os.path.join(os.sep, ICONS_DIR) + tmp_icons_dir = os.path.join(artifact, ICONS_DIR + ".tmp") + artifact_icons_dir = os.path.join(artifact, ICONS_DIR) + + make_directory(tmp_icons_dir) + copy_directory(host_icons_dir, tmp_icons_dir) + Gtk4UpdateIconCache.update(tmp_icons_dir) + + make_directory(artifact_icons_dir) + move_file( + os.path.join(tmp_icons_dir, ICONS_CACHE_FILENAME), + os.path.join(artifact_icons_dir, ICONS_CACHE_FILENAME), + ) + remove_directory(tmp_icons_dir) def build_artifact(system: str, arguments: List[str]) -> str: diff --git a/src/sysext_utils/workflows.py b/src/sysext_utils/workflows.py index e236412..e6eb49c 100644 --- a/src/sysext_utils/workflows.py +++ b/src/sysext_utils/workflows.py @@ -40,7 +40,7 @@ from .utilities import ( build_tree, add_tree, remove_tree, - build_integrations_artifact, + build_integration_artifact, ) @@ -71,13 +71,13 @@ def run_image_build_workflow( def run_image_integration_workflow( name: str, runtime: bool, - integrations: List[str], + skip_integration: bool, ) -> None: - if not integrations: + if skip_integration: return remove_tree(name) - artifact = build_integrations_artifact(integrations) + artifact = build_integration_artifact() build_tree(name, artifact, False) add_tree(name, artifact, runtime) @@ -85,7 +85,7 @@ def run_image_integration_workflow( def run_image_add_workflow( image: str, runtime: bool, - integrations: List[str], + skip_integration: bool, ) -> None: name = get_name_from_input_sysext_image(image) @@ -95,7 +95,7 @@ def run_image_add_workflow( run_image_integration_workflow( get_name_for_integrations_sysext(name), runtime, - integrations, + skip_integration, ) print(f"Successfully added {image}") diff --git a/tests/test_workflows.py b/tests/test_workflows.py index 7c6bd26..351938f 100644 --- a/tests/test_workflows.py +++ b/tests/test_workflows.py @@ -24,7 +24,7 @@ # SPDX-License-Identifier: MIT from sysext_utils import config -from sysext_utils.definitions import ImageFormat, IntegrationType, BuildSystemType +from sysext_utils.definitions import ImageFormat, BuildSystemType from sysext_utils.workflows import ( run_image_build_workflow, run_image_add_workflow, @@ -104,25 +104,34 @@ def test_image_build_work_with_ignore_release(): def test_image_add_workflow(): - run_image_add_workflow(image="image", runtime=False, integrations=[]) + run_image_add_workflow( + image="image", + runtime=False, + skip_integration=False, + ) def test_image_add_workflow_with_remote_image(): - run_image_add_workflow(image="https://image", runtime=False, integrations=[]) + run_image_add_workflow( + image="https://image", + runtime=False, + skip_integration=False, + ) def test_image_add_workflow_with_runtime_flag(): - run_image_add_workflow(image="image", runtime=True, integrations=[]) + run_image_add_workflow( + image="image", + runtime=True, + skip_integration=False, + ) -def test_image_add_workflow_with_integrations(): +def test_image_add_workflow_without_integrations(): run_image_add_workflow( image="image", runtime=True, - integrations=[ - IntegrationType.SCHEMAS, - IntegrationType.ICONS, - ], + skip_integration=True, ) -- GitLab From 367b67e5eecfa6ee6e3d4ba044bb4900caad847e Mon Sep 17 00:00:00 2001 From: Martin Abente Lahaye Date: Wed, 5 Jun 2024 13:21:53 -0400 Subject: [PATCH 32/46] utilities: Make integration sysext global --- src/sysext_utils/definitions.py | 1 + src/sysext_utils/helpers.py | 8 ++++---- src/sysext_utils/utilities.py | 3 ++- src/sysext_utils/workflows.py | 18 ++++++------------ 4 files changed, 13 insertions(+), 17 deletions(-) diff --git a/src/sysext_utils/definitions.py b/src/sysext_utils/definitions.py index 8d49f51..bda0919 100644 --- a/src/sysext_utils/definitions.py +++ b/src/sysext_utils/definitions.py @@ -37,6 +37,7 @@ METADATA_DIR = os.path.join("usr", "lib", "extension-release.d") METADATA_FILENAME = "extension-release.%s" SYSEXT_IMAGE_SUFFIX = ".sysext.raw" WORKSPACE_DIR = os.path.join(os.getcwd(), ".sysext-utils") +INTEGRATION_SYSEXT_NAME = "integration.sysext" class ImageFormat(StrEnum): diff --git a/src/sysext_utils/helpers.py b/src/sysext_utils/helpers.py index 19c41e5..5b98d6e 100644 --- a/src/sysext_utils/helpers.py +++ b/src/sysext_utils/helpers.py @@ -80,10 +80,6 @@ def get_name_from_input_sysext_image(image: str) -> str: return filename.removesuffix(SYSEXT_IMAGE_SUFFIX) -def get_name_for_integrations_sysext(name: str) -> str: - return f"{name}.integrations" - - def get_tree_path_for_active_sysext_name(name: str, runtime: bool) -> str: if runtime: return os.path.join(RUN_EXTENSIONS_DIR, name) @@ -99,6 +95,10 @@ def get_path_for_artifact() -> str: return os.path.join(WORKSPACE_DIR, "artifact") +def get_path_for_integration_artifact() -> str: + return os.path.join(WORKSPACE_DIR, "integration") + + def get_path_for_build() -> str: return os.path.join(WORKSPACE_DIR, "build") diff --git a/src/sysext_utils/utilities.py b/src/sysext_utils/utilities.py index feb597e..191c797 100644 --- a/src/sysext_utils/utilities.py +++ b/src/sysext_utils/utilities.py @@ -55,6 +55,7 @@ from .helpers import ( get_tree_path_for_active_sysext_name, get_path_for_input_sysext_name, get_path_for_artifact, + get_path_for_integration_artifact, get_path_for_build, make_directory, copy_directory, @@ -142,7 +143,7 @@ def remove_tree(name: str) -> None: def build_integration_artifact() -> str: - artifact = get_path_for_artifact() + artifact = get_path_for_integration_artifact() remove_directory(artifact) make_directory(artifact) diff --git a/src/sysext_utils/workflows.py b/src/sysext_utils/workflows.py index e6eb49c..d151fe1 100644 --- a/src/sysext_utils/workflows.py +++ b/src/sysext_utils/workflows.py @@ -25,10 +25,9 @@ from typing import List -from .definitions import WORKSPACE_DIR +from .definitions import WORKSPACE_DIR, INTEGRATION_SYSEXT_NAME from .helpers import ( get_name_from_input_sysext_image, - get_name_for_integrations_sysext, ensure_priviledged, remove_directory, ) @@ -69,17 +68,16 @@ def run_image_build_workflow( def run_image_integration_workflow( - name: str, runtime: bool, skip_integration: bool, ) -> None: if skip_integration: return - remove_tree(name) + remove_tree(INTEGRATION_SYSEXT_NAME) artifact = build_integration_artifact() - build_tree(name, artifact, False) - add_tree(name, artifact, runtime) + build_tree(INTEGRATION_SYSEXT_NAME, artifact, False) + add_tree(INTEGRATION_SYSEXT_NAME, artifact, runtime) def run_image_add_workflow( @@ -92,11 +90,7 @@ def run_image_add_workflow( ensure_priviledged() remove_image(name) add_image(name, image, runtime) - run_image_integration_workflow( - get_name_for_integrations_sysext(name), - runtime, - skip_integration, - ) + run_image_integration_workflow(runtime, skip_integration) print(f"Successfully added {image}") @@ -104,7 +98,7 @@ def run_image_add_workflow( def run_image_remove_workflow(name: str) -> None: ensure_priviledged() remove_image(name) - remove_tree(get_name_for_integrations_sysext(name)) + remove_tree(INTEGRATION_SYSEXT_NAME) print(f"Successfully removed {name}") -- GitLab From f17d55dda2ee9a0cfce7c849f9b930cefc788ac7 Mon Sep 17 00:00:00 2001 From: Martin Abente Lahaye Date: Wed, 5 Jun 2024 14:51:31 -0400 Subject: [PATCH 33/46] workflows: Handle de-integrations properly --- src/sysext_utils/definitions.py | 3 ++- src/sysext_utils/helpers.py | 42 ++++++++++++++++++++++++++++----- src/sysext_utils/utilities.py | 15 +++++++++--- src/sysext_utils/workflows.py | 18 +++++++++++++- 4 files changed, 67 insertions(+), 11 deletions(-) diff --git a/src/sysext_utils/definitions.py b/src/sysext_utils/definitions.py index bda0919..b98f9f1 100644 --- a/src/sysext_utils/definitions.py +++ b/src/sysext_utils/definitions.py @@ -34,7 +34,8 @@ SCHEMAS_DIR = os.path.join("usr", "share", "glib-2.0", "schemas") ICONS_DIR = os.path.join("usr", "share", "icons", "hicolor") ICONS_CACHE_FILENAME = "icon-theme.cache" METADATA_DIR = os.path.join("usr", "lib", "extension-release.d") -METADATA_FILENAME = "extension-release.%s" +METADATA_PREFIX = "extension-release." +METADATA_CUSTOM_FIELD = "SYSEXT_UTILS_VERSION" SYSEXT_IMAGE_SUFFIX = ".sysext.raw" WORKSPACE_DIR = os.path.join(os.getcwd(), ".sysext-utils") INTEGRATION_SYSEXT_NAME = "integration.sysext" diff --git a/src/sysext_utils/helpers.py b/src/sysext_utils/helpers.py index 5b98d6e..eb4801e 100644 --- a/src/sysext_utils/helpers.py +++ b/src/sysext_utils/helpers.py @@ -28,26 +28,30 @@ import re import shutil from urllib.parse import urlparse -from typing import Dict +from typing import Dict, List from . import config +from .version import version from .errors import NotPriviledgedError, EmptyDirectoryError from .definitions import ( RUN_EXTENSIONS_DIR, VAR_EXTENSIONS_DIR, + METADATA_DIR, + METADATA_PREFIX, WORKSPACE_DIR, OS_RELEASE_PATH, SYSEXT_IMAGE_SUFFIX, + METADATA_CUSTOM_FIELD, DryStepType, ) -def get_release_data() -> Dict[str, str]: +def get_release_data(path: str) -> Dict[str, str]: data = {} try: - with open(OS_RELEASE_PATH, encoding="utf-8") as f: + with open(path, encoding="utf-8") as f: data = dict([l.strip().split("=", 1) for l in f.readlines()]) except FileNotFoundError: pass @@ -56,11 +60,37 @@ def get_release_data() -> Dict[str, str]: def get_sysext_metadata(ignore_release: bool) -> str: + metadata = { + f"{METADATA_CUSTOM_FIELD}": version, + } + if ignore_release: - return "ID=_any" + metadata["ID"] = "_any" + else: + release = get_release_data(OS_RELEASE_PATH) + metadata["ID"] = release.get("ID", "_any") + metadata["VERSION_ID"] = release.get("VERSION_ID", "_any") + + return "\n".join([f"{k}={v}" for k, v in metadata.items()]) + + +def get_active_sysext_names() -> List[str]: + names: List[str] = [] + + path = os.path.join(os.sep, METADATA_DIR) + + if not os.path.exists(path): + return names + + for filename in os.listdir(path): + release = get_release_data(os.path.join(path, filename)) + + if not release.get(METADATA_CUSTOM_FIELD): + continue + + names.append(filename.removeprefix(METADATA_PREFIX)) - data = get_release_data() - return f"ID={data['ID']}\nVERSION_ID={data['VERSION_ID']}" + return names def is_url(string: str) -> bool: diff --git a/src/sysext_utils/utilities.py b/src/sysext_utils/utilities.py index 191c797..11dbaf1 100644 --- a/src/sysext_utils/utilities.py +++ b/src/sysext_utils/utilities.py @@ -25,7 +25,7 @@ import os -from typing import List +from typing import List, Tuple from .errors import UnsupportedImageFormatError, UnsupportedBuildSystemError from .definitions import ( @@ -36,7 +36,7 @@ from .definitions import ( ICONS_DIR, ICONS_CACHE_FILENAME, METADATA_DIR, - METADATA_FILENAME, + METADATA_PREFIX, ) from .commands import ( SystemdRepart, @@ -95,7 +95,7 @@ def build_image( def build_tree(name: str, artifact: str, ignore_release: bool) -> None: metadata_directory = os.path.join(artifact, METADATA_DIR) - metadata_path = os.path.join(metadata_directory, METADATA_FILENAME % name) + metadata_path = os.path.join(metadata_directory, f"{METADATA_PREFIX}{name}") metadata_content = get_sysext_metadata(ignore_release) make_directory(metadata_directory) @@ -142,6 +142,15 @@ def remove_tree(name: str) -> None: SystemdSysext.refresh() +def check_tree(name: str) -> Tuple[bool, bool]: + if os.path.exists(get_tree_path_for_active_sysext_name(name, runtime=True)): + return True, True + if os.path.exists(get_tree_path_for_active_sysext_name(name, runtime=False)): + return True, False + + return False, False + + def build_integration_artifact() -> str: artifact = get_path_for_integration_artifact() diff --git a/src/sysext_utils/workflows.py b/src/sysext_utils/workflows.py index d151fe1..bd5dca9 100644 --- a/src/sysext_utils/workflows.py +++ b/src/sysext_utils/workflows.py @@ -28,6 +28,7 @@ from typing import List from .definitions import WORKSPACE_DIR, INTEGRATION_SYSEXT_NAME from .helpers import ( get_name_from_input_sysext_image, + get_active_sysext_names, ensure_priviledged, remove_directory, ) @@ -39,6 +40,7 @@ from .utilities import ( build_tree, add_tree, remove_tree, + check_tree, build_integration_artifact, ) @@ -95,10 +97,24 @@ def run_image_add_workflow( print(f"Successfully added {image}") +def run_image_deintegration_workflow() -> None: + exists, runtime = check_tree(INTEGRATION_SYSEXT_NAME) + + if not exists: + return + + remove_tree(INTEGRATION_SYSEXT_NAME) + + if not get_active_sysext_names(): + return + + run_image_integration_workflow(runtime, False) + + def run_image_remove_workflow(name: str) -> None: ensure_priviledged() remove_image(name) - remove_tree(INTEGRATION_SYSEXT_NAME) + run_image_deintegration_workflow() print(f"Successfully removed {name}") -- GitLab From f4155895d139091c6f8f1b36b15ce5fe0f42ef71 Mon Sep 17 00:00:00 2001 From: Martin Abente Lahaye Date: Wed, 5 Jun 2024 15:18:30 -0400 Subject: [PATCH 34/46] main: Make integration steps mandatory There's no reliable way to have a global integration sysext and to keep integration steps optional. See the following example with verbs: 1. Add a.sysext.raw --skip-integration # will skip integration sysext 2. Add b.sysext.raw # will create integration sysext 3. Add c.sysext.raw # will create integration sysext 4. Remove c # will create integration sysext 5. Remove b # will create integration sysext At step 5, there's no reliable way of knowing which of the remaining extensions required integration, and therefore will always integrate until the last extension is removed and so the integration extension. Let's just accept this for now and always integrate. --- README.md | 2 +- src/sysext_utils/main.py | 7 +------ src/sysext_utils/workflows.py | 21 +++++---------------- tests/test_workflows.py | 24 ++++-------------------- 4 files changed, 11 insertions(+), 43 deletions(-) diff --git a/README.md b/README.md index 04567f1..a4be242 100644 --- a/README.md +++ b/README.md @@ -33,7 +33,7 @@ $ sysext-utils --dry build sdk --format=ddi --private_key=key --certificate=cert ```bash $ sysext-utils --dry add sdk.sysext.raw -$ sysext-utils --dry add sdk.sysext.raw --skip-integration +$ sysext-utils --dry add sdk.sysext.raw $ sysext-utils --dry add https://os.gnome.org/sysext/sdk.sysext.raw --runtime $ sysext-utils --dry remove sdk ``` diff --git a/src/sysext_utils/main.py b/src/sysext_utils/main.py index 12654c2..2851134 100644 --- a/src/sysext_utils/main.py +++ b/src/sysext_utils/main.py @@ -79,7 +79,7 @@ def _handle_image_add( args: argparse.Namespace, ) -> None: try: - run_image_add_workflow(args.image, args.runtime, args.skip_integration) + run_image_add_workflow(args.image, args.runtime) except HandledError as e: sys.exit(str(e)) finally: @@ -141,11 +141,6 @@ def main(): help="add the sysext image under /run/extensions/ instead", action="store_true", ) - image_add_parser.add_argument( - "--skip-integration", - help="skip integration steps", - action="store_true", - ) # Arguments for removing a sysext image from the host OS diff --git a/src/sysext_utils/workflows.py b/src/sysext_utils/workflows.py index bd5dca9..db0c5fa 100644 --- a/src/sysext_utils/workflows.py +++ b/src/sysext_utils/workflows.py @@ -69,46 +69,35 @@ def run_image_build_workflow( print(f"Successfully built {image}") -def run_image_integration_workflow( - runtime: bool, - skip_integration: bool, -) -> None: - if skip_integration: - return - +def run_image_integration_workflow(runtime: bool) -> None: remove_tree(INTEGRATION_SYSEXT_NAME) artifact = build_integration_artifact() build_tree(INTEGRATION_SYSEXT_NAME, artifact, False) add_tree(INTEGRATION_SYSEXT_NAME, artifact, runtime) -def run_image_add_workflow( - image: str, - runtime: bool, - skip_integration: bool, -) -> None: +def run_image_add_workflow(image: str, runtime: bool) -> None: name = get_name_from_input_sysext_image(image) ensure_priviledged() remove_image(name) add_image(name, image, runtime) - run_image_integration_workflow(runtime, skip_integration) + run_image_integration_workflow(runtime) print(f"Successfully added {image}") def run_image_deintegration_workflow() -> None: exists, runtime = check_tree(INTEGRATION_SYSEXT_NAME) + remove_tree(INTEGRATION_SYSEXT_NAME) if not exists: return - remove_tree(INTEGRATION_SYSEXT_NAME) - if not get_active_sysext_names(): return - run_image_integration_workflow(runtime, False) + run_image_integration_workflow(runtime) def run_image_remove_workflow(name: str) -> None: diff --git a/tests/test_workflows.py b/tests/test_workflows.py index 351938f..60bdb14 100644 --- a/tests/test_workflows.py +++ b/tests/test_workflows.py @@ -104,35 +104,19 @@ def test_image_build_work_with_ignore_release(): def test_image_add_workflow(): - run_image_add_workflow( - image="image", - runtime=False, - skip_integration=False, - ) + run_image_add_workflow(image="image", runtime=False) def test_image_add_workflow_with_remote_image(): - run_image_add_workflow( - image="https://image", - runtime=False, - skip_integration=False, - ) + run_image_add_workflow(image="https://image", runtime=False) def test_image_add_workflow_with_runtime_flag(): - run_image_add_workflow( - image="image", - runtime=True, - skip_integration=False, - ) + run_image_add_workflow(image="image", runtime=True) def test_image_add_workflow_without_integrations(): - run_image_add_workflow( - image="image", - runtime=True, - skip_integration=True, - ) + run_image_add_workflow(image="image", runtime=True) def test_image_remove_workflow(): -- GitLab From 311cddd8e0050c8fe2dd7c687b618c040230b21c Mon Sep 17 00:00:00 2001 From: Martin Abente Lahaye Date: Wed, 5 Jun 2024 16:07:25 -0400 Subject: [PATCH 35/46] workflows: Ensure that all extensions are either runtime or persistent We can't have a global integration sysext if we mmix different runtime settings. See the following example with verbs: 1. Add a.sysext.raw --runtime # integration sysext goes to /run/extensions/ 2. Add b.sysext.raw # integration sysext goes to /var/lib/extensions/ 3. Remove b # integration sysext goes to /run/lib/extensions/ 4. Reboot # integration sysext for "a" is still around but not "a" At step 4, we would be in a state where the the "a" sysext is gone, due to the roobot, but its integration files would still be around and something will break. --- src/sysext_utils/errors.py | 5 +++++ src/sysext_utils/helpers.py | 29 +++++++++++++++++++++++++++-- src/sysext_utils/utilities.py | 11 +---------- src/sysext_utils/workflows.py | 9 ++++++--- 4 files changed, 39 insertions(+), 15 deletions(-) diff --git a/src/sysext_utils/errors.py b/src/sysext_utils/errors.py index e5faef9..065e4c8 100644 --- a/src/sysext_utils/errors.py +++ b/src/sysext_utils/errors.py @@ -63,3 +63,8 @@ class UnsupportedImageFormatError(HandledError): class UnsupportedBuildSystemError(HandledError): def __init__(self, system: str): super().__init__(f"Build system not supported: {system}") + + +class IncompatibleSettingError(HandledError): + def __init__(self): + super().__init__("All extensions must be either runtime or persistent") diff --git a/src/sysext_utils/helpers.py b/src/sysext_utils/helpers.py index eb4801e..9ede2cb 100644 --- a/src/sysext_utils/helpers.py +++ b/src/sysext_utils/helpers.py @@ -28,12 +28,16 @@ import re import shutil from urllib.parse import urlparse -from typing import Dict, List +from typing import Dict, List, Tuple from . import config from .version import version -from .errors import NotPriviledgedError, EmptyDirectoryError +from .errors import ( + NotPriviledgedError, + EmptyDirectoryError, + IncompatibleSettingError, +) from .definitions import ( RUN_EXTENSIONS_DIR, VAR_EXTENSIONS_DIR, @@ -43,6 +47,7 @@ from .definitions import ( OS_RELEASE_PATH, SYSEXT_IMAGE_SUFFIX, METADATA_CUSTOM_FIELD, + INTEGRATION_SYSEXT_NAME, DryStepType, ) @@ -117,6 +122,15 @@ def get_tree_path_for_active_sysext_name(name: str, runtime: bool) -> str: return os.path.join(VAR_EXTENSIONS_DIR, name) +def get_tree_runtime_setting_for_sysext_name(name: str) -> Tuple[bool, bool]: + if os.path.exists(get_tree_path_for_active_sysext_name(name, runtime=True)): + return True, True + elif os.path.exists(get_tree_path_for_active_sysext_name(name, runtime=False)): + return True, False + + return False, False + + def get_image_path_for_active_sysext_name(name: str, runtime: bool) -> str: return get_tree_path_for_active_sysext_name(name, runtime) + ".raw" @@ -187,3 +201,14 @@ def ensure_priviledged() -> None: def ensure_directory(path: str) -> None: if not config.dry_run and (not os.path.isdir(path) or not os.listdir(path)): raise EmptyDirectoryError(path) + + +def ensure_setting(runtime: bool) -> None: + _exists, _runtime = get_tree_runtime_setting_for_sysext_name( + INTEGRATION_SYSEXT_NAME + ) + + if not _exists: + return + if _runtime != runtime: + raise IncompatibleSettingError() diff --git a/src/sysext_utils/utilities.py b/src/sysext_utils/utilities.py index 11dbaf1..15d9a7b 100644 --- a/src/sysext_utils/utilities.py +++ b/src/sysext_utils/utilities.py @@ -25,7 +25,7 @@ import os -from typing import List, Tuple +from typing import List from .errors import UnsupportedImageFormatError, UnsupportedBuildSystemError from .definitions import ( @@ -142,15 +142,6 @@ def remove_tree(name: str) -> None: SystemdSysext.refresh() -def check_tree(name: str) -> Tuple[bool, bool]: - if os.path.exists(get_tree_path_for_active_sysext_name(name, runtime=True)): - return True, True - if os.path.exists(get_tree_path_for_active_sysext_name(name, runtime=False)): - return True, False - - return False, False - - def build_integration_artifact() -> str: artifact = get_path_for_integration_artifact() diff --git a/src/sysext_utils/workflows.py b/src/sysext_utils/workflows.py index db0c5fa..f0699eb 100644 --- a/src/sysext_utils/workflows.py +++ b/src/sysext_utils/workflows.py @@ -28,8 +28,10 @@ from typing import List from .definitions import WORKSPACE_DIR, INTEGRATION_SYSEXT_NAME from .helpers import ( get_name_from_input_sysext_image, + get_tree_runtime_setting_for_sysext_name, get_active_sysext_names, ensure_priviledged, + ensure_setting, remove_directory, ) from .utilities import ( @@ -40,7 +42,6 @@ from .utilities import ( build_tree, add_tree, remove_tree, - check_tree, build_integration_artifact, ) @@ -80,6 +81,7 @@ def run_image_add_workflow(image: str, runtime: bool) -> None: name = get_name_from_input_sysext_image(image) ensure_priviledged() + ensure_setting(runtime) remove_image(name) add_image(name, image, runtime) run_image_integration_workflow(runtime) @@ -88,12 +90,13 @@ def run_image_add_workflow(image: str, runtime: bool) -> None: def run_image_deintegration_workflow() -> None: - exists, runtime = check_tree(INTEGRATION_SYSEXT_NAME) - remove_tree(INTEGRATION_SYSEXT_NAME) + exists, runtime = get_tree_runtime_setting_for_sysext_name(INTEGRATION_SYSEXT_NAME) if not exists: return + remove_tree(INTEGRATION_SYSEXT_NAME) + if not get_active_sysext_names(): return -- GitLab From d509b14152e1d4ba6a9912e934f9ac47cf8069b0 Mon Sep 17 00:00:00 2001 From: Martin Abente Lahaye Date: Wed, 5 Jun 2024 16:42:20 -0400 Subject: [PATCH 36/46] definitions: Fix integration sysext name --- src/sysext_utils/definitions.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/sysext_utils/definitions.py b/src/sysext_utils/definitions.py index b98f9f1..6ed4af4 100644 --- a/src/sysext_utils/definitions.py +++ b/src/sysext_utils/definitions.py @@ -38,7 +38,7 @@ METADATA_PREFIX = "extension-release." METADATA_CUSTOM_FIELD = "SYSEXT_UTILS_VERSION" SYSEXT_IMAGE_SUFFIX = ".sysext.raw" WORKSPACE_DIR = os.path.join(os.getcwd(), ".sysext-utils") -INTEGRATION_SYSEXT_NAME = "integration.sysext" +INTEGRATION_SYSEXT_NAME = "sysext-utils-integration" class ImageFormat(StrEnum): -- GitLab From 5b6b36d0cf1759bb2ed1a919eebf7f9ad1e7fcf3 Mon Sep 17 00:00:00 2001 From: Martin Abente Lahaye Date: Thu, 6 Jun 2024 07:57:34 -0400 Subject: [PATCH 37/46] main: Add a new install verb that combines both build and add verbs --- README.md | 8 +++ src/sysext_utils/main.py | 92 ++++++++++++++++++++++++++++++++++- src/sysext_utils/workflows.py | 4 +- 3 files changed, 101 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index a4be242..45288a5 100644 --- a/README.md +++ b/README.md @@ -38,6 +38,14 @@ $ sysext-utils --dry add https://os.gnome.org/sysext/sdk.sysext.raw --runtime $ sysext-utils --dry remove sdk ``` +### Combine build and add steps + +```bash +$ sysext-utils --dry install sdk meson -Dsysprof=enabled +$ sysext-utils --dry install sdk bst sdk/gtk.bst sdk/libadwaita.bst +$ sysext-utils --dry install sdk import ./destdir +``` + ## License MIT License diff --git a/src/sysext_utils/main.py b/src/sysext_utils/main.py index 2851134..6afa184 100644 --- a/src/sysext_utils/main.py +++ b/src/sysext_utils/main.py @@ -30,7 +30,7 @@ import argparse from . import config from .errors import HandledError -from .definitions import ImageFormat, BuildSystemType +from .definitions import ImageFormat, BuildSystemType, WORKSPACE_DIR from .version import version from .workflows import ( run_image_build_workflow, @@ -46,7 +46,7 @@ def _name_validator(name: str) -> str: return name -def _handle_image_build( +def _build_args_validator( parser: argparse.ArgumentParser, args: argparse.Namespace, ) -> None: @@ -57,6 +57,13 @@ def _handle_image_build( if args.system == BuildSystemType.IMPORT and len(args.arguments) != 1: parser.error("A single path is required") + +def _handle_image_build( + parser: argparse.ArgumentParser, + args: argparse.Namespace, +) -> None: + _build_args_validator(parser, args) + try: run_image_build_workflow( args.name, @@ -96,6 +103,32 @@ def _handle_image_remove( sys.exit(str(e)) +def _handle_image_install( + parser: argparse.ArgumentParser, + args: argparse.Namespace, +) -> None: + _build_args_validator(parser, args) + + try: + run_image_add_workflow( + run_image_build_workflow( + args.name, + WORKSPACE_DIR, + args.format, + args.private_key, + args.certificate, + args.ignore_release, + args.system, + args.arguments, + ), + args.runtime, + ) + except HandledError as e: + sys.exit(str(e)) + finally: + run_cleanup_workflow() + + def main(): parser = argparse.ArgumentParser( prog="sysext-utils", @@ -208,6 +241,60 @@ def main(): default=[], ) + # Arguments for building and adding an image + + image_install_parser = subparser.add_parser( + "install", + help="build and add a sysext image out of a project", + ) + image_install_parser.add_argument( + "name", + help="what to name the sysext e.g., sdk", + type=_name_validator, + ) + image_install_parser.add_argument( + "--format", + help="format to be used for the image creation", + choices=[str(ImageFormat.DDI), str(ImageFormat.COMPRESSED)], + default=ImageFormat.COMPRESSED, + ) + image_install_parser.add_argument( + "--private_key", + help="path to the private key file e.g., files/boot-keys/SYSEXT.key", + default="", + ) + image_install_parser.add_argument( + "--certificate", + help="path to the certificate file, e.g., files/boot-keys/SYSEXT.crt", + default="", + ) + image_install_parser.add_argument( + "--ignore-release", + help="ignore the host release data and use 'ID=_any' instead", + action="store_true", + ) + image_install_parser.add_argument( + "--runtime", + help="add the sysext image under /run/extensions/ instead", + action="store_true", + ) + image_install_parser.add_argument( + "system", + help="build system to use e.g., bst", + choices=[ + str(BuildSystemType.BST), + str(BuildSystemType.MESON), + str(BuildSystemType.IMPORT), + ], + default=BuildSystemType.BST, + ) + image_install_parser.add_argument( + "arguments", + help="list of build-system specific arguments", + nargs=argparse.REMAINDER, + default=[], + ) + args = parser.parse_args() # Handle global flags @@ -221,6 +308,7 @@ def main(): "build": _handle_image_build, "add": _handle_image_add, "remove": _handle_image_remove, + "install": _handle_image_install, } handlers[args.operation](parser, args) diff --git a/src/sysext_utils/workflows.py b/src/sysext_utils/workflows.py index f0699eb..2ae5f5e 100644 --- a/src/sysext_utils/workflows.py +++ b/src/sysext_utils/workflows.py @@ -55,7 +55,7 @@ def run_image_build_workflow( ignore_release: bool, system: str, arguments: List[str], -) -> None: +) -> str: artifact = build_artifact(system, arguments) image = build_image( name, @@ -69,6 +69,8 @@ def run_image_build_workflow( print(f"Successfully built {image}") + return image + def run_image_integration_workflow(runtime: bool) -> None: remove_tree(INTEGRATION_SYSEXT_NAME) -- GitLab From d1917e6bb9b73f81b406deae3bdd605684458cb9 Mon Sep 17 00:00:00 2001 From: Martin Abente Lahaye Date: Thu, 6 Jun 2024 08:20:41 -0400 Subject: [PATCH 38/46] utilities: Don't setup meson build directory --- README.md | 8 ++++---- src/sysext_utils/commands.py | 4 ---- src/sysext_utils/helpers.py | 4 ---- src/sysext_utils/main.py | 2 ++ src/sysext_utils/utilities.py | 6 +----- tests/test_workflows.py | 6 +++--- 6 files changed, 10 insertions(+), 20 deletions(-) diff --git a/README.md b/README.md index 45288a5..820fe4b 100644 --- a/README.md +++ b/README.md @@ -22,11 +22,11 @@ $ python -m pip install . ### Build sysext images out of a project ```bash -$ sysext-utils --dry build sdk meson -Dsysprof=enabled +$ sysext-utils --dry build sdk meson ./build $ sysext-utils --dry build sdk bst sdk/gtk.bst sdk/libadwaita.bst $ sysext-utils --dry build sdk import ./destdir -$ sysext-utils --dry build sdk --ignore-release meson -$ sysext-utils --dry build sdk --format=ddi --private_key=key --certificate=cert meson +$ sysext-utils --dry build sdk --ignore-release meson ./build +$ sysext-utils --dry build sdk --format=ddi --private_key=key --certificate=cert meson ./build ``` ### Manage sysext images @@ -41,7 +41,7 @@ $ sysext-utils --dry remove sdk ### Combine build and add steps ```bash -$ sysext-utils --dry install sdk meson -Dsysprof=enabled +$ sysext-utils --dry install sdk meson ./build $ sysext-utils --dry install sdk bst sdk/gtk.bst sdk/libadwaita.bst $ sysext-utils --dry install sdk import ./destdir ``` diff --git a/src/sysext_utils/commands.py b/src/sysext_utils/commands.py index 44885bd..6f2ca6d 100644 --- a/src/sysext_utils/commands.py +++ b/src/sysext_utils/commands.py @@ -197,10 +197,6 @@ class Bst(BaseCommand): class Meson(BaseCommand): - @classmethod - def setup(cls, build_directory: str, arguments: List[str]) -> None: - cls.run(["setup", build_directory, "--prefix", "/usr"] + arguments) - @classmethod def compile(cls, build_directory: str) -> None: cls.run(["compile", "-C", build_directory]) diff --git a/src/sysext_utils/helpers.py b/src/sysext_utils/helpers.py index 9ede2cb..f0b3cbb 100644 --- a/src/sysext_utils/helpers.py +++ b/src/sysext_utils/helpers.py @@ -143,10 +143,6 @@ def get_path_for_integration_artifact() -> str: return os.path.join(WORKSPACE_DIR, "integration") -def get_path_for_build() -> str: - return os.path.join(WORKSPACE_DIR, "build") - - def make_directory(directory: str) -> None: if config.dry_run: print([str(DryStepType.MAKING), directory]) diff --git a/src/sysext_utils/main.py b/src/sysext_utils/main.py index 6afa184..deed440 100644 --- a/src/sysext_utils/main.py +++ b/src/sysext_utils/main.py @@ -56,6 +56,8 @@ def _build_args_validator( parser.error("One or more elements are required") if args.system == BuildSystemType.IMPORT and len(args.arguments) != 1: parser.error("A single path is required") + if args.system == BuildSystemType.MESON and len(args.arguments) != 1: + parser.error("Build directory is required") def _handle_image_build( diff --git a/src/sysext_utils/utilities.py b/src/sysext_utils/utilities.py index 15d9a7b..c31f0ef 100644 --- a/src/sysext_utils/utilities.py +++ b/src/sysext_utils/utilities.py @@ -56,7 +56,6 @@ from .helpers import ( get_path_for_input_sysext_name, get_path_for_artifact, get_path_for_integration_artifact, - get_path_for_build, make_directory, copy_directory, remove_directory, @@ -209,11 +208,8 @@ def build_artifact_bst_elements(directory: str, elements: List[str]) -> None: def build_artifact_meson_project(directory: str, arguments: List[str]) -> None: - build_directory = get_path_for_build() + build_directory = arguments[0] - remove_directory(build_directory) - - Meson.setup(build_directory, arguments) Meson.compile(build_directory) Meson.install(build_directory, directory) diff --git a/tests/test_workflows.py b/tests/test_workflows.py index 60bdb14..0fae61c 100644 --- a/tests/test_workflows.py +++ b/tests/test_workflows.py @@ -73,7 +73,7 @@ def test_image_build_workflow_with_meson(): certificate="certificate", ignore_release=False, system=BuildSystemType.MESON, - arguments=[], + arguments=["build"], ) @@ -86,7 +86,7 @@ def test_image_build_workflow_with_compressed_format(): certificate="certificate", ignore_release=False, system=BuildSystemType.MESON, - arguments=[], + arguments=["build"], ) @@ -99,7 +99,7 @@ def test_image_build_work_with_ignore_release(): certificate="certificate", ignore_release=True, system=BuildSystemType.MESON, - arguments=[], + arguments=["build"], ) -- GitLab From 08be0ece1be0d3e059b839049c88a181d329fa73 Mon Sep 17 00:00:00 2001 From: Martin Abente Lahaye Date: Thu, 6 Jun 2024 10:14:45 -0400 Subject: [PATCH 39/46] main: Don't specify individual bst elements only the workspace --- README.md | 4 ++-- src/sysext_utils/commands.py | 18 ++++++++++++------ src/sysext_utils/main.py | 4 ++-- src/sysext_utils/utilities.py | 8 ++++---- tests/test_workflows.py | 4 +--- 5 files changed, 21 insertions(+), 17 deletions(-) diff --git a/README.md b/README.md index 820fe4b..69ffe70 100644 --- a/README.md +++ b/README.md @@ -23,7 +23,7 @@ $ python -m pip install . ```bash $ sysext-utils --dry build sdk meson ./build -$ sysext-utils --dry build sdk bst sdk/gtk.bst sdk/libadwaita.bst +$ sysext-utils --dry build sdk bst ./workspace $ sysext-utils --dry build sdk import ./destdir $ sysext-utils --dry build sdk --ignore-release meson ./build $ sysext-utils --dry build sdk --format=ddi --private_key=key --certificate=cert meson ./build @@ -42,7 +42,7 @@ $ sysext-utils --dry remove sdk ```bash $ sysext-utils --dry install sdk meson ./build -$ sysext-utils --dry install sdk bst sdk/gtk.bst sdk/libadwaita.bst +$ sysext-utils --dry install sdk bst ./workspace $ sysext-utils --dry install sdk import ./destdir ``` diff --git a/src/sysext_utils/commands.py b/src/sysext_utils/commands.py index 6f2ca6d..c09bc7d 100644 --- a/src/sysext_utils/commands.py +++ b/src/sysext_utils/commands.py @@ -25,8 +25,6 @@ import subprocess -from typing import List - from . import config from .errors import CommandNotAvailableError, CommandFailedError @@ -174,20 +172,28 @@ class SystemdSysext(BaseCommand): class Bst(BaseCommand): @classmethod - def build(cls, elements: List[str]) -> None: - cls.run(["build", "--retry-failed"] + elements) + def build(cls, workspace: str) -> None: + cls.run( + [ + "--directory", + workspace, + "build", + "--retry-failed", + ] + ) @classmethod - def artifact_checkout(cls, element: str, directory: str) -> None: + def artifact_checkout(cls, workspace: str, directory: str) -> None: cls.run( [ + "--directory", + workspace, "artifact", "checkout", "--force", "--deps=none", "--directory", directory, - element, ] ) diff --git a/src/sysext_utils/main.py b/src/sysext_utils/main.py index deed440..67b3fa1 100644 --- a/src/sysext_utils/main.py +++ b/src/sysext_utils/main.py @@ -52,8 +52,8 @@ def _build_args_validator( ) -> None: if args.format == ImageFormat.DDI and not (args.private_key and args.certificate): parser.error("DDI image creation require private_key and certificate") - if args.system == BuildSystemType.BST and not args.arguments: - parser.error("One or more elements are required") + if args.system == BuildSystemType.BST and len(args.arguments) != 1: + parser.error("Workspace directory is required") if args.system == BuildSystemType.IMPORT and len(args.arguments) != 1: parser.error("A single path is required") if args.system == BuildSystemType.MESON and len(args.arguments) != 1: diff --git a/src/sysext_utils/utilities.py b/src/sysext_utils/utilities.py index c31f0ef..84ad84d 100644 --- a/src/sysext_utils/utilities.py +++ b/src/sysext_utils/utilities.py @@ -200,11 +200,11 @@ def build_artifact(system: str, arguments: List[str]) -> str: return artifact -def build_artifact_bst_elements(directory: str, elements: List[str]) -> None: - Bst.build(elements) +def build_artifact_bst_elements(directory: str, arguments: List[str]) -> None: + workspace_directory = arguments[0] - for element in elements: - Bst.artifact_checkout(element, directory) + Bst.build(workspace_directory) + Bst.artifact_checkout(workspace_directory, directory) def build_artifact_meson_project(directory: str, arguments: List[str]) -> None: diff --git a/tests/test_workflows.py b/tests/test_workflows.py index 0fae61c..aacf4fa 100644 --- a/tests/test_workflows.py +++ b/tests/test_workflows.py @@ -45,9 +45,7 @@ def test_image_build_workflow(): certificate="certificate", ignore_release=False, system=BuildSystemType.BST, - arguments=[ - "element/one.bst", - ], + arguments=["workspace"], ) -- GitLab From e6493c37e10cf9fccd99f1e30aa64e1ee236b391 Mon Sep 17 00:00:00 2001 From: Martin Abente Lahaye Date: Thu, 6 Jun 2024 10:55:12 -0400 Subject: [PATCH 40/46] main: Split utils into different tools --- README.md | 23 ++-- pyproject.toml | 5 +- src/sysext_utils/main.py | 230 ++++++++++++++++----------------------- 3 files changed, 111 insertions(+), 147 deletions(-) diff --git a/README.md b/README.md index 69ffe70..ad7cce5 100644 --- a/README.md +++ b/README.md @@ -22,28 +22,27 @@ $ python -m pip install . ### Build sysext images out of a project ```bash -$ sysext-utils --dry build sdk meson ./build -$ sysext-utils --dry build sdk bst ./workspace -$ sysext-utils --dry build sdk import ./destdir -$ sysext-utils --dry build sdk --ignore-release meson ./build -$ sysext-utils --dry build sdk --format=ddi --private_key=key --certificate=cert meson ./build +$ sysext-build sdk meson ./build +$ sysext-build sdk bst ./workspace +$ sysext-build sdk import ./destdir +$ sysext-build sdk --verbose --ignore-release meson ./build +$ sysext-build sdk --dry --format=ddi --private_key=key --certificate=cert meson ./build ``` ### Manage sysext images ```bash -$ sysext-utils --dry add sdk.sysext.raw -$ sysext-utils --dry add sdk.sysext.raw -$ sysext-utils --dry add https://os.gnome.org/sysext/sdk.sysext.raw --runtime -$ sysext-utils --dry remove sdk +$ sysext-add sdk.sysext.raw +$ sysext-add https://os.gnome.org/sysext/sdk.sysext.raw --runtime +$ sysext-remove sdk ``` ### Combine build and add steps ```bash -$ sysext-utils --dry install sdk meson ./build -$ sysext-utils --dry install sdk bst ./workspace -$ sysext-utils --dry install sdk import ./destdir +$ sysext-install sdk meson ./build +$ sysext-install sdk bst ./workspace +$ sysext-install sdk import ./destdir ``` ## License diff --git a/pyproject.toml b/pyproject.toml index 1eb91d3..6a1e8ef 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -8,7 +8,10 @@ dynamic = ["version"] requires-python = ">=3.8" [project.scripts] -sysext-utils = "sysext_utils:main.main" +sysext-add = "sysext_utils:main.add" +sysext-remove = "sysext_utils:main.remove" +sysext-build = "sysext_utils:main.build" +sysext-install = "sysext_utils:main.install" [tool.setuptools_scm] version_file = "src/sysext_utils/version.py" diff --git a/src/sysext_utils/main.py b/src/sysext_utils/main.py index 67b3fa1..4471b78 100644 --- a/src/sysext_utils/main.py +++ b/src/sysext_utils/main.py @@ -60,81 +60,10 @@ def _build_args_validator( parser.error("Build directory is required") -def _handle_image_build( - parser: argparse.ArgumentParser, - args: argparse.Namespace, -) -> None: - _build_args_validator(parser, args) - - try: - run_image_build_workflow( - args.name, - args.directory, - args.format, - args.private_key, - args.certificate, - args.ignore_release, - args.system, - args.arguments, - ) - except HandledError as e: - sys.exit(str(e)) - finally: - run_cleanup_workflow() - - -def _handle_image_add( - parser: argparse.ArgumentParser, - args: argparse.Namespace, -) -> None: - try: - run_image_add_workflow(args.image, args.runtime) - except HandledError as e: - sys.exit(str(e)) - finally: - run_cleanup_workflow() - - -def _handle_image_remove( - parser: argparse.ArgumentParser, - args: argparse.Namespace, -) -> None: - try: - run_image_remove_workflow(args.name) - except HandledError as e: - sys.exit(str(e)) - - -def _handle_image_install( - parser: argparse.ArgumentParser, - args: argparse.Namespace, -) -> None: - _build_args_validator(parser, args) - - try: - run_image_add_workflow( - run_image_build_workflow( - args.name, - WORKSPACE_DIR, - args.format, - args.private_key, - args.certificate, - args.ignore_release, - args.system, - args.arguments, - ), - args.runtime, - ) - except HandledError as e: - sys.exit(str(e)) - finally: - run_cleanup_workflow() - - -def main(): +def _build_base_parser(name: str, description: str) -> argparse.ArgumentParser: parser = argparse.ArgumentParser( - prog="sysext-utils", - description="A tool to develop and test system components on immutable OSes", + prog=name, + description=description, ) parser.add_argument( "--version", @@ -153,80 +82,91 @@ def main(): action="store_true", ) - # Arguments for the different supported utilities + return parser - subparser = parser.add_subparsers( - dest="operation", - help="a supported operation, e.g., build", - required=True, - ) - # Arguments for adding an image to the host +def _handle_global_args( + parser: argparse.ArgumentParser, + args: argparse.Namespace, +) -> None: + config.dry_run = args.dry + config.verbose = args.verbose - image_add_parser = subparser.add_parser( - "add", - help="add and activate a sysext image onto host OS", - ) - image_add_parser.add_argument( + +def add(): + parser = _build_base_parser("sysext-add", "add an image to the system") + parser.add_argument( "image", help="path or URL to the sysext image e.g., sdk.sysext.raw", ) - image_add_parser.add_argument( + parser.add_argument( "--runtime", help="add the sysext image under /run/extensions/ instead", action="store_true", ) - # Arguments for removing a sysext image from the host OS + args = parser.parse_args() + _handle_global_args(parser, args) - image_remove_parser = subparser.add_parser( - "remove", - help="deactivate and remove a sysext image from the host OS", - ) - image_remove_parser.add_argument( + try: + run_image_add_workflow(args.image, args.runtime) + except HandledError as e: + sys.exit(str(e)) + finally: + run_cleanup_workflow() + + +def remove(): + parser = _build_base_parser("sysext-remove", "remove an image from the system") + parser.add_argument( "name", help="name of the the sysext e.g., sdk", type=_name_validator, ) - # Arguments for building an image + args = parser.parse_args() + _handle_global_args(parser, args) + + try: + run_image_remove_workflow(args.name) + except HandledError as e: + sys.exit(str(e)) + - image_build_parser = subparser.add_parser( - "build", - help="build a sysext image out of a project", - ) - image_build_parser.add_argument( +def build(): + parser = _build_base_parser("sysext-build", "build an image from a given directory") + parser.add_argument( "name", help="what to name the sysext e.g., sdk", type=_name_validator, ) - image_build_parser.add_argument( + parser.add_argument( "--format", help="format to be used for the image creation", choices=[str(ImageFormat.DDI), str(ImageFormat.COMPRESSED)], default=ImageFormat.COMPRESSED, ) - image_build_parser.add_argument( + parser.add_argument( "--private_key", help="path to the private key file e.g., files/boot-keys/SYSEXT.key", default="", ) - image_build_parser.add_argument( + parser.add_argument( "--certificate", help="path to the certificate file, e.g., files/boot-keys/SYSEXT.crt", default="", ) - image_build_parser.add_argument( + parser.add_argument( "--directory", help="where to place the image, e.g., ./images/. Defaults to $PWD", default=os.path.abspath(os.getcwd()), ) - image_build_parser.add_argument( + parser.add_argument( "--ignore-release", help="ignore the host release data and use 'ID=_any' instead", action="store_true", ) - image_build_parser.add_argument( + parser.add_argument( "system", help="build system to use e.g., bst", choices=[ @@ -236,51 +176,68 @@ def main(): ], default=BuildSystemType.BST, ) - image_build_parser.add_argument( + parser.add_argument( "arguments", help="list of build-system specific arguments", nargs=argparse.REMAINDER, default=[], ) - # Arguments for building and adding an image + args = parser.parse_args() + _handle_global_args(parser, args) + _build_args_validator(parser, args) - image_install_parser = subparser.add_parser( - "install", - help="build and add a sysext image out of a project", - ) - image_install_parser.add_argument( + try: + run_image_build_workflow( + args.name, + args.directory, + args.format, + args.private_key, + args.certificate, + args.ignore_release, + args.system, + args.arguments, + ) + except HandledError as e: + sys.exit(str(e)) + finally: + run_cleanup_workflow() + + +def install(): + parser = _build_base_parser("sysext-install", "build and add an image") + parser.add_argument( "name", help="what to name the sysext e.g., sdk", type=_name_validator, ) - image_install_parser.add_argument( + parser.add_argument( "--format", help="format to be used for the image creation", choices=[str(ImageFormat.DDI), str(ImageFormat.COMPRESSED)], default=ImageFormat.COMPRESSED, ) - image_install_parser.add_argument( + parser.add_argument( "--private_key", help="path to the private key file e.g., files/boot-keys/SYSEXT.key", default="", ) - image_install_parser.add_argument( + parser.add_argument( "--certificate", help="path to the certificate file, e.g., files/boot-keys/SYSEXT.crt", default="", ) - image_install_parser.add_argument( + parser.add_argument( "--ignore-release", help="ignore the host release data and use 'ID=_any' instead", action="store_true", ) - image_install_parser.add_argument( + parser.add_argument( "--runtime", help="add the sysext image under /run/extensions/ instead", action="store_true", ) - image_install_parser.add_argument( + parser.add_argument( "system", help="build system to use e.g., bst", choices=[ @@ -290,7 +247,7 @@ def main(): ], default=BuildSystemType.BST, ) - image_install_parser.add_argument( + parser.add_argument( "arguments", help="list of build-system specific arguments", nargs=argparse.REMAINDER, @@ -298,19 +255,24 @@ def main(): ) args = parser.parse_args() + _handle_global_args(parser, args) + _build_args_validator(parser, args) - # Handle global flags - - config.dry_run = args.dry - config.verbose = args.verbose - - # Handle supported operations - - handlers = { - "build": _handle_image_build, - "add": _handle_image_add, - "remove": _handle_image_remove, - "install": _handle_image_install, - } - - handlers[args.operation](parser, args) + try: + run_image_add_workflow( + run_image_build_workflow( + args.name, + WORKSPACE_DIR, + args.format, + args.private_key, + args.certificate, + args.ignore_release, + args.system, + args.arguments, + ), + args.runtime, + ) + except HandledError as e: + sys.exit(str(e)) + finally: + run_cleanup_workflow() -- GitLab From d7adbf4a78bd5dfec90a97279bad9b4428664b5f Mon Sep 17 00:00:00 2001 From: Martin Abente Lahaye Date: Thu, 6 Jun 2024 13:50:29 -0400 Subject: [PATCH 41/46] main: Make build system an optional argument and source directory mandatory --- README.md | 16 +++++----- src/sysext_utils/main.py | 56 +++++++++++++++-------------------- src/sysext_utils/utilities.py | 45 ++++++++++++---------------- src/sysext_utils/workflows.py | 10 +++---- tests/test_errors.py | 4 +-- tests/test_workflows.py | 20 ++++++------- 6 files changed, 67 insertions(+), 84 deletions(-) diff --git a/README.md b/README.md index ad7cce5..fbfa1f4 100644 --- a/README.md +++ b/README.md @@ -22,11 +22,11 @@ $ python -m pip install . ### Build sysext images out of a project ```bash -$ sysext-build sdk meson ./build -$ sysext-build sdk bst ./workspace -$ sysext-build sdk import ./destdir -$ sysext-build sdk --verbose --ignore-release meson ./build -$ sysext-build sdk --dry --format=ddi --private_key=key --certificate=cert meson ./build +$ sysext-build sdk ./build --system=meson +$ sysext-build sdk ./workspace --system=bst +$ sysext-build sdk ./destdir --system=import +$ sysext-build sdk ./build --system=meson --verbose --ignore-release --dry +$ sysext-build sdk ./build --system=meson --format=ddi --private_key=key --certificate=cert ``` ### Manage sysext images @@ -40,9 +40,9 @@ $ sysext-remove sdk ### Combine build and add steps ```bash -$ sysext-install sdk meson ./build -$ sysext-install sdk bst ./workspace -$ sysext-install sdk import ./destdir +$ sysext-install sdk ./build --system=meson +$ sysext-install sdk ./workspace --system=bst +$ sysext-install sdk ./destdir --system=import ``` ## License diff --git a/src/sysext_utils/main.py b/src/sysext_utils/main.py index 4471b78..ab439d4 100644 --- a/src/sysext_utils/main.py +++ b/src/sysext_utils/main.py @@ -52,12 +52,6 @@ def _build_args_validator( ) -> None: if args.format == ImageFormat.DDI and not (args.private_key and args.certificate): parser.error("DDI image creation require private_key and certificate") - if args.system == BuildSystemType.BST and len(args.arguments) != 1: - parser.error("Workspace directory is required") - if args.system == BuildSystemType.IMPORT and len(args.arguments) != 1: - parser.error("A single path is required") - if args.system == BuildSystemType.MESON and len(args.arguments) != 1: - parser.error("Build directory is required") def _build_base_parser(name: str, description: str) -> argparse.ArgumentParser: @@ -140,6 +134,16 @@ def build(): help="what to name the sysext e.g., sdk", type=_name_validator, ) + parser.add_argument( + "source", + help="directory to build the image from e.g., ./destdir/", + default=os.path.abspath(os.getcwd()), + ) + parser.add_argument( + "--destination", + help="directory to write the image to e.g., ./images/. Defaults to $PWD", + default=os.path.abspath(os.getcwd()), + ) parser.add_argument( "--format", help="format to be used for the image creation", @@ -156,31 +160,20 @@ def build(): help="path to the certificate file, e.g., files/boot-keys/SYSEXT.crt", default="", ) - parser.add_argument( - "--directory", - help="where to place the image, e.g., ./images/. Defaults to $PWD", - default=os.path.abspath(os.getcwd()), - ) parser.add_argument( "--ignore-release", help="ignore the host release data and use 'ID=_any' instead", action="store_true", ) parser.add_argument( - "system", - help="build system to use e.g., bst", + "--system", + help="build system to use e.g., import", choices=[ str(BuildSystemType.BST), str(BuildSystemType.MESON), str(BuildSystemType.IMPORT), ], - default=BuildSystemType.BST, - ) - parser.add_argument( - "arguments", - help="list of build-system specific arguments", - nargs=argparse.REMAINDER, - default=[], + default=BuildSystemType.IMPORT, ) args = parser.parse_args() @@ -190,13 +183,13 @@ def build(): try: run_image_build_workflow( args.name, - args.directory, + args.source, + args.destination, args.format, args.private_key, args.certificate, args.ignore_release, args.system, - args.arguments, ) except HandledError as e: sys.exit(str(e)) @@ -211,6 +204,11 @@ def install(): help="what to name the sysext e.g., sdk", type=_name_validator, ) + parser.add_argument( + "source", + help="directory to build the image from e.g., ./destdir/", + default=os.path.abspath(os.getcwd()), + ) parser.add_argument( "--format", help="format to be used for the image creation", @@ -238,20 +236,14 @@ def install(): action="store_true", ) parser.add_argument( - "system", - help="build system to use e.g., bst", + "--system", + help="build system to use e.g., import", choices=[ str(BuildSystemType.BST), str(BuildSystemType.MESON), str(BuildSystemType.IMPORT), ], - default=BuildSystemType.BST, - ) - parser.add_argument( - "arguments", - help="list of build-system specific arguments", - nargs=argparse.REMAINDER, - default=[], + default=BuildSystemType.IMPORT, ) args = parser.parse_args() @@ -262,13 +254,13 @@ def install(): run_image_add_workflow( run_image_build_workflow( args.name, + args.source, WORKSPACE_DIR, args.format, args.private_key, args.certificate, args.ignore_release, args.system, - args.arguments, ), args.runtime, ) diff --git a/src/sysext_utils/utilities.py b/src/sysext_utils/utilities.py index 84ad84d..784e72e 100644 --- a/src/sysext_utils/utilities.py +++ b/src/sysext_utils/utilities.py @@ -25,8 +25,6 @@ import os -from typing import List - from .errors import UnsupportedImageFormatError, UnsupportedBuildSystemError from .definitions import ( ImageFormat, @@ -69,16 +67,16 @@ from .helpers import ( def build_image( name: str, artifact: str, - directory: str, + destination: str, image_format: str, private_key: str, certificate: str, ignore_release: bool, ) -> str: - image = get_path_for_input_sysext_name(name, directory) + image = get_path_for_input_sysext_name(name, destination) remove_file(image) - make_directory(directory) + make_directory(destination) build_tree(name, artifact, ignore_release) @@ -180,41 +178,36 @@ def build_integration_icons(artifact: str) -> None: remove_directory(tmp_icons_dir) -def build_artifact(system: str, arguments: List[str]) -> str: - artifact = get_path_for_artifact() +def build_artifact(source: str, system: str) -> str: + destination = get_path_for_artifact() - remove_directory(artifact) - make_directory(artifact) + remove_directory(destination) + make_directory(destination) if system == BuildSystemType.BST: - build_artifact_bst_elements(artifact, arguments) + build_artifact_bst_elements(source, destination) elif system == BuildSystemType.MESON: - build_artifact_meson_project(artifact, arguments) + build_artifact_meson_project(source, destination) elif system == BuildSystemType.IMPORT: - build_artifact_import_directory(artifact, arguments) + build_artifact_import_directory(source, destination) else: raise UnsupportedBuildSystemError(system) - ensure_directory(artifact) - - return artifact - + ensure_directory(destination) -def build_artifact_bst_elements(directory: str, arguments: List[str]) -> None: - workspace_directory = arguments[0] + return destination - Bst.build(workspace_directory) - Bst.artifact_checkout(workspace_directory, directory) +def build_artifact_bst_elements(workspace: str, directory: str) -> None: + Bst.build(workspace) + Bst.artifact_checkout(workspace, directory) -def build_artifact_meson_project(directory: str, arguments: List[str]) -> None: - build_directory = arguments[0] +def build_artifact_meson_project(build_directory: str, destination: str) -> None: Meson.compile(build_directory) - Meson.install(build_directory, directory) + Meson.install(build_directory, destination) -def build_artifact_import_directory(directory: str, arguments: List[str]) -> None: - source = arguments[0] +def build_artifact_import_directory(source: str, destination: str) -> None: ensure_directory(source) - copy_directory(source, directory) + copy_directory(source, destination) diff --git a/src/sysext_utils/workflows.py b/src/sysext_utils/workflows.py index 2ae5f5e..2726eb5 100644 --- a/src/sysext_utils/workflows.py +++ b/src/sysext_utils/workflows.py @@ -23,8 +23,6 @@ # # SPDX-License-Identifier: MIT -from typing import List - from .definitions import WORKSPACE_DIR, INTEGRATION_SYSEXT_NAME from .helpers import ( get_name_from_input_sysext_image, @@ -48,19 +46,19 @@ from .utilities import ( def run_image_build_workflow( name: str, - directory: str, + source: str, + destination: str, image_format: str, private_key: str, certificate: str, ignore_release: bool, system: str, - arguments: List[str], ) -> str: - artifact = build_artifact(system, arguments) + artifact = build_artifact(source, system) image = build_image( name, artifact, - directory, + destination, image_format, private_key, certificate, diff --git a/tests/test_errors.py b/tests/test_errors.py index a2aad84..73e0c7e 100644 --- a/tests/test_errors.py +++ b/tests/test_errors.py @@ -72,7 +72,7 @@ def test_unsupported_image_format_error(): build_image( name="name", artifact="artifact", - directory="directory", + destination="destination", image_format="BOGUS", private_key=None, certificate=None, @@ -87,8 +87,8 @@ def test_unsupported_build_system_error(): with pytest.raises(UnsupportedBuildSystemError) as _: build_artifact( + source="source", system="BOGUS", - arguments=[], ) config.dry_run = False diff --git a/tests/test_workflows.py b/tests/test_workflows.py index aacf4fa..88f6dc3 100644 --- a/tests/test_workflows.py +++ b/tests/test_workflows.py @@ -39,65 +39,65 @@ def setup_module(): def test_image_build_workflow(): run_image_build_workflow( name="name", - directory="directory", + source="source", + destination="destination", image_format=ImageFormat.DDI, private_key="private_key", certificate="certificate", ignore_release=False, system=BuildSystemType.BST, - arguments=["workspace"], ) def test_image_build_workflow_with_import(): run_image_build_workflow( name="name", - directory="directory", + source="source", + destination="destination", image_format=ImageFormat.DDI, private_key="private_key", certificate="certificate", ignore_release=False, system=BuildSystemType.IMPORT, - arguments=["./destdir"], ) def test_image_build_workflow_with_meson(): run_image_build_workflow( name="name", - directory="directory", + source="source", + destination="destination", image_format=ImageFormat.DDI, private_key="private_key", certificate="certificate", ignore_release=False, system=BuildSystemType.MESON, - arguments=["build"], ) def test_image_build_workflow_with_compressed_format(): run_image_build_workflow( name="name", - directory="directory", + source="source", + destination="destination", image_format=ImageFormat.COMPRESSED, private_key="private_key", certificate="certificate", ignore_release=False, system=BuildSystemType.MESON, - arguments=["build"], ) def test_image_build_work_with_ignore_release(): run_image_build_workflow( name="name", - directory="directory", + source="source", + destination="destination", image_format=ImageFormat.COMPRESSED, private_key="private_key", certificate="certificate", ignore_release=True, system=BuildSystemType.MESON, - arguments=["build"], ) -- GitLab From 8426a0dfe58403a0f8af21ba90049a0ddd7f7eed Mon Sep 17 00:00:00 2001 From: Martin Abente Lahaye Date: Thu, 6 Jun 2024 14:31:37 -0400 Subject: [PATCH 42/46] main: Determine the build system from source or fail --- README.md | 2 ++ src/sysext_utils/definitions.py | 2 ++ src/sysext_utils/helpers.py | 12 ++++++++++++ src/sysext_utils/main.py | 6 ++++-- src/sysext_utils/utilities.py | 3 +++ 5 files changed, 23 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index fbfa1f4..ae2943e 100644 --- a/README.md +++ b/README.md @@ -22,6 +22,7 @@ $ python -m pip install . ### Build sysext images out of a project ```bash +$ sysext-build sdk ./build $ sysext-build sdk ./build --system=meson $ sysext-build sdk ./workspace --system=bst $ sysext-build sdk ./destdir --system=import @@ -40,6 +41,7 @@ $ sysext-remove sdk ### Combine build and add steps ```bash +$ sysext-install sdk ./build $ sysext-install sdk ./build --system=meson $ sysext-install sdk ./workspace --system=bst $ sysext-install sdk ./destdir --system=import diff --git a/src/sysext_utils/definitions.py b/src/sysext_utils/definitions.py index 6ed4af4..4128592 100644 --- a/src/sysext_utils/definitions.py +++ b/src/sysext_utils/definitions.py @@ -56,6 +56,8 @@ class DryStepType(StrEnum): class BuildSystemType(StrEnum): + AUTO = "auto" BST = "bst" MESON = "meson" IMPORT = "import" + UNKOWN = "unknown" diff --git a/src/sysext_utils/helpers.py b/src/sysext_utils/helpers.py index f0b3cbb..9cf61a8 100644 --- a/src/sysext_utils/helpers.py +++ b/src/sysext_utils/helpers.py @@ -48,6 +48,7 @@ from .definitions import ( SYSEXT_IMAGE_SUFFIX, METADATA_CUSTOM_FIELD, INTEGRATION_SYSEXT_NAME, + BuildSystemType, DryStepType, ) @@ -102,6 +103,17 @@ def is_url(string: str) -> bool: return re.match("http[s]?://", string) is not None +def guess_system_from_source(source: str) -> str: + if os.path.exists(os.path.join(source, "meson-info")): + return BuildSystemType.MESON + elif os.path.exists(os.path.join(source, ".bstproject.yaml")): + return BuildSystemType.BST + elif os.path.exists(os.path.join(source, "usr")): + return BuildSystemType.IMPORT + + return BuildSystemType.UNKOWN + + def get_path_for_input_sysext_name(name: str, directory: str) -> str: return os.path.join(directory, f"{name}.sysext.raw") diff --git a/src/sysext_utils/main.py b/src/sysext_utils/main.py index ab439d4..f1a96f0 100644 --- a/src/sysext_utils/main.py +++ b/src/sysext_utils/main.py @@ -169,11 +169,12 @@ def build(): "--system", help="build system to use e.g., import", choices=[ + str(BuildSystemType.AUTO), str(BuildSystemType.BST), str(BuildSystemType.MESON), str(BuildSystemType.IMPORT), ], - default=BuildSystemType.IMPORT, + default=BuildSystemType.AUTO, ) args = parser.parse_args() @@ -239,11 +240,12 @@ def install(): "--system", help="build system to use e.g., import", choices=[ + str(BuildSystemType.AUTO), str(BuildSystemType.BST), str(BuildSystemType.MESON), str(BuildSystemType.IMPORT), ], - default=BuildSystemType.IMPORT, + default=BuildSystemType.AUTO, ) args = parser.parse_args() diff --git a/src/sysext_utils/utilities.py b/src/sysext_utils/utilities.py index 784e72e..e9b6b1e 100644 --- a/src/sysext_utils/utilities.py +++ b/src/sysext_utils/utilities.py @@ -54,6 +54,7 @@ from .helpers import ( get_path_for_input_sysext_name, get_path_for_artifact, get_path_for_integration_artifact, + guess_system_from_source, make_directory, copy_directory, remove_directory, @@ -184,6 +185,8 @@ def build_artifact(source: str, system: str) -> str: remove_directory(destination) make_directory(destination) + if system == BuildSystemType.AUTO: + system = guess_system_from_source(source) if system == BuildSystemType.BST: build_artifact_bst_elements(source, destination) elif system == BuildSystemType.MESON: -- GitLab From 1e037d853e43627e32c6ea3c59ee23d61c8e067c Mon Sep 17 00:00:00 2001 From: Martin Abente Lahaye Date: Thu, 6 Jun 2024 14:51:16 -0400 Subject: [PATCH 43/46] helpers: Cleanup --- src/sysext_utils/helpers.py | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/src/sysext_utils/helpers.py b/src/sysext_utils/helpers.py index 9cf61a8..776c770 100644 --- a/src/sysext_utils/helpers.py +++ b/src/sysext_utils/helpers.py @@ -212,11 +212,7 @@ def ensure_directory(path: str) -> None: def ensure_setting(runtime: bool) -> None: - _exists, _runtime = get_tree_runtime_setting_for_sysext_name( - INTEGRATION_SYSEXT_NAME - ) + exists, _runtime = get_tree_runtime_setting_for_sysext_name(INTEGRATION_SYSEXT_NAME) - if not _exists: - return - if _runtime != runtime: + if exists and _runtime != runtime: raise IncompatibleSettingError() -- GitLab From 10570344fa9be2214ac8e833b8f577a38ec98f72 Mon Sep 17 00:00:00 2001 From: Martin Abente Lahaye Date: Thu, 6 Jun 2024 14:51:32 -0400 Subject: [PATCH 44/46] workflows: Cleanup --- src/sysext_utils/workflows.py | 13 ++++--------- 1 file changed, 4 insertions(+), 9 deletions(-) diff --git a/src/sysext_utils/workflows.py b/src/sysext_utils/workflows.py index 2726eb5..1729716 100644 --- a/src/sysext_utils/workflows.py +++ b/src/sysext_utils/workflows.py @@ -92,15 +92,10 @@ def run_image_add_workflow(image: str, runtime: bool) -> None: def run_image_deintegration_workflow() -> None: exists, runtime = get_tree_runtime_setting_for_sysext_name(INTEGRATION_SYSEXT_NAME) - if not exists: - return - - remove_tree(INTEGRATION_SYSEXT_NAME) - - if not get_active_sysext_names(): - return - - run_image_integration_workflow(runtime) + if exists: + remove_tree(INTEGRATION_SYSEXT_NAME) + if get_active_sysext_names(): + run_image_integration_workflow(runtime) def run_image_remove_workflow(name: str) -> None: -- GitLab From 1235eed4e65328fe18b1f60ba1a751a05babdf0f Mon Sep 17 00:00:00 2001 From: Martin Abente Lahaye Date: Fri, 7 Jun 2024 07:29:22 -0400 Subject: [PATCH 45/46] main: Make all extensions not persistent by default --- README.md | 2 +- src/sysext_utils/errors.py | 2 +- src/sysext_utils/helpers.py | 24 ++++++++++++------------ src/sysext_utils/main.py | 12 ++++++------ src/sysext_utils/utilities.py | 20 ++++++++++---------- src/sysext_utils/workflows.py | 18 +++++++++--------- tests/test_workflows.py | 10 +++++----- 7 files changed, 44 insertions(+), 44 deletions(-) diff --git a/README.md b/README.md index ae2943e..52f7da5 100644 --- a/README.md +++ b/README.md @@ -34,7 +34,7 @@ $ sysext-build sdk ./build --system=meson --format=ddi --private_key=key --certi ```bash $ sysext-add sdk.sysext.raw -$ sysext-add https://os.gnome.org/sysext/sdk.sysext.raw --runtime +$ sysext-add https://os.gnome.org/sysext/sdk.sysext.raw --persistent $ sysext-remove sdk ``` diff --git a/src/sysext_utils/errors.py b/src/sysext_utils/errors.py index 065e4c8..41ea73a 100644 --- a/src/sysext_utils/errors.py +++ b/src/sysext_utils/errors.py @@ -67,4 +67,4 @@ class UnsupportedBuildSystemError(HandledError): class IncompatibleSettingError(HandledError): def __init__(self): - super().__init__("All extensions must be either runtime or persistent") + super().__init__("Extensions must be either all persistent or runtime") diff --git a/src/sysext_utils/helpers.py b/src/sysext_utils/helpers.py index 776c770..299ecd5 100644 --- a/src/sysext_utils/helpers.py +++ b/src/sysext_utils/helpers.py @@ -127,24 +127,24 @@ def get_name_from_input_sysext_image(image: str) -> str: return filename.removesuffix(SYSEXT_IMAGE_SUFFIX) -def get_tree_path_for_active_sysext_name(name: str, runtime: bool) -> str: - if runtime: - return os.path.join(RUN_EXTENSIONS_DIR, name) +def get_tree_path_for_active_sysext_name(name: str, persistent: bool) -> str: + if persistent: + return os.path.join(VAR_EXTENSIONS_DIR, name) - return os.path.join(VAR_EXTENSIONS_DIR, name) + return os.path.join(RUN_EXTENSIONS_DIR, name) -def get_tree_runtime_setting_for_sysext_name(name: str) -> Tuple[bool, bool]: - if os.path.exists(get_tree_path_for_active_sysext_name(name, runtime=True)): +def get_tree_setting_for_sysext_name(name: str) -> Tuple[bool, bool]: + if os.path.exists(get_tree_path_for_active_sysext_name(name, persistent=True)): return True, True - elif os.path.exists(get_tree_path_for_active_sysext_name(name, runtime=False)): + elif os.path.exists(get_tree_path_for_active_sysext_name(name, persistent=False)): return True, False return False, False -def get_image_path_for_active_sysext_name(name: str, runtime: bool) -> str: - return get_tree_path_for_active_sysext_name(name, runtime) + ".raw" +def get_image_path_for_active_sysext_name(name: str, persistent: bool) -> str: + return get_tree_path_for_active_sysext_name(name, persistent) + ".raw" def get_path_for_artifact() -> str: @@ -211,8 +211,8 @@ def ensure_directory(path: str) -> None: raise EmptyDirectoryError(path) -def ensure_setting(runtime: bool) -> None: - exists, _runtime = get_tree_runtime_setting_for_sysext_name(INTEGRATION_SYSEXT_NAME) +def ensure_setting(persistent: bool) -> None: + exists, _persistent = get_tree_setting_for_sysext_name(INTEGRATION_SYSEXT_NAME) - if exists and _runtime != runtime: + if exists and _persistent != persistent: raise IncompatibleSettingError() diff --git a/src/sysext_utils/main.py b/src/sysext_utils/main.py index f1a96f0..38b5e9e 100644 --- a/src/sysext_utils/main.py +++ b/src/sysext_utils/main.py @@ -94,8 +94,8 @@ def add(): help="path or URL to the sysext image e.g., sdk.sysext.raw", ) parser.add_argument( - "--runtime", - help="add the sysext image under /run/extensions/ instead", + "--persistent", + help="add the image under /var/lib/extensions/ to persist after reboots", action="store_true", ) @@ -103,7 +103,7 @@ def add(): _handle_global_args(parser, args) try: - run_image_add_workflow(args.image, args.runtime) + run_image_add_workflow(args.image, args.persistent) except HandledError as e: sys.exit(str(e)) finally: @@ -232,8 +232,8 @@ def install(): action="store_true", ) parser.add_argument( - "--runtime", - help="add the sysext image under /run/extensions/ instead", + "--persistent", + help="add the image under /var/lib/extensions/ to persist after reboots", action="store_true", ) parser.add_argument( @@ -264,7 +264,7 @@ def install(): args.ignore_release, args.system, ), - args.runtime, + args.persistent, ) except HandledError as e: sys.exit(str(e)) diff --git a/src/sysext_utils/utilities.py b/src/sysext_utils/utilities.py index e9b6b1e..4b104b0 100644 --- a/src/sysext_utils/utilities.py +++ b/src/sysext_utils/utilities.py @@ -100,42 +100,42 @@ def build_tree(name: str, artifact: str, ignore_release: bool) -> None: write_file(metadata_content, metadata_path) -def add_image(name: str, image: str, runtime: bool) -> None: +def add_image(name: str, image: str, persistent: bool) -> None: if is_url(image): Importctl.pull_raw(name, image) else: Importctl.import_raw(name, image) # See https://github.com/systemd/systemd/issues/32938 - if runtime is True: + if not persistent: make_directory(RUN_EXTENSIONS_DIR) move_file( - get_image_path_for_active_sysext_name(name, runtime=False), - get_image_path_for_active_sysext_name(name, runtime=True), + get_image_path_for_active_sysext_name(name, persistent=True), + get_image_path_for_active_sysext_name(name, persistent=False), ) SystemdSysext.refresh() -def add_tree(name: str, artifact: str, runtime: bool) -> None: +def add_tree(name: str, artifact: str, persistent: bool) -> None: copy_directory( artifact, - get_tree_path_for_active_sysext_name(name, runtime=runtime), + get_tree_path_for_active_sysext_name(name, persistent=persistent), ) SystemdSysext.refresh() def remove_image(name: str) -> None: - remove_file(get_image_path_for_active_sysext_name(name, runtime=True)) - remove_file(get_image_path_for_active_sysext_name(name, runtime=False)) + remove_file(get_image_path_for_active_sysext_name(name, persistent=True)) + remove_file(get_image_path_for_active_sysext_name(name, persistent=False)) SystemdSysext.refresh() def remove_tree(name: str) -> None: - remove_directory(get_tree_path_for_active_sysext_name(name, runtime=True)) - remove_directory(get_tree_path_for_active_sysext_name(name, runtime=False)) + remove_directory(get_tree_path_for_active_sysext_name(name, persistent=True)) + remove_directory(get_tree_path_for_active_sysext_name(name, persistent=False)) SystemdSysext.refresh() diff --git a/src/sysext_utils/workflows.py b/src/sysext_utils/workflows.py index 1729716..1bea6b8 100644 --- a/src/sysext_utils/workflows.py +++ b/src/sysext_utils/workflows.py @@ -26,7 +26,7 @@ from .definitions import WORKSPACE_DIR, INTEGRATION_SYSEXT_NAME from .helpers import ( get_name_from_input_sysext_image, - get_tree_runtime_setting_for_sysext_name, + get_tree_setting_for_sysext_name, get_active_sysext_names, ensure_priviledged, ensure_setting, @@ -70,32 +70,32 @@ def run_image_build_workflow( return image -def run_image_integration_workflow(runtime: bool) -> None: +def run_image_integration_workflow(persistent: bool) -> None: remove_tree(INTEGRATION_SYSEXT_NAME) artifact = build_integration_artifact() build_tree(INTEGRATION_SYSEXT_NAME, artifact, False) - add_tree(INTEGRATION_SYSEXT_NAME, artifact, runtime) + add_tree(INTEGRATION_SYSEXT_NAME, artifact, persistent) -def run_image_add_workflow(image: str, runtime: bool) -> None: +def run_image_add_workflow(image: str, persistent: bool) -> None: name = get_name_from_input_sysext_image(image) ensure_priviledged() - ensure_setting(runtime) + ensure_setting(persistent) remove_image(name) - add_image(name, image, runtime) - run_image_integration_workflow(runtime) + add_image(name, image, persistent) + run_image_integration_workflow(persistent) print(f"Successfully added {image}") def run_image_deintegration_workflow() -> None: - exists, runtime = get_tree_runtime_setting_for_sysext_name(INTEGRATION_SYSEXT_NAME) + exists, persistent = get_tree_setting_for_sysext_name(INTEGRATION_SYSEXT_NAME) if exists: remove_tree(INTEGRATION_SYSEXT_NAME) if get_active_sysext_names(): - run_image_integration_workflow(runtime) + run_image_integration_workflow(persistent) def run_image_remove_workflow(name: str) -> None: diff --git a/tests/test_workflows.py b/tests/test_workflows.py index 88f6dc3..70220ba 100644 --- a/tests/test_workflows.py +++ b/tests/test_workflows.py @@ -102,19 +102,19 @@ def test_image_build_work_with_ignore_release(): def test_image_add_workflow(): - run_image_add_workflow(image="image", runtime=False) + run_image_add_workflow(image="image", persistent=False) def test_image_add_workflow_with_remote_image(): - run_image_add_workflow(image="https://image", runtime=False) + run_image_add_workflow(image="https://image", persistent=False) -def test_image_add_workflow_with_runtime_flag(): - run_image_add_workflow(image="image", runtime=True) +def test_image_add_workflow_with_persistent_flag(): + run_image_add_workflow(image="image", persistent=True) def test_image_add_workflow_without_integrations(): - run_image_add_workflow(image="image", runtime=True) + run_image_add_workflow(image="image", persistent=False) def test_image_remove_workflow(): -- GitLab From b82181c05304fabfaf35863ce6362367889dcddf Mon Sep 17 00:00:00 2001 From: Martin Abente Lahaye Date: Fri, 7 Jun 2024 11:20:43 -0400 Subject: [PATCH 46/46] README: Update --- README.md | 120 +++++++++++++++++++++++++------ docker/Dockerfile | 7 ++ docker/README.md | 16 +++++ tests/data/example/usr/bin/works | 1 + 4 files changed, 124 insertions(+), 20 deletions(-) create mode 100644 docker/Dockerfile create mode 100644 docker/README.md create mode 100755 tests/data/example/usr/bin/works diff --git a/README.md b/README.md index 52f7da5..4de5cc8 100644 --- a/README.md +++ b/README.md @@ -1,14 +1,29 @@ # sysext-utils -A tool that provides a better experience for developing and testing system components on GNOME OS (and eventually other immutable OSes). It facilitates the steps of the workflow described in this [proposal](https://discourse.gnome.org/t/towards-a-better-way-to-hack-and-test-your-system-components/21075). +sysext-utils is a collection of tools, built on top of [systemd-sysext](https://www.freedesktop.org/software/systemd/man/latest/systemd-sysext.html), that aims to provide a better experience developing and testing system components for [GNOME OS](https://os.gnome.org). + +The basic idea is to facilitate building and managing system extensions that can be overlaid on top of the OS. This way, one can add modified or new components to its system to test. Check out the [original proposal](https://discourse.gnome.org/t/towards-a-better-way-to-hack-and-test-your-system-components/21075) for the rationale behind this project. + +To achieve this, sysext-utils provides the following tools: + +* sysext-build: creates an extension out of a [Meson](https://mesonbuild.com/) or [BuildStream](https://buildstream.build/) project, or an OS tree directory. +* sysext-add: imports, activates and integrates an extension. +* sysext-remove: deactivates and deintegrates an extension. +* sysext-install: combines both *sysext-build* and *sysext-add* functionalities into one, for convenience. + +Although this project is primarily designed for GNOME OS, the tool itself and its principles could be expanded to any other OS with systemd version 256 or newer. ## Requirements -* [Install](https://gitlab.gnome.org/GNOME/gnome-build-meta/-/wikis/gnome_os/Start-GNOME-OS#getting-the-images) the sysupdate variant of GNOME OS. -* Turn that installation in to a development environment by using systemd-sysupdate to [import](https://gitlab.gnome.org/GNOME/gnome-build-meta/-/wikis/gnome_os/Install-Software#install-developer-tools-and-system-extensions-with-systemd-sysext) the development tree sysext. +* An installation of [GNOME OS](https://gitlab.gnome.org/GNOME/gnome-build-meta/-/wikis/gnome_os/Start-GNOME-OS#getting-the-images) with the sysupdate variant. +* The development tree [sysext](https://gitlab.gnome.org/GNOME/gnome-build-meta/-/wikis/gnome_os/Install-Software#install-developer-tools-and-system-extensions-with-systemd-sysext) of GNOME OS, to pull all required dependencies. + +Alternatively, the core GNOME OS OCI image can be used to build extensions with *sysext-build*. A few extra dependencies might be needed, depending on the target build system and extension image format. See this [example](./docker). ## Installation +To install sysext-utils simply follow these steps: + ```bash $ git clone https://gitlab.gnome.org/tchx84/sysext-utils.git $ cd sysext-utils @@ -17,36 +32,101 @@ $ source env/bin/activate $ python -m pip install . ``` -## Examples +## Usage + +### Building extensions + +*sysext-build* can facilitate the creation of extensions out of projects with different build systems. + +#### Meson + +```bash +$ git clone https://gitlab.gnome.org/GNOME/gtk.git +$ cd gtk +$ meson setup ./build --prefix /usr +# do development +$ sysext-build example ./build +# obtain the example.sysext.raw extension image +``` + +#### BuildStream + +```bash +$ git clone https://gitlab.gnome.org/GNOME/gnome-build-meta +$ cd gnome-build-meta +$ bst workspace open --directory ./workspace sdk/gtk.bst +# do development +$ sysext-build example ./workspace +# obtain the example.sysext.raw extension image +``` + +#### An OS tree directory -### Build sysext images out of a project +```bash +$ git clone https://gitlab.gnome.org/tchx84/sysext-utils.git +$ cd sysext-utils +# do development and use another build system to produce an OS tree directory +$ sysext-build example ./tests/data/example +# obtain the example.sysext.raw extension image +``` + +#### Notes + +* The first argument to *sysext-build* is the name to be given to the extension. The name can be any valid filename. +* The second argument must be a directory. *sysext-build* will try to detect the proper build system to use. If needed, one can force a build system by using the `--system` optional argument. +* By default, *sysext-build* will use *mksquashfs* to make the extension images. Images can also be signed by specifying the `--format=ddi` optional argument and providing a private key and certificate. +* By default, *sysext-build* will set metadata to ensure that the extension can only be used in identical systems. This behavior can be overridden by using the `--ignore-release` optional argument. This is useful when building extensions in a separate environment like in a CI pipeline. +* See `sysext-build --help` for more details. + +### Managing extensions + +Whether the extension is available locally or remotely, it can be added to the system as follows: + +#### Adding an extension ```bash -$ sysext-build sdk ./build -$ sysext-build sdk ./build --system=meson -$ sysext-build sdk ./workspace --system=bst -$ sysext-build sdk ./destdir --system=import -$ sysext-build sdk ./build --system=meson --verbose --ignore-release --dry -$ sysext-build sdk ./build --system=meson --format=ddi --private_key=key --certificate=cert +$ sysext-add example.sysext.raw +$ sysext-add https://os.gnome.org/sysext/example.sysext.raw +# do testing ``` -### Manage sysext images +#### Removing an extension ```bash -$ sysext-add sdk.sysext.raw -$ sysext-add https://os.gnome.org/sysext/sdk.sysext.raw --persistent -$ sysext-remove sdk +$ sysext-remove example ``` -### Combine build and add steps +#### Notes + +* Adding an extension will run integration steps like recompiling the glib schemas and updating the icons cache. These integration artifacts are added in a separate extension which is global for all extensions. +* Removing an extension will also run these integration steps. The global integration extension will be removed along with the last extension. +* By default, extensions are added under `/run/extensions`. This means that the extension will be removed after a system reboot, which is a safer default behavior. This can be overridden by using the `--persistent` optional argument. +* See `sysext-add --help` and `sysext-remove --help` for more details. + +### Facilitating local development + +Constantly building and adding extensions can be tedious when testing things locally, therefore a *sysext-install* convenience tool is included to do both in a single step. + +#### A single step ```bash -$ sysext-install sdk ./build -$ sysext-install sdk ./build --system=meson -$ sysext-install sdk ./workspace --system=bst -$ sysext-install sdk ./destdir --system=import +$ git clone https://gitlab.gnome.org/GNOME/gtk.git +$ cd gtk +$ meson setup ./build --prefix /usr +# do development +$ sysext-install example ./build +# do testing ``` +#### Notes + +* *sysext-install* combines both *sysext-build* and *sysext-add* functionalities and arguments and therefore follows the exact same defaults. +* See `sysext-install --help` for more details. + +## What else is missing? + +See the list of [issues](https://gitlab.gnome.org/tchx84/sysext-utils/-/issues) for opportunities to improve this tool. + ## License MIT License diff --git a/docker/Dockerfile b/docker/Dockerfile new file mode 100644 index 0000000..7da764a --- /dev/null +++ b/docker/Dockerfile @@ -0,0 +1,7 @@ +FROM quay.io/gnome_infrastructure/gnome-build-meta:core-nightly + +RUN git clone --branch v6.1.1 --single-branch https://github.com/plougher/squashfs-tools.git && \ + (cd squashfs-tools/squashfs-tools && sudo make install INSTALL_PREFIX=/usr) + +RUN git clone --branch main --single-branch https://gitlab.gnome.org/tchx84/sysext-utils.git && \ + (cd sysext-utils && sudo python -m pip install .) diff --git a/docker/README.md b/docker/README.md new file mode 100644 index 0000000..691a1f5 --- /dev/null +++ b/docker/README.md @@ -0,0 +1,16 @@ +# sysext-utils Toolbx + +This directory contains a sample Dockerfile based on the [core](https://gitlab.gnome.org/GNOME/gnome-build-meta) GNOME OS OCI image, plus *mksquashfs* and *sysext-utils* itself. This can create a suitable Toolbx environment to create images that are not signed, out of any GNOME project with the meson build system. + +## Build it + +```bash +$ podman build . -t sysext-utils:latest +$ toolbox create -c sysext-utils -i sysext-utils +``` + +## Use it + +```bash +$ toolbox enter sysext-utils +``` diff --git a/tests/data/example/usr/bin/works b/tests/data/example/usr/bin/works new file mode 100755 index 0000000..6744650 --- /dev/null +++ b/tests/data/example/usr/bin/works @@ -0,0 +1 @@ +echo "yes" -- GitLab