From 81e312326250b0d378997fcd0c87d9e956dd9e71 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?G=C3=A1bor=20Stefanik?= Date: Tue, 26 Dec 2023 04:28:47 +0100 Subject: [PATCH] Rename tmpdir to target and always preserve it, dropping --preserve There is nothing temporary about our "tmpdir" - its sole purpose is to contain the final product of the bootstrap process. Thus, removing it at the end of bootstrap amounts to doing the entire process for nothing. To remedy this, --tmpdir is renamed --target, keeping the -t short form, and defaulting to "target" instead of "tmp" to make its purpose clearer. The --preserve option is removed, as the target is now always preserved. --- .github/workflows/bwrap.yml | 16 +++++----- .github/workflows/lint.yml | 2 +- lib/generator.py | 60 ++++++++++++++++++------------------ lib/{tmpdir.py => target.py} | 32 +++++++------------ rootfs.py | 60 +++++++++++++++++------------------- 5 files changed, 79 insertions(+), 91 deletions(-) rename lib/{tmpdir.py => target.py} (76%) diff --git a/.github/workflows/bwrap.yml b/.github/workflows/bwrap.yml index 776ea20..66cea84 100644 --- a/.github/workflows/bwrap.yml +++ b/.github/workflows/bwrap.yml @@ -43,15 +43,15 @@ jobs: distfiles key: cache-${{ hashFiles('steps/*/sources') }} - name: Run bootstrap - run: ./rootfs.py --bwrap --external-sources --build-kernels --preserve --cores 2 --internal-ci pass1 + run: ./rootfs.py --bwrap --external-sources --build-kernels --cores 2 --internal-ci pass1 - name: Archive created packages if: failure() # archive failed builds progress uses: actions/upload-artifact@v3 with: name: packages - path: tmp/external/repo/** + path: target/external/repo/** - name: Tar pass1 image - run: tar -cf pass1_image.tar tmp + run: tar -cf pass1_image.tar target - name: Archive pass1_image uses: actions/upload-artifact@v3 with: @@ -96,15 +96,15 @@ jobs: distfiles key: cache-${{ hashFiles('steps/*/sources') }} - name: Run bootstrap - run: ./rootfs.py --bwrap --external-sources --build-kernels --preserve --cores 2 --internal-ci pass2 + run: ./rootfs.py --bwrap --external-sources --build-kernels --cores 2 --internal-ci pass2 - name: Archive created packages if: failure() # archive failed builds progress uses: actions/upload-artifact@v3 with: name: internal_packages_pass2 - path: tmp/external/repo/** + path: target/external/repo/** - name: Tar pass2 image - run: tar -cf pass2_image.tar tmp + run: tar -cf pass2_image.tar target - name: Archive pass2_image uses: actions/upload-artifact@v3 with: @@ -149,10 +149,10 @@ jobs: distfiles key: cache-${{ hashFiles('steps/*/sources') }} - name: Run bootstrap - run: ./rootfs.py --bwrap --external-sources --build-kernels --preserve --cores 2 --internal-ci pass3 + run: ./rootfs.py --bwrap --external-sources --build-kernels --cores 2 --internal-ci pass3 - name: Archive created packages if: always() # archive both failed and successful builds uses: actions/upload-artifact@v3 with: name: packages - path: tmp/external/repo/** + path: target/external/repo/** diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index e2000ab..b8fada9 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -21,7 +21,7 @@ jobs: - name: Install pylint run: sudo pip3 install pylint - name: pylint - run: pylint rootfs.py lib/utils.py lib/generator.py lib/tmpdir.py --disable=duplicate-code + run: pylint rootfs.py lib/utils.py lib/generator.py lib/target.py --disable=duplicate-code shellcheck: name: Lint shell files diff --git a/lib/generator.py b/lib/generator.py index 326101a..0f16e2f 100755 --- a/lib/generator.py +++ b/lib/generator.py @@ -29,26 +29,26 @@ class Generator(): self.external_sources = external_sources self.repo_path = repo_path self.source_manifest = self.get_source_manifest(not self.external_sources) - self.tmp_dir = None + self.target_dir = None self.external_dir = None - def reuse(self, tmpdir): + def reuse(self, target): """ Reuse a previously prepared bwrap environment for further stages. """ - self.tmp_dir = tmpdir.path - self.external_dir = os.path.join(self.tmp_dir, 'external') + self.target_dir = target.path + self.external_dir = os.path.join(self.target_dir, 'external') self.distfiles() - def prepare(self, tmpdir, using_kernel=False, kernel_bootstrap=False, target_size=0): + def prepare(self, target, using_kernel=False, kernel_bootstrap=False, target_size=0): """ Prepare basic media of live-bootstrap. /steps -- contains steps to be built / -- contains seed to allow steps to be built, containing custom scripts and stage0-posix """ - self.tmp_dir = tmpdir.path - self.external_dir = os.path.join(self.tmp_dir, 'external') + self.target_dir = target.path + self.external_dir = os.path.join(self.target_dir, 'external') # We use ext3 here; ext4 actually has a variety of extensions that # have been added with varying levels of recency @@ -59,33 +59,33 @@ class Generator(): # argument matrix ... or we could just use ext3 instead which # is effectively universally the same if kernel_bootstrap: - init_path = os.path.join(self.tmp_dir, 'init') + init_path = os.path.join(self.target_dir, 'init') os.mkdir(init_path) - self.tmp_dir = init_path + self.target_dir = init_path if self.repo_path or self.external_sources: - tmpdir.add_disk("external", filesystem="ext3") - tmpdir.mount_disk("external", "external") + target.add_disk("external", filesystem="ext3") + target.mount_disk("external", "external") else: - self.external_dir = os.path.join(self.tmp_dir, 'external') + self.external_dir = os.path.join(self.target_dir, 'external') elif using_kernel: - self.tmp_dir = os.path.join(self.tmp_dir, 'disk') - tmpdir.add_disk("disk", + self.target_dir = os.path.join(self.target_dir, 'disk') + target.add_disk("disk", filesystem="ext3", size=(target_size + "M") if target_size else "16G", bootable=True) - tmpdir.mount_disk("disk", "disk") - self.external_dir = os.path.join(self.tmp_dir, 'external') + target.mount_disk("disk", "disk") + self.external_dir = os.path.join(self.target_dir, 'external') os.makedirs(self.external_dir, exist_ok=True) if self.early_preseed: # Extract tar containing preseed with tarfile.open(self.early_preseed, "r") as seed: - seed.extractall(self.tmp_dir) + seed.extractall(self.target_dir) shutil.copy2(os.path.join(self.git_dir, 'seed', 'preseeded.kaem'), - os.path.join(self.tmp_dir, 'kaem.x86')) + os.path.join(self.target_dir, 'kaem.x86')) else: self.stage0_posix() self.seed() @@ -101,25 +101,25 @@ class Generator(): shutil.copytree(self.repo_path, repo_dir) if kernel_bootstrap: - self.create_builder_hex0_disk_image(self.tmp_dir + '.img', target_size) + self.create_builder_hex0_disk_image(self.target_dir + '.img', target_size) if kernel_bootstrap and (self.external_sources or self.repo_path): - tmpdir.umount_disk('external') + target.umount_disk('external') elif using_kernel: - tmpdir.umount_disk('disk') + target.umount_disk('disk') def steps(self): """Copy in steps.""" self.get_packages() - shutil.copytree(os.path.join(self.git_dir, 'steps'), os.path.join(self.tmp_dir, 'steps')) + shutil.copytree(os.path.join(self.git_dir, 'steps'), os.path.join(self.target_dir, 'steps')) def stage0_posix(self): """Copy in all of the stage0-posix""" stage0_posix_base_dir = os.path.join(self.git_dir, 'seed', 'stage0-posix') for entry in os.listdir(stage0_posix_base_dir): orig = os.path.join(stage0_posix_base_dir, entry) - target = os.path.join(self.tmp_dir, entry) + target = os.path.join(self.target_dir, entry) if os.path.isfile(orig): shutil.copy2(orig, target) else: @@ -128,14 +128,14 @@ class Generator(): arch = stage0_arch_map.get(self.arch, self.arch) kaem_optional_seed = os.path.join(self.git_dir, 'seed', 'stage0-posix', 'bootstrap-seeds', 'POSIX', arch, 'kaem-optional-seed') - shutil.copy2(kaem_optional_seed, os.path.join(self.tmp_dir, 'init')) + shutil.copy2(kaem_optional_seed, os.path.join(self.target_dir, 'init')) def seed(self): """Copy in extra seed files""" seed_dir = os.path.join(self.git_dir, 'seed') for entry in os.listdir(seed_dir): if os.path.isfile(os.path.join(seed_dir, entry)): - shutil.copy2(os.path.join(seed_dir, entry), os.path.join(self.tmp_dir, entry)) + shutil.copy2(os.path.join(seed_dir, entry), os.path.join(self.target_dir, entry)) @staticmethod def add_fiwix_files(file_list_path, dirpath): @@ -149,14 +149,14 @@ class Generator(): def create_fiwix_file_list(self): """Create a list of files to populate Fiwix file system""" - file_list_path = os.path.join(self.tmp_dir, 'steps', 'lwext4-1.0.0-lb1', + file_list_path = os.path.join(self.target_dir, 'steps', 'lwext4-1.0.0-lb1', 'files', 'fiwix-file-list.txt') - shutil.copyfile(os.path.join(self.tmp_dir, 'steps', 'lwext4-1.0.0-lb1', + shutil.copyfile(os.path.join(self.target_dir, 'steps', 'lwext4-1.0.0-lb1', 'files', 'early-artifacts-needed-after-fiwix.txt'), file_list_path) save_cwd = os.getcwd() - os.chdir(self.tmp_dir) + os.chdir(self.target_dir) self.add_fiwix_files(file_list_path, 'steps') self.add_fiwix_files(file_list_path, 'distfiles') os.chdir(save_cwd) @@ -170,7 +170,7 @@ class Generator(): shutil.copy2(os.path.join(self.distfiles_dir, file), os.path.join(out, file)) - early_distfile_dir = os.path.join(self.tmp_dir, 'external', 'distfiles') + early_distfile_dir = os.path.join(self.target_dir, 'external', 'distfiles') main_distfile_dir = os.path.join(self.external_dir, 'distfiles') if early_distfile_dir != main_distfile_dir: @@ -217,7 +217,7 @@ class Generator(): """Append srcfs file system to disk image""" save_cwd = os.getcwd() - os.chdir(self.tmp_dir) + os.chdir(self.target_dir) self.output_tree(image_file, '.') # Add commands to kick off stage0-posix diff --git a/lib/tmpdir.py b/lib/target.py similarity index 76% rename from lib/tmpdir.py rename to lib/target.py index 954235c..db4b474 100644 --- a/lib/tmpdir.py +++ b/lib/target.py @@ -4,34 +4,32 @@ # SPDX-License-Identifier: GPL-3.0-or-later """ -Contains a class that represents a tmpdir +Contains a class that represents a target directory """ import enum import getpass import os -import shutil from lib.utils import mount, umount, create_disk, run_as_root -class TmpType(enum.Enum): - """Different types of tmpdirs we can have""" +class TargetType(enum.Enum): + """Different types of target dirs we can have""" NONE = 0 TMPFS = 1 -class Tmpdir: +class Target: """ - Represents a tmpdir + Represents a target directory """ _disks = {} _disk_filesystems = {} _mountpoints = {} - def __init__(self, preserve, path="tmp"): + def __init__(self, path="target"): self.path = os.path.abspath(path) - self.preserve = preserve - self._type = TmpType.NONE + self._type = TargetType.NONE if not os.path.exists(self.path): os.mkdir(self.path) @@ -41,23 +39,15 @@ class Tmpdir: print(f"Unmounting {path}") umount(path) - if not self.preserve: - for disk in self._disks.values(): - print(f"Detaching {disk}") - run_as_root("losetup", "-d", disk) - - if self._type == TmpType.TMPFS: - print(f"Unmounting tmpdir from {self.path}") - umount(self.path) - - print(f"Removing {self.path}") - shutil.rmtree(self.path, ignore_errors=True) + for disk in self._disks.values(): + print(f"Detaching {disk}") + run_as_root("losetup", "-d", disk) def tmpfs(self, size="8G"): """Mount a tmpfs""" print(f"Mounting tmpfs on {self.path}") mount("tmpfs", self.path, "tmpfs", f"size={size}") - self._type = TmpType.TMPFS + self._type = TargetType.TMPFS # pylint: disable=too-many-arguments def add_disk(self, diff --git a/rootfs.py b/rootfs.py index 18834ab..a08d27d 100755 --- a/rootfs.py +++ b/rootfs.py @@ -17,7 +17,7 @@ import argparse import os from lib.utils import run, run_as_root -from lib.tmpdir import Tmpdir +from lib.target import Target from lib.generator import Generator, stage0_arch_map def create_configuration_file(args): @@ -58,11 +58,9 @@ def main(): action="store_true") parser.add_argument("-bw", "--bwrap", help="Run inside a bwrap sandbox", action="store_true") - parser.add_argument("-p", "--preserve", help="Do not remove temporary dir", - action="store_true") - parser.add_argument("-t", "--tmpdir", help="Temporary directory", - default="tmp") - parser.add_argument("--tmpfs", help="Use a tmpfs on tmpdir", + parser.add_argument("-t", "--target", help="Target directory", + default="target") + parser.add_argument("--tmpfs", help="Use a tmpfs on target", action="store_true") parser.add_argument("--tmpfs-size", help="Size of the tmpfs", default="8G") @@ -128,7 +126,7 @@ def main(): if args.arch != "x86": print("Only x86 is supported at the moment, other arches are for development only.") - # Tmp validation + # Tmpfs validation if args.bwrap and args.tmpfs: raise ValueError("tmpfs cannot be used with bwrap.") @@ -156,19 +154,19 @@ def main(): with open(os.path.join('steps', 'bootstrap.cfg'), 'a', encoding='UTF-8'): pass - # tmpdir - tmpdir = Tmpdir(path=args.tmpdir, preserve=args.preserve) + # target + target = Target(path=args.target) if args.tmpfs: - tmpdir.tmpfs(size=args.tmpfs_size) + target.tmpfs(size=args.tmpfs_size) generator = Generator(arch=args.arch, external_sources=args.external_sources, repo_path=args.repo, early_preseed=args.early_preseed) - bootstrap(args, generator, tmpdir, args.target_size) + bootstrap(args, generator, target, args.target_size) -def bootstrap(args, generator, tmpdir, size): +def bootstrap(args, generator, target, size): """Kick off bootstrap process.""" print(f"Bootstrapping {args.arch}") if args.chroot: @@ -179,28 +177,28 @@ print(shutil.which('chroot')) chroot_binary = run_as_root('python3', '-c', find_chroot, capture_output=True).stdout.decode().strip() - generator.prepare(tmpdir, using_kernel=False) + generator.prepare(target, using_kernel=False) arch = stage0_arch_map.get(args.arch, args.arch) init = os.path.join(os.sep, 'bootstrap-seeds', 'POSIX', arch, 'kaem-optional-seed') - run_as_root('env', '-i', 'PATH=/bin', chroot_binary, generator.tmp_dir, init) + run_as_root('env', '-i', 'PATH=/bin', chroot_binary, generator.target_dir, init) elif args.bwrap: init = '/init' if not args.internal_ci or args.internal_ci == "pass1": - generator.prepare(tmpdir, using_kernel=False) + generator.prepare(target, using_kernel=False) arch = stage0_arch_map.get(args.arch, args.arch) init = os.path.join(os.sep, 'bootstrap-seeds', 'POSIX', arch, 'kaem-optional-seed') else: - generator.reuse(tmpdir) + generator.reuse(target) run('env', '-i', 'bwrap', '--unshare-user', '--uid', '0', '--gid', '0', '--unshare-net' if args.external_sources else None, '--setenv', 'PATH', '/usr/bin', - '--bind', generator.tmp_dir, '/', + '--bind', generator.target_dir, '/', '--dir', '/dev', '--dev-bind', '/dev/null', '/dev/null', '--dev-bind', '/dev/zero', '/dev/zero', @@ -216,44 +214,44 @@ print(shutil.which('chroot')) elif args.bare_metal: if args.kernel: - generator.prepare(tmpdir, using_kernel=True, target_size=size) - image_path = os.path.join(args.tmpdir, os.path.relpath(generator.tmp_dir, args.tmpdir)) + generator.prepare(target, using_kernel=True, target_size=size) + path = os.path.join(args.target, os.path.relpath(generator.target_dir, args.target)) print("Please:") - print(f" 1. Take {image_path}/initramfs and your kernel, boot using this.") - print(f" 2. Take {image_path}/disk.img and put this on a writable storage medium.") + print(f" 1. Take {path}/initramfs and your kernel, boot using this.") + print(f" 2. Take {path}/disk.img and put this on a writable storage medium.") else: - generator.prepare(tmpdir, kernel_bootstrap=True, target_size=size) - image_path = os.path.join(args.tmpdir, os.path.relpath(generator.tmp_dir, args.tmpdir)) + generator.prepare(target, kernel_bootstrap=True, target_size=size) + path = os.path.join(args.target, os.path.relpath(generator.target_dir, args.target)) print("Please:") - print(f" 1. Take {image_path}.img and write it to a boot drive and then boot it.") + print(f" 1. Take {path}.img and write it to a boot drive and then boot it.") else: if args.kernel: - generator.prepare(tmpdir, using_kernel=True, target_size=size) + generator.prepare(target, using_kernel=True, target_size=size) run(args.qemu_cmd, '-enable-kvm', '-m', str(args.qemu_ram) + 'M', '-smp', str(args.cores), '-no-reboot', - '-drive', 'file=' + tmpdir.get_disk("disk") + ',format=raw', - '-drive', 'file=' + tmpdir.get_disk("external") + ',format=raw', + '-drive', 'file=' + target.get_disk("disk") + ',format=raw', + '-drive', 'file=' + target.get_disk("external") + ',format=raw', '-nic', 'user,ipv6=off,model=e1000', '-kernel', args.kernel, '-nographic', '-append', 'console=ttyS0 root=/dev/sda1 rootfstype=ext3 init=/init rw') else: - generator.prepare(tmpdir, kernel_bootstrap=True, target_size=size) + generator.prepare(target, kernel_bootstrap=True, target_size=size) arg_list = [ '-enable-kvm', '-m', str(args.qemu_ram) + 'M', '-smp', str(args.cores), '-no-reboot', - '-drive', 'file=' + generator.tmp_dir + '.img' + ',format=raw' + '-drive', 'file=' + generator.target_dir + '.img' + ',format=raw' ] - if tmpdir.get_disk("external") is not None: + if target.get_disk("external") is not None: arg_list += [ - '-drive', 'file=' + tmpdir.get_disk("external") + ',format=raw', + '-drive', 'file=' + target.get_disk("external") + ',format=raw', ] arg_list += [ '-machine', 'kernel-irqchip=split',