diff --git a/.github/workflows/bwrap.yml b/.github/workflows/bwrap.yml index 776ea20..66cea84 100644 --- a/.github/workflows/bwrap.yml +++ b/.github/workflows/bwrap.yml @@ -43,15 +43,15 @@ jobs: distfiles key: cache-${{ hashFiles('steps/*/sources') }} - name: Run bootstrap - run: ./rootfs.py --bwrap --external-sources --build-kernels --preserve --cores 2 --internal-ci pass1 + run: ./rootfs.py --bwrap --external-sources --build-kernels --cores 2 --internal-ci pass1 - name: Archive created packages if: failure() # archive failed builds progress uses: actions/upload-artifact@v3 with: name: packages - path: tmp/external/repo/** + path: target/external/repo/** - name: Tar pass1 image - run: tar -cf pass1_image.tar tmp + run: tar -cf pass1_image.tar target - name: Archive pass1_image uses: actions/upload-artifact@v3 with: @@ -96,15 +96,15 @@ jobs: distfiles key: cache-${{ hashFiles('steps/*/sources') }} - name: Run bootstrap - run: ./rootfs.py --bwrap --external-sources --build-kernels --preserve --cores 2 --internal-ci pass2 + run: ./rootfs.py --bwrap --external-sources --build-kernels --cores 2 --internal-ci pass2 - name: Archive created packages if: failure() # archive failed builds progress uses: actions/upload-artifact@v3 with: name: internal_packages_pass2 - path: tmp/external/repo/** + path: target/external/repo/** - name: Tar pass2 image - run: tar -cf pass2_image.tar tmp + run: tar -cf pass2_image.tar target - name: Archive pass2_image uses: actions/upload-artifact@v3 with: @@ -149,10 +149,10 @@ jobs: distfiles key: cache-${{ hashFiles('steps/*/sources') }} - name: Run bootstrap - run: ./rootfs.py --bwrap --external-sources --build-kernels --preserve --cores 2 --internal-ci pass3 + run: ./rootfs.py --bwrap --external-sources --build-kernels --cores 2 --internal-ci pass3 - name: Archive created packages if: always() # archive both failed and successful builds uses: actions/upload-artifact@v3 with: name: packages - path: tmp/external/repo/** + path: target/external/repo/** diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index e2000ab..b8fada9 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -21,7 +21,7 @@ jobs: - name: Install pylint run: sudo pip3 install pylint - name: pylint - run: pylint rootfs.py lib/utils.py lib/generator.py lib/tmpdir.py --disable=duplicate-code + run: pylint rootfs.py lib/utils.py lib/generator.py lib/target.py --disable=duplicate-code shellcheck: name: Lint shell files diff --git a/lib/generator.py b/lib/generator.py index 326101a..0f16e2f 100755 --- a/lib/generator.py +++ b/lib/generator.py @@ -29,26 +29,26 @@ class Generator(): self.external_sources = external_sources self.repo_path = repo_path self.source_manifest = self.get_source_manifest(not self.external_sources) - self.tmp_dir = None + self.target_dir = None self.external_dir = None - def reuse(self, tmpdir): + def reuse(self, target): """ Reuse a previously prepared bwrap environment for further stages. """ - self.tmp_dir = tmpdir.path - self.external_dir = os.path.join(self.tmp_dir, 'external') + self.target_dir = target.path + self.external_dir = os.path.join(self.target_dir, 'external') self.distfiles() - def prepare(self, tmpdir, using_kernel=False, kernel_bootstrap=False, target_size=0): + def prepare(self, target, using_kernel=False, kernel_bootstrap=False, target_size=0): """ Prepare basic media of live-bootstrap. /steps -- contains steps to be built / -- contains seed to allow steps to be built, containing custom scripts and stage0-posix """ - self.tmp_dir = tmpdir.path - self.external_dir = os.path.join(self.tmp_dir, 'external') + self.target_dir = target.path + self.external_dir = os.path.join(self.target_dir, 'external') # We use ext3 here; ext4 actually has a variety of extensions that # have been added with varying levels of recency @@ -59,33 +59,33 @@ class Generator(): # argument matrix ... or we could just use ext3 instead which # is effectively universally the same if kernel_bootstrap: - init_path = os.path.join(self.tmp_dir, 'init') + init_path = os.path.join(self.target_dir, 'init') os.mkdir(init_path) - self.tmp_dir = init_path + self.target_dir = init_path if self.repo_path or self.external_sources: - tmpdir.add_disk("external", filesystem="ext3") - tmpdir.mount_disk("external", "external") + target.add_disk("external", filesystem="ext3") + target.mount_disk("external", "external") else: - self.external_dir = os.path.join(self.tmp_dir, 'external') + self.external_dir = os.path.join(self.target_dir, 'external') elif using_kernel: - self.tmp_dir = os.path.join(self.tmp_dir, 'disk') - tmpdir.add_disk("disk", + self.target_dir = os.path.join(self.target_dir, 'disk') + target.add_disk("disk", filesystem="ext3", size=(target_size + "M") if target_size else "16G", bootable=True) - tmpdir.mount_disk("disk", "disk") - self.external_dir = os.path.join(self.tmp_dir, 'external') + target.mount_disk("disk", "disk") + self.external_dir = os.path.join(self.target_dir, 'external') os.makedirs(self.external_dir, exist_ok=True) if self.early_preseed: # Extract tar containing preseed with tarfile.open(self.early_preseed, "r") as seed: - seed.extractall(self.tmp_dir) + seed.extractall(self.target_dir) shutil.copy2(os.path.join(self.git_dir, 'seed', 'preseeded.kaem'), - os.path.join(self.tmp_dir, 'kaem.x86')) + os.path.join(self.target_dir, 'kaem.x86')) else: self.stage0_posix() self.seed() @@ -101,25 +101,25 @@ class Generator(): shutil.copytree(self.repo_path, repo_dir) if kernel_bootstrap: - self.create_builder_hex0_disk_image(self.tmp_dir + '.img', target_size) + self.create_builder_hex0_disk_image(self.target_dir + '.img', target_size) if kernel_bootstrap and (self.external_sources or self.repo_path): - tmpdir.umount_disk('external') + target.umount_disk('external') elif using_kernel: - tmpdir.umount_disk('disk') + target.umount_disk('disk') def steps(self): """Copy in steps.""" self.get_packages() - shutil.copytree(os.path.join(self.git_dir, 'steps'), os.path.join(self.tmp_dir, 'steps')) + shutil.copytree(os.path.join(self.git_dir, 'steps'), os.path.join(self.target_dir, 'steps')) def stage0_posix(self): """Copy in all of the stage0-posix""" stage0_posix_base_dir = os.path.join(self.git_dir, 'seed', 'stage0-posix') for entry in os.listdir(stage0_posix_base_dir): orig = os.path.join(stage0_posix_base_dir, entry) - target = os.path.join(self.tmp_dir, entry) + target = os.path.join(self.target_dir, entry) if os.path.isfile(orig): shutil.copy2(orig, target) else: @@ -128,14 +128,14 @@ class Generator(): arch = stage0_arch_map.get(self.arch, self.arch) kaem_optional_seed = os.path.join(self.git_dir, 'seed', 'stage0-posix', 'bootstrap-seeds', 'POSIX', arch, 'kaem-optional-seed') - shutil.copy2(kaem_optional_seed, os.path.join(self.tmp_dir, 'init')) + shutil.copy2(kaem_optional_seed, os.path.join(self.target_dir, 'init')) def seed(self): """Copy in extra seed files""" seed_dir = os.path.join(self.git_dir, 'seed') for entry in os.listdir(seed_dir): if os.path.isfile(os.path.join(seed_dir, entry)): - shutil.copy2(os.path.join(seed_dir, entry), os.path.join(self.tmp_dir, entry)) + shutil.copy2(os.path.join(seed_dir, entry), os.path.join(self.target_dir, entry)) @staticmethod def add_fiwix_files(file_list_path, dirpath): @@ -149,14 +149,14 @@ class Generator(): def create_fiwix_file_list(self): """Create a list of files to populate Fiwix file system""" - file_list_path = os.path.join(self.tmp_dir, 'steps', 'lwext4-1.0.0-lb1', + file_list_path = os.path.join(self.target_dir, 'steps', 'lwext4-1.0.0-lb1', 'files', 'fiwix-file-list.txt') - shutil.copyfile(os.path.join(self.tmp_dir, 'steps', 'lwext4-1.0.0-lb1', + shutil.copyfile(os.path.join(self.target_dir, 'steps', 'lwext4-1.0.0-lb1', 'files', 'early-artifacts-needed-after-fiwix.txt'), file_list_path) save_cwd = os.getcwd() - os.chdir(self.tmp_dir) + os.chdir(self.target_dir) self.add_fiwix_files(file_list_path, 'steps') self.add_fiwix_files(file_list_path, 'distfiles') os.chdir(save_cwd) @@ -170,7 +170,7 @@ class Generator(): shutil.copy2(os.path.join(self.distfiles_dir, file), os.path.join(out, file)) - early_distfile_dir = os.path.join(self.tmp_dir, 'external', 'distfiles') + early_distfile_dir = os.path.join(self.target_dir, 'external', 'distfiles') main_distfile_dir = os.path.join(self.external_dir, 'distfiles') if early_distfile_dir != main_distfile_dir: @@ -217,7 +217,7 @@ class Generator(): """Append srcfs file system to disk image""" save_cwd = os.getcwd() - os.chdir(self.tmp_dir) + os.chdir(self.target_dir) self.output_tree(image_file, '.') # Add commands to kick off stage0-posix diff --git a/lib/tmpdir.py b/lib/target.py similarity index 76% rename from lib/tmpdir.py rename to lib/target.py index 954235c..db4b474 100644 --- a/lib/tmpdir.py +++ b/lib/target.py @@ -4,34 +4,32 @@ # SPDX-License-Identifier: GPL-3.0-or-later """ -Contains a class that represents a tmpdir +Contains a class that represents a target directory """ import enum import getpass import os -import shutil from lib.utils import mount, umount, create_disk, run_as_root -class TmpType(enum.Enum): - """Different types of tmpdirs we can have""" +class TargetType(enum.Enum): + """Different types of target dirs we can have""" NONE = 0 TMPFS = 1 -class Tmpdir: +class Target: """ - Represents a tmpdir + Represents a target directory """ _disks = {} _disk_filesystems = {} _mountpoints = {} - def __init__(self, preserve, path="tmp"): + def __init__(self, path="target"): self.path = os.path.abspath(path) - self.preserve = preserve - self._type = TmpType.NONE + self._type = TargetType.NONE if not os.path.exists(self.path): os.mkdir(self.path) @@ -41,23 +39,15 @@ class Tmpdir: print(f"Unmounting {path}") umount(path) - if not self.preserve: - for disk in self._disks.values(): - print(f"Detaching {disk}") - run_as_root("losetup", "-d", disk) - - if self._type == TmpType.TMPFS: - print(f"Unmounting tmpdir from {self.path}") - umount(self.path) - - print(f"Removing {self.path}") - shutil.rmtree(self.path, ignore_errors=True) + for disk in self._disks.values(): + print(f"Detaching {disk}") + run_as_root("losetup", "-d", disk) def tmpfs(self, size="8G"): """Mount a tmpfs""" print(f"Mounting tmpfs on {self.path}") mount("tmpfs", self.path, "tmpfs", f"size={size}") - self._type = TmpType.TMPFS + self._type = TargetType.TMPFS # pylint: disable=too-many-arguments def add_disk(self, diff --git a/rootfs.py b/rootfs.py index 18834ab..a08d27d 100755 --- a/rootfs.py +++ b/rootfs.py @@ -17,7 +17,7 @@ import argparse import os from lib.utils import run, run_as_root -from lib.tmpdir import Tmpdir +from lib.target import Target from lib.generator import Generator, stage0_arch_map def create_configuration_file(args): @@ -58,11 +58,9 @@ def main(): action="store_true") parser.add_argument("-bw", "--bwrap", help="Run inside a bwrap sandbox", action="store_true") - parser.add_argument("-p", "--preserve", help="Do not remove temporary dir", - action="store_true") - parser.add_argument("-t", "--tmpdir", help="Temporary directory", - default="tmp") - parser.add_argument("--tmpfs", help="Use a tmpfs on tmpdir", + parser.add_argument("-t", "--target", help="Target directory", + default="target") + parser.add_argument("--tmpfs", help="Use a tmpfs on target", action="store_true") parser.add_argument("--tmpfs-size", help="Size of the tmpfs", default="8G") @@ -128,7 +126,7 @@ def main(): if args.arch != "x86": print("Only x86 is supported at the moment, other arches are for development only.") - # Tmp validation + # Tmpfs validation if args.bwrap and args.tmpfs: raise ValueError("tmpfs cannot be used with bwrap.") @@ -156,19 +154,19 @@ def main(): with open(os.path.join('steps', 'bootstrap.cfg'), 'a', encoding='UTF-8'): pass - # tmpdir - tmpdir = Tmpdir(path=args.tmpdir, preserve=args.preserve) + # target + target = Target(path=args.target) if args.tmpfs: - tmpdir.tmpfs(size=args.tmpfs_size) + target.tmpfs(size=args.tmpfs_size) generator = Generator(arch=args.arch, external_sources=args.external_sources, repo_path=args.repo, early_preseed=args.early_preseed) - bootstrap(args, generator, tmpdir, args.target_size) + bootstrap(args, generator, target, args.target_size) -def bootstrap(args, generator, tmpdir, size): +def bootstrap(args, generator, target, size): """Kick off bootstrap process.""" print(f"Bootstrapping {args.arch}") if args.chroot: @@ -179,28 +177,28 @@ print(shutil.which('chroot')) chroot_binary = run_as_root('python3', '-c', find_chroot, capture_output=True).stdout.decode().strip() - generator.prepare(tmpdir, using_kernel=False) + generator.prepare(target, using_kernel=False) arch = stage0_arch_map.get(args.arch, args.arch) init = os.path.join(os.sep, 'bootstrap-seeds', 'POSIX', arch, 'kaem-optional-seed') - run_as_root('env', '-i', 'PATH=/bin', chroot_binary, generator.tmp_dir, init) + run_as_root('env', '-i', 'PATH=/bin', chroot_binary, generator.target_dir, init) elif args.bwrap: init = '/init' if not args.internal_ci or args.internal_ci == "pass1": - generator.prepare(tmpdir, using_kernel=False) + generator.prepare(target, using_kernel=False) arch = stage0_arch_map.get(args.arch, args.arch) init = os.path.join(os.sep, 'bootstrap-seeds', 'POSIX', arch, 'kaem-optional-seed') else: - generator.reuse(tmpdir) + generator.reuse(target) run('env', '-i', 'bwrap', '--unshare-user', '--uid', '0', '--gid', '0', '--unshare-net' if args.external_sources else None, '--setenv', 'PATH', '/usr/bin', - '--bind', generator.tmp_dir, '/', + '--bind', generator.target_dir, '/', '--dir', '/dev', '--dev-bind', '/dev/null', '/dev/null', '--dev-bind', '/dev/zero', '/dev/zero', @@ -216,44 +214,44 @@ print(shutil.which('chroot')) elif args.bare_metal: if args.kernel: - generator.prepare(tmpdir, using_kernel=True, target_size=size) - image_path = os.path.join(args.tmpdir, os.path.relpath(generator.tmp_dir, args.tmpdir)) + generator.prepare(target, using_kernel=True, target_size=size) + path = os.path.join(args.target, os.path.relpath(generator.target_dir, args.target)) print("Please:") - print(f" 1. Take {image_path}/initramfs and your kernel, boot using this.") - print(f" 2. Take {image_path}/disk.img and put this on a writable storage medium.") + print(f" 1. Take {path}/initramfs and your kernel, boot using this.") + print(f" 2. Take {path}/disk.img and put this on a writable storage medium.") else: - generator.prepare(tmpdir, kernel_bootstrap=True, target_size=size) - image_path = os.path.join(args.tmpdir, os.path.relpath(generator.tmp_dir, args.tmpdir)) + generator.prepare(target, kernel_bootstrap=True, target_size=size) + path = os.path.join(args.target, os.path.relpath(generator.target_dir, args.target)) print("Please:") - print(f" 1. Take {image_path}.img and write it to a boot drive and then boot it.") + print(f" 1. Take {path}.img and write it to a boot drive and then boot it.") else: if args.kernel: - generator.prepare(tmpdir, using_kernel=True, target_size=size) + generator.prepare(target, using_kernel=True, target_size=size) run(args.qemu_cmd, '-enable-kvm', '-m', str(args.qemu_ram) + 'M', '-smp', str(args.cores), '-no-reboot', - '-drive', 'file=' + tmpdir.get_disk("disk") + ',format=raw', - '-drive', 'file=' + tmpdir.get_disk("external") + ',format=raw', + '-drive', 'file=' + target.get_disk("disk") + ',format=raw', + '-drive', 'file=' + target.get_disk("external") + ',format=raw', '-nic', 'user,ipv6=off,model=e1000', '-kernel', args.kernel, '-nographic', '-append', 'console=ttyS0 root=/dev/sda1 rootfstype=ext3 init=/init rw') else: - generator.prepare(tmpdir, kernel_bootstrap=True, target_size=size) + generator.prepare(target, kernel_bootstrap=True, target_size=size) arg_list = [ '-enable-kvm', '-m', str(args.qemu_ram) + 'M', '-smp', str(args.cores), '-no-reboot', - '-drive', 'file=' + generator.tmp_dir + '.img' + ',format=raw' + '-drive', 'file=' + generator.target_dir + '.img' + ',format=raw' ] - if tmpdir.get_disk("external") is not None: + if target.get_disk("external") is not None: arg_list += [ - '-drive', 'file=' + tmpdir.get_disk("external") + ',format=raw', + '-drive', 'file=' + target.get_disk("external") + ',format=raw', ] arg_list += [ '-machine', 'kernel-irqchip=split',