diff --git a/.cirrus.yml b/.cirrus.yml index 67b85e6..2c748f4 100644 --- a/.cirrus.yml +++ b/.cirrus.yml @@ -13,7 +13,7 @@ pylint_task: - apt-get -y clean - pip3 install pylint check_script: - - pylint rootfs.py sysa.py sysb.py sysc.py lib/utils.py lib/sysgeneral.py --disable=duplicate-code + - pylint rootfs.py sysa.py sysc.py lib/utils.py lib/sysgeneral.py lib/tmpdir.py --disable=duplicate-code shell_lint_task: container: diff --git a/lib/sysgeneral.py b/lib/sysgeneral.py index 5bc13c4..7a41e1c 100644 --- a/lib/sysgeneral.py +++ b/lib/sysgeneral.py @@ -4,20 +4,17 @@ This file contains a few functions to be shared by all Sys* classes """ # SPDX-FileCopyrightText: 2022-2023 Dor Askayo -# SPDX-FileCopyrightText: 2021-22 fosslinux +# SPDX-FileCopyrightText: 2021-23 fosslinux # SPDX-FileCopyrightText: 2021 Andrius Štikonas # SPDX-License-Identifier: GPL-3.0-or-later import os -import shutil import hashlib import glob import subprocess import requests -from lib.utils import mount, umount - class SysGeneral: """ A class from which all Sys* class are extended. @@ -25,38 +22,12 @@ class SysGeneral: """ # All of these are variables defined in the individual Sys* classes - preserve_tmp = None - tmp_dir = None cache_dir = None base_dir = None git_dir = None sys_dir = None initramfs_path = None - mounted_tmpfs = False - - def __del__(self): - if not self.preserve_tmp: - self.remove_tmp() - - def remove_tmp(self): - """Remove the tmp directory""" - if self.tmp_dir is None: - return - - if self.mounted_tmpfs: - print(f"Unmounting tmpfs from {self.tmp_dir}") - umount(self.tmp_dir) - - print(f"Removing {self.tmp_dir}") - shutil.rmtree(self.tmp_dir, ignore_errors=True) - - def mount_tmpfs(self): - """Mount the tmpfs for this sysx""" - if not os.path.isdir(self.tmp_dir): - os.mkdir(self.tmp_dir) - print(f"Mounting tmpfs on {self.tmp_dir}") - mount('tmpfs', self.tmp_dir, 'tmpfs', 'size=8G') - self.mounted_tmpfs = True + tmp_dir = None def check_file(self, file_name, expected_hash): """Check hash of downloaded source file.""" diff --git a/lib/tmpdir.py b/lib/tmpdir.py new file mode 100644 index 0000000..50179ef --- /dev/null +++ b/lib/tmpdir.py @@ -0,0 +1,99 @@ +#!/usr/bin/env python3 + +# SPDX-FileCopyrightText: 2023 fosslinux +# SPDX-License-Identifier: GPL-3.0-or-later + +""" +Contains a class that represents a tmpdir +""" + +import enum +import getpass +import os +import shutil + +from lib.utils import mount, umount, create_disk, run + +class TmpType(enum.Enum): + """Different types of tmpdirs we can have""" + NONE = 0 + TMPFS = 1 + +class Tmpdir: + """ + Represents a tmpdir + """ + + _syses = {} + _disks = {} + _disk_filesystems = {} + _mountpoints = {} + + def __init__(self, preserve, path="tmp"): + self.path = os.path.abspath(path) + self.preserve = preserve + self._type = TmpType.NONE + + if not os.path.exists(self.path): + os.mkdir(self.path) + + def __del__(self): + for path in self._mountpoints: + print(f"Unmounting {path}") + umount(path) + + if not self.preserve: + for disk in self._disks.values(): + print(f"Detaching {disk}") + run("sudo", "losetup", "-d", disk) + + if self._type == TmpType.TMPFS: + print(f"Unmounting tmpdir from {self.path}") + umount(self.path) + + print(f"Removing {self.path}") + shutil.rmtree(self.path, ignore_errors=True) + + def tmpfs(self, size="8G"): + """Mount a tmpfs""" + print(f"Mounting tmpfs on {self.path}") + mount("tmpfs", self.path, "tmpfs", f"size={size}") + self._type = TmpType.TMPFS + + def add_sys(self, name, subdir=None): + """Create a subdirectory and register a sys""" + if subdir is None: + subdir = name + sys_path = os.path.join(self.path, name) + if not os.path.exists(sys_path): + os.mkdir(sys_path) + return sys_path + + def add_disk(self, name, size="8G", filesystem="ext4"): + """Add a disk""" + disk_path = os.path.join(self.path, f"{name}.img") + self._disks[name] = create_disk(disk_path, "msdos", filesystem, size) + self._disk_filesystems[name] = filesystem + # Allow executing user to access it + run("sudo", "chown", getpass.getuser(), self._disks[name]) + + def mount_disk(self, name, mountpoint=None): + """Mount the disk""" + if mountpoint is None: + mountpoint = f"{name}_mnt" + mountpoint = os.path.join(self.path, mountpoint) + os.mkdir(mountpoint) + mount(self._disks[name] + "p1", mountpoint, self._disk_filesystems[name]) + # Allow executing user to access it + run("sudo", "chown", getpass.getuser(), mountpoint) + self._mountpoints[name] = mountpoint + return mountpoint + + def umount_disk(self, name): + """Unmount a disk""" + umount(self._mountpoints[name]) + del self._mountpoints[name] + + def get_disk(self, name): + """Get the path to a device of a disk""" + return self._disks[name] diff --git a/lib/utils.py b/lib/utils.py index 8b3733d..12ba2cb 100755 --- a/lib/utils.py +++ b/lib/utils.py @@ -5,7 +5,7 @@ This file contains a few self-contained helper functions # SPDX-License-Identifier: GPL-3.0-or-later # SPDX-FileCopyrightText: 2021 Andrius Štikonas -# SPDX-FileCopyrightText: 2021-22 fosslinux +# SPDX-FileCopyrightText: 2021-23 fosslinux import os import shutil diff --git a/rootfs.py b/rootfs.py index 82729fd..f174fab 100755 --- a/rootfs.py +++ b/rootfs.py @@ -11,17 +11,16 @@ you can run bootstap inside chroot. # SPDX-FileCopyrightText: 2021 Andrius Štikonas # SPDX-FileCopyrightText: 2021 Bastian Bittorf # SPDX-FileCopyrightText: 2021 Melg Eight -# SPDX-FileCopyrightText: 2021-22 fosslinux +# SPDX-FileCopyrightText: 2021-23 fosslinux import argparse import os -import shutil from sysa import SysA -from sysb import SysB from sysc import SysC from lib.utils import run from lib.sysgeneral import stage0_arch_map +from lib.tmpdir import Tmpdir def create_configuration_file(args): """ @@ -54,7 +53,12 @@ def main(): action="store_true") parser.add_argument("-p", "--preserve", help="Do not remove temporary dir", action="store_true") - parser.add_argument("-t", "--tmpdir", help="Temporary directory") + parser.add_argument("-t", "--tmpdir", help="Temporary directory", + default="tmp") + parser.add_argument("--tmpfs", help="Use a tmpfs on tmpdir", + action="store_true") + parser.add_argument("--tmpfs-size", help="Size of the tmpfs", + default="8G") parser.add_argument("--force-timestamps", help="Force all files timestamps to be 0 unix time", action="store_true") @@ -82,13 +86,12 @@ def main(): parser.add_argument("-qk", "--kernel", help="Kernel to use (default is ./kernel)", default="kernel") - parser.add_argument("-m", "--minikernel", help="Use minikernel", - action="store_true") parser.add_argument("-b", "--bare-metal", help="Build images for bare metal", action="store_true") args = parser.parse_args() + # Mode validation def check_types(): count = 0 if args.qemu: @@ -97,24 +100,28 @@ def main(): count += 1 if args.bwrap: count += 1 - if args.minikernel: - count += 1 if args.bare_metal: count += 1 return count if check_types() > 1: - raise ValueError("No more than one of qemu, chroot, bwrap, minikernel, bare metal " + raise ValueError("No more than one of qemu, chroot, bwrap, bare metal" "may be used.") if check_types() == 0: - raise ValueError("One of qemu, chroot, bwrap, minikernel or bare metal must be selected.") - - if args.bare_metal: - args.no_create_config = True + raise ValueError("One of qemu, chroot, bwrap, or bare metal must be selected.") + # Arch validation if args.arch != "x86": raise ValueError("Only x86 is supported at the moment.") + # Tmp validation + if args.bwrap and args.tmpfs: + raise ValueError("tmpfs cannot be used writh bwrap.") + + # bootstrap.cfg + if args.bare_metal: + args.no_create_config = True + try: os.remove(os.path.join('sysa', 'bootstrap.cfg')) except FileNotFoundError: @@ -125,20 +132,21 @@ def main(): with open(os.path.join('sysa', 'bootstrap.cfg'), 'a', encoding='UTF-8'): pass - system_c = SysC(arch=args.arch, preserve_tmp=args.preserve, - tmpdir=args.tmpdir, external_sources=args.external_sources) - system_b = SysB(arch=args.arch, preserve_tmp=args.preserve) - system_a = SysA(arch=args.arch, preserve_tmp=args.preserve, - early_preseed=args.early_preseed, tmpdir=args.tmpdir, - external_sources=args.external_sources, - sysb_dir=system_b.sys_dir, sysc_dir=system_c.sys_dir) + # tmpdir + tmpdir = Tmpdir(path=args.tmpdir, preserve=args.preserve) + if args.tmpfs: + tmpdir.tmpfs(size=args.tmpfs_size) - if args.tmpdir is not None: - os.makedirs(args.tmpdir, exist_ok=True) + # sys + system_c = SysC(arch=args.arch, tmpdir=tmpdir, + external_sources=args.external_sources) + system_a = SysA(arch=args.arch, early_preseed=args.early_preseed, + tmpdir=tmpdir, external_sources=args.external_sources, + repo_path=args.repo) - bootstrap(args, system_a, system_b, system_c) + bootstrap(args, system_a, system_c, tmpdir) -def bootstrap(args, system_a, system_b, system_c): +def bootstrap(args, system_a, system_c, tmpdir): """Kick off bootstrap process.""" print(f"Bootstrapping {args.arch} -- SysA") if args.chroot: @@ -149,25 +157,17 @@ print(shutil.which('chroot')) chroot_binary = run('sudo', 'python3', '-c', find_chroot, capture_output=True).stdout.decode().strip() - system_c.prepare(mount_tmpfs=True, - create_disk_image=False) - system_a.prepare(mount_tmpfs=True, - create_initramfs=False, - repo_path=args.repo) + system_c.prepare(create_disk_image=False) + system_a.prepare(create_initramfs=False) - # sysa arch = stage0_arch_map.get(args.arch, args.arch) init = os.path.join(os.sep, 'bootstrap-seeds', 'POSIX', arch, 'kaem-optional-seed') run('sudo', 'env', '-i', 'PATH=/bin', chroot_binary, system_a.tmp_dir, init) elif args.bwrap: - system_c.prepare(mount_tmpfs=False, - create_disk_image=False) - system_a.prepare(mount_tmpfs=False, - create_initramfs=False, - repo_path=args.repo) + system_c.prepare(create_disk_image=False) + system_a.prepare(create_initramfs=False) - # sysa arch = stage0_arch_map.get(args.arch, args.arch) init = os.path.join(os.sep, 'bootstrap-seeds', 'POSIX', arch, 'kaem-optional-seed') run('bwrap', '--unshare-user', @@ -192,55 +192,23 @@ print(shutil.which('chroot')) '--tmpfs', '/sysc_image/tmp', init) - elif args.minikernel: - if os.path.isdir('kritis-linux'): - shutil.rmtree('kritis-linux') - - system_c.prepare(mount_tmpfs=True, - create_disk_image=True) - system_a.prepare(mount_tmpfs=True, - create_initramfs=True, - repo_path=args.repo) - - run('git', 'clone', - '--depth', '1', '--branch', 'v0.7', - 'https://github.com/bittorf/kritis-linux.git') - run('kritis-linux/ci_helper.sh', - '--private', - '--multi', '1', - '--repeat', '1', - '--arch', args.arch, - '--qemucpu', '486', - '--kernel', '3.18.140', - '--features', 'kflock,highrestimers', - # Hack to add -hda /dev/blah - '--ramsize', str(args.qemu_ram) + 'M -hda ' + system_b.dev_name, - '--initrd', system_a.initramfs_path, - '--log', '/tmp/bootstrap.log') - elif args.bare_metal: - system_c.prepare(mount_tmpfs=True, - create_disk_image=True) - system_a.prepare(mount_tmpfs=True, - create_initramfs=True, - repo_path=args.repo) + system_c.prepare(create_disk_image=True) + system_a.prepare(create_initramfs=True) print("Please:") - print(" 1. Take sysa/tmp/initramfs and your kernel, boot using this.") - print(" 2. Take sysc/tmp/disk.img and put this on a writable storage medium.") + print(" 1. Take tmp/sysa/initramfs and your kernel, boot using this.") + print(" 2. Take tmp/sysc/disk.img and put this on a writable storage medium.") else: - system_c.prepare(mount_tmpfs=True, - create_disk_image=True) - system_a.prepare(mount_tmpfs=True, - create_initramfs=True, - repo_path=args.repo) + system_c.prepare(create_disk_image=True) + system_a.prepare(create_initramfs=True) run(args.qemu_cmd, '-enable-kvm', '-m', str(args.qemu_ram) + 'M', '-no-reboot', - '-hda', system_c.dev_name, + '-hda', tmpdir.get_disk("sysc"), '-nic', 'user,ipv6=off,model=e1000', '-kernel', args.kernel, '-initrd', system_a.initramfs_path, diff --git a/sysa.py b/sysa.py index dd709c9..9fc1efd 100755 --- a/sysa.py +++ b/sysa.py @@ -4,7 +4,7 @@ # SPDX-FileCopyrightText: 2022-2023 Dor Askayo # SPDX-FileCopyrightText: 2021 Andrius Štikonas # SPDX-FileCopyrightText: 2021 Melg Eight -# SPDX-FileCopyrightText: 2021-22 fosslinux +# SPDX-FileCopyrightText: 2021-23 fosslinux import os from distutils.dir_util import copy_tree @@ -22,37 +22,26 @@ class SysA(SysGeneral): git_dir = os.path.dirname(os.path.join(__file__)) sys_dir = os.path.join(git_dir, 'sysa') + sysb_dir = os.path.join(git_dir, 'sysb') + sysc_dir = os.path.join(git_dir, 'sysc') cache_dir = os.path.join(sys_dir, 'distfiles') # pylint: disable=too-many-arguments - def __init__(self, arch, preserve_tmp, external_sources, - early_preseed, tmpdir, sysb_dir, sysc_dir): + def __init__(self, tmpdir, arch, external_sources, + early_preseed, repo_path): self.arch = arch - self.preserve_tmp = preserve_tmp self.early_preseed = early_preseed - - if tmpdir is None: - self.tmp_dir = os.path.join(self.git_dir, 'tmp') - else: - self.tmp_dir = os.path.join(tmpdir, 'sysa') - self.sysa_dir = os.path.join(self.tmp_dir, 'sysa') - self.base_dir = self.sysa_dir - - self.sysb_dir = sysb_dir - self.sysc_dir = sysc_dir self.external_sources = external_sources + self.repo_path = repo_path - def prepare(self, mount_tmpfs, create_initramfs, repo_path): + self.tmp_dir = tmpdir.add_sys("sysa") + + def prepare(self, create_initramfs): """ Prepare directory structure for System A. We create an empty tmp directory, unpack stage0-posix. Rest of the files are unpacked into more structured directory /sysa """ - if mount_tmpfs: - self.mount_tmpfs() - else: - os.mkdir(self.tmp_dir) - if self.early_preseed: # Extract tar containing preseed with tarfile.open(self.early_preseed, "r") as seed: @@ -69,9 +58,9 @@ class SysA(SysGeneral): self.sysc(create_initramfs) - if repo_path: + if self.repo_path: repo_dir = os.path.join(self.tmp_dir, 'usr', 'src', 'repo-preseeded') - shutil.copytree(repo_path, repo_dir) + shutil.copytree(self.repo_path, repo_dir) if create_initramfs: self.make_initramfs() diff --git a/sysb.py b/sysb.py deleted file mode 100755 index d4752f4..0000000 --- a/sysb.py +++ /dev/null @@ -1,21 +0,0 @@ -#!/usr/bin/env python3 -"""System B""" -# SPDX-License-Identifier: GPL-3.0-or-later -# SPDX-FileCopyrightText: 2021 Andrius Štikonas -# SPDX-FileCopyrightText: 2021-22 fosslinux - -import os - -from lib.sysgeneral import SysGeneral - -class SysB(SysGeneral): - """ - Class responsible for preparing sources for System B. - """ - - git_dir = os.path.dirname(os.path.join(__file__)) - sys_dir = os.path.join(git_dir, 'sysb') - - def __init__(self, arch, preserve_tmp): - self.arch = arch - self.preserve_tmp = preserve_tmp diff --git a/sysc.py b/sysc.py index 0851f25..e587668 100755 --- a/sysc.py +++ b/sysc.py @@ -2,13 +2,12 @@ """System C""" # SPDX-License-Identifier: GPL-3.0-or-later # SPDX-FileCopyrightText: 2022-2023 Dor Askayo -# SPDX-FileCopyrightText: 2021-22 fosslinux +# SPDX-FileCopyrightText: 2021-23 fosslinux # SPDX-FileCopyrightText: 2021 Andrius Štikonas import os -import getpass -from lib.utils import mount, umount, create_disk, run, copytree +from lib.utils import copytree from lib.sysgeneral import SysGeneral # pylint: disable=consider-using-with @@ -21,60 +20,30 @@ class SysC(SysGeneral): git_dir = os.path.dirname(os.path.join(__file__)) sys_dir = os.path.join(git_dir, 'sysc') cache_dir = os.path.join(sys_dir, 'distfiles') - dev_name = None - def __init__(self, arch, preserve_tmp, tmpdir, external_sources): + def __init__(self, tmpdir, arch, external_sources): self.arch = arch - self.preserve_tmp = preserve_tmp self.external_sources = external_sources + self._tmpdir = tmpdir - if tmpdir is None: - self.tmp_dir = os.path.join(self.sys_dir, 'tmp') - else: - self.tmp_dir = os.path.join(tmpdir, 'sysc') + self.tmp_dir = tmpdir.add_sys("sysc") - def __del__(self): - if not self.preserve_tmp: - if self.dev_name is not None: - print(f"Detaching {self.dev_name}") - run('sudo', 'losetup', '-d', self.dev_name) - - super().__del__() - - def prepare(self, mount_tmpfs, create_disk_image): + def prepare(self, create_disk_image): """ Prepare directory structure for System C. """ - if mount_tmpfs: - self.mount_tmpfs() - else: - os.mkdir(self.tmp_dir) - - rootfs_dir = None - if create_disk_image: - # Create + mount a disk for QEMU to use - disk_path = os.path.join(self.tmp_dir, 'disk.img') - if self.external_sources: - self.dev_name = create_disk(disk_path, "msdos", "ext4", '8G') - rootfs_dir = os.path.join(self.tmp_dir, 'mnt') - os.mkdir(rootfs_dir) - mount(self.dev_name + "p1", rootfs_dir, 'ext4') - else: - self.dev_name = create_disk(disk_path, "none", "ext4", '8G') - # Use chown to allow executing user to access it - run('sudo', 'chown', getpass.getuser(), self.dev_name) - if self.external_sources: - run('sudo', 'chown', getpass.getuser(), rootfs_dir) - else: - rootfs_dir = self.tmp_dir + self._tmpdir.add_disk("sysc") if self.external_sources: + if create_disk_image: + rootfs_dir = self._tmpdir.mount_disk("sysc") + else: + rootfs_dir = self.tmp_dir source_manifest = self.get_source_manifest() self.get_packages(source_manifest) copytree(self.cache_dir, os.path.join(rootfs_dir, "distfiles")) - # Unmount tmp/mnt if it was mounted - if create_disk_image and self.external_sources: - umount(rootfs_dir) + if create_disk_image: + self._tmpdir.umount_disk("sysc")