Rework tmpdir & associated a bit.

- Split out tmpdir logic into a separate entity & add the appropriate
  arguments and checks.
- sysb can be removed since there is now no associated logic.
- Move disk/etc logic into tmpdir.py.

.cirrus.yml

@@ -13,7 +13,7 @@ pylint_task:
     - apt-get -y clean
     - pip3 install pylint
   check_script:
-    - pylint rootfs.py sysa.py sysb.py sysc.py lib/utils.py lib/sysgeneral.py --disable=duplicate-code
+    - pylint rootfs.py sysa.py sysc.py lib/utils.py lib/sysgeneral.py lib/tmpdir.py --disable=duplicate-code
 
 shell_lint_task:
   container:

lib/sysgeneral.py

@@ -4,20 +4,17 @@ This file contains a few functions to be shared by all Sys* classes
 """
 
 # SPDX-FileCopyrightText: 2022-2023 Dor Askayo <dor.askayo@gmail.com>
-# SPDX-FileCopyrightText: 2021-22 fosslinux <fosslinux@aussies.space>
+# SPDX-FileCopyrightText: 2021-23 fosslinux <fosslinux@aussies.space>
 # SPDX-FileCopyrightText: 2021 Andrius Štikonas <andrius@stikonas.eu>
 # SPDX-License-Identifier: GPL-3.0-or-later
 
 import os
-import shutil
 import hashlib
 import glob
 import subprocess
 
 import requests
 
-from lib.utils import mount, umount
-
 class SysGeneral:
     """
     A class from which all Sys* class are extended.
@@ -25,38 +22,12 @@ class SysGeneral:
     """
 
     # All of these are variables defined in the individual Sys* classes
-    preserve_tmp = None
-    tmp_dir = None
     cache_dir = None
     base_dir = None
     git_dir = None
     sys_dir = None
     initramfs_path = None
-    mounted_tmpfs = False
+    tmp_dir = None
-
-    def __del__(self):
-        if not self.preserve_tmp:
-            self.remove_tmp()
-
-    def remove_tmp(self):
-        """Remove the tmp directory"""
-        if self.tmp_dir is None:
-            return
-
-        if self.mounted_tmpfs:
-            print(f"Unmounting tmpfs from {self.tmp_dir}")
-            umount(self.tmp_dir)
-
-        print(f"Removing {self.tmp_dir}")
-        shutil.rmtree(self.tmp_dir, ignore_errors=True)
-
-    def mount_tmpfs(self):
-        """Mount the tmpfs for this sysx"""
-        if not os.path.isdir(self.tmp_dir):
-            os.mkdir(self.tmp_dir)
-        print(f"Mounting tmpfs on {self.tmp_dir}")
-        mount('tmpfs', self.tmp_dir, 'tmpfs', 'size=8G')
-        self.mounted_tmpfs = True
 
     def check_file(self, file_name, expected_hash):
         """Check hash of downloaded source file."""

lib/tmpdir.py

@@ -0,0 +1,99 @@
+#!/usr/bin/env python3
+
+# SPDX-FileCopyrightText: 2023 fosslinux <fosslinux@aussies.space>
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+"""
+Contains a class that represents a tmpdir
+"""
+
+import enum
+import getpass
+import os
+import shutil
+
+from lib.utils import mount, umount, create_disk, run
+
+class TmpType(enum.Enum):
+    """Different types of tmpdirs we can have"""
+    NONE = 0
+    TMPFS = 1
+
+class Tmpdir:
+    """
+    Represents a tmpdir
+    """
+
+    _syses = {}
+    _disks = {}
+    _disk_filesystems = {}
+    _mountpoints = {}
+
+    def __init__(self, preserve, path="tmp"):
+        self.path = os.path.abspath(path)
+        self.preserve = preserve
+        self._type = TmpType.NONE
+
+        if not os.path.exists(self.path):
+            os.mkdir(self.path)
+
+    def __del__(self):
+        for path in self._mountpoints:
+            print(f"Unmounting {path}")
+            umount(path)
+
+        if not self.preserve:
+            for disk in self._disks.values():
+                print(f"Detaching {disk}")
+                run("sudo", "losetup", "-d", disk)
+
+            if self._type == TmpType.TMPFS:
+                print(f"Unmounting tmpdir from {self.path}")
+                umount(self.path)
+
+            print(f"Removing {self.path}")
+            shutil.rmtree(self.path, ignore_errors=True)
+
+    def tmpfs(self, size="8G"):
+        """Mount a tmpfs"""
+        print(f"Mounting tmpfs on {self.path}")
+        mount("tmpfs", self.path, "tmpfs", f"size={size}")
+        self._type = TmpType.TMPFS
+
+    def add_sys(self, name, subdir=None):
+        """Create a subdirectory and register a sys"""
+        if subdir is None:
+            subdir = name
+        sys_path = os.path.join(self.path, name)
+        if not os.path.exists(sys_path):
+            os.mkdir(sys_path)
+        return sys_path
+
+    def add_disk(self, name, size="8G", filesystem="ext4"):
+        """Add a disk"""
+        disk_path = os.path.join(self.path, f"{name}.img")
+        self._disks[name] = create_disk(disk_path, "msdos", filesystem, size)
+        self._disk_filesystems[name] = filesystem
+        # Allow executing user to access it
+        run("sudo", "chown", getpass.getuser(), self._disks[name])
+
+    def mount_disk(self, name, mountpoint=None):
+        """Mount the disk"""
+        if mountpoint is None:
+            mountpoint = f"{name}_mnt"
+        mountpoint = os.path.join(self.path, mountpoint)
+        os.mkdir(mountpoint)
+        mount(self._disks[name] + "p1", mountpoint, self._disk_filesystems[name])
+        # Allow executing user to access it
+        run("sudo", "chown", getpass.getuser(), mountpoint)
+        self._mountpoints[name] = mountpoint
+        return mountpoint
+
+    def umount_disk(self, name):
+        """Unmount a disk"""
+        umount(self._mountpoints[name])
+        del self._mountpoints[name]
+
+    def get_disk(self, name):
+        """Get the path to a device of a disk"""
+        return self._disks[name]

lib/utils.py

@@ -5,7 +5,7 @@ This file contains a few self-contained helper functions
 
 # SPDX-License-Identifier: GPL-3.0-or-later
 # SPDX-FileCopyrightText: 2021 Andrius Štikonas <andrius@stikonas.eu>
-# SPDX-FileCopyrightText: 2021-22 fosslinux <fosslinux@aussies.space>
+# SPDX-FileCopyrightText: 2021-23 fosslinux <fosslinux@aussies.space>
 
 import os
 import shutil

rootfs.py

@@ -11,17 +11,16 @@ you can run bootstap inside chroot.
 # SPDX-FileCopyrightText: 2021 Andrius Štikonas <andrius@stikonas.eu>
 # SPDX-FileCopyrightText: 2021 Bastian Bittorf <bb@npl.de>
 # SPDX-FileCopyrightText: 2021 Melg Eight <public.melg8@gmail.com>
-# SPDX-FileCopyrightText: 2021-22 fosslinux <fosslinux@aussies.space>
+# SPDX-FileCopyrightText: 2021-23 fosslinux <fosslinux@aussies.space>
 
 import argparse
 import os
-import shutil
 
 from sysa import SysA
-from sysb import SysB
 from sysc import SysC
 from lib.utils import run
 from lib.sysgeneral import stage0_arch_map
+from lib.tmpdir import Tmpdir
 
 def create_configuration_file(args):
     """
@@ -54,7 +53,12 @@ def main():
                         action="store_true")
     parser.add_argument("-p", "--preserve", help="Do not remove temporary dir",
                         action="store_true")
-    parser.add_argument("-t", "--tmpdir", help="Temporary directory")
+    parser.add_argument("-t", "--tmpdir", help="Temporary directory",
+                        default="tmp")
+    parser.add_argument("--tmpfs", help="Use a tmpfs on tmpdir",
+                        action="store_true")
+    parser.add_argument("--tmpfs-size", help="Size of the tmpfs",
+                        default="8G")
     parser.add_argument("--force-timestamps",
                         help="Force all files timestamps to be 0 unix time",
                         action="store_true")
@@ -82,13 +86,12 @@ def main():
     parser.add_argument("-qk", "--kernel", help="Kernel to use (default is ./kernel)",
                         default="kernel")
 
-    parser.add_argument("-m", "--minikernel", help="Use minikernel",
-                        action="store_true")
     parser.add_argument("-b", "--bare-metal", help="Build images for bare metal",
                         action="store_true")
 
     args = parser.parse_args()
 
+    # Mode validation
     def check_types():
         count = 0
         if args.qemu:
@@ -97,24 +100,28 @@ def main():
             count += 1
         if args.bwrap:
             count += 1
-        if args.minikernel:
-            count += 1
         if args.bare_metal:
             count += 1
         return count
 
     if check_types() > 1:
-        raise ValueError("No more than one of qemu, chroot, bwrap, minikernel, bare metal "
+        raise ValueError("No more than one of qemu, chroot, bwrap, bare metal"
                          "may be used.")
     if check_types() == 0:
-        raise ValueError("One of qemu, chroot, bwrap, minikernel or bare metal must be selected.")
+        raise ValueError("One of qemu, chroot, bwrap, or bare metal must be selected.")
-
-    if args.bare_metal:
-        args.no_create_config = True
 
+    # Arch validation
     if args.arch != "x86":
         raise ValueError("Only x86 is supported at the moment.")
 
+    # Tmp validation
+    if args.bwrap and args.tmpfs:
+        raise ValueError("tmpfs cannot be used writh bwrap.")
+
+    # bootstrap.cfg
+    if args.bare_metal:
+        args.no_create_config = True
+
     try:
         os.remove(os.path.join('sysa', 'bootstrap.cfg'))
     except FileNotFoundError:
@@ -125,20 +132,21 @@ def main():
         with open(os.path.join('sysa', 'bootstrap.cfg'), 'a', encoding='UTF-8'):
             pass
 
-    system_c = SysC(arch=args.arch, preserve_tmp=args.preserve,
+    # tmpdir
-                    tmpdir=args.tmpdir, external_sources=args.external_sources)
+    tmpdir = Tmpdir(path=args.tmpdir, preserve=args.preserve)
-    system_b = SysB(arch=args.arch, preserve_tmp=args.preserve)
+    if args.tmpfs:
-    system_a = SysA(arch=args.arch, preserve_tmp=args.preserve,
+        tmpdir.tmpfs(size=args.tmpfs_size)
-                    early_preseed=args.early_preseed, tmpdir=args.tmpdir,
-                    external_sources=args.external_sources,
-                    sysb_dir=system_b.sys_dir, sysc_dir=system_c.sys_dir)
 
-    if args.tmpdir is not None:
+    # sys
-        os.makedirs(args.tmpdir, exist_ok=True)
+    system_c = SysC(arch=args.arch, tmpdir=tmpdir,
+                    external_sources=args.external_sources)
+    system_a = SysA(arch=args.arch, early_preseed=args.early_preseed,
+                    tmpdir=tmpdir, external_sources=args.external_sources,
+                    repo_path=args.repo)
 
-    bootstrap(args, system_a, system_b, system_c)
+    bootstrap(args, system_a, system_c, tmpdir)
 
-def bootstrap(args, system_a, system_b, system_c):
+def bootstrap(args, system_a, system_c, tmpdir):
     """Kick off bootstrap process."""
     print(f"Bootstrapping {args.arch} -- SysA")
     if args.chroot:
@@ -149,25 +157,17 @@ print(shutil.which('chroot'))
         chroot_binary = run('sudo', 'python3', '-c', find_chroot,
                             capture_output=True).stdout.decode().strip()
 
-        system_c.prepare(mount_tmpfs=True,
+        system_c.prepare(create_disk_image=False)
-                         create_disk_image=False)
+        system_a.prepare(create_initramfs=False)
-        system_a.prepare(mount_tmpfs=True,
-                         create_initramfs=False,
-                         repo_path=args.repo)
 
-        # sysa
         arch = stage0_arch_map.get(args.arch, args.arch)
         init = os.path.join(os.sep, 'bootstrap-seeds', 'POSIX', arch, 'kaem-optional-seed')
         run('sudo', 'env', '-i', 'PATH=/bin', chroot_binary, system_a.tmp_dir, init)
 
     elif args.bwrap:
-        system_c.prepare(mount_tmpfs=False,
+        system_c.prepare(create_disk_image=False)
-                         create_disk_image=False)
+        system_a.prepare(create_initramfs=False)
-        system_a.prepare(mount_tmpfs=False,
-                         create_initramfs=False,
-                         repo_path=args.repo)
 
-        # sysa
         arch = stage0_arch_map.get(args.arch, args.arch)
         init = os.path.join(os.sep, 'bootstrap-seeds', 'POSIX', arch, 'kaem-optional-seed')
         run('bwrap', '--unshare-user',
@@ -192,55 +192,23 @@ print(shutil.which('chroot'))
                      '--tmpfs', '/sysc_image/tmp',
                      init)
 
-    elif args.minikernel:
-        if os.path.isdir('kritis-linux'):
-            shutil.rmtree('kritis-linux')
-
-        system_c.prepare(mount_tmpfs=True,
-                         create_disk_image=True)
-        system_a.prepare(mount_tmpfs=True,
-                         create_initramfs=True,
-                         repo_path=args.repo)
-
-        run('git', 'clone',
-            '--depth', '1', '--branch', 'v0.7',
-            'https://github.com/bittorf/kritis-linux.git')
-        run('kritis-linux/ci_helper.sh',
-            '--private',
-            '--multi', '1',
-            '--repeat', '1',
-            '--arch', args.arch,
-            '--qemucpu', '486',
-            '--kernel', '3.18.140',
-            '--features', 'kflock,highrestimers',
-            # Hack to add -hda /dev/blah
-            '--ramsize', str(args.qemu_ram) + 'M -hda ' + system_b.dev_name,
-            '--initrd', system_a.initramfs_path,
-            '--log', '/tmp/bootstrap.log')
-
     elif args.bare_metal:
-        system_c.prepare(mount_tmpfs=True,
+        system_c.prepare(create_disk_image=True)
-                         create_disk_image=True)
+        system_a.prepare(create_initramfs=True)
-        system_a.prepare(mount_tmpfs=True,
-                         create_initramfs=True,
-                         repo_path=args.repo)
 
         print("Please:")
-        print("  1. Take sysa/tmp/initramfs and your kernel, boot using this.")
+        print("  1. Take tmp/sysa/initramfs and your kernel, boot using this.")
-        print("  2. Take sysc/tmp/disk.img and put this on a writable storage medium.")
+        print("  2. Take tmp/sysc/disk.img and put this on a writable storage medium.")
 
     else:
-        system_c.prepare(mount_tmpfs=True,
+        system_c.prepare(create_disk_image=True)
-                         create_disk_image=True)
+        system_a.prepare(create_initramfs=True)
-        system_a.prepare(mount_tmpfs=True,
-                         create_initramfs=True,
-                         repo_path=args.repo)
 
         run(args.qemu_cmd,
             '-enable-kvm',
             '-m', str(args.qemu_ram) + 'M',
             '-no-reboot',
-            '-hda', system_c.dev_name,
+            '-hda', tmpdir.get_disk("sysc"),
             '-nic', 'user,ipv6=off,model=e1000',
             '-kernel', args.kernel,
             '-initrd', system_a.initramfs_path,

sysa.py

@@ -4,7 +4,7 @@
 # SPDX-FileCopyrightText: 2022-2023 Dor Askayo <dor.askayo@gmail.com>
 # SPDX-FileCopyrightText: 2021 Andrius Štikonas <andrius@stikonas.eu>
 # SPDX-FileCopyrightText: 2021 Melg Eight <public.melg8@gmail.com>
-# SPDX-FileCopyrightText: 2021-22 fosslinux <fosslinux@aussies.space>
+# SPDX-FileCopyrightText: 2021-23 fosslinux <fosslinux@aussies.space>
 
 import os
 from distutils.dir_util import copy_tree
@@ -22,37 +22,26 @@ class SysA(SysGeneral):
 
     git_dir = os.path.dirname(os.path.join(__file__))
     sys_dir = os.path.join(git_dir, 'sysa')
+    sysb_dir = os.path.join(git_dir, 'sysb')
+    sysc_dir = os.path.join(git_dir, 'sysc')
     cache_dir = os.path.join(sys_dir, 'distfiles')
 
     # pylint: disable=too-many-arguments
-    def __init__(self, arch, preserve_tmp, external_sources,
+    def __init__(self, tmpdir, arch, external_sources,
-                 early_preseed, tmpdir, sysb_dir, sysc_dir):
+                 early_preseed, repo_path):
         self.arch = arch
-        self.preserve_tmp = preserve_tmp
         self.early_preseed = early_preseed
-
-        if tmpdir is None:
-            self.tmp_dir = os.path.join(self.git_dir, 'tmp')
-        else:
-            self.tmp_dir = os.path.join(tmpdir, 'sysa')
-        self.sysa_dir = os.path.join(self.tmp_dir, 'sysa')
-        self.base_dir = self.sysa_dir
-
-        self.sysb_dir = sysb_dir
-        self.sysc_dir = sysc_dir
         self.external_sources = external_sources
+        self.repo_path = repo_path
 
-    def prepare(self, mount_tmpfs, create_initramfs, repo_path):
+        self.tmp_dir = tmpdir.add_sys("sysa")
+
+    def prepare(self, create_initramfs):
         """
         Prepare directory structure for System A.
         We create an empty tmp directory, unpack stage0-posix.
         Rest of the files are unpacked into more structured directory /sysa
         """
-        if mount_tmpfs:
-            self.mount_tmpfs()
-        else:
-            os.mkdir(self.tmp_dir)
-
         if self.early_preseed:
             # Extract tar containing preseed
             with tarfile.open(self.early_preseed, "r") as seed:
@@ -69,9 +58,9 @@ class SysA(SysGeneral):
 
         self.sysc(create_initramfs)
 
-        if repo_path:
+        if self.repo_path:
             repo_dir = os.path.join(self.tmp_dir, 'usr', 'src', 'repo-preseeded')
-            shutil.copytree(repo_path, repo_dir)
+            shutil.copytree(self.repo_path, repo_dir)
 
         if create_initramfs:
             self.make_initramfs()

sysb.py

@@ -1,21 +0,0 @@
-#!/usr/bin/env python3
-"""System B"""
-# SPDX-License-Identifier: GPL-3.0-or-later
-# SPDX-FileCopyrightText: 2021 Andrius Štikonas <andrius@stikonas.eu>
-# SPDX-FileCopyrightText: 2021-22 fosslinux <fosslinux@aussies.space>
-
-import os
-
-from lib.sysgeneral import SysGeneral
-
-class SysB(SysGeneral):
-    """
-    Class responsible for preparing sources for System B.
-    """
-
-    git_dir = os.path.dirname(os.path.join(__file__))
-    sys_dir = os.path.join(git_dir, 'sysb')
-
-    def __init__(self, arch, preserve_tmp):
-        self.arch = arch
-        self.preserve_tmp = preserve_tmp

sysc.py

@@ -2,13 +2,12 @@
 """System C"""
 # SPDX-License-Identifier: GPL-3.0-or-later
 # SPDX-FileCopyrightText: 2022-2023 Dor Askayo <dor.askayo@gmail.com>
-# SPDX-FileCopyrightText: 2021-22 fosslinux <fosslinux@aussies.space>
+# SPDX-FileCopyrightText: 2021-23 fosslinux <fosslinux@aussies.space>
 # SPDX-FileCopyrightText: 2021 Andrius Štikonas <andrius@stikonas.eu>
 
 import os
-import getpass
 
-from lib.utils import mount, umount, create_disk, run, copytree
+from lib.utils import copytree
 from lib.sysgeneral import SysGeneral
 
 # pylint: disable=consider-using-with
@@ -21,60 +20,30 @@ class SysC(SysGeneral):
     git_dir = os.path.dirname(os.path.join(__file__))
     sys_dir = os.path.join(git_dir, 'sysc')
     cache_dir = os.path.join(sys_dir, 'distfiles')
-    dev_name = None
 
-    def __init__(self, arch, preserve_tmp, tmpdir, external_sources):
+    def __init__(self, tmpdir, arch, external_sources):
         self.arch = arch
-        self.preserve_tmp = preserve_tmp
         self.external_sources = external_sources
+        self._tmpdir = tmpdir
 
-        if tmpdir is None:
+        self.tmp_dir = tmpdir.add_sys("sysc")
-            self.tmp_dir = os.path.join(self.sys_dir, 'tmp')
-        else:
-            self.tmp_dir = os.path.join(tmpdir, 'sysc')
 
-    def __del__(self):
+    def prepare(self, create_disk_image):
-        if not self.preserve_tmp:
-            if self.dev_name is not None:
-                print(f"Detaching {self.dev_name}")
-                run('sudo', 'losetup', '-d', self.dev_name)
-
-        super().__del__()
-
-    def prepare(self, mount_tmpfs, create_disk_image):
         """
         Prepare directory structure for System C.
         """
-        if mount_tmpfs:
-            self.mount_tmpfs()
-        else:
-            os.mkdir(self.tmp_dir)
-
-        rootfs_dir = None
-
         if create_disk_image:
-            # Create + mount a disk for QEMU to use
+            self._tmpdir.add_disk("sysc")
-            disk_path = os.path.join(self.tmp_dir, 'disk.img')
-            if self.external_sources:
-                self.dev_name = create_disk(disk_path, "msdos", "ext4", '8G')
-                rootfs_dir = os.path.join(self.tmp_dir, 'mnt')
-                os.mkdir(rootfs_dir)
-                mount(self.dev_name + "p1", rootfs_dir, 'ext4')
-            else:
-                self.dev_name = create_disk(disk_path, "none", "ext4", '8G')
-            # Use chown to allow executing user to access it
-            run('sudo', 'chown', getpass.getuser(), self.dev_name)
-            if self.external_sources:
-                run('sudo', 'chown', getpass.getuser(), rootfs_dir)
-        else:
-            rootfs_dir = self.tmp_dir
 
         if self.external_sources:
+            if create_disk_image:
+                rootfs_dir = self._tmpdir.mount_disk("sysc")
+            else:
+                rootfs_dir = self.tmp_dir
             source_manifest = self.get_source_manifest()
             self.get_packages(source_manifest)
 
             copytree(self.cache_dir, os.path.join(rootfs_dir, "distfiles"))
 
-        # Unmount tmp/mnt if it was mounted
+            if create_disk_image:
-        if create_disk_image and self.external_sources:
+                self._tmpdir.umount_disk("sysc")
-            umount(rootfs_dir)