From dc62d8242cafddb438e149fcd81e3272eea73da1 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?G=C3=A1bor=20Stefanik?= <netrolller.3d@gmail.com>
Date: Sun, 17 Dec 2023 20:50:47 +0100
Subject: [PATCH 01/13] Create external.img only when needed

---
 lib/generator.py | 2 --
 1 file changed, 2 deletions(-)

diff --git a/lib/generator.py b/lib/generator.py
index e1dbac5..3d33650 100755
--- a/lib/generator.py
+++ b/lib/generator.py
@@ -57,8 +57,6 @@ class Generator():
             if self.repo_path or self.external_sources:
                 self.tmpdir.add_disk("external", filesystem="ext3")
                 self.tmpdir.mount_disk("external", "external")
-            else:
-                self.tmpdir.add_disk("external", tabletype="none")
         elif using_kernel:
             self.tmp_dir = os.path.join(self.tmp_dir, 'disk')
             self.tmpdir.add_disk("disk", filesystem="ext3")

From b4d9c5e7cb4bb3e1bee9e6105c90c9c0e3b0eb35 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?G=C3=A1bor=20Stefanik?= <netrolller.3d@gmail.com>
Date: Sun, 17 Dec 2023 21:59:00 +0100
Subject: [PATCH 02/13] Disk creation improvements

* Support specifying the size of the target disk image for qemu
* For bare metal, only pad the image to the next megabyte
* Use truncate() to extend images, instead of writing zeros (faster)
* Return None from get_disk() with nonexistent name
* Leave 1MiB on non-boot disks, or 1GiB on boot disks, unpartitioned
  (for proper 4K alignment and to help preserve the srcfs or boot
  partition creation)
* Fix qemu invocation when an external.img is not used
* Make -qr work with kernel bootstrap (will need kexec-fiwix fix)
---
 lib/generator.py | 24 ++++++++++++++----------
 lib/tmpdir.py    | 17 ++++++++++++++---
 lib/utils.py     |  5 +++--
 rootfs.py        | 42 ++++++++++++++++++++++++++++++------------
 4 files changed, 61 insertions(+), 27 deletions(-)

diff --git a/lib/generator.py b/lib/generator.py
index 3d33650..a5cadc1 100755
--- a/lib/generator.py
+++ b/lib/generator.py
@@ -33,7 +33,7 @@ class Generator():
         self.tmp_dir = tmpdir.path
         self.external_dir = os.path.join(self.tmp_dir, 'external')
 
-    def prepare(self, using_kernel=False, kernel_bootstrap=False):
+    def prepare(self, using_kernel=False, kernel_bootstrap=False, target_size=0):
         """
         Prepare basic media of live-bootstrap.
         /steps -- contains steps to be built
@@ -57,9 +57,14 @@ class Generator():
             if self.repo_path or self.external_sources:
                 self.tmpdir.add_disk("external", filesystem="ext3")
                 self.tmpdir.mount_disk("external", "external")
+            else:
+                self.external_dir = os.path.join(self.tmp_dir, 'external')
         elif using_kernel:
             self.tmp_dir = os.path.join(self.tmp_dir, 'disk')
-            self.tmpdir.add_disk("disk", filesystem="ext3")
+            self.tmpdir.add_disk("disk",
+                            filesystem="ext3",
+                            size=(target_size + "M") if target_size else "16G",
+                            bootable=True)
             self.tmpdir.mount_disk("disk", "disk")
             self.external_dir = os.path.join(self.tmp_dir, 'external')
 
@@ -86,7 +91,7 @@ class Generator():
             shutil.copytree(self.repo_path, repo_dir)
 
         if kernel_bootstrap:
-            self.create_builder_hex0_disk_image(os.path.join(self.tmp_dir, 'disk.img'))
+            self.create_builder_hex0_disk_image(os.path.join(self.tmp_dir, 'disk.img'), target_size)
 
         if kernel_bootstrap and (self.external_sources or self.repo_path):
             self.tmpdir.umount_disk('external')
@@ -149,7 +154,7 @@ class Generator():
     def distfiles(self):
         """Copy in distfiles"""
         def copy_no_network_distfiles(out):
-            # Note that no network == no disk for kernel bootstrap mode
+            # Note that "no disk" implies "no network" for kernel bootstrap mode
             pre_src_path = os.path.join(self.git_dir, 'steps', 'pre-network-sources')
             with open(pre_src_path, 'r', encoding="utf-8") as source_list:
                 for file in source_list.readlines():
@@ -222,7 +227,7 @@ class Generator():
 
         os.chdir(save_cwd)
 
-    def create_builder_hex0_disk_image(self, image_file_name):
+    def create_builder_hex0_disk_image(self, image_file_name, size):
         """Create builder-hex0 disk image"""
         shutil.copyfile(os.path.join('seed', 'stage0-posix', 'bootstrap-seeds',
                                      'NATIVE', 'x86', 'builder-hex0-x86-stage1.img'),
@@ -250,11 +255,10 @@ class Generator():
             image_file.write(b'\0' * round_up)
         current_size += round_up
 
-        # fill file with zeros up to desired size, one megabyte at a time
-        with open(image_file_name, 'ab') as image_file:
-            while current_size < 16384 * megabyte:
-                image_file.write(b'\0' * megabyte)
-                current_size += megabyte
+        # extend file up to desired size
+        if current_size < size * megabyte:
+            with open(image_file_name, 'ab') as image_file:
+                image_file.truncate(size * megabyte)
 
     def check_file(self, file_name, expected_hash):
         """Check hash of downloaded source file."""
diff --git a/lib/tmpdir.py b/lib/tmpdir.py
index f72286f..954235c 100644
--- a/lib/tmpdir.py
+++ b/lib/tmpdir.py
@@ -60,10 +60,21 @@ class Tmpdir:
         self._type = TmpType.TMPFS
 
     # pylint: disable=too-many-arguments
-    def add_disk(self, name, size="16G", filesystem="ext4", tabletype="msdos", mkfs_args=None):
+    def add_disk(self,
+                 name,
+                 size="16G",
+                 filesystem="ext4",
+                 tabletype="msdos",
+                 bootable=False,
+                 mkfs_args=None):
         """Add a disk"""
         disk_path = os.path.join(self.path, f"{name}.img")
-        self._disks[name] = create_disk(disk_path, tabletype, filesystem, size, mkfs_args=mkfs_args)
+        self._disks[name] = create_disk(disk_path,
+                                        tabletype,
+                                        filesystem,
+                                        size,
+                                        bootable,
+                                        mkfs_args)
         self._disk_filesystems[name] = filesystem
         # Allow executing user to access it
         run_as_root("chown", getpass.getuser(), self._disks[name])
@@ -87,4 +98,4 @@ class Tmpdir:
 
     def get_disk(self, name):
         """Get the path to a device of a disk"""
-        return self._disks[name]
+        return self._disks.get(name)
diff --git a/lib/utils.py b/lib/utils.py
index a80d656..d88ae4d 100755
--- a/lib/utils.py
+++ b/lib/utils.py
@@ -31,7 +31,8 @@ def run_as_root(*args, **kwargs):
         return run("sudo", *args, **kwargs)
     return run(*args, **kwargs)
 
-def create_disk(image, disk_type, fs_type, size, mkfs_args=None):
+# pylint: disable=too-many-arguments
+def create_disk(image, disk_type, fs_type, size, bootable=False, mkfs_args=None):
     """Create a disk image, with a filesystem on it"""
     if mkfs_args is None:
         mkfs_args = []
@@ -42,7 +43,7 @@ def create_disk(image, disk_type, fs_type, size, mkfs_args=None):
     # Create the partition
     if disk_type != "none":
         run_as_root('parted', '--script', image, 'mklabel', disk_type, 'mkpart',
-                'primary', fs_type, '0%', '100%')
+                'primary', fs_type, '1GiB' if bootable else '1MiB', '100%')
         run_as_root('partprobe', loop_dev)
         run_as_root('mkfs.' + fs_type, loop_dev + "p1", *mkfs_args)
     return loop_dev
diff --git a/rootfs.py b/rootfs.py
index 648b1d0..d278294 100755
--- a/rootfs.py
+++ b/rootfs.py
@@ -39,7 +39,7 @@ def create_configuration_file(args):
             if args.repo or args.external_sources:
                 config.write("DISK=sdb1\n")
             else:
-                config.write("DISK=sdb\n")
+                config.write("DISK=sda\n")
             config.write("KERNEL_BOOTSTRAP=True\n")
         else:
             config.write("DISK=sda1\n")
@@ -98,6 +98,8 @@ def main():
                         default="qemu-system-x86_64")
     parser.add_argument("-qr", "--qemu-ram", help="Memory (in megabytes) allocated to QEMU VM",
                         default=4096)
+    parser.add_argument("-qs", "--target-size", help="Size of the target image (for QEMU only)",
+                        default="16G")
     parser.add_argument("-qk", "--kernel", help="Custom sysa kernel to use")
 
     parser.add_argument("-b", "--bare-metal", help="Build images for bare metal",
@@ -136,6 +138,15 @@ def main():
     if int(args.cores) < 1:
         raise ValueError("Must use one or more cores.")
 
+    # Target image size validation
+    if args.qemu:
+        if int(str(args.target_size).rstrip('gGmM')) < 1:
+            raise ValueError("Please specify a positive target size for qemu.")
+        args.target_size = (int(str(args.target_size).rstrip('gGmM')) *
+            (1024 if str(args.target_size).lower().endswith('g') else 1))
+    else:
+        args.target_size = 0
+
     # bootstrap.cfg
     try:
         os.remove(os.path.join('sysa', 'bootstrap.cfg'))
@@ -158,9 +169,9 @@ def main():
                           repo_path=args.repo,
                           early_preseed=args.early_preseed)
 
-    bootstrap(args, generator, tmpdir)
+    bootstrap(args, generator, tmpdir, args.target_size)
 
-def bootstrap(args, generator, tmpdir):
+def bootstrap(args, generator, tmpdir, size):
     """Kick off bootstrap process."""
     print(f"Bootstrapping {args.arch} -- SysA")
     if args.chroot:
@@ -224,18 +235,18 @@ print(shutil.which('chroot'))
 
     elif args.bare_metal:
         if args.kernel:
-            generator.prepare(using_kernel=True)
+            generator.prepare(using_kernel=True, target_size=size)
             print("Please:")
             print("  1. Take tmp/initramfs and your kernel, boot using this.")
             print("  2. Take tmp/disk.img and put this on a writable storage medium.")
         else:
-            generator.prepare(kernel_bootstrap=True)
+            generator.prepare(kernel_bootstrap=True, target_size=size)
             print("Please:")
             print("  1. Take tmp/disk.img and write it to a boot drive and then boot it.")
 
     else:
         if args.kernel:
-            generator.prepare(using_kernel=True)
+            generator.prepare(using_kernel=True, target_size=size)
 
             run(args.qemu_cmd,
                 '-enable-kvm',
@@ -249,17 +260,24 @@ print(shutil.which('chroot'))
                 '-nographic',
                 '-append', 'console=ttyS0 root=/dev/sda1 rootfstype=ext3 init=/init rw')
         else:
-            generator.prepare(kernel_bootstrap=True)
-            run(args.qemu_cmd,
+            generator.prepare(kernel_bootstrap=True, target_size=size)
+            arg_list = [
                 '-enable-kvm',
-                '-m', "4G",
+                '-m', str(args.qemu_ram) + 'M',
                 '-smp', str(args.cores),
                 '-no-reboot',
-                '-drive', 'file=' + os.path.join(generator.tmp_dir, 'disk.img') + ',format=raw',
-                '-drive', 'file=' + tmpdir.get_disk("external") + ',format=raw',
+                '-drive', 'file=' + os.path.join(generator.tmp_dir, 'disk.img') + ',format=raw'
+            ]
+            if tmpdir.get_disk("external") is not None:
+                arg_list += [
+                    '-drive', 'file=' + tmpdir.get_disk("external") + ',format=raw',
+                ]
+            arg_list += [
                 '-machine', 'kernel-irqchip=split',
                 '-nic', 'user,ipv6=off,model=e1000',
-                '-nographic')
+                '-nographic'
+            ]
+            run(args.qemu_cmd, *arg_list)
 
 if __name__ == "__main__":
     main()

From c188185ad4525c5679ebc38f2819a1ecd629c202 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?G=C3=A1bor=20Stefanik?= <netrolller.3d@gmail.com>
Date: Sun, 17 Dec 2023 22:09:04 +0100
Subject: [PATCH 03/13] Remove residual references to sysa/sysb/sysc from
 rootfs.py

---
 rootfs.py | 16 ++++++++--------
 1 file changed, 8 insertions(+), 8 deletions(-)

diff --git a/rootfs.py b/rootfs.py
index d278294..98a5794 100755
--- a/rootfs.py
+++ b/rootfs.py
@@ -30,7 +30,6 @@ def create_configuration_file(args):
     with open(config_path, "w", encoding="utf_8") as config:
         config.write(f"FORCE_TIMESTAMPS={args.force_timestamps}\n")
         config.write(f"CHROOT={args.chroot or args.bwrap}\n")
-        config.write(f"CHROOT_ONLY_SYSA={args.bwrap}\n")
         config.write(f"UPDATE_CHECKSUMS={args.update_checksums}\n")
         config.write(f"JOBS={args.cores}\n")
         config.write(f"INTERNAL_CI={args.internal_ci}\n")
@@ -100,7 +99,7 @@ def main():
                         default=4096)
     parser.add_argument("-qs", "--target-size", help="Size of the target image (for QEMU only)",
                         default="16G")
-    parser.add_argument("-qk", "--kernel", help="Custom sysa kernel to use")
+    parser.add_argument("-qk", "--kernel", help="Custom early kernel to use")
 
     parser.add_argument("-b", "--bare-metal", help="Build images for bare metal",
                         action="store_true")
@@ -149,13 +148,13 @@ def main():
 
     # bootstrap.cfg
     try:
-        os.remove(os.path.join('sysa', 'bootstrap.cfg'))
+        os.remove(os.path.join('steps', 'bootstrap.cfg'))
     except FileNotFoundError:
         pass
     if not args.no_create_config:
         create_configuration_file(args)
     else:
-        with open(os.path.join('sysa', 'bootstrap.cfg'), 'a', encoding='UTF-8'):
+        with open(os.path.join('steps', 'bootstrap.cfg'), 'a', encoding='UTF-8'):
             pass
 
     # tmpdir
@@ -173,7 +172,7 @@ def main():
 
 def bootstrap(args, generator, tmpdir, size):
     """Kick off bootstrap process."""
-    print(f"Bootstrapping {args.arch} -- SysA")
+    print(f"Bootstrapping {args.arch}")
     if args.chroot:
         find_chroot = """
 import shutil
@@ -211,15 +210,16 @@ print(shutil.which('chroot'))
                          init)
 
         if not args.internal_ci or args.internal_ci == "pass2" or args.internal_ci == "pass3":
-            shutil.copy2(os.path.join('sysa', 'bootstrap.cfg'),
-                         os.path.join('tmp', 'sysa', 'sysc_image', 'usr', 'src', 'bootstrap.cfg'))
+            os.makedirs(os.path.join(generator.tmp_dir, 'stage2', 'steps'), exist_ok=True)
+            shutil.copy2(os.path.join('steps', 'bootstrap.cfg'),
+                         os.path.join(generator.tmp_dir, 'stage2', 'steps', 'bootstrap.cfg'))
             run('bwrap', '--unshare-user',
                          '--uid', '0',
                          '--gid', '0',
                          '--unshare-net' if args.external_sources else None,
                          '--clearenv',
                          '--setenv', 'PATH', '/usr/bin',
-                         '--bind', generator.tmp_dir + "/sysc_image", '/',
+                         '--bind', os.path.join(generator.tmp_dir, "stage2"), '/',
                          '--dir', '/dev',
                          '--dev-bind', '/dev/null', '/dev/null',
                          '--dev-bind', '/dev/zero', '/dev/zero',

From faad907fba73a3d081b905d380e96b9fe3c1030b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?G=C3=A1bor=20Stefanik?= <netrolller.3d@gmail.com>
Date: Sun, 17 Dec 2023 22:11:02 +0100
Subject: [PATCH 04/13] Fix source_manifest.py in light of the recent refactor

There's no sysa or sysc anymore.
---
 source_manifest.py | 20 ++------------------
 1 file changed, 2 insertions(+), 18 deletions(-)

diff --git a/source_manifest.py b/source_manifest.py
index 6fa4e35..81a1ba9 100755
--- a/source_manifest.py
+++ b/source_manifest.py
@@ -9,27 +9,11 @@ for the bootstrapping process.
 
 import argparse
 
-from sysa import SysA
-from sysc import SysC
+from lib.generator import Generator
 
 def main():
     """Generate a source manifest for a system"""
-    parser = argparse.ArgumentParser()
-
-    parser.add_argument("-s", "--system",
-                        help="Generate source manifest for the specified systems",
-                        choices=["sysa", "sysc"],
-                        nargs="+",
-                        action="extend",
-                        required=True)
-
-    args = parser.parse_args()
-
-    if "sysa" in args.system:
-        print(SysA.get_source_manifest())
-
-    if "sysc" in args.system:
-        print(SysC.get_source_manifest())
+    print(Generator.get_source_manifest())
 
 if __name__ == "__main__":
     main()

From 8d193df8231b9ce4737614ba039008c40a73b6cf Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?G=C3=A1bor=20Stefanik?= <netrolller.3d@gmail.com>
Date: Sun, 17 Dec 2023 22:22:13 +0100
Subject: [PATCH 05/13] Fix printed image paths for bare-metal bootstrap

---
 rootfs.py | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/rootfs.py b/rootfs.py
index 98a5794..172065b 100755
--- a/rootfs.py
+++ b/rootfs.py
@@ -236,13 +236,15 @@ print(shutil.which('chroot'))
     elif args.bare_metal:
         if args.kernel:
             generator.prepare(using_kernel=True, target_size=size)
+            image_path = os.path.join(args.tmpdir, os.path.relpath(generator.tmp_dir, args.tmpdir))
             print("Please:")
-            print("  1. Take tmp/initramfs and your kernel, boot using this.")
-            print("  2. Take tmp/disk.img and put this on a writable storage medium.")
+            print(f"  1. Take {image_path}/initramfs and your kernel, boot using this.")
+            print(f"  2. Take {image_path}/disk.img and put this on a writable storage medium.")
         else:
             generator.prepare(kernel_bootstrap=True, target_size=size)
+            image_path = os.path.join(args.tmpdir, os.path.relpath(generator.tmp_dir, args.tmpdir))
             print("Please:")
-            print("  1. Take tmp/disk.img and write it to a boot drive and then boot it.")
+            print(f"  1. Take {image_path}/disk.img and write it to a boot drive and then boot it.")
 
     else:
         if args.kernel:

From a68ae62f9e50781eb123f0fcc4542f75707244dc Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?G=C3=A1bor=20Stefanik?= <netrolller.3d@gmail.com>
Date: Sun, 17 Dec 2023 22:58:30 +0100
Subject: [PATCH 06/13] Download distfiles only when needed, based on manifest

Unless --external-sources is given, only download distfiles that
need to be included in srcfs. The rest will be downloaded anyway
by the bootstrap system once it gets network access.

To accomplish this, instead of searching steps for sources files,
we now parse steps/manifest. As a side effect, source_manifest.py
now outputs source files in the order they appear in the manifest.
---
 lib/generator.py | 21 +++++++++++++++------
 1 file changed, 15 insertions(+), 6 deletions(-)

diff --git a/lib/generator.py b/lib/generator.py
index a5cadc1..54e3859 100755
--- a/lib/generator.py
+++ b/lib/generator.py
@@ -100,7 +100,7 @@ class Generator():
 
     def steps(self):
         """Copy in steps."""
-        source_manifest = self.get_source_manifest()
+        source_manifest = self.get_source_manifest(not self.external_sources)
         self.get_packages(source_manifest)
 
         shutil.copytree(os.path.join(self.git_dir, 'steps'), os.path.join(self.tmp_dir, 'steps'))
@@ -307,7 +307,7 @@ this script the next time")
             self.check_file(path, line[0])
 
     @classmethod
-    def get_source_manifest(cls):
+    def get_source_manifest(cls, pre_network=False):
         """
         Generate a source manifest for the system.
         """
@@ -316,9 +316,16 @@ this script the next time")
 
         # Find all source files
         steps_dir = os.path.join(cls.git_dir, 'steps')
-        for file in os.listdir(steps_dir):
-            if os.path.isdir(os.path.join(steps_dir, file)):
-                sourcef = os.path.join(steps_dir, file, "sources")
+        with open(os.path.join(steps_dir, 'manifest'), 'r', encoding="utf_8") as file:
+            for line in file:
+                if pre_network and line.strip().startswith("improve: ") and "network" in line:
+                    break
+
+                if not line.strip().startswith("build: "):
+                    continue
+
+                step = line.split(" ")[1].split("#")[0].strip()
+                sourcef = os.path.join(steps_dir, step, "sources")
                 if os.path.exists(sourcef):
                     # Read sources from the source file
                     with open(sourcef, "r", encoding="utf_8") as sources:
@@ -331,7 +338,9 @@ this script the next time")
                                 # Automatically determine file name based on URL.
                                 file_name = os.path.basename(line[0])
 
-                            manifest_lines.append(f"{line[1]} {directory} {line[0]} {file_name}")
+                            manifest_line = f"{line[1]} {directory} {line[0]} {file_name}"
+                            if manifest_line not in manifest_lines:
+                                manifest_lines.append(manifest_line)
 
         return "\n".join(manifest_lines)
 

From 3305f2a41b13a1bac3e23ea0ddee069c7e3909c7 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?G=C3=A1bor=20Stefanik?= <netrolller.3d@gmail.com>
Date: Sun, 17 Dec 2023 23:08:42 +0100
Subject: [PATCH 07/13] Use manifest to deduce pre-network sources list

No need to maintain a separate pre-network-sources file anymore,
the list is instead derived from the bootstrap manifest via the
source manifest.
---
 lib/generator.py          | 40 ++++++++----------
 source_manifest.py        |  2 +-
 steps/pre-network-sources | 86 ---------------------------------------
 3 files changed, 19 insertions(+), 109 deletions(-)
 delete mode 100644 steps/pre-network-sources

diff --git a/lib/generator.py b/lib/generator.py
index 54e3859..1aec17f 100755
--- a/lib/generator.py
+++ b/lib/generator.py
@@ -32,6 +32,7 @@ class Generator():
         self.tmpdir = tmpdir
         self.tmp_dir = tmpdir.path
         self.external_dir = os.path.join(self.tmp_dir, 'external')
+        self.source_manifest = self.get_source_manifest(not self.external_sources)
 
     def prepare(self, using_kernel=False, kernel_bootstrap=False, target_size=0):
         """
@@ -100,8 +101,7 @@ class Generator():
 
     def steps(self):
         """Copy in steps."""
-        source_manifest = self.get_source_manifest(not self.external_sources)
-        self.get_packages(source_manifest)
+        self.get_packages()
 
         shutil.copytree(os.path.join(self.git_dir, 'steps'), os.path.join(self.tmp_dir, 'steps'))
 
@@ -155,12 +155,10 @@ class Generator():
         """Copy in distfiles"""
         def copy_no_network_distfiles(out):
             # Note that "no disk" implies "no network" for kernel bootstrap mode
-            pre_src_path = os.path.join(self.git_dir, 'steps', 'pre-network-sources')
-            with open(pre_src_path, 'r', encoding="utf-8") as source_list:
-                for file in source_list.readlines():
-                    file = file.strip()
-                    shutil.copy2(os.path.join(self.distfiles_dir, file),
-                                 os.path.join(out, file))
+            for file in self.source_manifest:
+                file = file[3].strip()
+                shutil.copy2(os.path.join(self.distfiles_dir, file),
+                             os.path.join(out, file))
 
         early_distfile_dir = os.path.join(self.tmp_dir, 'external', 'distfiles')
         main_distfile_dir = os.path.join(self.external_dir, 'distfiles')
@@ -298,11 +296,9 @@ this script the next time")
                 raise requests.HTTPError("Download failed.")
         return abs_file_name
 
-    def get_packages(self, source_manifest):
+    def get_packages(self):
         """Prepare remaining sources"""
-        for line in source_manifest.split("\n"):
-            line = line.strip().split(" ")
-
+        for line in self.source_manifest:
             path = self.download_file(line[2], line[1], line[3])
             self.check_file(path, line[0])
 
@@ -311,7 +307,7 @@ this script the next time")
         """
         Generate a source manifest for the system.
         """
-        manifest_lines = []
+        entries = []
         directory = os.path.relpath(cls.distfiles_dir, cls.git_dir)
 
         # Find all source files
@@ -329,20 +325,20 @@ this script the next time")
                 if os.path.exists(sourcef):
                     # Read sources from the source file
                     with open(sourcef, "r", encoding="utf_8") as sources:
-                        for line in sources.readlines():
-                            line = line.strip().split(" ")
+                        for source in sources.readlines():
+                            source = source.strip().split(" ")
 
-                            if len(line) > 2:
-                                file_name = line[2]
+                            if len(source) > 2:
+                                file_name = source[2]
                             else:
                                 # Automatically determine file name based on URL.
-                                file_name = os.path.basename(line[0])
+                                file_name = os.path.basename(source[0])
 
-                            manifest_line = f"{line[1]} {directory} {line[0]} {file_name}"
-                            if manifest_line not in manifest_lines:
-                                manifest_lines.append(manifest_line)
+                            entry = (source[1], directory, source[0], file_name)
+                            if entry not in entries:
+                                entries.append(entry)
 
-        return "\n".join(manifest_lines)
+        return entries
 
 stage0_arch_map = {
     "amd64": "AMD64",
diff --git a/source_manifest.py b/source_manifest.py
index 81a1ba9..a972538 100755
--- a/source_manifest.py
+++ b/source_manifest.py
@@ -13,7 +13,7 @@ from lib.generator import Generator
 
 def main():
     """Generate a source manifest for a system"""
-    print(Generator.get_source_manifest())
+    print('\n'.join(map(' '.join, Generator.get_source_manifest())))
 
 if __name__ == "__main__":
     main()
diff --git a/steps/pre-network-sources b/steps/pre-network-sources
deleted file mode 100644
index 8b54d0d..0000000
--- a/steps/pre-network-sources
+++ /dev/null
@@ -1,86 +0,0 @@
-mes-0.25.tar.gz
-nyacc-1.00.2.tar.gz
-tcc-0.9.26.tar.gz
-tcc-0.9.27.tar.bz2
-fiwix-1.4.0-lb3.tar.gz
-lwext4-1.0.0-lb1.tar.gz
-make-3.82.tar.bz2
-patch-2.5.9.tar.gz
-gzip-1.2.4.tar.gz
-tar-1.12.tar.gz
-sed-4.0.9.tar.gz
-bzip2-1.0.8.tar.gz
-coreutils-5.0.tar.bz2
-heirloom-devtools-070527.tar.bz2
-bash-2.05b.tar.gz
-flex-2.5.11.tar.gz
-tcc-0.9.27.tar.bz2
-musl-1.1.24.tar.gz
-tcc-0.9.27.tar.bz2
-musl-1.1.24.tar.gz
-tcc-0.9.27.tar.bz2
-sed-4.0.9.tar.gz
-bzip2-1.0.8.tar.gz
-m4-1.4.7.tar.gz
-flex-2.6.4.tar.gz
-bison-3.4.1.tar.gz
-bison-3.4.1.tar.gz
-bison-3.4.1.tar.gz
-grep-2.4.tar.gz
-diffutils-2.7.tar.gz
-coreutils-5.0.tar.bz2
-coreutils-6.10.tar.gz
-gawk-3.0.4.tar.gz
-perl-5.000.tar.gz
-perl-5.003.tar.gz
-perl5.004_05.tar.gz
-perl5.005_03.tar.gz
-perl-5.6.2.tar.gz
-autoconf-2.52.tar.bz2
-automake-1.6.3.tar.bz2
-automake-1.6.3.tar.bz2
-autoconf-2.53.tar.bz2
-automake-1.7.tar.bz2
-autoconf-2.54.tar.bz2
-autoconf-2.55.tar.bz2
-automake-1.7.8.tar.bz2
-autoconf-2.57.tar.bz2
-autoconf-2.59.tar.bz2
-automake-1.8.5.tar.bz2
-help2man-1.36.4.tar.gz
-autoconf-2.61.tar.bz2
-automake-1.9.6.tar.bz2
-automake-1.10.3.tar.bz2
-autoconf-2.64.tar.bz2
-automake-1.11.2.tar.bz2
-autoconf-2.69.tar.gz
-libtool-2.2.4.tar.bz2
-automake-1.15.1.tar.gz
-binutils-2.30.tar.bz2
-musl-1.1.24.tar.gz
-tcc-0.9.27.tar.bz2
-gcc-core-4.0.4.tar.bz2
-automake-1.16.3.tar.gz
-findutils-4.2.33.tar.gz
-gnulib-8e128e.tar.gz
-musl-1.2.4.tar.gz
-gcc-core-4.0.4.tar.bz2
-automake-1.16.3.tar.gz
-util-linux-2.19.1.tar.gz
-e2fsprogs-1.45.7.tar.gz
-CaseFolding.txt
-DerivedAge.txt
-DerivedCombiningClass.txt
-DerivedCoreProperties.txt
-NormalizationCorrections.txt
-NormalizationTest.txt
-UnicodeData.txt
-v10.0.1.tar.gz
-kbd-1.15.tar.gz
-make-3.82.tar.bz2
-ed-1.4.tar.gz
-bc-1.07.1.tar.gz
-v2.0.22.tar.gz
-linux-4.9.10.tar.gz
-deblob-4.9
-curl-7.88.1.tar.bz2

From 55d3c36e09437d150a2a8876f25daaa3b9281148 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?G=C3=A1bor=20Stefanik?= <netrolller.3d@gmail.com>
Date: Sun, 17 Dec 2023 23:27:02 +0100
Subject: [PATCH 08/13] Move kernel-bootstrap source image out of the directory
 it's generated from

This way, an incomplete version of the image itself won't get included
in srcfs anymore.
---
 lib/generator.py | 2 +-
 rootfs.py        | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/lib/generator.py b/lib/generator.py
index 1aec17f..bd532ad 100755
--- a/lib/generator.py
+++ b/lib/generator.py
@@ -92,7 +92,7 @@ class Generator():
             shutil.copytree(self.repo_path, repo_dir)
 
         if kernel_bootstrap:
-            self.create_builder_hex0_disk_image(os.path.join(self.tmp_dir, 'disk.img'), target_size)
+            self.create_builder_hex0_disk_image(self.tmp_dir + '.img', target_size)
 
         if kernel_bootstrap and (self.external_sources or self.repo_path):
             self.tmpdir.umount_disk('external')
diff --git a/rootfs.py b/rootfs.py
index 172065b..d202cb2 100755
--- a/rootfs.py
+++ b/rootfs.py
@@ -244,7 +244,7 @@ print(shutil.which('chroot'))
             generator.prepare(kernel_bootstrap=True, target_size=size)
             image_path = os.path.join(args.tmpdir, os.path.relpath(generator.tmp_dir, args.tmpdir))
             print("Please:")
-            print(f"  1. Take {image_path}/disk.img and write it to a boot drive and then boot it.")
+            print(f"  1. Take {image_path}.img and write it to a boot drive and then boot it.")
 
     else:
         if args.kernel:
@@ -268,7 +268,7 @@ print(shutil.which('chroot'))
                 '-m', str(args.qemu_ram) + 'M',
                 '-smp', str(args.cores),
                 '-no-reboot',
-                '-drive', 'file=' + os.path.join(generator.tmp_dir, 'disk.img') + ',format=raw'
+                '-drive', 'file=' + generator.tmp_dir + '.img' + ',format=raw'
             ]
             if tmpdir.get_disk("external") is not None:
                 arg_list += [

From 32dc4c702b38718a01088fe856d309346f4e0f8a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?G=C3=A1bor=20Stefanik?= <netrolller.3d@gmail.com>
Date: Sun, 17 Dec 2023 23:55:27 +0100
Subject: [PATCH 09/13] More verbose error message on HTTP errors

---
 lib/generator.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/lib/generator.py b/lib/generator.py
index bd532ad..428a541 100755
--- a/lib/generator.py
+++ b/lib/generator.py
@@ -293,7 +293,8 @@ this script the next time")
                 with open(abs_file_name, 'wb') as target_file:
                     target_file.write(response.raw.read())
             else:
-                raise requests.HTTPError("Download failed.")
+                raise requests.HTTPError("Download failed: HTTP " +
+                    response.status_code + " " + response.reason)
         return abs_file_name
 
     def get_packages(self):

From b45e1f81aebdb5c40e154c7d0902a63102b22ebd Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?G=C3=A1bor=20Stefanik?= <netrolller.3d@gmail.com>
Date: Mon, 18 Dec 2023 17:22:52 +0100
Subject: [PATCH 10/13] Fix failure with --external-sources

---
 lib/generator.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/lib/generator.py b/lib/generator.py
index 428a541..f6df890 100755
--- a/lib/generator.py
+++ b/lib/generator.py
@@ -168,7 +168,6 @@ class Generator():
             copy_no_network_distfiles(early_distfile_dir)
 
         if self.external_sources:
-            os.mkdir(main_distfile_dir)
             shutil.copytree(self.distfiles_dir, main_distfile_dir)
         else:
             os.mkdir(main_distfile_dir)

From be1333ee8bebb6f064f4062f4d0627764a144ef7 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?G=C3=A1bor=20Stefanik?= <netrolller.3d@gmail.com>
Date: Fri, 22 Dec 2023 12:47:09 +0100
Subject: [PATCH 11/13] Make tmpdir a method parameter of prepare()

Fixes pylint errors.
---
 lib/generator.py | 27 ++++++++++++++-------------
 rootfs.py        | 15 +++++++--------
 2 files changed, 21 insertions(+), 21 deletions(-)

diff --git a/lib/generator.py b/lib/generator.py
index f6df890..9acd031 100755
--- a/lib/generator.py
+++ b/lib/generator.py
@@ -14,6 +14,7 @@ import shutil
 import tarfile
 import requests
 
+# pylint: disable=too-many-instance-attributes
 class Generator():
     """
     Class responsible for generating the basic media to be consumed.
@@ -22,25 +23,25 @@ class Generator():
     git_dir = os.path.join(os.path.dirname(os.path.join(__file__)), '..')
     distfiles_dir = os.path.join(git_dir, 'distfiles')
 
-    # pylint: disable=too-many-arguments
-    def __init__(self, tmpdir, arch, external_sources,
-                 early_preseed, repo_path):
+    def __init__(self, arch, external_sources, early_preseed, repo_path):
         self.arch = arch
         self.early_preseed = early_preseed
         self.external_sources = external_sources
         self.repo_path = repo_path
-        self.tmpdir = tmpdir
-        self.tmp_dir = tmpdir.path
-        self.external_dir = os.path.join(self.tmp_dir, 'external')
         self.source_manifest = self.get_source_manifest(not self.external_sources)
+        self.tmp_dir = None
+        self.external_dir = None
 
-    def prepare(self, using_kernel=False, kernel_bootstrap=False, target_size=0):
+    def prepare(self, tmpdir, using_kernel=False, kernel_bootstrap=False, target_size=0):
         """
         Prepare basic media of live-bootstrap.
         /steps -- contains steps to be built
         / -- contains seed to allow steps to be built, containing custom
              scripts and stage0-posix
         """
+        self.tmp_dir = tmpdir.path
+        self.external_dir = os.path.join(self.tmp_dir, 'external')
+
         # We use ext3 here; ext4 actually has a variety of extensions that
         # have been added with varying levels of recency
         # Linux 4.9.10 does not support a bunch of them
@@ -56,17 +57,17 @@ class Generator():
             self.tmp_dir = init_path
 
             if self.repo_path or self.external_sources:
-                self.tmpdir.add_disk("external", filesystem="ext3")
-                self.tmpdir.mount_disk("external", "external")
+                tmpdir.add_disk("external", filesystem="ext3")
+                tmpdir.mount_disk("external", "external")
             else:
                 self.external_dir = os.path.join(self.tmp_dir, 'external')
         elif using_kernel:
             self.tmp_dir = os.path.join(self.tmp_dir, 'disk')
-            self.tmpdir.add_disk("disk",
+            tmpdir.add_disk("disk",
                             filesystem="ext3",
                             size=(target_size + "M") if target_size else "16G",
                             bootable=True)
-            self.tmpdir.mount_disk("disk", "disk")
+            tmpdir.mount_disk("disk", "disk")
             self.external_dir = os.path.join(self.tmp_dir, 'external')
 
         os.makedirs(self.external_dir, exist_ok=True)
@@ -95,9 +96,9 @@ class Generator():
             self.create_builder_hex0_disk_image(self.tmp_dir + '.img', target_size)
 
         if kernel_bootstrap and (self.external_sources or self.repo_path):
-            self.tmpdir.umount_disk('external')
+            tmpdir.umount_disk('external')
         elif using_kernel:
-            self.tmpdir.umount_disk('disk')
+            tmpdir.umount_disk('disk')
 
     def steps(self):
         """Copy in steps."""
diff --git a/rootfs.py b/rootfs.py
index d202cb2..8641d02 100755
--- a/rootfs.py
+++ b/rootfs.py
@@ -162,8 +162,7 @@ def main():
     if args.tmpfs:
         tmpdir.tmpfs(size=args.tmpfs_size)
 
-    generator = Generator(tmpdir=tmpdir,
-                          arch=args.arch,
+    generator = Generator(arch=args.arch,
                           external_sources=args.external_sources,
                           repo_path=args.repo,
                           early_preseed=args.early_preseed)
@@ -181,7 +180,7 @@ print(shutil.which('chroot'))
         chroot_binary = run_as_root('python3', '-c', find_chroot,
                                     capture_output=True).stdout.decode().strip()
 
-        generator.prepare(using_kernel=False)
+        generator.prepare(tmpdir, using_kernel=False)
 
         arch = stage0_arch_map.get(args.arch, args.arch)
         init = os.path.join(os.sep, 'bootstrap-seeds', 'POSIX', arch, 'kaem-optional-seed')
@@ -189,7 +188,7 @@ print(shutil.which('chroot'))
 
     elif args.bwrap:
         if not args.internal_ci or args.internal_ci == "pass1":
-            generator.prepare(using_kernel=False)
+            generator.prepare(tmpdir, using_kernel=False)
 
             arch = stage0_arch_map.get(args.arch, args.arch)
             init = os.path.join(os.sep, 'bootstrap-seeds', 'POSIX', arch, 'kaem-optional-seed')
@@ -235,20 +234,20 @@ print(shutil.which('chroot'))
 
     elif args.bare_metal:
         if args.kernel:
-            generator.prepare(using_kernel=True, target_size=size)
+            generator.prepare(tmpdir, using_kernel=True, target_size=size)
             image_path = os.path.join(args.tmpdir, os.path.relpath(generator.tmp_dir, args.tmpdir))
             print("Please:")
             print(f"  1. Take {image_path}/initramfs and your kernel, boot using this.")
             print(f"  2. Take {image_path}/disk.img and put this on a writable storage medium.")
         else:
-            generator.prepare(kernel_bootstrap=True, target_size=size)
+            generator.prepare(tmpdir, kernel_bootstrap=True, target_size=size)
             image_path = os.path.join(args.tmpdir, os.path.relpath(generator.tmp_dir, args.tmpdir))
             print("Please:")
             print(f"  1. Take {image_path}.img and write it to a boot drive and then boot it.")
 
     else:
         if args.kernel:
-            generator.prepare(using_kernel=True, target_size=size)
+            generator.prepare(tmpdir, using_kernel=True, target_size=size)
 
             run(args.qemu_cmd,
                 '-enable-kvm',
@@ -262,7 +261,7 @@ print(shutil.which('chroot'))
                 '-nographic',
                 '-append', 'console=ttyS0 root=/dev/sda1 rootfstype=ext3 init=/init rw')
         else:
-            generator.prepare(kernel_bootstrap=True, target_size=size)
+            generator.prepare(tmpdir, kernel_bootstrap=True, target_size=size)
             arg_list = [
                 '-enable-kvm',
                 '-m', str(args.qemu_ram) + 'M',

From ab9455f9180494840eaf672cae14f9f11a6f163b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?G=C3=A1bor=20Stefanik?= <netrolller.3d@gmail.com>
Date: Fri, 22 Dec 2023 12:54:12 +0100
Subject: [PATCH 12/13] Fix warnings in existing code revealed by newer pylint

---
 lib/generator.py | 25 ++++++++++++++-----------
 1 file changed, 14 insertions(+), 11 deletions(-)

diff --git a/lib/generator.py b/lib/generator.py
index 9acd031..82ecd1e 100755
--- a/lib/generator.py
+++ b/lib/generator.py
@@ -109,13 +109,13 @@ class Generator():
     def stage0_posix(self):
         """Copy in all of the stage0-posix"""
         stage0_posix_base_dir = os.path.join(self.git_dir, 'seed', 'stage0-posix')
-        for f in os.listdir(stage0_posix_base_dir):
-            orig = os.path.join(stage0_posix_base_dir, f)
-            to = os.path.join(self.tmp_dir, f)
+        for entry in os.listdir(stage0_posix_base_dir):
+            orig = os.path.join(stage0_posix_base_dir, entry)
+            target = os.path.join(self.tmp_dir, entry)
             if os.path.isfile(orig):
-                shutil.copy2(orig, to)
+                shutil.copy2(orig, target)
             else:
-                shutil.copytree(orig, to)
+                shutil.copytree(orig, target)
 
         arch = stage0_arch_map.get(self.arch, self.arch)
         kaem_optional_seed = os.path.join(self.git_dir, 'seed', 'stage0-posix', 'bootstrap-seeds',
@@ -125,11 +125,12 @@ class Generator():
     def seed(self):
         """Copy in extra seed files"""
         seed_dir = os.path.join(self.git_dir, 'seed')
-        for f in os.listdir(seed_dir):
-            if os.path.isfile(os.path.join(seed_dir, f)):
-                shutil.copy2(os.path.join(seed_dir, f), os.path.join(self.tmp_dir, f))
+        for entry in os.listdir(seed_dir):
+            if os.path.isfile(os.path.join(seed_dir, entry)):
+                shutil.copy2(os.path.join(seed_dir, entry), os.path.join(self.tmp_dir, entry))
 
-    def add_fiwix_files(self, file_list_path, dirpath):
+    @staticmethod
+    def add_fiwix_files(file_list_path, dirpath):
         """Add files to the list to populate Fiwix file system"""
         for root, _, filepaths in os.walk(dirpath):
             if 'stage0-posix' in root:
@@ -258,7 +259,8 @@ class Generator():
             with open(image_file_name, 'ab') as image_file:
                 image_file.truncate(size * megabyte)
 
-    def check_file(self, file_name, expected_hash):
+    @staticmethod
+    def check_file(file_name, expected_hash):
         """Check hash of downloaded source file."""
         with open(file_name, "rb") as downloaded_file:
             downloaded_content = downloaded_file.read() # read entire file as bytes
@@ -271,7 +273,8 @@ actual:   {readable_hash}\n\
 When in doubt, try deleting the file in question -- it will be downloaded again when running \
 this script the next time")
 
-    def download_file(self, url, directory, file_name):
+    @staticmethod
+    def download_file(url, directory, file_name):
         """
         Download a single source archive.
         """

From 529ea4cae19361acdb2e64b4f667d568ba4b64df Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?G=C3=A1bor=20Stefanik?= <netrolller.3d@gmail.com>
Date: Fri, 22 Dec 2023 13:12:40 +0100
Subject: [PATCH 13/13] Explain significance of build-bash and improve-network
 directives

---
 steps/manifest | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/steps/manifest b/steps/manifest
index 4d475a1..b1ebe68 100644
--- a/steps/manifest
+++ b/steps/manifest
@@ -16,6 +16,13 @@
 # - jump: jump (usually) to a new kernel, executes a script with that name
 #   eg, jump: fiwix
 #
+# The following directives have special significance:
+# - build directives beginning with "bash" (as well as jumps) trigger the generation of
+#   a new script
+# - the first improve directive containing "network" is used by generator.py to deduce
+#   what source files need to be downloaded in advance (files referenced after that will
+#   be downloaded during bootstrap, unless --external-sources is given)
+#
 # Other features:
 # - predicate; based on variables set in bootstrap.cfg, require for something to execute
 #   must be enclosed in brackets with spaces padded