Disk creation improvements

* Support specifying the size of the target disk image for qemu
* For bare metal, only pad the image to the next megabyte
* Use truncate() to extend images, instead of writing zeros (faster)
* Return None from get_disk() with nonexistent name
* Leave 1MiB on non-boot disks, or 1GiB on boot disks, unpartitioned
  (for proper 4K alignment and to help preserve the srcfs or boot
  partition creation)
* Fix qemu invocation when an external.img is not used
* Make -qr work with kernel bootstrap (will need kexec-fiwix fix)

lib/generator.py

@@ -33,7 +33,7 @@ class Generator():
         self.tmp_dir = tmpdir.path
         self.external_dir = os.path.join(self.tmp_dir, 'external')
 
-    def prepare(self, using_kernel=False, kernel_bootstrap=False):
+    def prepare(self, using_kernel=False, kernel_bootstrap=False, target_size=0):
         """
         Prepare basic media of live-bootstrap.
         /steps -- contains steps to be built
@@ -57,9 +57,14 @@ class Generator():
             if self.repo_path or self.external_sources:
                 self.tmpdir.add_disk("external", filesystem="ext3")
                 self.tmpdir.mount_disk("external", "external")
+            else:
+                self.external_dir = os.path.join(self.tmp_dir, 'external')
         elif using_kernel:
             self.tmp_dir = os.path.join(self.tmp_dir, 'disk')
-            self.tmpdir.add_disk("disk", filesystem="ext3")
+            self.tmpdir.add_disk("disk",
+                            filesystem="ext3",
+                            size=(target_size + "M") if target_size else "16G",
+                            bootable=True)
             self.tmpdir.mount_disk("disk", "disk")
             self.external_dir = os.path.join(self.tmp_dir, 'external')
 
@@ -86,7 +91,7 @@ class Generator():
             shutil.copytree(self.repo_path, repo_dir)
 
         if kernel_bootstrap:
-            self.create_builder_hex0_disk_image(os.path.join(self.tmp_dir, 'disk.img'))
+            self.create_builder_hex0_disk_image(os.path.join(self.tmp_dir, 'disk.img'), target_size)
 
         if kernel_bootstrap and (self.external_sources or self.repo_path):
             self.tmpdir.umount_disk('external')
@@ -149,7 +154,7 @@ class Generator():
     def distfiles(self):
         """Copy in distfiles"""
         def copy_no_network_distfiles(out):
-            # Note that no network == no disk for kernel bootstrap mode
+            # Note that "no disk" implies "no network" for kernel bootstrap mode
             pre_src_path = os.path.join(self.git_dir, 'steps', 'pre-network-sources')
             with open(pre_src_path, 'r', encoding="utf-8") as source_list:
                 for file in source_list.readlines():
@@ -222,7 +227,7 @@ class Generator():
 
         os.chdir(save_cwd)
 
-    def create_builder_hex0_disk_image(self, image_file_name):
+    def create_builder_hex0_disk_image(self, image_file_name, size):
         """Create builder-hex0 disk image"""
         shutil.copyfile(os.path.join('seed', 'stage0-posix', 'bootstrap-seeds',
                                      'NATIVE', 'x86', 'builder-hex0-x86-stage1.img'),
@@ -250,11 +255,10 @@ class Generator():
             image_file.write(b'\0' * round_up)
         current_size += round_up
 
-        # fill file with zeros up to desired size, one megabyte at a time
+        # extend file up to desired size
-        with open(image_file_name, 'ab') as image_file:
+        if current_size < size * megabyte:
-            while current_size < 16384 * megabyte:
+            with open(image_file_name, 'ab') as image_file:
-                image_file.write(b'\0' * megabyte)
+                image_file.truncate(size * megabyte)
-                current_size += megabyte
 
     def check_file(self, file_name, expected_hash):
         """Check hash of downloaded source file."""

lib/tmpdir.py

@@ -60,10 +60,21 @@ class Tmpdir:
         self._type = TmpType.TMPFS
 
     # pylint: disable=too-many-arguments
-    def add_disk(self, name, size="16G", filesystem="ext4", tabletype="msdos", mkfs_args=None):
+    def add_disk(self,
+                 name,
+                 size="16G",
+                 filesystem="ext4",
+                 tabletype="msdos",
+                 bootable=False,
+                 mkfs_args=None):
         """Add a disk"""
         disk_path = os.path.join(self.path, f"{name}.img")
-        self._disks[name] = create_disk(disk_path, tabletype, filesystem, size, mkfs_args=mkfs_args)
+        self._disks[name] = create_disk(disk_path,
+                                        tabletype,
+                                        filesystem,
+                                        size,
+                                        bootable,
+                                        mkfs_args)
         self._disk_filesystems[name] = filesystem
         # Allow executing user to access it
         run_as_root("chown", getpass.getuser(), self._disks[name])
@@ -87,4 +98,4 @@ class Tmpdir:
 
     def get_disk(self, name):
         """Get the path to a device of a disk"""
-        return self._disks[name]
+        return self._disks.get(name)

lib/utils.py

@@ -31,7 +31,8 @@ def run_as_root(*args, **kwargs):
         return run("sudo", *args, **kwargs)
     return run(*args, **kwargs)
 
-def create_disk(image, disk_type, fs_type, size, mkfs_args=None):
+# pylint: disable=too-many-arguments
+def create_disk(image, disk_type, fs_type, size, bootable=False, mkfs_args=None):
     """Create a disk image, with a filesystem on it"""
     if mkfs_args is None:
         mkfs_args = []
@@ -42,7 +43,7 @@ def create_disk(image, disk_type, fs_type, size, mkfs_args=None):
     # Create the partition
     if disk_type != "none":
         run_as_root('parted', '--script', image, 'mklabel', disk_type, 'mkpart',
-                'primary', fs_type, '0%', '100%')
+                'primary', fs_type, '1GiB' if bootable else '1MiB', '100%')
         run_as_root('partprobe', loop_dev)
         run_as_root('mkfs.' + fs_type, loop_dev + "p1", *mkfs_args)
     return loop_dev

rootfs.py

@@ -39,7 +39,7 @@ def create_configuration_file(args):
             if args.repo or args.external_sources:
                 config.write("DISK=sdb1\n")
             else:
-                config.write("DISK=sdb\n")
+                config.write("DISK=sda\n")
             config.write("KERNEL_BOOTSTRAP=True\n")
         else:
             config.write("DISK=sda1\n")
@@ -98,6 +98,8 @@ def main():
                         default="qemu-system-x86_64")
     parser.add_argument("-qr", "--qemu-ram", help="Memory (in megabytes) allocated to QEMU VM",
                         default=4096)
+    parser.add_argument("-qs", "--target-size", help="Size of the target image (for QEMU only)",
+                        default="16G")
     parser.add_argument("-qk", "--kernel", help="Custom sysa kernel to use")
 
     parser.add_argument("-b", "--bare-metal", help="Build images for bare metal",
@@ -136,6 +138,15 @@ def main():
     if int(args.cores) < 1:
         raise ValueError("Must use one or more cores.")
 
+    # Target image size validation
+    if args.qemu:
+        if int(str(args.target_size).rstrip('gGmM')) < 1:
+            raise ValueError("Please specify a positive target size for qemu.")
+        args.target_size = (int(str(args.target_size).rstrip('gGmM')) *
+            (1024 if str(args.target_size).lower().endswith('g') else 1))
+    else:
+        args.target_size = 0
+
     # bootstrap.cfg
     try:
         os.remove(os.path.join('sysa', 'bootstrap.cfg'))
@@ -158,9 +169,9 @@ def main():
                           repo_path=args.repo,
                           early_preseed=args.early_preseed)
 
-    bootstrap(args, generator, tmpdir)
+    bootstrap(args, generator, tmpdir, args.target_size)
 
-def bootstrap(args, generator, tmpdir):
+def bootstrap(args, generator, tmpdir, size):
     """Kick off bootstrap process."""
     print(f"Bootstrapping {args.arch} -- SysA")
     if args.chroot:
@@ -224,18 +235,18 @@ print(shutil.which('chroot'))
 
     elif args.bare_metal:
         if args.kernel:
-            generator.prepare(using_kernel=True)
+            generator.prepare(using_kernel=True, target_size=size)
             print("Please:")
             print("  1. Take tmp/initramfs and your kernel, boot using this.")
             print("  2. Take tmp/disk.img and put this on a writable storage medium.")
         else:
-            generator.prepare(kernel_bootstrap=True)
+            generator.prepare(kernel_bootstrap=True, target_size=size)
             print("Please:")
             print("  1. Take tmp/disk.img and write it to a boot drive and then boot it.")
 
     else:
         if args.kernel:
-            generator.prepare(using_kernel=True)
+            generator.prepare(using_kernel=True, target_size=size)
 
             run(args.qemu_cmd,
                 '-enable-kvm',
@@ -249,17 +260,24 @@ print(shutil.which('chroot'))
                 '-nographic',
                 '-append', 'console=ttyS0 root=/dev/sda1 rootfstype=ext3 init=/init rw')
         else:
-            generator.prepare(kernel_bootstrap=True)
+            generator.prepare(kernel_bootstrap=True, target_size=size)
-            run(args.qemu_cmd,
+            arg_list = [
                 '-enable-kvm',
-                '-m', "4G",
+                '-m', str(args.qemu_ram) + 'M',
                 '-smp', str(args.cores),
                 '-no-reboot',
-                '-drive', 'file=' + os.path.join(generator.tmp_dir, 'disk.img') + ',format=raw',
+                '-drive', 'file=' + os.path.join(generator.tmp_dir, 'disk.img') + ',format=raw'
-                '-drive', 'file=' + tmpdir.get_disk("external") + ',format=raw',
+            ]
+            if tmpdir.get_disk("external") is not None:
+                arg_list += [
+                    '-drive', 'file=' + tmpdir.get_disk("external") + ',format=raw',
+                ]
+            arg_list += [
                 '-machine', 'kernel-irqchip=split',
                 '-nic', 'user,ipv6=off,model=e1000',
-                '-nographic')
+                '-nographic'
+            ]
+            run(args.qemu_cmd, *arg_list)
 
 if __name__ == "__main__":
     main()