diff --git a/lib/generator.py b/lib/generator.py index e1dbac5..82ecd1e 100755 --- a/lib/generator.py +++ b/lib/generator.py @@ -14,6 +14,7 @@ import shutil import tarfile import requests +# pylint: disable=too-many-instance-attributes class Generator(): """ Class responsible for generating the basic media to be consumed. @@ -22,24 +23,25 @@ class Generator(): git_dir = os.path.join(os.path.dirname(os.path.join(__file__)), '..') distfiles_dir = os.path.join(git_dir, 'distfiles') - # pylint: disable=too-many-arguments - def __init__(self, tmpdir, arch, external_sources, - early_preseed, repo_path): + def __init__(self, arch, external_sources, early_preseed, repo_path): self.arch = arch self.early_preseed = early_preseed self.external_sources = external_sources self.repo_path = repo_path - self.tmpdir = tmpdir - self.tmp_dir = tmpdir.path - self.external_dir = os.path.join(self.tmp_dir, 'external') + self.source_manifest = self.get_source_manifest(not self.external_sources) + self.tmp_dir = None + self.external_dir = None - def prepare(self, using_kernel=False, kernel_bootstrap=False): + def prepare(self, tmpdir, using_kernel=False, kernel_bootstrap=False, target_size=0): """ Prepare basic media of live-bootstrap. /steps -- contains steps to be built / -- contains seed to allow steps to be built, containing custom scripts and stage0-posix """ + self.tmp_dir = tmpdir.path + self.external_dir = os.path.join(self.tmp_dir, 'external') + # We use ext3 here; ext4 actually has a variety of extensions that # have been added with varying levels of recency # Linux 4.9.10 does not support a bunch of them @@ -55,14 +57,17 @@ class Generator(): self.tmp_dir = init_path if self.repo_path or self.external_sources: - self.tmpdir.add_disk("external", filesystem="ext3") - self.tmpdir.mount_disk("external", "external") + tmpdir.add_disk("external", filesystem="ext3") + tmpdir.mount_disk("external", "external") else: - self.tmpdir.add_disk("external", tabletype="none") + self.external_dir = os.path.join(self.tmp_dir, 'external') elif using_kernel: self.tmp_dir = os.path.join(self.tmp_dir, 'disk') - self.tmpdir.add_disk("disk", filesystem="ext3") - self.tmpdir.mount_disk("disk", "disk") + tmpdir.add_disk("disk", + filesystem="ext3", + size=(target_size + "M") if target_size else "16G", + bootable=True) + tmpdir.mount_disk("disk", "disk") self.external_dir = os.path.join(self.tmp_dir, 'external') os.makedirs(self.external_dir, exist_ok=True) @@ -88,30 +93,29 @@ class Generator(): shutil.copytree(self.repo_path, repo_dir) if kernel_bootstrap: - self.create_builder_hex0_disk_image(os.path.join(self.tmp_dir, 'disk.img')) + self.create_builder_hex0_disk_image(self.tmp_dir + '.img', target_size) if kernel_bootstrap and (self.external_sources or self.repo_path): - self.tmpdir.umount_disk('external') + tmpdir.umount_disk('external') elif using_kernel: - self.tmpdir.umount_disk('disk') + tmpdir.umount_disk('disk') def steps(self): """Copy in steps.""" - source_manifest = self.get_source_manifest() - self.get_packages(source_manifest) + self.get_packages() shutil.copytree(os.path.join(self.git_dir, 'steps'), os.path.join(self.tmp_dir, 'steps')) def stage0_posix(self): """Copy in all of the stage0-posix""" stage0_posix_base_dir = os.path.join(self.git_dir, 'seed', 'stage0-posix') - for f in os.listdir(stage0_posix_base_dir): - orig = os.path.join(stage0_posix_base_dir, f) - to = os.path.join(self.tmp_dir, f) + for entry in os.listdir(stage0_posix_base_dir): + orig = os.path.join(stage0_posix_base_dir, entry) + target = os.path.join(self.tmp_dir, entry) if os.path.isfile(orig): - shutil.copy2(orig, to) + shutil.copy2(orig, target) else: - shutil.copytree(orig, to) + shutil.copytree(orig, target) arch = stage0_arch_map.get(self.arch, self.arch) kaem_optional_seed = os.path.join(self.git_dir, 'seed', 'stage0-posix', 'bootstrap-seeds', @@ -121,11 +125,12 @@ class Generator(): def seed(self): """Copy in extra seed files""" seed_dir = os.path.join(self.git_dir, 'seed') - for f in os.listdir(seed_dir): - if os.path.isfile(os.path.join(seed_dir, f)): - shutil.copy2(os.path.join(seed_dir, f), os.path.join(self.tmp_dir, f)) + for entry in os.listdir(seed_dir): + if os.path.isfile(os.path.join(seed_dir, entry)): + shutil.copy2(os.path.join(seed_dir, entry), os.path.join(self.tmp_dir, entry)) - def add_fiwix_files(self, file_list_path, dirpath): + @staticmethod + def add_fiwix_files(file_list_path, dirpath): """Add files to the list to populate Fiwix file system""" for root, _, filepaths in os.walk(dirpath): if 'stage0-posix' in root: @@ -151,13 +156,11 @@ class Generator(): def distfiles(self): """Copy in distfiles""" def copy_no_network_distfiles(out): - # Note that no network == no disk for kernel bootstrap mode - pre_src_path = os.path.join(self.git_dir, 'steps', 'pre-network-sources') - with open(pre_src_path, 'r', encoding="utf-8") as source_list: - for file in source_list.readlines(): - file = file.strip() - shutil.copy2(os.path.join(self.distfiles_dir, file), - os.path.join(out, file)) + # Note that "no disk" implies "no network" for kernel bootstrap mode + for file in self.source_manifest: + file = file[3].strip() + shutil.copy2(os.path.join(self.distfiles_dir, file), + os.path.join(out, file)) early_distfile_dir = os.path.join(self.tmp_dir, 'external', 'distfiles') main_distfile_dir = os.path.join(self.external_dir, 'distfiles') @@ -167,7 +170,6 @@ class Generator(): copy_no_network_distfiles(early_distfile_dir) if self.external_sources: - os.mkdir(main_distfile_dir) shutil.copytree(self.distfiles_dir, main_distfile_dir) else: os.mkdir(main_distfile_dir) @@ -224,7 +226,7 @@ class Generator(): os.chdir(save_cwd) - def create_builder_hex0_disk_image(self, image_file_name): + def create_builder_hex0_disk_image(self, image_file_name, size): """Create builder-hex0 disk image""" shutil.copyfile(os.path.join('seed', 'stage0-posix', 'bootstrap-seeds', 'NATIVE', 'x86', 'builder-hex0-x86-stage1.img'), @@ -252,13 +254,13 @@ class Generator(): image_file.write(b'\0' * round_up) current_size += round_up - # fill file with zeros up to desired size, one megabyte at a time - with open(image_file_name, 'ab') as image_file: - while current_size < 16384 * megabyte: - image_file.write(b'\0' * megabyte) - current_size += megabyte + # extend file up to desired size + if current_size < size * megabyte: + with open(image_file_name, 'ab') as image_file: + image_file.truncate(size * megabyte) - def check_file(self, file_name, expected_hash): + @staticmethod + def check_file(file_name, expected_hash): """Check hash of downloaded source file.""" with open(file_name, "rb") as downloaded_file: downloaded_content = downloaded_file.read() # read entire file as bytes @@ -271,7 +273,8 @@ actual: {readable_hash}\n\ When in doubt, try deleting the file in question -- it will be downloaded again when running \ this script the next time") - def download_file(self, url, directory, file_name): + @staticmethod + def download_file(url, directory, file_name): """ Download a single source archive. """ @@ -293,45 +296,53 @@ this script the next time") with open(abs_file_name, 'wb') as target_file: target_file.write(response.raw.read()) else: - raise requests.HTTPError("Download failed.") + raise requests.HTTPError("Download failed: HTTP " + + response.status_code + " " + response.reason) return abs_file_name - def get_packages(self, source_manifest): + def get_packages(self): """Prepare remaining sources""" - for line in source_manifest.split("\n"): - line = line.strip().split(" ") - + for line in self.source_manifest: path = self.download_file(line[2], line[1], line[3]) self.check_file(path, line[0]) @classmethod - def get_source_manifest(cls): + def get_source_manifest(cls, pre_network=False): """ Generate a source manifest for the system. """ - manifest_lines = [] + entries = [] directory = os.path.relpath(cls.distfiles_dir, cls.git_dir) # Find all source files steps_dir = os.path.join(cls.git_dir, 'steps') - for file in os.listdir(steps_dir): - if os.path.isdir(os.path.join(steps_dir, file)): - sourcef = os.path.join(steps_dir, file, "sources") + with open(os.path.join(steps_dir, 'manifest'), 'r', encoding="utf_8") as file: + for line in file: + if pre_network and line.strip().startswith("improve: ") and "network" in line: + break + + if not line.strip().startswith("build: "): + continue + + step = line.split(" ")[1].split("#")[0].strip() + sourcef = os.path.join(steps_dir, step, "sources") if os.path.exists(sourcef): # Read sources from the source file with open(sourcef, "r", encoding="utf_8") as sources: - for line in sources.readlines(): - line = line.strip().split(" ") + for source in sources.readlines(): + source = source.strip().split(" ") - if len(line) > 2: - file_name = line[2] + if len(source) > 2: + file_name = source[2] else: # Automatically determine file name based on URL. - file_name = os.path.basename(line[0]) + file_name = os.path.basename(source[0]) - manifest_lines.append(f"{line[1]} {directory} {line[0]} {file_name}") + entry = (source[1], directory, source[0], file_name) + if entry not in entries: + entries.append(entry) - return "\n".join(manifest_lines) + return entries stage0_arch_map = { "amd64": "AMD64", diff --git a/lib/tmpdir.py b/lib/tmpdir.py index f72286f..954235c 100644 --- a/lib/tmpdir.py +++ b/lib/tmpdir.py @@ -60,10 +60,21 @@ class Tmpdir: self._type = TmpType.TMPFS # pylint: disable=too-many-arguments - def add_disk(self, name, size="16G", filesystem="ext4", tabletype="msdos", mkfs_args=None): + def add_disk(self, + name, + size="16G", + filesystem="ext4", + tabletype="msdos", + bootable=False, + mkfs_args=None): """Add a disk""" disk_path = os.path.join(self.path, f"{name}.img") - self._disks[name] = create_disk(disk_path, tabletype, filesystem, size, mkfs_args=mkfs_args) + self._disks[name] = create_disk(disk_path, + tabletype, + filesystem, + size, + bootable, + mkfs_args) self._disk_filesystems[name] = filesystem # Allow executing user to access it run_as_root("chown", getpass.getuser(), self._disks[name]) @@ -87,4 +98,4 @@ class Tmpdir: def get_disk(self, name): """Get the path to a device of a disk""" - return self._disks[name] + return self._disks.get(name) diff --git a/lib/utils.py b/lib/utils.py index a80d656..d88ae4d 100755 --- a/lib/utils.py +++ b/lib/utils.py @@ -31,7 +31,8 @@ def run_as_root(*args, **kwargs): return run("sudo", *args, **kwargs) return run(*args, **kwargs) -def create_disk(image, disk_type, fs_type, size, mkfs_args=None): +# pylint: disable=too-many-arguments +def create_disk(image, disk_type, fs_type, size, bootable=False, mkfs_args=None): """Create a disk image, with a filesystem on it""" if mkfs_args is None: mkfs_args = [] @@ -42,7 +43,7 @@ def create_disk(image, disk_type, fs_type, size, mkfs_args=None): # Create the partition if disk_type != "none": run_as_root('parted', '--script', image, 'mklabel', disk_type, 'mkpart', - 'primary', fs_type, '0%', '100%') + 'primary', fs_type, '1GiB' if bootable else '1MiB', '100%') run_as_root('partprobe', loop_dev) run_as_root('mkfs.' + fs_type, loop_dev + "p1", *mkfs_args) return loop_dev diff --git a/rootfs.py b/rootfs.py index 648b1d0..8641d02 100755 --- a/rootfs.py +++ b/rootfs.py @@ -30,7 +30,6 @@ def create_configuration_file(args): with open(config_path, "w", encoding="utf_8") as config: config.write(f"FORCE_TIMESTAMPS={args.force_timestamps}\n") config.write(f"CHROOT={args.chroot or args.bwrap}\n") - config.write(f"CHROOT_ONLY_SYSA={args.bwrap}\n") config.write(f"UPDATE_CHECKSUMS={args.update_checksums}\n") config.write(f"JOBS={args.cores}\n") config.write(f"INTERNAL_CI={args.internal_ci}\n") @@ -39,7 +38,7 @@ def create_configuration_file(args): if args.repo or args.external_sources: config.write("DISK=sdb1\n") else: - config.write("DISK=sdb\n") + config.write("DISK=sda\n") config.write("KERNEL_BOOTSTRAP=True\n") else: config.write("DISK=sda1\n") @@ -98,7 +97,9 @@ def main(): default="qemu-system-x86_64") parser.add_argument("-qr", "--qemu-ram", help="Memory (in megabytes) allocated to QEMU VM", default=4096) - parser.add_argument("-qk", "--kernel", help="Custom sysa kernel to use") + parser.add_argument("-qs", "--target-size", help="Size of the target image (for QEMU only)", + default="16G") + parser.add_argument("-qk", "--kernel", help="Custom early kernel to use") parser.add_argument("-b", "--bare-metal", help="Build images for bare metal", action="store_true") @@ -136,15 +137,24 @@ def main(): if int(args.cores) < 1: raise ValueError("Must use one or more cores.") + # Target image size validation + if args.qemu: + if int(str(args.target_size).rstrip('gGmM')) < 1: + raise ValueError("Please specify a positive target size for qemu.") + args.target_size = (int(str(args.target_size).rstrip('gGmM')) * + (1024 if str(args.target_size).lower().endswith('g') else 1)) + else: + args.target_size = 0 + # bootstrap.cfg try: - os.remove(os.path.join('sysa', 'bootstrap.cfg')) + os.remove(os.path.join('steps', 'bootstrap.cfg')) except FileNotFoundError: pass if not args.no_create_config: create_configuration_file(args) else: - with open(os.path.join('sysa', 'bootstrap.cfg'), 'a', encoding='UTF-8'): + with open(os.path.join('steps', 'bootstrap.cfg'), 'a', encoding='UTF-8'): pass # tmpdir @@ -152,17 +162,16 @@ def main(): if args.tmpfs: tmpdir.tmpfs(size=args.tmpfs_size) - generator = Generator(tmpdir=tmpdir, - arch=args.arch, + generator = Generator(arch=args.arch, external_sources=args.external_sources, repo_path=args.repo, early_preseed=args.early_preseed) - bootstrap(args, generator, tmpdir) + bootstrap(args, generator, tmpdir, args.target_size) -def bootstrap(args, generator, tmpdir): +def bootstrap(args, generator, tmpdir, size): """Kick off bootstrap process.""" - print(f"Bootstrapping {args.arch} -- SysA") + print(f"Bootstrapping {args.arch}") if args.chroot: find_chroot = """ import shutil @@ -171,7 +180,7 @@ print(shutil.which('chroot')) chroot_binary = run_as_root('python3', '-c', find_chroot, capture_output=True).stdout.decode().strip() - generator.prepare(using_kernel=False) + generator.prepare(tmpdir, using_kernel=False) arch = stage0_arch_map.get(args.arch, args.arch) init = os.path.join(os.sep, 'bootstrap-seeds', 'POSIX', arch, 'kaem-optional-seed') @@ -179,7 +188,7 @@ print(shutil.which('chroot')) elif args.bwrap: if not args.internal_ci or args.internal_ci == "pass1": - generator.prepare(using_kernel=False) + generator.prepare(tmpdir, using_kernel=False) arch = stage0_arch_map.get(args.arch, args.arch) init = os.path.join(os.sep, 'bootstrap-seeds', 'POSIX', arch, 'kaem-optional-seed') @@ -200,15 +209,16 @@ print(shutil.which('chroot')) init) if not args.internal_ci or args.internal_ci == "pass2" or args.internal_ci == "pass3": - shutil.copy2(os.path.join('sysa', 'bootstrap.cfg'), - os.path.join('tmp', 'sysa', 'sysc_image', 'usr', 'src', 'bootstrap.cfg')) + os.makedirs(os.path.join(generator.tmp_dir, 'stage2', 'steps'), exist_ok=True) + shutil.copy2(os.path.join('steps', 'bootstrap.cfg'), + os.path.join(generator.tmp_dir, 'stage2', 'steps', 'bootstrap.cfg')) run('bwrap', '--unshare-user', '--uid', '0', '--gid', '0', '--unshare-net' if args.external_sources else None, '--clearenv', '--setenv', 'PATH', '/usr/bin', - '--bind', generator.tmp_dir + "/sysc_image", '/', + '--bind', os.path.join(generator.tmp_dir, "stage2"), '/', '--dir', '/dev', '--dev-bind', '/dev/null', '/dev/null', '--dev-bind', '/dev/zero', '/dev/zero', @@ -224,18 +234,20 @@ print(shutil.which('chroot')) elif args.bare_metal: if args.kernel: - generator.prepare(using_kernel=True) + generator.prepare(tmpdir, using_kernel=True, target_size=size) + image_path = os.path.join(args.tmpdir, os.path.relpath(generator.tmp_dir, args.tmpdir)) print("Please:") - print(" 1. Take tmp/initramfs and your kernel, boot using this.") - print(" 2. Take tmp/disk.img and put this on a writable storage medium.") + print(f" 1. Take {image_path}/initramfs and your kernel, boot using this.") + print(f" 2. Take {image_path}/disk.img and put this on a writable storage medium.") else: - generator.prepare(kernel_bootstrap=True) + generator.prepare(tmpdir, kernel_bootstrap=True, target_size=size) + image_path = os.path.join(args.tmpdir, os.path.relpath(generator.tmp_dir, args.tmpdir)) print("Please:") - print(" 1. Take tmp/disk.img and write it to a boot drive and then boot it.") + print(f" 1. Take {image_path}.img and write it to a boot drive and then boot it.") else: if args.kernel: - generator.prepare(using_kernel=True) + generator.prepare(tmpdir, using_kernel=True, target_size=size) run(args.qemu_cmd, '-enable-kvm', @@ -249,17 +261,24 @@ print(shutil.which('chroot')) '-nographic', '-append', 'console=ttyS0 root=/dev/sda1 rootfstype=ext3 init=/init rw') else: - generator.prepare(kernel_bootstrap=True) - run(args.qemu_cmd, + generator.prepare(tmpdir, kernel_bootstrap=True, target_size=size) + arg_list = [ '-enable-kvm', - '-m', "4G", + '-m', str(args.qemu_ram) + 'M', '-smp', str(args.cores), '-no-reboot', - '-drive', 'file=' + os.path.join(generator.tmp_dir, 'disk.img') + ',format=raw', - '-drive', 'file=' + tmpdir.get_disk("external") + ',format=raw', + '-drive', 'file=' + generator.tmp_dir + '.img' + ',format=raw' + ] + if tmpdir.get_disk("external") is not None: + arg_list += [ + '-drive', 'file=' + tmpdir.get_disk("external") + ',format=raw', + ] + arg_list += [ '-machine', 'kernel-irqchip=split', '-nic', 'user,ipv6=off,model=e1000', - '-nographic') + '-nographic' + ] + run(args.qemu_cmd, *arg_list) if __name__ == "__main__": main() diff --git a/source_manifest.py b/source_manifest.py index 6fa4e35..a972538 100755 --- a/source_manifest.py +++ b/source_manifest.py @@ -9,27 +9,11 @@ for the bootstrapping process. import argparse -from sysa import SysA -from sysc import SysC +from lib.generator import Generator def main(): """Generate a source manifest for a system""" - parser = argparse.ArgumentParser() - - parser.add_argument("-s", "--system", - help="Generate source manifest for the specified systems", - choices=["sysa", "sysc"], - nargs="+", - action="extend", - required=True) - - args = parser.parse_args() - - if "sysa" in args.system: - print(SysA.get_source_manifest()) - - if "sysc" in args.system: - print(SysC.get_source_manifest()) + print('\n'.join(map(' '.join, Generator.get_source_manifest()))) if __name__ == "__main__": main() diff --git a/steps/manifest b/steps/manifest index 4d475a1..b1ebe68 100644 --- a/steps/manifest +++ b/steps/manifest @@ -16,6 +16,13 @@ # - jump: jump (usually) to a new kernel, executes a script with that name # eg, jump: fiwix # +# The following directives have special significance: +# - build directives beginning with "bash" (as well as jumps) trigger the generation of +# a new script +# - the first improve directive containing "network" is used by generator.py to deduce +# what source files need to be downloaded in advance (files referenced after that will +# be downloaded during bootstrap, unless --external-sources is given) +# # Other features: # - predicate; based on variables set in bootstrap.cfg, require for something to execute # must be enclosed in brackets with spaces padded diff --git a/steps/pre-network-sources b/steps/pre-network-sources deleted file mode 100644 index 8b54d0d..0000000 --- a/steps/pre-network-sources +++ /dev/null @@ -1,86 +0,0 @@ -mes-0.25.tar.gz -nyacc-1.00.2.tar.gz -tcc-0.9.26.tar.gz -tcc-0.9.27.tar.bz2 -fiwix-1.4.0-lb3.tar.gz -lwext4-1.0.0-lb1.tar.gz -make-3.82.tar.bz2 -patch-2.5.9.tar.gz -gzip-1.2.4.tar.gz -tar-1.12.tar.gz -sed-4.0.9.tar.gz -bzip2-1.0.8.tar.gz -coreutils-5.0.tar.bz2 -heirloom-devtools-070527.tar.bz2 -bash-2.05b.tar.gz -flex-2.5.11.tar.gz -tcc-0.9.27.tar.bz2 -musl-1.1.24.tar.gz -tcc-0.9.27.tar.bz2 -musl-1.1.24.tar.gz -tcc-0.9.27.tar.bz2 -sed-4.0.9.tar.gz -bzip2-1.0.8.tar.gz -m4-1.4.7.tar.gz -flex-2.6.4.tar.gz -bison-3.4.1.tar.gz -bison-3.4.1.tar.gz -bison-3.4.1.tar.gz -grep-2.4.tar.gz -diffutils-2.7.tar.gz -coreutils-5.0.tar.bz2 -coreutils-6.10.tar.gz -gawk-3.0.4.tar.gz -perl-5.000.tar.gz -perl-5.003.tar.gz -perl5.004_05.tar.gz -perl5.005_03.tar.gz -perl-5.6.2.tar.gz -autoconf-2.52.tar.bz2 -automake-1.6.3.tar.bz2 -automake-1.6.3.tar.bz2 -autoconf-2.53.tar.bz2 -automake-1.7.tar.bz2 -autoconf-2.54.tar.bz2 -autoconf-2.55.tar.bz2 -automake-1.7.8.tar.bz2 -autoconf-2.57.tar.bz2 -autoconf-2.59.tar.bz2 -automake-1.8.5.tar.bz2 -help2man-1.36.4.tar.gz -autoconf-2.61.tar.bz2 -automake-1.9.6.tar.bz2 -automake-1.10.3.tar.bz2 -autoconf-2.64.tar.bz2 -automake-1.11.2.tar.bz2 -autoconf-2.69.tar.gz -libtool-2.2.4.tar.bz2 -automake-1.15.1.tar.gz -binutils-2.30.tar.bz2 -musl-1.1.24.tar.gz -tcc-0.9.27.tar.bz2 -gcc-core-4.0.4.tar.bz2 -automake-1.16.3.tar.gz -findutils-4.2.33.tar.gz -gnulib-8e128e.tar.gz -musl-1.2.4.tar.gz -gcc-core-4.0.4.tar.bz2 -automake-1.16.3.tar.gz -util-linux-2.19.1.tar.gz -e2fsprogs-1.45.7.tar.gz -CaseFolding.txt -DerivedAge.txt -DerivedCombiningClass.txt -DerivedCoreProperties.txt -NormalizationCorrections.txt -NormalizationTest.txt -UnicodeData.txt -v10.0.1.tar.gz -kbd-1.15.tar.gz -make-3.82.tar.bz2 -ed-1.4.tar.gz -bc-1.07.1.tar.gz -v2.0.22.tar.gz -linux-4.9.10.tar.gz -deblob-4.9 -curl-7.88.1.tar.bz2