From 05c13dd64ede6a8f1f83d7a16b304d07e1379b51 Mon Sep 17 00:00:00 2001 From: fosslinux Date: Wed, 8 Nov 2023 11:30:20 +1100 Subject: [PATCH] Modify rootfs.py infrastructure to support the new layout --- lib/generator.py | 336 ++++++++++++++++++ lib/sysgeneral.py | 132 ------- lib/tmpdir.py | 14 +- lib/utils.py | 6 +- rootfs.py | 65 ++-- steps/env | 2 +- .../files/make_fiwix_initrd.c | 2 +- sysa.py | 227 ------------ 8 files changed, 374 insertions(+), 410 deletions(-) create mode 100755 lib/generator.py delete mode 100644 lib/sysgeneral.py delete mode 100755 sysa.py diff --git a/lib/generator.py b/lib/generator.py new file mode 100755 index 0000000..d17a792 --- /dev/null +++ b/lib/generator.py @@ -0,0 +1,336 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-3.0-or-later +# SPDX-FileCopyrightText: 2022-2023 Dor Askayo +# SPDX-FileCopyrightText: 2021 Andrius Štikonas +# SPDX-FileCopyrightText: 2021 Melg Eight +# SPDX-FileCopyrightText: 2021-23 fosslinux + +import glob +import hashlib +import os +import shutil +import subprocess +import tarfile +import requests + +class Generator(): + """ + Class responsible for generating the basic media to be consumed. + """ + + git_dir = os.path.join(os.path.dirname(os.path.join(__file__)), '..') + distfiles_dir = os.path.join(git_dir, 'distfiles') + + # pylint: disable=too-many-arguments + def __init__(self, tmpdir, arch, external_sources, + early_preseed, repo_path): + self.arch = arch + self.early_preseed = early_preseed + self.external_sources = external_sources + self.repo_path = repo_path + self.tmpdir = tmpdir + self.tmp_dir = tmpdir.path + + def prepare(self, using_kernel=False, kernel_bootstrap=False): + """ + Prepare basic media of live-bootstrap. + /steps -- contains steps to be built + / -- contains seed to allow steps to be built, containing custom + scripts and stage0-posix + """ + self.external_dir = os.path.join(self.tmp_dir, 'external') + # We use ext3 here; ext4 actually has a variety of extensions that + # have been added with varying levels of recency + # Linux 4.9.10 does not support a bunch of them + # Attempting to disable extensions that a particular e2fsprogs + # is *unaware* of causes the filesystem creation to fail + # We could hypothetically detect e2fsprogs version and create an + # argument matrix ... or we could just use ext3 instead which + # is effectively universally the same + if kernel_bootstrap: + init_path = os.path.join(self.tmp_dir, 'init') + + os.mkdir(init_path) + self.tmp_dir = init_path + + if self.repo_path or self.external_sources: + self.tmpdir.add_disk("external", filesystem="ext3") + self.tmpdir.mount_disk("external", "external") + else: + self.tmpdir.add_disk("external", tabletype="none") + elif using_kernel: + self.tmp_dir = os.path.join(self.tmp_dir, 'disk') + self.tmpdir.add_disk("disk", filesystem="ext3") + self.tmpdir.mount_disk("disk", "disk") + self.external_dir = os.path.join(self.tmp_dir, 'external') + + os.makedirs(self.external_dir, exist_ok=True) + + if self.early_preseed: + # Extract tar containing preseed + with tarfile.open(self.early_preseed, "r") as seed: + seed.extractall(self.tmp_dir) + shutil.copy2(os.path.join(self.git_dir, 'seed', 'preseeded.kaem'), + os.path.join(self.tmp_dir, 'kaem.x86')) + else: + self.stage0_posix() + self.seed() + + self.steps() + + self.distfiles() + + self.create_fiwix_file_list() + + if self.repo_path: + repo_dir = os.path.join(self.external_dir, 'repo-preseeded') + shutil.copytree(self.repo_path, repo_dir) + + if kernel_bootstrap: + self.create_builder_hex0_disk_image(os.path.join(self.tmp_dir, 'disk.img')) + + if kernel_bootstrap and (self.external_sources or self.repo_path): + self.tmpdir.umount_disk('external') + elif using_kernel: + self.tmpdir.umount_disk('disk') + + def steps(self): + """Copy in steps.""" + source_manifest = self.get_source_manifest() + self.get_packages(source_manifest) + + shutil.copytree(os.path.join(self.git_dir, 'steps'), os.path.join(self.tmp_dir, 'steps')) + + def stage0_posix(self): + """Copy in all of the stage0-posix""" + stage0_posix_base_dir = os.path.join(self.git_dir, 'seed', 'stage0-posix') + for f in os.listdir(stage0_posix_base_dir): + orig = os.path.join(stage0_posix_base_dir, f) + to = os.path.join(self.tmp_dir, f) + if os.path.isfile(orig): + shutil.copy2(orig, to) + else: + shutil.copytree(orig, to) + + arch = stage0_arch_map.get(self.arch, self.arch) + kaem_optional_seed = os.path.join(self.git_dir, 'seed', 'stage0-posix', 'bootstrap-seeds', + 'POSIX', arch, 'kaem-optional-seed') + shutil.copy2(kaem_optional_seed, os.path.join(self.tmp_dir, 'init')) + + def seed(self): + """Copy in extra seed files""" + seed_dir = os.path.join(self.git_dir, 'seed') + for f in os.listdir(seed_dir): + if os.path.isfile(os.path.join(seed_dir, f)): + shutil.copy2(os.path.join(seed_dir, f), os.path.join(self.tmp_dir, f)) + + def add_fiwix_files(self, file_list_path, dirpath): + """Add files to the list to populate Fiwix file system""" + for root, _, filepaths in os.walk(dirpath): + if 'stage0-posix' in root: + continue + with open(file_list_path, 'a', encoding="utf-8") as file_list: + for filepath in filepaths: + file_list.write(f"/{os.path.join(root, filepath)}\n") + + def create_fiwix_file_list(self): + """Create a list of files to populate Fiwix file system""" + file_list_path = os.path.join(self.tmp_dir, 'steps', 'lwext4-1.0.0-lb1', + 'files', 'fiwix-file-list.txt') + shutil.copyfile(os.path.join(self.tmp_dir, 'steps', 'lwext4-1.0.0-lb1', + 'files', 'early-artifacts-needed-after-fiwix.txt'), + file_list_path) + + save_cwd = os.getcwd() + os.chdir(self.tmp_dir) + self.add_fiwix_files(file_list_path, 'steps') + self.add_fiwix_files(file_list_path, 'distfiles') + os.chdir(save_cwd) + + def distfiles(self): + """Copy in distfiles""" + def copy_no_network_distfiles(out): + # Note that no network == no disk for kernel bootstrap mode + with open(os.path.join(self.git_dir, 'steps', 'pre-network-sources'), 'r') as source_list: + for file in source_list.readlines(): + file = file.strip() + shutil.copy2(os.path.join(self.distfiles_dir, file), + os.path.join(out, file)) + + early_distfile_dir = os.path.join(self.tmp_dir, 'external', 'distfiles') + main_distfile_dir = os.path.join(self.external_dir, 'distfiles') + + if early_distfile_dir != main_distfile_dir: + os.makedirs(early_distfile_dir) + copy_no_network_distfiles(early_distfile_dir) + + if self.external_sources: + os.mkdir(main_distfile_dir) + shutil.copytree(self.distfiles_dir, main_distfile_dir) + else: + os.mkdir(main_distfile_dir) + copy_no_network_distfiles(main_distfile_dir) + + @staticmethod + def output_dir(srcfs_file, dirpath): + """Add a directory to srcfs file system""" + srcline = f"src 0 {dirpath}\n" + srcfs_file.write(srcline.encode()) + + @staticmethod + def output_file(srcfs_file, filepath): + """Add a file to srcfs file system""" + srcline = f"src {os.path.getsize(filepath)} {filepath}\n" + srcfs_file.write(srcline.encode()) + with open(filepath, 'rb') as srcfile: + srcfs_file.write(srcfile.read()) + + def output_tree(self, srcfs_file, treepath): + """Add a tree of files to srcfs file system""" + self.output_dir(srcfs_file, treepath) + for root, dirs, files in os.walk(treepath): + if ".git" in root: + continue + for dirpath in dirs: + if ".git" in dirpath: + continue + self.output_dir(srcfs_file, os.path.join(root, dirpath)) + + for filepath in files: + if ".git" in filepath: + continue + self.output_file(srcfs_file, os.path.join(root, filepath)) + + def append_srcfs(self, image_file): + """Append srcfs file system to disk image""" + save_cwd = os.getcwd() + + os.chdir(self.tmp_dir) + self.output_tree(image_file, '.') + + # Add commands to kick off stage0-posix + cmd = ' '.join(['hex0', + './bootstrap-seeds/POSIX/x86/hex0_x86.hex0', + './bootstrap-seeds/POSIX/x86/hex0-seed\n']) + image_file.write(cmd.encode()) + cmd = ' '.join(['hex0', + './bootstrap-seeds/POSIX/x86/kaem-minimal.hex0', + './bootstrap-seeds/POSIX/x86/kaem-optional-seed\n']) + image_file.write(cmd.encode()) + cmd = ' '.join(['./bootstrap-seeds/POSIX/x86/kaem-optional-seed', './kaem.x86\n']) + image_file.write(cmd.encode()) + + os.chdir(save_cwd) + + def create_builder_hex0_disk_image(self, image_file_name): + """Create builder-hex0 disk image""" + shutil.copyfile(os.path.join('seed', 'stage0-posix', 'bootstrap-seeds', + 'NATIVE', 'x86', 'builder-hex0-x86-stage1.img'), + image_file_name) + + with open(image_file_name, 'ab') as image_file: + # Append stage2 hex0 source + with open(os.path.join('kernel-bootstrap', 'builder-hex0-x86-stage2.hex0'), + encoding="utf-8") as infile: + image_file.write(infile.read().encode()) + # Pad to next sector + current_size = os.stat(image_file_name).st_size + while current_size % 512 != 0: + image_file.write(b'\0') + current_size += 1 + self.append_srcfs(image_file) + + current_size = os.stat(image_file_name).st_size + + megabyte = 1024 * 1024 + # fill file with zeros up to next megabyte + extra = current_size % megabyte + round_up = megabyte - extra + with open(image_file_name, 'ab') as image_file: + image_file.write(b'\0' * round_up) + current_size += round_up + + # fill file with zeros up to desired size, one megabyte at a time + with open(image_file_name, 'ab') as image_file: + while current_size < 16384 * megabyte: + image_file.write(b'\0' * megabyte) + current_size += megabyte + + def check_file(self, file_name, expected_hash): + """Check hash of downloaded source file.""" + with open(file_name, "rb") as downloaded_file: + downloaded_content = downloaded_file.read() # read entire file as bytes + readable_hash = hashlib.sha256(downloaded_content).hexdigest() + if expected_hash == readable_hash: + return + raise ValueError(f"Checksum mismatch for file {os.path.basename(file_name)}:\n\ +expected: {expected_hash}\n\ +actual: {readable_hash}\n\ +When in doubt, try deleting the file in question -- it will be downloaded again when running \ +this script the next time") + + def download_file(self, url, directory, file_name): + """ + Download a single source archive. + """ + abs_file_name = os.path.join(directory, file_name) + + # Create a directory for downloaded file + if not os.path.isdir(directory): + os.mkdir(directory) + + # Actually download the file + headers = { + "Accept-Encoding": "identity" + } + if not os.path.isfile(abs_file_name): + print(f"Downloading: {file_name}") + response = requests.get(url, allow_redirects=True, stream=True, + headers=headers, timeout=20) + if response.status_code == 200: + with open(abs_file_name, 'wb') as target_file: + target_file.write(response.raw.read()) + else: + raise requests.HTTPError("Download failed.") + return abs_file_name + + def get_packages(self, source_manifest): + """Prepare remaining sources""" + for line in source_manifest.split("\n"): + line = line.strip().split(" ") + + path = self.download_file(line[2], line[1], line[3]) + self.check_file(path, line[0]) + + @classmethod + def get_source_manifest(cls): + """ + Generate a source manifest for the system. + """ + manifest_lines = [] + directory = os.path.relpath(cls.distfiles_dir, cls.git_dir) + + # Find all source files + steps_dir = os.path.join(cls.git_dir, 'steps') + for file in os.listdir(steps_dir): + if os.path.isdir(os.path.join(steps_dir, file)): + sourcef = os.path.join(steps_dir, file, "sources") + if os.path.exists(sourcef): + # Read sources from the source file + with open(sourcef, "r", encoding="utf_8") as sources: + for line in sources.readlines(): + line = line.strip().split(" ") + + if len(line) > 2: + file_name = line[2] + else: + # Automatically determine file name based on URL. + file_name = os.path.basename(line[0]) + + manifest_lines.append(f"{line[1]} {directory} {line[0]} {file_name}") + + return "\n".join(manifest_lines) + +stage0_arch_map = { + "amd64": "AMD64", +} diff --git a/lib/sysgeneral.py b/lib/sysgeneral.py deleted file mode 100644 index 1938645..0000000 --- a/lib/sysgeneral.py +++ /dev/null @@ -1,132 +0,0 @@ -#!/usr/bin/env python3 -""" -This file contains a few functions to be shared by all Sys* classes -""" - -# SPDX-FileCopyrightText: 2022-2023 Dor Askayo -# SPDX-FileCopyrightText: 2021-23 fosslinux -# SPDX-FileCopyrightText: 2021 Andrius Štikonas -# SPDX-License-Identifier: GPL-3.0-or-later - -import os -import hashlib -import glob -import subprocess - -import requests - -class SysGeneral: - """ - A class from which all Sys* class are extended. - Contains functions used in all Sys* - """ - - # All of these are variables defined in the individual Sys* classes - cache_dir = None - base_dir = None - git_dir = None - sys_dir = None - initramfs_path = None - tmp_dir = None - - def check_file(self, file_name, expected_hash): - """Check hash of downloaded source file.""" - with open(file_name, "rb") as downloaded_file: - downloaded_content = downloaded_file.read() # read entire file as bytes - readable_hash = hashlib.sha256(downloaded_content).hexdigest() - if expected_hash == readable_hash: - return - raise ValueError(f"Checksum mismatch for file {os.path.basename(file_name)}:\n\ -expected: {expected_hash}\n\ -actual: {readable_hash}\n\ -When in doubt, try deleting the file in question -- it will be downloaded again when running \ -this script the next time") - - def download_file(self, url, directory, file_name): - """ - Download a single source archive. - """ - abs_file_name = os.path.join(directory, file_name) - - # Create a directory for downloaded file - if not os.path.isdir(directory): - os.mkdir(directory) - - # Actually download the file - headers = { - "Accept-Encoding": "identity" - } - if not os.path.isfile(abs_file_name): - print(f"Downloading: {file_name}") - response = requests.get(url, allow_redirects=True, stream=True, - headers=headers, timeout=20) - if response.status_code == 200: - with open(abs_file_name, 'wb') as target_file: - target_file.write(response.raw.read()) - else: - raise requests.HTTPError("Download failed.") - return abs_file_name - - def get_packages(self, source_manifest): - """Prepare remaining sources""" - for line in source_manifest.split("\n"): - line = line.strip().split(" ") - - path = self.download_file(line[2], line[1], line[3]) - self.check_file(path, line[0]) - - @classmethod - def get_source_manifest(cls): - """ - Generate a source manifest for the system. - """ - manifest_lines = [] - directory = os.path.relpath(cls.cache_dir, cls.git_dir) - - # Find all source files - for file in os.listdir(cls.sys_dir): - if os.path.isdir(os.path.join(cls.sys_dir, file)): - sourcef = os.path.join(cls.sys_dir, file, "sources") - if os.path.exists(sourcef): - # Read sources from the source file - with open(sourcef, "r", encoding="utf_8") as sources: - for line in sources.readlines(): - line = line.strip().split(" ") - - if len(line) > 2: - file_name = line[2] - else: - # Automatically determine file name based on URL. - file_name = os.path.basename(line[0]) - - manifest_lines.append(f"{line[1]} {directory} {line[0]} {file_name}") - - return "\n".join(manifest_lines) - - def make_initramfs(self): - """Package binary bootstrap seeds and sources into initramfs.""" - self.initramfs_path = os.path.join(self.tmp_dir, 'initramfs') - - # Create a list of files to go within the initramfs - file_list = glob.glob(os.path.join(self.tmp_dir, '**'), recursive=True) - - # Use built-in removeprefix once we can use Python 3.9 - def remove_prefix(text, prefix): - if text.startswith(prefix): - return text[len(prefix):] - return text # or whatever - - file_list = [remove_prefix(f, self.tmp_dir + os.sep) for f in file_list] - - # Create the initramfs - with open(self.initramfs_path, "w", encoding="utf_8") as initramfs: - # pylint: disable=consider-using-with - cpio = subprocess.Popen( - ["cpio", "--format", "newc", "--create", - "--directory", self.tmp_dir], - stdin=subprocess.PIPE, stdout=initramfs) - cpio.communicate(input='\n'.join(file_list).encode()) - -stage0_arch_map = { - "amd64": "AMD64", -} diff --git a/lib/tmpdir.py b/lib/tmpdir.py index ee607da..d372d04 100644 --- a/lib/tmpdir.py +++ b/lib/tmpdir.py @@ -24,7 +24,6 @@ class Tmpdir: Represents a tmpdir """ - _syses = {} _disks = {} _disk_filesystems = {} _mountpoints = {} @@ -60,19 +59,10 @@ class Tmpdir: mount("tmpfs", self.path, "tmpfs", f"size={size}") self._type = TmpType.TMPFS - def add_sys(self, name, subdir=None): - """Create a subdirectory and register a sys""" - if subdir is None: - subdir = name - sys_path = os.path.join(self.path, name) - if not os.path.exists(sys_path): - os.mkdir(sys_path) - return sys_path - - def add_disk(self, name, size="16G", filesystem="ext4"): + def add_disk(self, name, size="16G", filesystem="ext4", tabletype="msdos", mkfs_args=[]): """Add a disk""" disk_path = os.path.join(self.path, f"{name}.img") - self._disks[name] = create_disk(disk_path, "msdos", filesystem, size) + self._disks[name] = create_disk(disk_path, tabletype, filesystem, size, mkfs_args=mkfs_args) self._disk_filesystems[name] = filesystem # Allow executing user to access it run_as_root("chown", getpass.getuser(), self._disks[name]) diff --git a/lib/utils.py b/lib/utils.py index 5c6bfe2..b734f88 100755 --- a/lib/utils.py +++ b/lib/utils.py @@ -31,7 +31,7 @@ def run_as_root(*args, **kwargs): return run("sudo", *args, **kwargs) return run(*args, **kwargs) -def create_disk(image, disk_type, fs_type, size): +def create_disk(image, disk_type, fs_type, size, mkfs_args=[]): """Create a disk image, with a filesystem on it""" run('truncate', '-s', size, image) # First find the device we will use, then actually use it @@ -40,9 +40,9 @@ def create_disk(image, disk_type, fs_type, size): # Create the partition if disk_type != "none": run_as_root('parted', '--script', image, 'mklabel', disk_type, 'mkpart', - 'primary', 'ext4', '0%', '100%') + 'primary', fs_type, '0%', '100%') run_as_root('partprobe', loop_dev) - run_as_root('mkfs.' + fs_type, loop_dev + "p1") + run_as_root('mkfs.' + fs_type, loop_dev + "p1", *mkfs_args) return loop_dev def mount(source, target, fs_type, options='', **kwargs): diff --git a/rootfs.py b/rootfs.py index f4a6d22..648b1d0 100755 --- a/rootfs.py +++ b/rootfs.py @@ -17,18 +17,16 @@ import argparse import os import shutil -from sysa import SysA -from sysc import SysC from lib.utils import run, run_as_root -from lib.sysgeneral import stage0_arch_map from lib.tmpdir import Tmpdir +from lib.generator import Generator, stage0_arch_map def create_configuration_file(args): """ Creates bootstrap.cfg file which would contain options used to customize bootstrap. """ - config_path = os.path.join('sysa', 'bootstrap.cfg') + config_path = os.path.join('steps', 'bootstrap.cfg') with open(config_path, "w", encoding="utf_8") as config: config.write(f"FORCE_TIMESTAMPS={args.force_timestamps}\n") config.write(f"CHROOT={args.chroot or args.bwrap}\n") @@ -38,7 +36,10 @@ def create_configuration_file(args): config.write(f"INTERNAL_CI={args.internal_ci}\n") config.write(f"BARE_METAL={args.bare_metal}\n") if (args.bare_metal or args.qemu) and not args.kernel: - config.write("DISK=sda\n") + if args.repo or args.external_sources: + config.write("DISK=sdb1\n") + else: + config.write("DISK=sdb\n") config.write("KERNEL_BOOTSTRAP=True\n") else: config.write("DISK=sda1\n") @@ -49,7 +50,7 @@ def create_configuration_file(args): def main(): """ A few command line arguments to customize bootstrap. - This function also creates SysA object which prepares directory + This function also creates object which prepares directory structure with bootstrap seeds and all sources. """ parser = argparse.ArgumentParser() @@ -151,16 +152,15 @@ def main(): if args.tmpfs: tmpdir.tmpfs(size=args.tmpfs_size) - # sys - system_c = SysC(arch=args.arch, tmpdir=tmpdir, - external_sources=args.external_sources) - system_a = SysA(arch=args.arch, early_preseed=args.early_preseed, - tmpdir=tmpdir, external_sources=args.external_sources, - repo_path=args.repo) + generator = Generator(tmpdir=tmpdir, + arch=args.arch, + external_sources=args.external_sources, + repo_path=args.repo, + early_preseed=args.early_preseed) - bootstrap(args, system_a, system_c, tmpdir) + bootstrap(args, generator, tmpdir) -def bootstrap(args, system_a, system_c, tmpdir): +def bootstrap(args, generator, tmpdir): """Kick off bootstrap process.""" print(f"Bootstrapping {args.arch} -- SysA") if args.chroot: @@ -171,17 +171,15 @@ print(shutil.which('chroot')) chroot_binary = run_as_root('python3', '-c', find_chroot, capture_output=True).stdout.decode().strip() - system_c.prepare(create_disk_image=False) - system_a.prepare(create_initramfs=False) + generator.prepare(using_kernel=False) arch = stage0_arch_map.get(args.arch, args.arch) init = os.path.join(os.sep, 'bootstrap-seeds', 'POSIX', arch, 'kaem-optional-seed') - run_as_root('env', '-i', 'PATH=/bin', chroot_binary, system_a.tmp_dir, init) + run_as_root('env', '-i', 'PATH=/bin', chroot_binary, generator.tmp_dir, init) elif args.bwrap: if not args.internal_ci or args.internal_ci == "pass1": - system_c.prepare(create_disk_image=False) - system_a.prepare(create_initramfs=False) + generator.prepare(using_kernel=False) arch = stage0_arch_map.get(args.arch, args.arch) init = os.path.join(os.sep, 'bootstrap-seeds', 'POSIX', arch, 'kaem-optional-seed') @@ -191,7 +189,7 @@ print(shutil.which('chroot')) '--unshare-net', '--clearenv', '--setenv', 'PATH', '/usr/bin', - '--bind', system_a.tmp_dir, '/', + '--bind', generator.tmp_dir, '/', '--dir', '/dev', '--dev-bind', '/dev/null', '/dev/null', '--dev-bind', '/dev/zero', '/dev/zero', @@ -210,7 +208,7 @@ print(shutil.which('chroot')) '--unshare-net' if args.external_sources else None, '--clearenv', '--setenv', 'PATH', '/usr/bin', - '--bind', system_a.tmp_dir + "/sysc_image", '/', + '--bind', generator.tmp_dir + "/sysc_image", '/', '--dir', '/dev', '--dev-bind', '/dev/null', '/dev/null', '--dev-bind', '/dev/zero', '/dev/zero', @@ -226,40 +224,39 @@ print(shutil.which('chroot')) elif args.bare_metal: if args.kernel: - system_c.prepare(create_disk_image=True) - system_a.prepare(create_initramfs=True) + generator.prepare(using_kernel=True) print("Please:") - print(" 1. Take tmp/sysa/initramfs and your kernel, boot using this.") - print(" 2. Take tmp/sysc/disk.img and put this on a writable storage medium.") + print(" 1. Take tmp/initramfs and your kernel, boot using this.") + print(" 2. Take tmp/disk.img and put this on a writable storage medium.") else: - system_a.prepare(create_initramfs=True, kernel_bootstrap=True) + generator.prepare(kernel_bootstrap=True) print("Please:") - print(" 1. Take tmp/sysa/sysa.img and write it to a boot drive and then boot it.") + print(" 1. Take tmp/disk.img and write it to a boot drive and then boot it.") else: if args.kernel: - system_c.prepare(create_disk_image=True) - system_a.prepare(create_initramfs=True) + generator.prepare(using_kernel=True) run(args.qemu_cmd, '-enable-kvm', '-m', str(args.qemu_ram) + 'M', '-smp', str(args.cores), '-no-reboot', - '-hda', tmpdir.get_disk("sysc"), + '-drive', 'file=' + tmpdir.get_disk("disk") + ',format=raw', + '-drive', 'file=' + tmpdir.get_disk("external") + ',format=raw', '-nic', 'user,ipv6=off,model=e1000', '-kernel', args.kernel, - '-initrd', system_a.initramfs_path, '-nographic', - '-append', 'console=ttyS0') + '-append', 'console=ttyS0 root=/dev/sda1 rootfstype=ext3 init=/init rw') else: - system_a.prepare(create_initramfs=True, kernel_bootstrap=True) + generator.prepare(kernel_bootstrap=True) run(args.qemu_cmd, '-enable-kvm', '-m', "4G", '-smp', str(args.cores), '-no-reboot', - '-drive', 'file=' + os.path.join(system_a.tmp_dir, 'sysa.img') + ',format=raw', + '-drive', 'file=' + os.path.join(generator.tmp_dir, 'disk.img') + ',format=raw', + '-drive', 'file=' + tmpdir.get_disk("external") + ',format=raw', '-machine', 'kernel-irqchip=split', '-nic', 'user,ipv6=off,model=e1000', '-nographic') diff --git a/steps/env b/steps/env index 0dca1f9..7ae4955 100644 --- a/steps/env +++ b/steps/env @@ -1,4 +1,4 @@ -DISTFILES=/distfiles +DISTFILES=/external/distfiles PREFIX=/usr BINDIR=${PREFIX}/bin LIBDIR=${PREFIX}/lib/mes diff --git a/steps/lwext4-1.0.0-lb1/files/make_fiwix_initrd.c b/steps/lwext4-1.0.0-lb1/files/make_fiwix_initrd.c index d900948..668f7ca 100644 --- a/steps/lwext4-1.0.0-lb1/files/make_fiwix_initrd.c +++ b/steps/lwext4-1.0.0-lb1/files/make_fiwix_initrd.c @@ -202,7 +202,7 @@ int main(int argc, char **argv) char zeros[BLOCK_SIZE]; unsigned int next_file_address; - + next_file_address = *((unsigned int *) 0x7F8D); printf("Starting fiwix.ext2 at addr 0x%08x\n", next_file_address); diff --git a/sysa.py b/sysa.py deleted file mode 100755 index dff2bfe..0000000 --- a/sysa.py +++ /dev/null @@ -1,227 +0,0 @@ -#!/usr/bin/env python3 -"""System A""" -# SPDX-License-Identifier: GPL-3.0-or-later -# SPDX-FileCopyrightText: 2022-2023 Dor Askayo -# SPDX-FileCopyrightText: 2021 Andrius Štikonas -# SPDX-FileCopyrightText: 2021 Melg Eight -# SPDX-FileCopyrightText: 2021-23 fosslinux - -import os -# pylint: disable=deprecated-module -from distutils.dir_util import copy_tree -import shutil -import tarfile - -from lib.sysgeneral import SysGeneral, stage0_arch_map - -# pylint: disable=consider-using-with -# pylint: disable=too-many-instance-attributes -class SysA(SysGeneral): - """ - Class responsible for preparing sources for System A. - """ - - git_dir = os.path.dirname(os.path.join(__file__)) - sys_dir = os.path.join(git_dir, 'sysa') - sysb_dir = os.path.join(git_dir, 'sysb') - sysc_dir = os.path.join(git_dir, 'sysc') - cache_dir = os.path.join(sys_dir, 'distfiles') - - # pylint: disable=too-many-arguments - def __init__(self, tmpdir, arch, external_sources, - early_preseed, repo_path): - self.arch = arch - self.early_preseed = early_preseed - self.external_sources = external_sources - self.repo_path = repo_path - - self.tmp_dir = tmpdir.add_sys("sysa") - - def prepare(self, create_initramfs, kernel_bootstrap=False): - """ - Prepare directory structure for System A. - We create an empty tmp directory, unpack stage0-posix. - Rest of the files are unpacked into more structured directory /sysa - """ - if self.early_preseed: - # Extract tar containing preseed - with tarfile.open(self.early_preseed, "r") as seed: - seed.extractall(self.tmp_dir) - shutil.copy2(os.path.join(self.sys_dir, 'base-preseeded.kaem'), - os.path.join(self.tmp_dir, 'kaem.x86')) - else: - self.stage0_posix() - - self.sysa() - - # sysb must be added to sysa as it is another initramfs stage - self.sysb() - - self.sysc(create_initramfs) - - if kernel_bootstrap: - self.create_fiwix_file_list() - self.create_builder_hex0_disk_image(os.path.join(self.tmp_dir, 'sysa.img')) - return - - if self.repo_path: - repo_dir = os.path.join(self.tmp_dir, 'usr', 'src', 'repo-preseeded') - shutil.copytree(self.repo_path, repo_dir) - - if create_initramfs: - self.make_initramfs() - - def sysa(self): - """Copy in sysa files for sysa.""" - source_manifest = self.get_source_manifest() - self.get_packages(source_manifest) - - shutil.copytree(self.sys_dir, os.path.join(self.tmp_dir, 'sysa'), - ignore=shutil.ignore_patterns('tmp')) - - def sysb(self): - """Copy in sysb files for sysb.""" - shutil.copytree(self.sysb_dir, os.path.join(self.tmp_dir, 'sysb'), - ignore=shutil.ignore_patterns('tmp')) - - def sysc(self, create_initramfs): - """Copy in sysc files for sysc.""" - if create_initramfs or not self.external_sources: - ignore = shutil.ignore_patterns('tmp', 'distfiles') - else: - ignore = shutil.ignore_patterns('tmp') - shutil.copytree(self.sysc_dir, os.path.join(self.tmp_dir, 'sysc'), - ignore=ignore) - - def stage0_posix(self): - """Copy in all of the stage0-posix""" - stage0_posix_base_dir = os.path.join(self.sys_dir, 'stage0-posix', 'src') - copy_tree(stage0_posix_base_dir, self.tmp_dir) - - arch = stage0_arch_map.get(self.arch, self.arch) - kaem_optional_seed = os.path.join(self.sys_dir, 'stage0-posix', 'src', 'bootstrap-seeds', - 'POSIX', arch, 'kaem-optional-seed') - shutil.copy2(kaem_optional_seed, os.path.join(self.tmp_dir, 'init')) - - # stage0-posix hook to continue running live-bootstrap - shutil.copy2(os.path.join(self.sys_dir, 'after.kaem'), - os.path.join(self.tmp_dir, 'after.kaem')) - - def add_fiwix_files(self, file_list_path, dirpath): - """Add files to the list to populate Fiwix file system""" - for root, _, filepaths in os.walk(dirpath): - if 'stage0-posix' in root: - continue - if root == os.path.join('sysc', 'distfiles'): - continue - with open(file_list_path, 'a', encoding="utf-8") as file_list: - for filepath in filepaths: - file_list.write(f"/{os.path.join(root, filepath)}\n") - - def create_fiwix_file_list(self): - """Create a list of files to populate Fiwix file system""" - file_list_path = os.path.join(self.tmp_dir, 'sysa', 'lwext4-1.0.0-lb1', - 'files', 'fiwix-file-list.txt') - shutil.copyfile(os.path.join(self.tmp_dir, 'sysa', 'lwext4-1.0.0-lb1', - 'files', 'early-artifacts-needed-after-fiwix.txt'), - file_list_path) - - save_cwd = os.getcwd() - self.add_fiwix_files(file_list_path, 'sysa') - self.add_fiwix_files(file_list_path, 'sysb') - self.add_fiwix_files(file_list_path, 'sysc') - os.chdir(save_cwd) - - @staticmethod - def output_dir(srcfs_file, dirpath): - """Add a directory to srcfs file system""" - srcline = f"src 0 {dirpath}\n" - srcfs_file.write(srcline.encode()) - - @staticmethod - def output_file(srcfs_file, filepath): - """Add a file to srcfs file system""" - srcline = f"src {os.path.getsize(filepath)} {filepath}\n" - srcfs_file.write(srcline.encode()) - with open(filepath, 'rb') as srcfile: - srcfs_file.write(srcfile.read()) - - def output_tree(self, srcfs_file, treepath): - """Add a tree of files to srcfs file system""" - self.output_dir(srcfs_file, treepath) - for root, dirs, files in os.walk(treepath): - if ".git" in root: - continue - for dirpath in dirs: - if ".git" in dirpath: - continue - self.output_dir(srcfs_file, os.path.join(root, dirpath)) - - for filepath in files: - if ".git" in filepath: - continue - self.output_file(srcfs_file, os.path.join(root, filepath)) - - def append_srcfs(self, image_file): - """Append srcfs file system to sysa disk image""" - save_cwd = os.getcwd() - - os.chdir(os.path.join(self.tmp_dir, 'sysa', 'stage0-posix', 'src')) - self.output_tree(image_file, '.') - - os.chdir(self.tmp_dir) - shutil.move(os.path.join('sysa', 'stage0-posix'), '.') - self.output_tree(image_file, 'sysa') - self.output_tree(image_file, 'sysb') - self.output_tree(image_file, 'sysc') - shutil.move('stage0-posix', 'sysa') - shutil.copyfile(os.path.join('sysa', 'after.kaem'), 'after.kaem') - self.output_file(image_file, 'after.kaem') - - # Add commands to kick off stage0-posix - cmd = ' '.join(['hex0', - './bootstrap-seeds/POSIX/x86/hex0_x86.hex0', - './bootstrap-seeds/POSIX/x86/hex0-seed\n']) - image_file.write(cmd.encode()) - cmd = ' '.join(['hex0', - './bootstrap-seeds/POSIX/x86/kaem-minimal.hex0', - './bootstrap-seeds/POSIX/x86/kaem-optional-seed\n']) - image_file.write(cmd.encode()) - cmd = ' '.join(['./bootstrap-seeds/POSIX/x86/kaem-optional-seed', './kaem.x86\n']) - image_file.write(cmd.encode()) - - os.chdir(save_cwd) - - def create_builder_hex0_disk_image(self, image_file_name): - """Create builder-hex0 disk image""" - shutil.copyfile(os.path.join('sysa', 'stage0-posix', 'src', 'bootstrap-seeds', - 'NATIVE', 'x86', 'builder-hex0-x86-stage1.img'), - image_file_name) - - with open(image_file_name, 'ab') as image_file: - # Append stage2 hex0 source - with open(os.path.join('kernel-bootstrap', 'builder-hex0-x86-stage2.hex0'), - encoding="utf-8") as infile: - image_file.write(infile.read().encode()) - # Pad to next sector - current_size = os.stat(image_file_name).st_size - while current_size % 512 != 0: - image_file.write(b'\0') - current_size += 1 - self.append_srcfs(image_file) - - current_size = os.stat(image_file_name).st_size - - megabyte = 1024 * 1024 - # fill file with zeros up to next megabyte - extra = current_size % megabyte - round_up = megabyte - extra - with open(image_file_name, 'ab') as image_file: - image_file.write(b'\0' * round_up) - current_size += round_up - - # fill file with zeros up to desired size, one megabyte at a time - with open(image_file_name, 'ab') as image_file: - while current_size < 16384 * megabyte: - image_file.write(b'\0' * megabyte) - current_size += megabyte