andrius
/
live-bootstrap
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Modify rootfs.py infrastructure to support the new layout

This commit is contained in:
fosslinux 2023-11-08 11:30:20 +11:00
parent 6ed2e09f3a
commit 05c13dd64e
8 changed files with 374 additions and 410 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

336
lib/generator.py Executable file
View File

@ -0,0 +1,336 @@
#!/usr/bin/env python3
# SPDX-License-Identifier: GPL-3.0-or-later
# SPDX-FileCopyrightText: 2022-2023 Dor Askayo <dor.askayo@gmail.com>
# SPDX-FileCopyrightText: 2021 Andrius Štikonas <andrius@stikonas.eu>
# SPDX-FileCopyrightText: 2021 Melg Eight <public.melg8@gmail.com>
# SPDX-FileCopyrightText: 2021-23 fosslinux <fosslinux@aussies.space>
import glob
import hashlib
import os
import shutil
import subprocess
import tarfile
import requests
class Generator():
"""
Class responsible for generating the basic media to be consumed.
"""
git_dir = os.path.join(os.path.dirname(os.path.join(__file__)), '..')
distfiles_dir = os.path.join(git_dir, 'distfiles')
# pylint: disable=too-many-arguments
def __init__(self, tmpdir, arch, external_sources,
early_preseed, repo_path):
self.arch = arch
self.early_preseed = early_preseed
self.external_sources = external_sources
self.repo_path = repo_path
self.tmpdir = tmpdir
self.tmp_dir = tmpdir.path
def prepare(self, using_kernel=False, kernel_bootstrap=False):
"""
Prepare basic media of live-bootstrap.
/steps -- contains steps to be built
/ -- contains seed to allow steps to be built, containing custom
scripts and stage0-posix
"""
self.external_dir = os.path.join(self.tmp_dir, 'external')
# We use ext3 here; ext4 actually has a variety of extensions that
# have been added with varying levels of recency
# Linux 4.9.10 does not support a bunch of them
# Attempting to disable extensions that a particular e2fsprogs
# is *unaware* of causes the filesystem creation to fail
# We could hypothetically detect e2fsprogs version and create an
# argument matrix ... or we could just use ext3 instead which
# is effectively universally the same
if kernel_bootstrap:
init_path = os.path.join(self.tmp_dir, 'init')
os.mkdir(init_path)
self.tmp_dir = init_path
if self.repo_path or self.external_sources:
self.tmpdir.add_disk("external", filesystem="ext3")
self.tmpdir.mount_disk("external", "external")
else:
self.tmpdir.add_disk("external", tabletype="none")
elif using_kernel:
self.tmp_dir = os.path.join(self.tmp_dir, 'disk')
self.tmpdir.add_disk("disk", filesystem="ext3")
self.tmpdir.mount_disk("disk", "disk")
self.external_dir = os.path.join(self.tmp_dir, 'external')
os.makedirs(self.external_dir, exist_ok=True)
if self.early_preseed:
# Extract tar containing preseed
with tarfile.open(self.early_preseed, "r") as seed:
seed.extractall(self.tmp_dir)
shutil.copy2(os.path.join(self.git_dir, 'seed', 'preseeded.kaem'),
os.path.join(self.tmp_dir, 'kaem.x86'))
else:
self.stage0_posix()
self.seed()
self.steps()
self.distfiles()
self.create_fiwix_file_list()
if self.repo_path:
repo_dir = os.path.join(self.external_dir, 'repo-preseeded')
shutil.copytree(self.repo_path, repo_dir)
if kernel_bootstrap:
self.create_builder_hex0_disk_image(os.path.join(self.tmp_dir, 'disk.img'))
if kernel_bootstrap and (self.external_sources or self.repo_path):
self.tmpdir.umount_disk('external')
elif using_kernel:
self.tmpdir.umount_disk('disk')
def steps(self):
"""Copy in steps."""
source_manifest = self.get_source_manifest()
self.get_packages(source_manifest)
shutil.copytree(os.path.join(self.git_dir, 'steps'), os.path.join(self.tmp_dir, 'steps'))
def stage0_posix(self):
"""Copy in all of the stage0-posix"""
stage0_posix_base_dir = os.path.join(self.git_dir, 'seed', 'stage0-posix')
for f in os.listdir(stage0_posix_base_dir):
orig = os.path.join(stage0_posix_base_dir, f)
to = os.path.join(self.tmp_dir, f)
if os.path.isfile(orig):
shutil.copy2(orig, to)
else:
shutil.copytree(orig, to)
arch = stage0_arch_map.get(self.arch, self.arch)
kaem_optional_seed = os.path.join(self.git_dir, 'seed', 'stage0-posix', 'bootstrap-seeds',
'POSIX', arch, 'kaem-optional-seed')
shutil.copy2(kaem_optional_seed, os.path.join(self.tmp_dir, 'init'))
def seed(self):
"""Copy in extra seed files"""
seed_dir = os.path.join(self.git_dir, 'seed')
for f in os.listdir(seed_dir):
if os.path.isfile(os.path.join(seed_dir, f)):
shutil.copy2(os.path.join(seed_dir, f), os.path.join(self.tmp_dir, f))
def add_fiwix_files(self, file_list_path, dirpath):
"""Add files to the list to populate Fiwix file system"""
for root, _, filepaths in os.walk(dirpath):
if 'stage0-posix' in root:
continue
with open(file_list_path, 'a', encoding="utf-8") as file_list:
for filepath in filepaths:
file_list.write(f"/{os.path.join(root, filepath)}\n")
def create_fiwix_file_list(self):
"""Create a list of files to populate Fiwix file system"""
file_list_path = os.path.join(self.tmp_dir, 'steps', 'lwext4-1.0.0-lb1',
'files', 'fiwix-file-list.txt')
shutil.copyfile(os.path.join(self.tmp_dir, 'steps', 'lwext4-1.0.0-lb1',
'files', 'early-artifacts-needed-after-fiwix.txt'),
file_list_path)
save_cwd = os.getcwd()
os.chdir(self.tmp_dir)
self.add_fiwix_files(file_list_path, 'steps')
self.add_fiwix_files(file_list_path, 'distfiles')
os.chdir(save_cwd)
def distfiles(self):
"""Copy in distfiles"""
def copy_no_network_distfiles(out):
# Note that no network == no disk for kernel bootstrap mode
with open(os.path.join(self.git_dir, 'steps', 'pre-network-sources'), 'r') as source_list:
for file in source_list.readlines():
file = file.strip()
shutil.copy2(os.path.join(self.distfiles_dir, file),
os.path.join(out, file))
early_distfile_dir = os.path.join(self.tmp_dir, 'external', 'distfiles')
main_distfile_dir = os.path.join(self.external_dir, 'distfiles')
if early_distfile_dir != main_distfile_dir:
os.makedirs(early_distfile_dir)
copy_no_network_distfiles(early_distfile_dir)
if self.external_sources:
os.mkdir(main_distfile_dir)
shutil.copytree(self.distfiles_dir, main_distfile_dir)
else:
os.mkdir(main_distfile_dir)
copy_no_network_distfiles(main_distfile_dir)
@staticmethod
def output_dir(srcfs_file, dirpath):
"""Add a directory to srcfs file system"""
srcline = f"src 0 {dirpath}\n"
srcfs_file.write(srcline.encode())
@staticmethod
def output_file(srcfs_file, filepath):
"""Add a file to srcfs file system"""
srcline = f"src {os.path.getsize(filepath)} {filepath}\n"
srcfs_file.write(srcline.encode())
with open(filepath, 'rb') as srcfile:
srcfs_file.write(srcfile.read())
def output_tree(self, srcfs_file, treepath):
"""Add a tree of files to srcfs file system"""
self.output_dir(srcfs_file, treepath)
for root, dirs, files in os.walk(treepath):
if ".git" in root:
continue
for dirpath in dirs:
if ".git" in dirpath:
continue
self.output_dir(srcfs_file, os.path.join(root, dirpath))
for filepath in files:
if ".git" in filepath:
continue
self.output_file(srcfs_file, os.path.join(root, filepath))
def append_srcfs(self, image_file):
"""Append srcfs file system to disk image"""
save_cwd = os.getcwd()
os.chdir(self.tmp_dir)
self.output_tree(image_file, '.')
# Add commands to kick off stage0-posix
cmd = ' '.join(['hex0',
'./bootstrap-seeds/POSIX/x86/hex0_x86.hex0',
'./bootstrap-seeds/POSIX/x86/hex0-seed\n'])
image_file.write(cmd.encode())
cmd = ' '.join(['hex0',
'./bootstrap-seeds/POSIX/x86/kaem-minimal.hex0',
'./bootstrap-seeds/POSIX/x86/kaem-optional-seed\n'])
image_file.write(cmd.encode())
cmd = ' '.join(['./bootstrap-seeds/POSIX/x86/kaem-optional-seed', './kaem.x86\n'])
image_file.write(cmd.encode())
os.chdir(save_cwd)
def create_builder_hex0_disk_image(self, image_file_name):
"""Create builder-hex0 disk image"""
shutil.copyfile(os.path.join('seed', 'stage0-posix', 'bootstrap-seeds',
'NATIVE', 'x86', 'builder-hex0-x86-stage1.img'),
image_file_name)
with open(image_file_name, 'ab') as image_file:
# Append stage2 hex0 source
with open(os.path.join('kernel-bootstrap', 'builder-hex0-x86-stage2.hex0'),
encoding="utf-8") as infile:
image_file.write(infile.read().encode())
# Pad to next sector
current_size = os.stat(image_file_name).st_size
while current_size % 512 != 0:
image_file.write(b'\0')
current_size += 1
self.append_srcfs(image_file)
current_size = os.stat(image_file_name).st_size
megabyte = 1024 * 1024
# fill file with zeros up to next megabyte
extra = current_size % megabyte
round_up = megabyte - extra
with open(image_file_name, 'ab') as image_file:
image_file.write(b'\0' * round_up)
current_size += round_up
# fill file with zeros up to desired size, one megabyte at a time
with open(image_file_name, 'ab') as image_file:
while current_size < 16384 * megabyte:
image_file.write(b'\0' * megabyte)
current_size += megabyte
def check_file(self, file_name, expected_hash):
"""Check hash of downloaded source file."""
with open(file_name, "rb") as downloaded_file:
downloaded_content = downloaded_file.read() # read entire file as bytes
readable_hash = hashlib.sha256(downloaded_content).hexdigest()
if expected_hash == readable_hash:
return
raise ValueError(f"Checksum mismatch for file {os.path.basename(file_name)}:\n\
expected: {expected_hash}\n\
actual: {readable_hash}\n\
When in doubt, try deleting the file in question -- it will be downloaded again when running \
this script the next time")
def download_file(self, url, directory, file_name):
"""
Download a single source archive.
"""
abs_file_name = os.path.join(directory, file_name)
# Create a directory for downloaded file
if not os.path.isdir(directory):
os.mkdir(directory)
# Actually download the file
headers = {
"Accept-Encoding": "identity"
}
if not os.path.isfile(abs_file_name):
print(f"Downloading: {file_name}")
response = requests.get(url, allow_redirects=True, stream=True,
headers=headers, timeout=20)
if response.status_code == 200:
with open(abs_file_name, 'wb') as target_file:
target_file.write(response.raw.read())
else:
raise requests.HTTPError("Download failed.")
return abs_file_name
def get_packages(self, source_manifest):
"""Prepare remaining sources"""
for line in source_manifest.split("\n"):
line = line.strip().split(" ")
path = self.download_file(line[2], line[1], line[3])
self.check_file(path, line[0])
@classmethod
def get_source_manifest(cls):
"""
Generate a source manifest for the system.
"""
manifest_lines = []
directory = os.path.relpath(cls.distfiles_dir, cls.git_dir)
# Find all source files
steps_dir = os.path.join(cls.git_dir, 'steps')
for file in os.listdir(steps_dir):
if os.path.isdir(os.path.join(steps_dir, file)):
sourcef = os.path.join(steps_dir, file, "sources")
if os.path.exists(sourcef):
# Read sources from the source file
with open(sourcef, "r", encoding="utf_8") as sources:
for line in sources.readlines():
line = line.strip().split(" ")
if len(line) > 2:
file_name = line[2]
else:
# Automatically determine file name based on URL.
file_name = os.path.basename(line[0])
manifest_lines.append(f"{line[1]} {directory} {line[0]} {file_name}")
return "\n".join(manifest_lines)
stage0_arch_map = {
"amd64": "AMD64",
}

132
lib/sysgeneral.py
View File

@ -1,132 +0,0 @@
#!/usr/bin/env python3
"""
This file contains a few functions to be shared by all Sys* classes
"""
# SPDX-FileCopyrightText: 2022-2023 Dor Askayo <dor.askayo@gmail.com>
# SPDX-FileCopyrightText: 2021-23 fosslinux <fosslinux@aussies.space>
# SPDX-FileCopyrightText: 2021 Andrius Štikonas <andrius@stikonas.eu>
# SPDX-License-Identifier: GPL-3.0-or-later
import os
import hashlib
import glob
import subprocess
import requests
class SysGeneral:
"""
A class from which all Sys* class are extended.
Contains functions used in all Sys*
"""
# All of these are variables defined in the individual Sys* classes
cache_dir = None
base_dir = None
git_dir = None
sys_dir = None
initramfs_path = None
tmp_dir = None
def check_file(self, file_name, expected_hash):
"""Check hash of downloaded source file."""
with open(file_name, "rb") as downloaded_file:
downloaded_content = downloaded_file.read() # read entire file as bytes
readable_hash = hashlib.sha256(downloaded_content).hexdigest()
if expected_hash == readable_hash:
return
raise ValueError(f"Checksum mismatch for file {os.path.basename(file_name)}:\n\
expected: {expected_hash}\n\
actual: {readable_hash}\n\
When in doubt, try deleting the file in question -- it will be downloaded again when running \
this script the next time")
def download_file(self, url, directory, file_name):
"""
Download a single source archive.
"""
abs_file_name = os.path.join(directory, file_name)
# Create a directory for downloaded file
if not os.path.isdir(directory):
os.mkdir(directory)
# Actually download the file
headers = {
"Accept-Encoding": "identity"
}
if not os.path.isfile(abs_file_name):
print(f"Downloading: {file_name}")
response = requests.get(url, allow_redirects=True, stream=True,
headers=headers, timeout=20)
if response.status_code == 200:
with open(abs_file_name, 'wb') as target_file:
target_file.write(response.raw.read())
else:
raise requests.HTTPError("Download failed.")
return abs_file_name
def get_packages(self, source_manifest):
"""Prepare remaining sources"""
for line in source_manifest.split("\n"):
line = line.strip().split(" ")
path = self.download_file(line[2], line[1], line[3])
self.check_file(path, line[0])
@classmethod
def get_source_manifest(cls):
"""
Generate a source manifest for the system.
"""
manifest_lines = []
directory = os.path.relpath(cls.cache_dir, cls.git_dir)
# Find all source files
for file in os.listdir(cls.sys_dir):
if os.path.isdir(os.path.join(cls.sys_dir, file)):
sourcef = os.path.join(cls.sys_dir, file, "sources")
if os.path.exists(sourcef):
# Read sources from the source file
with open(sourcef, "r", encoding="utf_8") as sources:
for line in sources.readlines():
line = line.strip().split(" ")
if len(line) > 2:
file_name = line[2]
else:
# Automatically determine file name based on URL.
file_name = os.path.basename(line[0])
manifest_lines.append(f"{line[1]} {directory} {line[0]} {file_name}")
return "\n".join(manifest_lines)
def make_initramfs(self):
"""Package binary bootstrap seeds and sources into initramfs."""
self.initramfs_path = os.path.join(self.tmp_dir, 'initramfs')
# Create a list of files to go within the initramfs
file_list = glob.glob(os.path.join(self.tmp_dir, '**'), recursive=True)
# Use built-in removeprefix once we can use Python 3.9
def remove_prefix(text, prefix):
if text.startswith(prefix):
return text[len(prefix):]
return text # or whatever
file_list = [remove_prefix(f, self.tmp_dir + os.sep) for f in file_list]
# Create the initramfs
with open(self.initramfs_path, "w", encoding="utf_8") as initramfs:
# pylint: disable=consider-using-with
cpio = subprocess.Popen(
["cpio", "--format", "newc", "--create",
"--directory", self.tmp_dir],
stdin=subprocess.PIPE, stdout=initramfs)
cpio.communicate(input='\n'.join(file_list).encode())
stage0_arch_map = {
"amd64": "AMD64",
}

14
lib/tmpdir.py
View File

@ -24,7 +24,6 @@ class Tmpdir:
Represents a tmpdir
"""
_syses = {}
_disks = {}
_disk_filesystems = {}
_mountpoints = {}
@ -60,19 +59,10 @@ class Tmpdir:
mount("tmpfs", self.path, "tmpfs", f"size={size}")
self._type = TmpType.TMPFS
def add_sys(self, name, subdir=None):
"""Create a subdirectory and register a sys"""
if subdir is None:
subdir = name
sys_path = os.path.join(self.path, name)
if not os.path.exists(sys_path):
os.mkdir(sys_path)
return sys_path
def add_disk(self, name, size="16G", filesystem="ext4"):
def add_disk(self, name, size="16G", filesystem="ext4", tabletype="msdos", mkfs_args=[]):
"""Add a disk"""
disk_path = os.path.join(self.path, f"{name}.img")
self._disks[name] = create_disk(disk_path, "msdos", filesystem, size)
self._disks[name] = create_disk(disk_path, tabletype, filesystem, size, mkfs_args=mkfs_args)
self._disk_filesystems[name] = filesystem
# Allow executing user to access it
run_as_root("chown", getpass.getuser(), self._disks[name])

6
lib/utils.py
View File

@ -31,7 +31,7 @@ def run_as_root(*args, **kwargs):
return run("sudo", *args, **kwargs)
return run(*args, **kwargs)
def create_disk(image, disk_type, fs_type, size):
def create_disk(image, disk_type, fs_type, size, mkfs_args=[]):
"""Create a disk image, with a filesystem on it"""
run('truncate', '-s', size, image)
# First find the device we will use, then actually use it
@ -40,9 +40,9 @@ def create_disk(image, disk_type, fs_type, size):
# Create the partition
if disk_type != "none":
run_as_root('parted', '--script', image, 'mklabel', disk_type, 'mkpart',
'primary', 'ext4', '0%', '100%')
'primary', fs_type, '0%', '100%')
run_as_root('partprobe', loop_dev)
run_as_root('mkfs.' + fs_type, loop_dev + "p1")
run_as_root('mkfs.' + fs_type, loop_dev + "p1", *mkfs_args)
return loop_dev
def mount(source, target, fs_type, options='', **kwargs):

65
rootfs.py
View File

@ -17,18 +17,16 @@ import argparse
import os
import shutil
from sysa import SysA
from sysc import SysC
from lib.utils import run, run_as_root
from lib.sysgeneral import stage0_arch_map
from lib.tmpdir import Tmpdir
from lib.generator import Generator, stage0_arch_map
def create_configuration_file(args):
"""
Creates bootstrap.cfg file which would contain options used to
customize bootstrap.
"""
config_path = os.path.join('sysa', 'bootstrap.cfg')
config_path = os.path.join('steps', 'bootstrap.cfg')
with open(config_path, "w", encoding="utf_8") as config:
config.write(f"FORCE_TIMESTAMPS={args.force_timestamps}\n")
config.write(f"CHROOT={args.chroot or args.bwrap}\n")
@ -38,7 +36,10 @@ def create_configuration_file(args):
config.write(f"INTERNAL_CI={args.internal_ci}\n")
config.write(f"BARE_METAL={args.bare_metal}\n")
if (args.bare_metal or args.qemu) and not args.kernel:
config.write("DISK=sda\n")
if args.repo or args.external_sources:
config.write("DISK=sdb1\n")
else:
config.write("DISK=sdb\n")
config.write("KERNEL_BOOTSTRAP=True\n")
else:
config.write("DISK=sda1\n")
@ -49,7 +50,7 @@ def create_configuration_file(args):
def main():
"""
A few command line arguments to customize bootstrap.
This function also creates SysA object which prepares directory
This function also creates object which prepares directory
structure with bootstrap seeds and all sources.
"""
parser = argparse.ArgumentParser()
@ -151,16 +152,15 @@ def main():
if args.tmpfs:
tmpdir.tmpfs(size=args.tmpfs_size)
# sys
system_c = SysC(arch=args.arch, tmpdir=tmpdir,
external_sources=args.external_sources)
system_a = SysA(arch=args.arch, early_preseed=args.early_preseed,
tmpdir=tmpdir, external_sources=args.external_sources,
repo_path=args.repo)
generator = Generator(tmpdir=tmpdir,
arch=args.arch,
external_sources=args.external_sources,
repo_path=args.repo,
early_preseed=args.early_preseed)
bootstrap(args, system_a, system_c, tmpdir)
bootstrap(args, generator, tmpdir)
def bootstrap(args, system_a, system_c, tmpdir):
def bootstrap(args, generator, tmpdir):
"""Kick off bootstrap process."""
print(f"Bootstrapping {args.arch} -- SysA")
if args.chroot:
@ -171,17 +171,15 @@ print(shutil.which('chroot'))
chroot_binary = run_as_root('python3', '-c', find_chroot,
capture_output=True).stdout.decode().strip()
system_c.prepare(create_disk_image=False)
system_a.prepare(create_initramfs=False)
generator.prepare(using_kernel=False)
arch = stage0_arch_map.get(args.arch, args.arch)
init = os.path.join(os.sep, 'bootstrap-seeds', 'POSIX', arch, 'kaem-optional-seed')
run_as_root('env', '-i', 'PATH=/bin', chroot_binary, system_a.tmp_dir, init)
run_as_root('env', '-i', 'PATH=/bin', chroot_binary, generator.tmp_dir, init)
elif args.bwrap:
if not args.internal_ci or args.internal_ci == "pass1":
system_c.prepare(create_disk_image=False)
system_a.prepare(create_initramfs=False)
generator.prepare(using_kernel=False)
arch = stage0_arch_map.get(args.arch, args.arch)
init = os.path.join(os.sep, 'bootstrap-seeds', 'POSIX', arch, 'kaem-optional-seed')
@ -191,7 +189,7 @@ print(shutil.which('chroot'))
'--unshare-net',
'--clearenv',
'--setenv', 'PATH', '/usr/bin',
'--bind', system_a.tmp_dir, '/',
'--bind', generator.tmp_dir, '/',
'--dir', '/dev',
'--dev-bind', '/dev/null', '/dev/null',
'--dev-bind', '/dev/zero', '/dev/zero',
@ -210,7 +208,7 @@ print(shutil.which('chroot'))
'--unshare-net' if args.external_sources else None,
'--clearenv',
'--setenv', 'PATH', '/usr/bin',
'--bind', system_a.tmp_dir + "/sysc_image", '/',
'--bind', generator.tmp_dir + "/sysc_image", '/',
'--dir', '/dev',
'--dev-bind', '/dev/null', '/dev/null',
'--dev-bind', '/dev/zero', '/dev/zero',
@ -226,40 +224,39 @@ print(shutil.which('chroot'))
elif args.bare_metal:
if args.kernel:
system_c.prepare(create_disk_image=True)
system_a.prepare(create_initramfs=True)
generator.prepare(using_kernel=True)
print("Please:")
print(" 1. Take tmp/sysa/initramfs and your kernel, boot using this.")
print(" 2. Take tmp/sysc/disk.img and put this on a writable storage medium.")
print(" 1. Take tmp/initramfs and your kernel, boot using this.")
print(" 2. Take tmp/disk.img and put this on a writable storage medium.")
else:
system_a.prepare(create_initramfs=True, kernel_bootstrap=True)
generator.prepare(kernel_bootstrap=True)
print("Please:")
print(" 1. Take tmp/sysa/sysa.img and write it to a boot drive and then boot it.")
print(" 1. Take tmp/disk.img and write it to a boot drive and then boot it.")
else:
if args.kernel:
system_c.prepare(create_disk_image=True)
system_a.prepare(create_initramfs=True)
generator.prepare(using_kernel=True)
run(args.qemu_cmd,
'-enable-kvm',
'-m', str(args.qemu_ram) + 'M',
'-smp', str(args.cores),
'-no-reboot',
'-hda', tmpdir.get_disk("sysc"),
'-drive', 'file=' + tmpdir.get_disk("disk") + ',format=raw',
'-drive', 'file=' + tmpdir.get_disk("external") + ',format=raw',
'-nic', 'user,ipv6=off,model=e1000',
'-kernel', args.kernel,
'-initrd', system_a.initramfs_path,
'-nographic',
'-append', 'console=ttyS0')
'-append', 'console=ttyS0 root=/dev/sda1 rootfstype=ext3 init=/init rw')
else:
system_a.prepare(create_initramfs=True, kernel_bootstrap=True)
generator.prepare(kernel_bootstrap=True)
run(args.qemu_cmd,
'-enable-kvm',
'-m', "4G",
'-smp', str(args.cores),
'-no-reboot',
'-drive', 'file=' + os.path.join(system_a.tmp_dir, 'sysa.img') + ',format=raw',
'-drive', 'file=' + os.path.join(generator.tmp_dir, 'disk.img') + ',format=raw',
'-drive', 'file=' + tmpdir.get_disk("external") + ',format=raw',
'-machine', 'kernel-irqchip=split',
'-nic', 'user,ipv6=off,model=e1000',
'-nographic')

2
steps/env
View File

@ -1,4 +1,4 @@
DISTFILES=/distfiles
DISTFILES=/external/distfiles
PREFIX=/usr
BINDIR=${PREFIX}/bin
LIBDIR=${PREFIX}/lib/mes

2
steps/lwext4-1.0.0-lb1/files/make_fiwix_initrd.c
View File

@ -202,7 +202,7 @@ int main(int argc, char **argv)
char zeros[BLOCK_SIZE];
unsigned int next_file_address;
next_file_address = *((unsigned int *) 0x7F8D);
printf("Starting fiwix.ext2 at addr 0x%08x\n", next_file_address);

227
sysa.py
View File

@ -1,227 +0,0 @@
#!/usr/bin/env python3
"""System A"""
# SPDX-License-Identifier: GPL-3.0-or-later
# SPDX-FileCopyrightText: 2022-2023 Dor Askayo <dor.askayo@gmail.com>
# SPDX-FileCopyrightText: 2021 Andrius Štikonas <andrius@stikonas.eu>
# SPDX-FileCopyrightText: 2021 Melg Eight <public.melg8@gmail.com>
# SPDX-FileCopyrightText: 2021-23 fosslinux <fosslinux@aussies.space>
import os
# pylint: disable=deprecated-module
from distutils.dir_util import copy_tree
import shutil
import tarfile
from lib.sysgeneral import SysGeneral, stage0_arch_map
# pylint: disable=consider-using-with
# pylint: disable=too-many-instance-attributes
class SysA(SysGeneral):
"""
Class responsible for preparing sources for System A.
"""
git_dir = os.path.dirname(os.path.join(__file__))
sys_dir = os.path.join(git_dir, 'sysa')
sysb_dir = os.path.join(git_dir, 'sysb')
sysc_dir = os.path.join(git_dir, 'sysc')
cache_dir = os.path.join(sys_dir, 'distfiles')
# pylint: disable=too-many-arguments
def __init__(self, tmpdir, arch, external_sources,
early_preseed, repo_path):
self.arch = arch
self.early_preseed = early_preseed
self.external_sources = external_sources
self.repo_path = repo_path
self.tmp_dir = tmpdir.add_sys("sysa")
def prepare(self, create_initramfs, kernel_bootstrap=False):
"""
Prepare directory structure for System A.
We create an empty tmp directory, unpack stage0-posix.
Rest of the files are unpacked into more structured directory /sysa
"""
if self.early_preseed:
# Extract tar containing preseed
with tarfile.open(self.early_preseed, "r") as seed:
seed.extractall(self.tmp_dir)
shutil.copy2(os.path.join(self.sys_dir, 'base-preseeded.kaem'),
os.path.join(self.tmp_dir, 'kaem.x86'))
else:
self.stage0_posix()
self.sysa()
# sysb must be added to sysa as it is another initramfs stage
self.sysb()
self.sysc(create_initramfs)
if kernel_bootstrap:
self.create_fiwix_file_list()
self.create_builder_hex0_disk_image(os.path.join(self.tmp_dir, 'sysa.img'))
return
if self.repo_path:
repo_dir = os.path.join(self.tmp_dir, 'usr', 'src', 'repo-preseeded')
shutil.copytree(self.repo_path, repo_dir)
if create_initramfs:
self.make_initramfs()
def sysa(self):
"""Copy in sysa files for sysa."""
source_manifest = self.get_source_manifest()
self.get_packages(source_manifest)
shutil.copytree(self.sys_dir, os.path.join(self.tmp_dir, 'sysa'),
ignore=shutil.ignore_patterns('tmp'))
def sysb(self):
"""Copy in sysb files for sysb."""
shutil.copytree(self.sysb_dir, os.path.join(self.tmp_dir, 'sysb'),
ignore=shutil.ignore_patterns('tmp'))
def sysc(self, create_initramfs):
"""Copy in sysc files for sysc."""
if create_initramfs or not self.external_sources:
ignore = shutil.ignore_patterns('tmp', 'distfiles')
else:
ignore = shutil.ignore_patterns('tmp')
shutil.copytree(self.sysc_dir, os.path.join(self.tmp_dir, 'sysc'),
ignore=ignore)
def stage0_posix(self):
"""Copy in all of the stage0-posix"""
stage0_posix_base_dir = os.path.join(self.sys_dir, 'stage0-posix', 'src')
copy_tree(stage0_posix_base_dir, self.tmp_dir)
arch = stage0_arch_map.get(self.arch, self.arch)
kaem_optional_seed = os.path.join(self.sys_dir, 'stage0-posix', 'src', 'bootstrap-seeds',
'POSIX', arch, 'kaem-optional-seed')
shutil.copy2(kaem_optional_seed, os.path.join(self.tmp_dir, 'init'))
# stage0-posix hook to continue running live-bootstrap
shutil.copy2(os.path.join(self.sys_dir, 'after.kaem'),
os.path.join(self.tmp_dir, 'after.kaem'))
def add_fiwix_files(self, file_list_path, dirpath):
"""Add files to the list to populate Fiwix file system"""
for root, _, filepaths in os.walk(dirpath):
if 'stage0-posix' in root:
continue
if root == os.path.join('sysc', 'distfiles'):
continue
with open(file_list_path, 'a', encoding="utf-8") as file_list:
for filepath in filepaths:
file_list.write(f"/{os.path.join(root, filepath)}\n")
def create_fiwix_file_list(self):
"""Create a list of files to populate Fiwix file system"""
file_list_path = os.path.join(self.tmp_dir, 'sysa', 'lwext4-1.0.0-lb1',
'files', 'fiwix-file-list.txt')
shutil.copyfile(os.path.join(self.tmp_dir, 'sysa', 'lwext4-1.0.0-lb1',
'files', 'early-artifacts-needed-after-fiwix.txt'),
file_list_path)
save_cwd = os.getcwd()
self.add_fiwix_files(file_list_path, 'sysa')
self.add_fiwix_files(file_list_path, 'sysb')
self.add_fiwix_files(file_list_path, 'sysc')
os.chdir(save_cwd)
@staticmethod
def output_dir(srcfs_file, dirpath):
"""Add a directory to srcfs file system"""
srcline = f"src 0 {dirpath}\n"
srcfs_file.write(srcline.encode())
@staticmethod
def output_file(srcfs_file, filepath):
"""Add a file to srcfs file system"""
srcline = f"src {os.path.getsize(filepath)} {filepath}\n"
srcfs_file.write(srcline.encode())
with open(filepath, 'rb') as srcfile:
srcfs_file.write(srcfile.read())
def output_tree(self, srcfs_file, treepath):
"""Add a tree of files to srcfs file system"""
self.output_dir(srcfs_file, treepath)
for root, dirs, files in os.walk(treepath):
if ".git" in root:
continue
for dirpath in dirs:
if ".git" in dirpath:
continue
self.output_dir(srcfs_file, os.path.join(root, dirpath))
for filepath in files:
if ".git" in filepath:
continue
self.output_file(srcfs_file, os.path.join(root, filepath))
def append_srcfs(self, image_file):
"""Append srcfs file system to sysa disk image"""
save_cwd = os.getcwd()
os.chdir(os.path.join(self.tmp_dir, 'sysa', 'stage0-posix', 'src'))
self.output_tree(image_file, '.')
os.chdir(self.tmp_dir)
shutil.move(os.path.join('sysa', 'stage0-posix'), '.')
self.output_tree(image_file, 'sysa')
self.output_tree(image_file, 'sysb')
self.output_tree(image_file, 'sysc')
shutil.move('stage0-posix', 'sysa')
shutil.copyfile(os.path.join('sysa', 'after.kaem'), 'after.kaem')
self.output_file(image_file, 'after.kaem')
# Add commands to kick off stage0-posix
cmd = ' '.join(['hex0',
'./bootstrap-seeds/POSIX/x86/hex0_x86.hex0',
'./bootstrap-seeds/POSIX/x86/hex0-seed\n'])
image_file.write(cmd.encode())
cmd = ' '.join(['hex0',
'./bootstrap-seeds/POSIX/x86/kaem-minimal.hex0',
'./bootstrap-seeds/POSIX/x86/kaem-optional-seed\n'])
image_file.write(cmd.encode())
cmd = ' '.join(['./bootstrap-seeds/POSIX/x86/kaem-optional-seed', './kaem.x86\n'])
image_file.write(cmd.encode())
os.chdir(save_cwd)
def create_builder_hex0_disk_image(self, image_file_name):
"""Create builder-hex0 disk image"""
shutil.copyfile(os.path.join('sysa', 'stage0-posix', 'src', 'bootstrap-seeds',
'NATIVE', 'x86', 'builder-hex0-x86-stage1.img'),
image_file_name)
with open(image_file_name, 'ab') as image_file:
# Append stage2 hex0 source
with open(os.path.join('kernel-bootstrap', 'builder-hex0-x86-stage2.hex0'),
encoding="utf-8") as infile:
image_file.write(infile.read().encode())
# Pad to next sector
current_size = os.stat(image_file_name).st_size
while current_size % 512 != 0:
image_file.write(b'\0')
current_size += 1
self.append_srcfs(image_file)
current_size = os.stat(image_file_name).st_size
megabyte = 1024 * 1024
# fill file with zeros up to next megabyte
extra = current_size % megabyte
round_up = megabyte - extra
with open(image_file_name, 'ab') as image_file:
image_file.write(b'\0' * round_up)
current_size += round_up
# fill file with zeros up to desired size, one megabyte at a time
with open(image_file_name, 'ab') as image_file:
while current_size < 16384 * megabyte:
image_file.write(b'\0' * megabyte)
current_size += megabyte