andrius
/
live-bootstrap
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Merge pull request #354 from Googulator/script-fixes 

Python script fixes and improvements
This commit is contained in:
fosslinux 2023-12-26 00:59:34 +00:00 committed by GitHub
parent 7d0e1c0e75 529ea4cae1
commit 6905852107
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
7 changed files with 142 additions and 195 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

129
lib/generator.py
View File

@ -14,6 +14,7 @@ import shutil
import tarfile
import requests
# pylint: disable=too-many-instance-attributes
class Generator():
"""
Class responsible for generating the basic media to be consumed.
@ -22,24 +23,25 @@ class Generator():
git_dir = os.path.join(os.path.dirname(os.path.join(__file__)), '..')
distfiles_dir = os.path.join(git_dir, 'distfiles')
# pylint: disable=too-many-arguments
def __init__(self, tmpdir, arch, external_sources,
early_preseed, repo_path):
def __init__(self, arch, external_sources, early_preseed, repo_path):
self.arch = arch
self.early_preseed = early_preseed
self.external_sources = external_sources
self.repo_path = repo_path
self.tmpdir = tmpdir
self.tmp_dir = tmpdir.path
self.external_dir = os.path.join(self.tmp_dir, 'external')
self.source_manifest = self.get_source_manifest(not self.external_sources)
self.tmp_dir = None
self.external_dir = None
def prepare(self, using_kernel=False, kernel_bootstrap=False):
def prepare(self, tmpdir, using_kernel=False, kernel_bootstrap=False, target_size=0):
"""
Prepare basic media of live-bootstrap.
/steps -- contains steps to be built
/ -- contains seed to allow steps to be built, containing custom
scripts and stage0-posix
"""
self.tmp_dir = tmpdir.path
self.external_dir = os.path.join(self.tmp_dir, 'external')
# We use ext3 here; ext4 actually has a variety of extensions that
# have been added with varying levels of recency
# Linux 4.9.10 does not support a bunch of them
@ -55,14 +57,17 @@ class Generator():
self.tmp_dir = init_path
if self.repo_path or self.external_sources:
self.tmpdir.add_disk("external", filesystem="ext3")
self.tmpdir.mount_disk("external", "external")
tmpdir.add_disk("external", filesystem="ext3")
tmpdir.mount_disk("external", "external")
else:
self.tmpdir.add_disk("external", tabletype="none")
self.external_dir = os.path.join(self.tmp_dir, 'external')
elif using_kernel:
self.tmp_dir = os.path.join(self.tmp_dir, 'disk')
self.tmpdir.add_disk("disk", filesystem="ext3")
self.tmpdir.mount_disk("disk", "disk")
tmpdir.add_disk("disk",
filesystem="ext3",
size=(target_size + "M") if target_size else "16G",
bootable=True)
tmpdir.mount_disk("disk", "disk")
self.external_dir = os.path.join(self.tmp_dir, 'external')
os.makedirs(self.external_dir, exist_ok=True)
@ -88,30 +93,29 @@ class Generator():
shutil.copytree(self.repo_path, repo_dir)
if kernel_bootstrap:
self.create_builder_hex0_disk_image(os.path.join(self.tmp_dir, 'disk.img'))
self.create_builder_hex0_disk_image(self.tmp_dir + '.img', target_size)
if kernel_bootstrap and (self.external_sources or self.repo_path):
self.tmpdir.umount_disk('external')
tmpdir.umount_disk('external')
elif using_kernel:
self.tmpdir.umount_disk('disk')
tmpdir.umount_disk('disk')
def steps(self):
"""Copy in steps."""
source_manifest = self.get_source_manifest()
self.get_packages(source_manifest)
self.get_packages()
shutil.copytree(os.path.join(self.git_dir, 'steps'), os.path.join(self.tmp_dir, 'steps'))
def stage0_posix(self):
"""Copy in all of the stage0-posix"""
stage0_posix_base_dir = os.path.join(self.git_dir, 'seed', 'stage0-posix')
for f in os.listdir(stage0_posix_base_dir):
orig = os.path.join(stage0_posix_base_dir, f)
to = os.path.join(self.tmp_dir, f)
for entry in os.listdir(stage0_posix_base_dir):
orig = os.path.join(stage0_posix_base_dir, entry)
target = os.path.join(self.tmp_dir, entry)
if os.path.isfile(orig):
shutil.copy2(orig, to)
shutil.copy2(orig, target)
else:
shutil.copytree(orig, to)
shutil.copytree(orig, target)
arch = stage0_arch_map.get(self.arch, self.arch)
kaem_optional_seed = os.path.join(self.git_dir, 'seed', 'stage0-posix', 'bootstrap-seeds',
@ -121,11 +125,12 @@ class Generator():
def seed(self):
"""Copy in extra seed files"""
seed_dir = os.path.join(self.git_dir, 'seed')
for f in os.listdir(seed_dir):
if os.path.isfile(os.path.join(seed_dir, f)):
shutil.copy2(os.path.join(seed_dir, f), os.path.join(self.tmp_dir, f))
for entry in os.listdir(seed_dir):
if os.path.isfile(os.path.join(seed_dir, entry)):
shutil.copy2(os.path.join(seed_dir, entry), os.path.join(self.tmp_dir, entry))
def add_fiwix_files(self, file_list_path, dirpath):
@staticmethod
def add_fiwix_files(file_list_path, dirpath):
"""Add files to the list to populate Fiwix file system"""
for root, _, filepaths in os.walk(dirpath):
if 'stage0-posix' in root:
@ -151,13 +156,11 @@ class Generator():
def distfiles(self):
"""Copy in distfiles"""
def copy_no_network_distfiles(out):
# Note that no network == no disk for kernel bootstrap mode
pre_src_path = os.path.join(self.git_dir, 'steps', 'pre-network-sources')
with open(pre_src_path, 'r', encoding="utf-8") as source_list:
for file in source_list.readlines():
file = file.strip()
shutil.copy2(os.path.join(self.distfiles_dir, file),
os.path.join(out, file))
# Note that "no disk" implies "no network" for kernel bootstrap mode
for file in self.source_manifest:
file = file[3].strip()
shutil.copy2(os.path.join(self.distfiles_dir, file),
os.path.join(out, file))
early_distfile_dir = os.path.join(self.tmp_dir, 'external', 'distfiles')
main_distfile_dir = os.path.join(self.external_dir, 'distfiles')
@ -167,7 +170,6 @@ class Generator():
copy_no_network_distfiles(early_distfile_dir)
if self.external_sources:
os.mkdir(main_distfile_dir)
shutil.copytree(self.distfiles_dir, main_distfile_dir)
else:
os.mkdir(main_distfile_dir)
@ -224,7 +226,7 @@ class Generator():
os.chdir(save_cwd)
def create_builder_hex0_disk_image(self, image_file_name):
def create_builder_hex0_disk_image(self, image_file_name, size):
"""Create builder-hex0 disk image"""
shutil.copyfile(os.path.join('seed', 'stage0-posix', 'bootstrap-seeds',
'NATIVE', 'x86', 'builder-hex0-x86-stage1.img'),
@ -252,13 +254,13 @@ class Generator():
image_file.write(b'\0' * round_up)
current_size += round_up
# fill file with zeros up to desired size, one megabyte at a time
with open(image_file_name, 'ab') as image_file:
while current_size < 16384 * megabyte:
image_file.write(b'\0' * megabyte)
current_size += megabyte
# extend file up to desired size
if current_size < size * megabyte:
with open(image_file_name, 'ab') as image_file:
image_file.truncate(size * megabyte)
def check_file(self, file_name, expected_hash):
@staticmethod
def check_file(file_name, expected_hash):
"""Check hash of downloaded source file."""
with open(file_name, "rb") as downloaded_file:
downloaded_content = downloaded_file.read() # read entire file as bytes
@ -271,7 +273,8 @@ actual: {readable_hash}\n\
When in doubt, try deleting the file in question -- it will be downloaded again when running \
this script the next time")
def download_file(self, url, directory, file_name):
@staticmethod
def download_file(url, directory, file_name):
"""
Download a single source archive.
"""
@ -293,45 +296,53 @@ this script the next time")
with open(abs_file_name, 'wb') as target_file:
target_file.write(response.raw.read())
else:
raise requests.HTTPError("Download failed.")
raise requests.HTTPError("Download failed: HTTP " +
response.status_code + " " + response.reason)
return abs_file_name
def get_packages(self, source_manifest):
def get_packages(self):
"""Prepare remaining sources"""
for line in source_manifest.split("\n"):
line = line.strip().split(" ")
for line in self.source_manifest:
path = self.download_file(line[2], line[1], line[3])
self.check_file(path, line[0])
@classmethod
def get_source_manifest(cls):
def get_source_manifest(cls, pre_network=False):
"""
Generate a source manifest for the system.
"""
manifest_lines = []
entries = []
directory = os.path.relpath(cls.distfiles_dir, cls.git_dir)
# Find all source files
steps_dir = os.path.join(cls.git_dir, 'steps')
for file in os.listdir(steps_dir):
if os.path.isdir(os.path.join(steps_dir, file)):
sourcef = os.path.join(steps_dir, file, "sources")
with open(os.path.join(steps_dir, 'manifest'), 'r', encoding="utf_8") as file:
for line in file:
if pre_network and line.strip().startswith("improve: ") and "network" in line:
break
if not line.strip().startswith("build: "):
continue
step = line.split(" ")[1].split("#")[0].strip()
sourcef = os.path.join(steps_dir, step, "sources")
if os.path.exists(sourcef):
# Read sources from the source file
with open(sourcef, "r", encoding="utf_8") as sources:
for line in sources.readlines():
line = line.strip().split(" ")
for source in sources.readlines():
source = source.strip().split(" ")
if len(line) > 2:
file_name = line[2]
if len(source) > 2:
file_name = source[2]
else:
# Automatically determine file name based on URL.
file_name = os.path.basename(line[0])
file_name = os.path.basename(source[0])
manifest_lines.append(f"{line[1]} {directory} {line[0]} {file_name}")
entry = (source[1], directory, source[0], file_name)
if entry not in entries:
entries.append(entry)
return "\n".join(manifest_lines)
return entries
stage0_arch_map = {
"amd64": "AMD64",

17
lib/tmpdir.py
View File

@ -60,10 +60,21 @@ class Tmpdir:
self._type = TmpType.TMPFS
# pylint: disable=too-many-arguments
def add_disk(self, name, size="16G", filesystem="ext4", tabletype="msdos", mkfs_args=None):
def add_disk(self,
name,
size="16G",
filesystem="ext4",
tabletype="msdos",
bootable=False,
mkfs_args=None):
"""Add a disk"""
disk_path = os.path.join(self.path, f"{name}.img")
self._disks[name] = create_disk(disk_path, tabletype, filesystem, size, mkfs_args=mkfs_args)
self._disks[name] = create_disk(disk_path,
tabletype,
filesystem,
size,
bootable,
mkfs_args)
self._disk_filesystems[name] = filesystem
# Allow executing user to access it
run_as_root("chown", getpass.getuser(), self._disks[name])
@ -87,4 +98,4 @@ class Tmpdir:
def get_disk(self, name):
"""Get the path to a device of a disk"""
return self._disks[name]
return self._disks.get(name)

5
lib/utils.py
View File

@ -31,7 +31,8 @@ def run_as_root(*args, **kwargs):
return run("sudo", *args, **kwargs)
return run(*args, **kwargs)
def create_disk(image, disk_type, fs_type, size, mkfs_args=None):
# pylint: disable=too-many-arguments
def create_disk(image, disk_type, fs_type, size, bootable=False, mkfs_args=None):
"""Create a disk image, with a filesystem on it"""
if mkfs_args is None:
mkfs_args = []
@ -42,7 +43,7 @@ def create_disk(image, disk_type, fs_type, size, mkfs_args=None):
# Create the partition
if disk_type != "none":
run_as_root('parted', '--script', image, 'mklabel', disk_type, 'mkpart',
'primary', fs_type, '0%', '100%')
'primary', fs_type, '1GiB' if bootable else '1MiB', '100%')
run_as_root('partprobe', loop_dev)
run_as_root('mkfs.' + fs_type, loop_dev + "p1", *mkfs_args)
return loop_dev

73
rootfs.py
View File

@ -30,7 +30,6 @@ def create_configuration_file(args):
with open(config_path, "w", encoding="utf_8") as config:
config.write(f"FORCE_TIMESTAMPS={args.force_timestamps}\n")
config.write(f"CHROOT={args.chroot or args.bwrap}\n")
config.write(f"CHROOT_ONLY_SYSA={args.bwrap}\n")
config.write(f"UPDATE_CHECKSUMS={args.update_checksums}\n")
config.write(f"JOBS={args.cores}\n")
config.write(f"INTERNAL_CI={args.internal_ci}\n")
@ -39,7 +38,7 @@ def create_configuration_file(args):
if args.repo or args.external_sources:
config.write("DISK=sdb1\n")
else:
config.write("DISK=sdb\n")
config.write("DISK=sda\n")
config.write("KERNEL_BOOTSTRAP=True\n")
else:
config.write("DISK=sda1\n")
@ -98,7 +97,9 @@ def main():
default="qemu-system-x86_64")
parser.add_argument("-qr", "--qemu-ram", help="Memory (in megabytes) allocated to QEMU VM",
default=4096)
parser.add_argument("-qk", "--kernel", help="Custom sysa kernel to use")
parser.add_argument("-qs", "--target-size", help="Size of the target image (for QEMU only)",
default="16G")
parser.add_argument("-qk", "--kernel", help="Custom early kernel to use")
parser.add_argument("-b", "--bare-metal", help="Build images for bare metal",
action="store_true")
@ -136,15 +137,24 @@ def main():
if int(args.cores) < 1:
raise ValueError("Must use one or more cores.")
# Target image size validation
if args.qemu:
if int(str(args.target_size).rstrip('gGmM')) < 1:
raise ValueError("Please specify a positive target size for qemu.")
args.target_size = (int(str(args.target_size).rstrip('gGmM')) *
(1024 if str(args.target_size).lower().endswith('g') else 1))
else:
args.target_size = 0
# bootstrap.cfg
try:
os.remove(os.path.join('sysa', 'bootstrap.cfg'))
os.remove(os.path.join('steps', 'bootstrap.cfg'))
except FileNotFoundError:
pass
if not args.no_create_config:
create_configuration_file(args)
else:
with open(os.path.join('sysa', 'bootstrap.cfg'), 'a', encoding='UTF-8'):
with open(os.path.join('steps', 'bootstrap.cfg'), 'a', encoding='UTF-8'):
pass
# tmpdir
@ -152,17 +162,16 @@ def main():
if args.tmpfs:
tmpdir.tmpfs(size=args.tmpfs_size)
generator = Generator(tmpdir=tmpdir,
arch=args.arch,
generator = Generator(arch=args.arch,
external_sources=args.external_sources,
repo_path=args.repo,
early_preseed=args.early_preseed)
bootstrap(args, generator, tmpdir)
bootstrap(args, generator, tmpdir, args.target_size)
def bootstrap(args, generator, tmpdir):
def bootstrap(args, generator, tmpdir, size):
"""Kick off bootstrap process."""
print(f"Bootstrapping {args.arch} -- SysA")
print(f"Bootstrapping {args.arch}")
if args.chroot:
find_chroot = """
import shutil
@ -171,7 +180,7 @@ print(shutil.which('chroot'))
chroot_binary = run_as_root('python3', '-c', find_chroot,
capture_output=True).stdout.decode().strip()
generator.prepare(using_kernel=False)
generator.prepare(tmpdir, using_kernel=False)
arch = stage0_arch_map.get(args.arch, args.arch)
init = os.path.join(os.sep, 'bootstrap-seeds', 'POSIX', arch, 'kaem-optional-seed')
@ -179,7 +188,7 @@ print(shutil.which('chroot'))
elif args.bwrap:
if not args.internal_ci or args.internal_ci == "pass1":
generator.prepare(using_kernel=False)
generator.prepare(tmpdir, using_kernel=False)
arch = stage0_arch_map.get(args.arch, args.arch)
init = os.path.join(os.sep, 'bootstrap-seeds', 'POSIX', arch, 'kaem-optional-seed')
@ -200,15 +209,16 @@ print(shutil.which('chroot'))
init)
if not args.internal_ci or args.internal_ci == "pass2" or args.internal_ci == "pass3":
shutil.copy2(os.path.join('sysa', 'bootstrap.cfg'),
os.path.join('tmp', 'sysa', 'sysc_image', 'usr', 'src', 'bootstrap.cfg'))
os.makedirs(os.path.join(generator.tmp_dir, 'stage2', 'steps'), exist_ok=True)
shutil.copy2(os.path.join('steps', 'bootstrap.cfg'),
os.path.join(generator.tmp_dir, 'stage2', 'steps', 'bootstrap.cfg'))
run('bwrap', '--unshare-user',
'--uid', '0',
'--gid', '0',
'--unshare-net' if args.external_sources else None,
'--clearenv',
'--setenv', 'PATH', '/usr/bin',
'--bind', generator.tmp_dir + "/sysc_image", '/',
'--bind', os.path.join(generator.tmp_dir, "stage2"), '/',
'--dir', '/dev',
'--dev-bind', '/dev/null', '/dev/null',
'--dev-bind', '/dev/zero', '/dev/zero',
@ -224,18 +234,20 @@ print(shutil.which('chroot'))
elif args.bare_metal:
if args.kernel:
generator.prepare(using_kernel=True)
generator.prepare(tmpdir, using_kernel=True, target_size=size)
image_path = os.path.join(args.tmpdir, os.path.relpath(generator.tmp_dir, args.tmpdir))
print("Please:")
print(" 1. Take tmp/initramfs and your kernel, boot using this.")
print(" 2. Take tmp/disk.img and put this on a writable storage medium.")
print(f" 1. Take {image_path}/initramfs and your kernel, boot using this.")
print(f" 2. Take {image_path}/disk.img and put this on a writable storage medium.")
else:
generator.prepare(kernel_bootstrap=True)
generator.prepare(tmpdir, kernel_bootstrap=True, target_size=size)
image_path = os.path.join(args.tmpdir, os.path.relpath(generator.tmp_dir, args.tmpdir))
print("Please:")
print(" 1. Take tmp/disk.img and write it to a boot drive and then boot it.")
print(f" 1. Take {image_path}.img and write it to a boot drive and then boot it.")
else:
if args.kernel:
generator.prepare(using_kernel=True)
generator.prepare(tmpdir, using_kernel=True, target_size=size)
run(args.qemu_cmd,
'-enable-kvm',
@ -249,17 +261,24 @@ print(shutil.which('chroot'))
'-nographic',
'-append', 'console=ttyS0 root=/dev/sda1 rootfstype=ext3 init=/init rw')
else:
generator.prepare(kernel_bootstrap=True)
run(args.qemu_cmd,
generator.prepare(tmpdir, kernel_bootstrap=True, target_size=size)
arg_list = [
'-enable-kvm',
'-m', "4G",
'-m', str(args.qemu_ram) + 'M',
'-smp', str(args.cores),
'-no-reboot',
'-drive', 'file=' + os.path.join(generator.tmp_dir, 'disk.img') + ',format=raw',
'-drive', 'file=' + tmpdir.get_disk("external") + ',format=raw',
'-drive', 'file=' + generator.tmp_dir + '.img' + ',format=raw'
]
if tmpdir.get_disk("external") is not None:
arg_list += [
'-drive', 'file=' + tmpdir.get_disk("external") + ',format=raw',
]
arg_list += [
'-machine', 'kernel-irqchip=split',
'-nic', 'user,ipv6=off,model=e1000',
'-nographic')
'-nographic'
]
run(args.qemu_cmd, *arg_list)
if __name__ == "__main__":
main()

20
source_manifest.py
View File

@ -9,27 +9,11 @@ for the bootstrapping process.
import argparse
from sysa import SysA
from sysc import SysC
from lib.generator import Generator
def main():
"""Generate a source manifest for a system"""
parser = argparse.ArgumentParser()
parser.add_argument("-s", "--system",
help="Generate source manifest for the specified systems",
choices=["sysa", "sysc"],
nargs="+",
action="extend",
required=True)
args = parser.parse_args()
if "sysa" in args.system:
print(SysA.get_source_manifest())
if "sysc" in args.system:
print(SysC.get_source_manifest())
print('\n'.join(map(' '.join, Generator.get_source_manifest())))
if __name__ == "__main__":
main()

7
steps/manifest
View File

@ -16,6 +16,13 @@
# - jump: jump (usually) to a new kernel, executes a script with that name
# eg, jump: fiwix
#
# The following directives have special significance:
# - build directives beginning with "bash" (as well as jumps) trigger the generation of
# a new script
# - the first improve directive containing "network" is used by generator.py to deduce
# what source files need to be downloaded in advance (files referenced after that will
# be downloaded during bootstrap, unless --external-sources is given)
#
# Other features:
# - predicate; based on variables set in bootstrap.cfg, require for something to execute
# must be enclosed in brackets with spaces padded

86
steps/pre-network-sources
View File

@ -1,86 +0,0 @@
mes-0.25.tar.gz
nyacc-1.00.2.tar.gz
tcc-0.9.26.tar.gz
tcc-0.9.27.tar.bz2
fiwix-1.4.0-lb3.tar.gz
lwext4-1.0.0-lb1.tar.gz
make-3.82.tar.bz2
patch-2.5.9.tar.gz
gzip-1.2.4.tar.gz
tar-1.12.tar.gz
sed-4.0.9.tar.gz
bzip2-1.0.8.tar.gz
coreutils-5.0.tar.bz2
heirloom-devtools-070527.tar.bz2
bash-2.05b.tar.gz
flex-2.5.11.tar.gz
tcc-0.9.27.tar.bz2
musl-1.1.24.tar.gz
tcc-0.9.27.tar.bz2
musl-1.1.24.tar.gz
tcc-0.9.27.tar.bz2
sed-4.0.9.tar.gz
bzip2-1.0.8.tar.gz
m4-1.4.7.tar.gz
flex-2.6.4.tar.gz
bison-3.4.1.tar.gz
bison-3.4.1.tar.gz
bison-3.4.1.tar.gz
grep-2.4.tar.gz
diffutils-2.7.tar.gz
coreutils-5.0.tar.bz2
coreutils-6.10.tar.gz
gawk-3.0.4.tar.gz
perl-5.000.tar.gz
perl-5.003.tar.gz
perl5.004_05.tar.gz
perl5.005_03.tar.gz
perl-5.6.2.tar.gz
autoconf-2.52.tar.bz2
automake-1.6.3.tar.bz2
automake-1.6.3.tar.bz2
autoconf-2.53.tar.bz2
automake-1.7.tar.bz2
autoconf-2.54.tar.bz2
autoconf-2.55.tar.bz2
automake-1.7.8.tar.bz2
autoconf-2.57.tar.bz2
autoconf-2.59.tar.bz2
automake-1.8.5.tar.bz2
help2man-1.36.4.tar.gz
autoconf-2.61.tar.bz2
automake-1.9.6.tar.bz2
automake-1.10.3.tar.bz2
autoconf-2.64.tar.bz2
automake-1.11.2.tar.bz2
autoconf-2.69.tar.gz
libtool-2.2.4.tar.bz2
automake-1.15.1.tar.gz
binutils-2.30.tar.bz2
musl-1.1.24.tar.gz
tcc-0.9.27.tar.bz2
gcc-core-4.0.4.tar.bz2
automake-1.16.3.tar.gz
findutils-4.2.33.tar.gz
gnulib-8e128e.tar.gz
musl-1.2.4.tar.gz
gcc-core-4.0.4.tar.bz2
automake-1.16.3.tar.gz
util-linux-2.19.1.tar.gz
e2fsprogs-1.45.7.tar.gz
CaseFolding.txt
DerivedAge.txt
DerivedCombiningClass.txt
DerivedCoreProperties.txt
NormalizationCorrections.txt
NormalizationTest.txt
UnicodeData.txt
v10.0.1.tar.gz
kbd-1.15.tar.gz
make-3.82.tar.bz2
ed-1.4.tar.gz
bc-1.07.1.tar.gz
v2.0.22.tar.gz
linux-4.9.10.tar.gz
deblob-4.9
curl-7.88.1.tar.bz2