Modify rootfs.py infrastructure to support the new layout
This commit is contained in:
parent
6ed2e09f3a
commit
05c13dd64e
|
@ -0,0 +1,336 @@
|
|||
#!/usr/bin/env python3
|
||||
# SPDX-License-Identifier: GPL-3.0-or-later
|
||||
# SPDX-FileCopyrightText: 2022-2023 Dor Askayo <dor.askayo@gmail.com>
|
||||
# SPDX-FileCopyrightText: 2021 Andrius Štikonas <andrius@stikonas.eu>
|
||||
# SPDX-FileCopyrightText: 2021 Melg Eight <public.melg8@gmail.com>
|
||||
# SPDX-FileCopyrightText: 2021-23 fosslinux <fosslinux@aussies.space>
|
||||
|
||||
import glob
|
||||
import hashlib
|
||||
import os
|
||||
import shutil
|
||||
import subprocess
|
||||
import tarfile
|
||||
import requests
|
||||
|
||||
class Generator():
|
||||
"""
|
||||
Class responsible for generating the basic media to be consumed.
|
||||
"""
|
||||
|
||||
git_dir = os.path.join(os.path.dirname(os.path.join(__file__)), '..')
|
||||
distfiles_dir = os.path.join(git_dir, 'distfiles')
|
||||
|
||||
# pylint: disable=too-many-arguments
|
||||
def __init__(self, tmpdir, arch, external_sources,
|
||||
early_preseed, repo_path):
|
||||
self.arch = arch
|
||||
self.early_preseed = early_preseed
|
||||
self.external_sources = external_sources
|
||||
self.repo_path = repo_path
|
||||
self.tmpdir = tmpdir
|
||||
self.tmp_dir = tmpdir.path
|
||||
|
||||
def prepare(self, using_kernel=False, kernel_bootstrap=False):
|
||||
"""
|
||||
Prepare basic media of live-bootstrap.
|
||||
/steps -- contains steps to be built
|
||||
/ -- contains seed to allow steps to be built, containing custom
|
||||
scripts and stage0-posix
|
||||
"""
|
||||
self.external_dir = os.path.join(self.tmp_dir, 'external')
|
||||
# We use ext3 here; ext4 actually has a variety of extensions that
|
||||
# have been added with varying levels of recency
|
||||
# Linux 4.9.10 does not support a bunch of them
|
||||
# Attempting to disable extensions that a particular e2fsprogs
|
||||
# is *unaware* of causes the filesystem creation to fail
|
||||
# We could hypothetically detect e2fsprogs version and create an
|
||||
# argument matrix ... or we could just use ext3 instead which
|
||||
# is effectively universally the same
|
||||
if kernel_bootstrap:
|
||||
init_path = os.path.join(self.tmp_dir, 'init')
|
||||
|
||||
os.mkdir(init_path)
|
||||
self.tmp_dir = init_path
|
||||
|
||||
if self.repo_path or self.external_sources:
|
||||
self.tmpdir.add_disk("external", filesystem="ext3")
|
||||
self.tmpdir.mount_disk("external", "external")
|
||||
else:
|
||||
self.tmpdir.add_disk("external", tabletype="none")
|
||||
elif using_kernel:
|
||||
self.tmp_dir = os.path.join(self.tmp_dir, 'disk')
|
||||
self.tmpdir.add_disk("disk", filesystem="ext3")
|
||||
self.tmpdir.mount_disk("disk", "disk")
|
||||
self.external_dir = os.path.join(self.tmp_dir, 'external')
|
||||
|
||||
os.makedirs(self.external_dir, exist_ok=True)
|
||||
|
||||
if self.early_preseed:
|
||||
# Extract tar containing preseed
|
||||
with tarfile.open(self.early_preseed, "r") as seed:
|
||||
seed.extractall(self.tmp_dir)
|
||||
shutil.copy2(os.path.join(self.git_dir, 'seed', 'preseeded.kaem'),
|
||||
os.path.join(self.tmp_dir, 'kaem.x86'))
|
||||
else:
|
||||
self.stage0_posix()
|
||||
self.seed()
|
||||
|
||||
self.steps()
|
||||
|
||||
self.distfiles()
|
||||
|
||||
self.create_fiwix_file_list()
|
||||
|
||||
if self.repo_path:
|
||||
repo_dir = os.path.join(self.external_dir, 'repo-preseeded')
|
||||
shutil.copytree(self.repo_path, repo_dir)
|
||||
|
||||
if kernel_bootstrap:
|
||||
self.create_builder_hex0_disk_image(os.path.join(self.tmp_dir, 'disk.img'))
|
||||
|
||||
if kernel_bootstrap and (self.external_sources or self.repo_path):
|
||||
self.tmpdir.umount_disk('external')
|
||||
elif using_kernel:
|
||||
self.tmpdir.umount_disk('disk')
|
||||
|
||||
def steps(self):
|
||||
"""Copy in steps."""
|
||||
source_manifest = self.get_source_manifest()
|
||||
self.get_packages(source_manifest)
|
||||
|
||||
shutil.copytree(os.path.join(self.git_dir, 'steps'), os.path.join(self.tmp_dir, 'steps'))
|
||||
|
||||
def stage0_posix(self):
|
||||
"""Copy in all of the stage0-posix"""
|
||||
stage0_posix_base_dir = os.path.join(self.git_dir, 'seed', 'stage0-posix')
|
||||
for f in os.listdir(stage0_posix_base_dir):
|
||||
orig = os.path.join(stage0_posix_base_dir, f)
|
||||
to = os.path.join(self.tmp_dir, f)
|
||||
if os.path.isfile(orig):
|
||||
shutil.copy2(orig, to)
|
||||
else:
|
||||
shutil.copytree(orig, to)
|
||||
|
||||
arch = stage0_arch_map.get(self.arch, self.arch)
|
||||
kaem_optional_seed = os.path.join(self.git_dir, 'seed', 'stage0-posix', 'bootstrap-seeds',
|
||||
'POSIX', arch, 'kaem-optional-seed')
|
||||
shutil.copy2(kaem_optional_seed, os.path.join(self.tmp_dir, 'init'))
|
||||
|
||||
def seed(self):
|
||||
"""Copy in extra seed files"""
|
||||
seed_dir = os.path.join(self.git_dir, 'seed')
|
||||
for f in os.listdir(seed_dir):
|
||||
if os.path.isfile(os.path.join(seed_dir, f)):
|
||||
shutil.copy2(os.path.join(seed_dir, f), os.path.join(self.tmp_dir, f))
|
||||
|
||||
def add_fiwix_files(self, file_list_path, dirpath):
|
||||
"""Add files to the list to populate Fiwix file system"""
|
||||
for root, _, filepaths in os.walk(dirpath):
|
||||
if 'stage0-posix' in root:
|
||||
continue
|
||||
with open(file_list_path, 'a', encoding="utf-8") as file_list:
|
||||
for filepath in filepaths:
|
||||
file_list.write(f"/{os.path.join(root, filepath)}\n")
|
||||
|
||||
def create_fiwix_file_list(self):
|
||||
"""Create a list of files to populate Fiwix file system"""
|
||||
file_list_path = os.path.join(self.tmp_dir, 'steps', 'lwext4-1.0.0-lb1',
|
||||
'files', 'fiwix-file-list.txt')
|
||||
shutil.copyfile(os.path.join(self.tmp_dir, 'steps', 'lwext4-1.0.0-lb1',
|
||||
'files', 'early-artifacts-needed-after-fiwix.txt'),
|
||||
file_list_path)
|
||||
|
||||
save_cwd = os.getcwd()
|
||||
os.chdir(self.tmp_dir)
|
||||
self.add_fiwix_files(file_list_path, 'steps')
|
||||
self.add_fiwix_files(file_list_path, 'distfiles')
|
||||
os.chdir(save_cwd)
|
||||
|
||||
def distfiles(self):
|
||||
"""Copy in distfiles"""
|
||||
def copy_no_network_distfiles(out):
|
||||
# Note that no network == no disk for kernel bootstrap mode
|
||||
with open(os.path.join(self.git_dir, 'steps', 'pre-network-sources'), 'r') as source_list:
|
||||
for file in source_list.readlines():
|
||||
file = file.strip()
|
||||
shutil.copy2(os.path.join(self.distfiles_dir, file),
|
||||
os.path.join(out, file))
|
||||
|
||||
early_distfile_dir = os.path.join(self.tmp_dir, 'external', 'distfiles')
|
||||
main_distfile_dir = os.path.join(self.external_dir, 'distfiles')
|
||||
|
||||
if early_distfile_dir != main_distfile_dir:
|
||||
os.makedirs(early_distfile_dir)
|
||||
copy_no_network_distfiles(early_distfile_dir)
|
||||
|
||||
if self.external_sources:
|
||||
os.mkdir(main_distfile_dir)
|
||||
shutil.copytree(self.distfiles_dir, main_distfile_dir)
|
||||
else:
|
||||
os.mkdir(main_distfile_dir)
|
||||
copy_no_network_distfiles(main_distfile_dir)
|
||||
|
||||
@staticmethod
|
||||
def output_dir(srcfs_file, dirpath):
|
||||
"""Add a directory to srcfs file system"""
|
||||
srcline = f"src 0 {dirpath}\n"
|
||||
srcfs_file.write(srcline.encode())
|
||||
|
||||
@staticmethod
|
||||
def output_file(srcfs_file, filepath):
|
||||
"""Add a file to srcfs file system"""
|
||||
srcline = f"src {os.path.getsize(filepath)} {filepath}\n"
|
||||
srcfs_file.write(srcline.encode())
|
||||
with open(filepath, 'rb') as srcfile:
|
||||
srcfs_file.write(srcfile.read())
|
||||
|
||||
def output_tree(self, srcfs_file, treepath):
|
||||
"""Add a tree of files to srcfs file system"""
|
||||
self.output_dir(srcfs_file, treepath)
|
||||
for root, dirs, files in os.walk(treepath):
|
||||
if ".git" in root:
|
||||
continue
|
||||
for dirpath in dirs:
|
||||
if ".git" in dirpath:
|
||||
continue
|
||||
self.output_dir(srcfs_file, os.path.join(root, dirpath))
|
||||
|
||||
for filepath in files:
|
||||
if ".git" in filepath:
|
||||
continue
|
||||
self.output_file(srcfs_file, os.path.join(root, filepath))
|
||||
|
||||
def append_srcfs(self, image_file):
|
||||
"""Append srcfs file system to disk image"""
|
||||
save_cwd = os.getcwd()
|
||||
|
||||
os.chdir(self.tmp_dir)
|
||||
self.output_tree(image_file, '.')
|
||||
|
||||
# Add commands to kick off stage0-posix
|
||||
cmd = ' '.join(['hex0',
|
||||
'./bootstrap-seeds/POSIX/x86/hex0_x86.hex0',
|
||||
'./bootstrap-seeds/POSIX/x86/hex0-seed\n'])
|
||||
image_file.write(cmd.encode())
|
||||
cmd = ' '.join(['hex0',
|
||||
'./bootstrap-seeds/POSIX/x86/kaem-minimal.hex0',
|
||||
'./bootstrap-seeds/POSIX/x86/kaem-optional-seed\n'])
|
||||
image_file.write(cmd.encode())
|
||||
cmd = ' '.join(['./bootstrap-seeds/POSIX/x86/kaem-optional-seed', './kaem.x86\n'])
|
||||
image_file.write(cmd.encode())
|
||||
|
||||
os.chdir(save_cwd)
|
||||
|
||||
def create_builder_hex0_disk_image(self, image_file_name):
|
||||
"""Create builder-hex0 disk image"""
|
||||
shutil.copyfile(os.path.join('seed', 'stage0-posix', 'bootstrap-seeds',
|
||||
'NATIVE', 'x86', 'builder-hex0-x86-stage1.img'),
|
||||
image_file_name)
|
||||
|
||||
with open(image_file_name, 'ab') as image_file:
|
||||
# Append stage2 hex0 source
|
||||
with open(os.path.join('kernel-bootstrap', 'builder-hex0-x86-stage2.hex0'),
|
||||
encoding="utf-8") as infile:
|
||||
image_file.write(infile.read().encode())
|
||||
# Pad to next sector
|
||||
current_size = os.stat(image_file_name).st_size
|
||||
while current_size % 512 != 0:
|
||||
image_file.write(b'\0')
|
||||
current_size += 1
|
||||
self.append_srcfs(image_file)
|
||||
|
||||
current_size = os.stat(image_file_name).st_size
|
||||
|
||||
megabyte = 1024 * 1024
|
||||
# fill file with zeros up to next megabyte
|
||||
extra = current_size % megabyte
|
||||
round_up = megabyte - extra
|
||||
with open(image_file_name, 'ab') as image_file:
|
||||
image_file.write(b'\0' * round_up)
|
||||
current_size += round_up
|
||||
|
||||
# fill file with zeros up to desired size, one megabyte at a time
|
||||
with open(image_file_name, 'ab') as image_file:
|
||||
while current_size < 16384 * megabyte:
|
||||
image_file.write(b'\0' * megabyte)
|
||||
current_size += megabyte
|
||||
|
||||
def check_file(self, file_name, expected_hash):
|
||||
"""Check hash of downloaded source file."""
|
||||
with open(file_name, "rb") as downloaded_file:
|
||||
downloaded_content = downloaded_file.read() # read entire file as bytes
|
||||
readable_hash = hashlib.sha256(downloaded_content).hexdigest()
|
||||
if expected_hash == readable_hash:
|
||||
return
|
||||
raise ValueError(f"Checksum mismatch for file {os.path.basename(file_name)}:\n\
|
||||
expected: {expected_hash}\n\
|
||||
actual: {readable_hash}\n\
|
||||
When in doubt, try deleting the file in question -- it will be downloaded again when running \
|
||||
this script the next time")
|
||||
|
||||
def download_file(self, url, directory, file_name):
|
||||
"""
|
||||
Download a single source archive.
|
||||
"""
|
||||
abs_file_name = os.path.join(directory, file_name)
|
||||
|
||||
# Create a directory for downloaded file
|
||||
if not os.path.isdir(directory):
|
||||
os.mkdir(directory)
|
||||
|
||||
# Actually download the file
|
||||
headers = {
|
||||
"Accept-Encoding": "identity"
|
||||
}
|
||||
if not os.path.isfile(abs_file_name):
|
||||
print(f"Downloading: {file_name}")
|
||||
response = requests.get(url, allow_redirects=True, stream=True,
|
||||
headers=headers, timeout=20)
|
||||
if response.status_code == 200:
|
||||
with open(abs_file_name, 'wb') as target_file:
|
||||
target_file.write(response.raw.read())
|
||||
else:
|
||||
raise requests.HTTPError("Download failed.")
|
||||
return abs_file_name
|
||||
|
||||
def get_packages(self, source_manifest):
|
||||
"""Prepare remaining sources"""
|
||||
for line in source_manifest.split("\n"):
|
||||
line = line.strip().split(" ")
|
||||
|
||||
path = self.download_file(line[2], line[1], line[3])
|
||||
self.check_file(path, line[0])
|
||||
|
||||
@classmethod
|
||||
def get_source_manifest(cls):
|
||||
"""
|
||||
Generate a source manifest for the system.
|
||||
"""
|
||||
manifest_lines = []
|
||||
directory = os.path.relpath(cls.distfiles_dir, cls.git_dir)
|
||||
|
||||
# Find all source files
|
||||
steps_dir = os.path.join(cls.git_dir, 'steps')
|
||||
for file in os.listdir(steps_dir):
|
||||
if os.path.isdir(os.path.join(steps_dir, file)):
|
||||
sourcef = os.path.join(steps_dir, file, "sources")
|
||||
if os.path.exists(sourcef):
|
||||
# Read sources from the source file
|
||||
with open(sourcef, "r", encoding="utf_8") as sources:
|
||||
for line in sources.readlines():
|
||||
line = line.strip().split(" ")
|
||||
|
||||
if len(line) > 2:
|
||||
file_name = line[2]
|
||||
else:
|
||||
# Automatically determine file name based on URL.
|
||||
file_name = os.path.basename(line[0])
|
||||
|
||||
manifest_lines.append(f"{line[1]} {directory} {line[0]} {file_name}")
|
||||
|
||||
return "\n".join(manifest_lines)
|
||||
|
||||
stage0_arch_map = {
|
||||
"amd64": "AMD64",
|
||||
}
|
|
@ -1,132 +0,0 @@
|
|||
#!/usr/bin/env python3
|
||||
"""
|
||||
This file contains a few functions to be shared by all Sys* classes
|
||||
"""
|
||||
|
||||
# SPDX-FileCopyrightText: 2022-2023 Dor Askayo <dor.askayo@gmail.com>
|
||||
# SPDX-FileCopyrightText: 2021-23 fosslinux <fosslinux@aussies.space>
|
||||
# SPDX-FileCopyrightText: 2021 Andrius Štikonas <andrius@stikonas.eu>
|
||||
# SPDX-License-Identifier: GPL-3.0-or-later
|
||||
|
||||
import os
|
||||
import hashlib
|
||||
import glob
|
||||
import subprocess
|
||||
|
||||
import requests
|
||||
|
||||
class SysGeneral:
|
||||
"""
|
||||
A class from which all Sys* class are extended.
|
||||
Contains functions used in all Sys*
|
||||
"""
|
||||
|
||||
# All of these are variables defined in the individual Sys* classes
|
||||
cache_dir = None
|
||||
base_dir = None
|
||||
git_dir = None
|
||||
sys_dir = None
|
||||
initramfs_path = None
|
||||
tmp_dir = None
|
||||
|
||||
def check_file(self, file_name, expected_hash):
|
||||
"""Check hash of downloaded source file."""
|
||||
with open(file_name, "rb") as downloaded_file:
|
||||
downloaded_content = downloaded_file.read() # read entire file as bytes
|
||||
readable_hash = hashlib.sha256(downloaded_content).hexdigest()
|
||||
if expected_hash == readable_hash:
|
||||
return
|
||||
raise ValueError(f"Checksum mismatch for file {os.path.basename(file_name)}:\n\
|
||||
expected: {expected_hash}\n\
|
||||
actual: {readable_hash}\n\
|
||||
When in doubt, try deleting the file in question -- it will be downloaded again when running \
|
||||
this script the next time")
|
||||
|
||||
def download_file(self, url, directory, file_name):
|
||||
"""
|
||||
Download a single source archive.
|
||||
"""
|
||||
abs_file_name = os.path.join(directory, file_name)
|
||||
|
||||
# Create a directory for downloaded file
|
||||
if not os.path.isdir(directory):
|
||||
os.mkdir(directory)
|
||||
|
||||
# Actually download the file
|
||||
headers = {
|
||||
"Accept-Encoding": "identity"
|
||||
}
|
||||
if not os.path.isfile(abs_file_name):
|
||||
print(f"Downloading: {file_name}")
|
||||
response = requests.get(url, allow_redirects=True, stream=True,
|
||||
headers=headers, timeout=20)
|
||||
if response.status_code == 200:
|
||||
with open(abs_file_name, 'wb') as target_file:
|
||||
target_file.write(response.raw.read())
|
||||
else:
|
||||
raise requests.HTTPError("Download failed.")
|
||||
return abs_file_name
|
||||
|
||||
def get_packages(self, source_manifest):
|
||||
"""Prepare remaining sources"""
|
||||
for line in source_manifest.split("\n"):
|
||||
line = line.strip().split(" ")
|
||||
|
||||
path = self.download_file(line[2], line[1], line[3])
|
||||
self.check_file(path, line[0])
|
||||
|
||||
@classmethod
|
||||
def get_source_manifest(cls):
|
||||
"""
|
||||
Generate a source manifest for the system.
|
||||
"""
|
||||
manifest_lines = []
|
||||
directory = os.path.relpath(cls.cache_dir, cls.git_dir)
|
||||
|
||||
# Find all source files
|
||||
for file in os.listdir(cls.sys_dir):
|
||||
if os.path.isdir(os.path.join(cls.sys_dir, file)):
|
||||
sourcef = os.path.join(cls.sys_dir, file, "sources")
|
||||
if os.path.exists(sourcef):
|
||||
# Read sources from the source file
|
||||
with open(sourcef, "r", encoding="utf_8") as sources:
|
||||
for line in sources.readlines():
|
||||
line = line.strip().split(" ")
|
||||
|
||||
if len(line) > 2:
|
||||
file_name = line[2]
|
||||
else:
|
||||
# Automatically determine file name based on URL.
|
||||
file_name = os.path.basename(line[0])
|
||||
|
||||
manifest_lines.append(f"{line[1]} {directory} {line[0]} {file_name}")
|
||||
|
||||
return "\n".join(manifest_lines)
|
||||
|
||||
def make_initramfs(self):
|
||||
"""Package binary bootstrap seeds and sources into initramfs."""
|
||||
self.initramfs_path = os.path.join(self.tmp_dir, 'initramfs')
|
||||
|
||||
# Create a list of files to go within the initramfs
|
||||
file_list = glob.glob(os.path.join(self.tmp_dir, '**'), recursive=True)
|
||||
|
||||
# Use built-in removeprefix once we can use Python 3.9
|
||||
def remove_prefix(text, prefix):
|
||||
if text.startswith(prefix):
|
||||
return text[len(prefix):]
|
||||
return text # or whatever
|
||||
|
||||
file_list = [remove_prefix(f, self.tmp_dir + os.sep) for f in file_list]
|
||||
|
||||
# Create the initramfs
|
||||
with open(self.initramfs_path, "w", encoding="utf_8") as initramfs:
|
||||
# pylint: disable=consider-using-with
|
||||
cpio = subprocess.Popen(
|
||||
["cpio", "--format", "newc", "--create",
|
||||
"--directory", self.tmp_dir],
|
||||
stdin=subprocess.PIPE, stdout=initramfs)
|
||||
cpio.communicate(input='\n'.join(file_list).encode())
|
||||
|
||||
stage0_arch_map = {
|
||||
"amd64": "AMD64",
|
||||
}
|
|
@ -24,7 +24,6 @@ class Tmpdir:
|
|||
Represents a tmpdir
|
||||
"""
|
||||
|
||||
_syses = {}
|
||||
_disks = {}
|
||||
_disk_filesystems = {}
|
||||
_mountpoints = {}
|
||||
|
@ -60,19 +59,10 @@ class Tmpdir:
|
|||
mount("tmpfs", self.path, "tmpfs", f"size={size}")
|
||||
self._type = TmpType.TMPFS
|
||||
|
||||
def add_sys(self, name, subdir=None):
|
||||
"""Create a subdirectory and register a sys"""
|
||||
if subdir is None:
|
||||
subdir = name
|
||||
sys_path = os.path.join(self.path, name)
|
||||
if not os.path.exists(sys_path):
|
||||
os.mkdir(sys_path)
|
||||
return sys_path
|
||||
|
||||
def add_disk(self, name, size="16G", filesystem="ext4"):
|
||||
def add_disk(self, name, size="16G", filesystem="ext4", tabletype="msdos", mkfs_args=[]):
|
||||
"""Add a disk"""
|
||||
disk_path = os.path.join(self.path, f"{name}.img")
|
||||
self._disks[name] = create_disk(disk_path, "msdos", filesystem, size)
|
||||
self._disks[name] = create_disk(disk_path, tabletype, filesystem, size, mkfs_args=mkfs_args)
|
||||
self._disk_filesystems[name] = filesystem
|
||||
# Allow executing user to access it
|
||||
run_as_root("chown", getpass.getuser(), self._disks[name])
|
||||
|
|
|
@ -31,7 +31,7 @@ def run_as_root(*args, **kwargs):
|
|||
return run("sudo", *args, **kwargs)
|
||||
return run(*args, **kwargs)
|
||||
|
||||
def create_disk(image, disk_type, fs_type, size):
|
||||
def create_disk(image, disk_type, fs_type, size, mkfs_args=[]):
|
||||
"""Create a disk image, with a filesystem on it"""
|
||||
run('truncate', '-s', size, image)
|
||||
# First find the device we will use, then actually use it
|
||||
|
@ -40,9 +40,9 @@ def create_disk(image, disk_type, fs_type, size):
|
|||
# Create the partition
|
||||
if disk_type != "none":
|
||||
run_as_root('parted', '--script', image, 'mklabel', disk_type, 'mkpart',
|
||||
'primary', 'ext4', '0%', '100%')
|
||||
'primary', fs_type, '0%', '100%')
|
||||
run_as_root('partprobe', loop_dev)
|
||||
run_as_root('mkfs.' + fs_type, loop_dev + "p1")
|
||||
run_as_root('mkfs.' + fs_type, loop_dev + "p1", *mkfs_args)
|
||||
return loop_dev
|
||||
|
||||
def mount(source, target, fs_type, options='', **kwargs):
|
||||
|
|
65
rootfs.py
65
rootfs.py
|
@ -17,18 +17,16 @@ import argparse
|
|||
import os
|
||||
import shutil
|
||||
|
||||
from sysa import SysA
|
||||
from sysc import SysC
|
||||
from lib.utils import run, run_as_root
|
||||
from lib.sysgeneral import stage0_arch_map
|
||||
from lib.tmpdir import Tmpdir
|
||||
from lib.generator import Generator, stage0_arch_map
|
||||
|
||||
def create_configuration_file(args):
|
||||
"""
|
||||
Creates bootstrap.cfg file which would contain options used to
|
||||
customize bootstrap.
|
||||
"""
|
||||
config_path = os.path.join('sysa', 'bootstrap.cfg')
|
||||
config_path = os.path.join('steps', 'bootstrap.cfg')
|
||||
with open(config_path, "w", encoding="utf_8") as config:
|
||||
config.write(f"FORCE_TIMESTAMPS={args.force_timestamps}\n")
|
||||
config.write(f"CHROOT={args.chroot or args.bwrap}\n")
|
||||
|
@ -38,7 +36,10 @@ def create_configuration_file(args):
|
|||
config.write(f"INTERNAL_CI={args.internal_ci}\n")
|
||||
config.write(f"BARE_METAL={args.bare_metal}\n")
|
||||
if (args.bare_metal or args.qemu) and not args.kernel:
|
||||
config.write("DISK=sda\n")
|
||||
if args.repo or args.external_sources:
|
||||
config.write("DISK=sdb1\n")
|
||||
else:
|
||||
config.write("DISK=sdb\n")
|
||||
config.write("KERNEL_BOOTSTRAP=True\n")
|
||||
else:
|
||||
config.write("DISK=sda1\n")
|
||||
|
@ -49,7 +50,7 @@ def create_configuration_file(args):
|
|||
def main():
|
||||
"""
|
||||
A few command line arguments to customize bootstrap.
|
||||
This function also creates SysA object which prepares directory
|
||||
This function also creates object which prepares directory
|
||||
structure with bootstrap seeds and all sources.
|
||||
"""
|
||||
parser = argparse.ArgumentParser()
|
||||
|
@ -151,16 +152,15 @@ def main():
|
|||
if args.tmpfs:
|
||||
tmpdir.tmpfs(size=args.tmpfs_size)
|
||||
|
||||
# sys
|
||||
system_c = SysC(arch=args.arch, tmpdir=tmpdir,
|
||||
external_sources=args.external_sources)
|
||||
system_a = SysA(arch=args.arch, early_preseed=args.early_preseed,
|
||||
tmpdir=tmpdir, external_sources=args.external_sources,
|
||||
repo_path=args.repo)
|
||||
generator = Generator(tmpdir=tmpdir,
|
||||
arch=args.arch,
|
||||
external_sources=args.external_sources,
|
||||
repo_path=args.repo,
|
||||
early_preseed=args.early_preseed)
|
||||
|
||||
bootstrap(args, system_a, system_c, tmpdir)
|
||||
bootstrap(args, generator, tmpdir)
|
||||
|
||||
def bootstrap(args, system_a, system_c, tmpdir):
|
||||
def bootstrap(args, generator, tmpdir):
|
||||
"""Kick off bootstrap process."""
|
||||
print(f"Bootstrapping {args.arch} -- SysA")
|
||||
if args.chroot:
|
||||
|
@ -171,17 +171,15 @@ print(shutil.which('chroot'))
|
|||
chroot_binary = run_as_root('python3', '-c', find_chroot,
|
||||
capture_output=True).stdout.decode().strip()
|
||||
|
||||
system_c.prepare(create_disk_image=False)
|
||||
system_a.prepare(create_initramfs=False)
|
||||
generator.prepare(using_kernel=False)
|
||||
|
||||
arch = stage0_arch_map.get(args.arch, args.arch)
|
||||
init = os.path.join(os.sep, 'bootstrap-seeds', 'POSIX', arch, 'kaem-optional-seed')
|
||||
run_as_root('env', '-i', 'PATH=/bin', chroot_binary, system_a.tmp_dir, init)
|
||||
run_as_root('env', '-i', 'PATH=/bin', chroot_binary, generator.tmp_dir, init)
|
||||
|
||||
elif args.bwrap:
|
||||
if not args.internal_ci or args.internal_ci == "pass1":
|
||||
system_c.prepare(create_disk_image=False)
|
||||
system_a.prepare(create_initramfs=False)
|
||||
generator.prepare(using_kernel=False)
|
||||
|
||||
arch = stage0_arch_map.get(args.arch, args.arch)
|
||||
init = os.path.join(os.sep, 'bootstrap-seeds', 'POSIX', arch, 'kaem-optional-seed')
|
||||
|
@ -191,7 +189,7 @@ print(shutil.which('chroot'))
|
|||
'--unshare-net',
|
||||
'--clearenv',
|
||||
'--setenv', 'PATH', '/usr/bin',
|
||||
'--bind', system_a.tmp_dir, '/',
|
||||
'--bind', generator.tmp_dir, '/',
|
||||
'--dir', '/dev',
|
||||
'--dev-bind', '/dev/null', '/dev/null',
|
||||
'--dev-bind', '/dev/zero', '/dev/zero',
|
||||
|
@ -210,7 +208,7 @@ print(shutil.which('chroot'))
|
|||
'--unshare-net' if args.external_sources else None,
|
||||
'--clearenv',
|
||||
'--setenv', 'PATH', '/usr/bin',
|
||||
'--bind', system_a.tmp_dir + "/sysc_image", '/',
|
||||
'--bind', generator.tmp_dir + "/sysc_image", '/',
|
||||
'--dir', '/dev',
|
||||
'--dev-bind', '/dev/null', '/dev/null',
|
||||
'--dev-bind', '/dev/zero', '/dev/zero',
|
||||
|
@ -226,40 +224,39 @@ print(shutil.which('chroot'))
|
|||
|
||||
elif args.bare_metal:
|
||||
if args.kernel:
|
||||
system_c.prepare(create_disk_image=True)
|
||||
system_a.prepare(create_initramfs=True)
|
||||
generator.prepare(using_kernel=True)
|
||||
print("Please:")
|
||||
print(" 1. Take tmp/sysa/initramfs and your kernel, boot using this.")
|
||||
print(" 2. Take tmp/sysc/disk.img and put this on a writable storage medium.")
|
||||
print(" 1. Take tmp/initramfs and your kernel, boot using this.")
|
||||
print(" 2. Take tmp/disk.img and put this on a writable storage medium.")
|
||||
else:
|
||||
system_a.prepare(create_initramfs=True, kernel_bootstrap=True)
|
||||
generator.prepare(kernel_bootstrap=True)
|
||||
print("Please:")
|
||||
print(" 1. Take tmp/sysa/sysa.img and write it to a boot drive and then boot it.")
|
||||
print(" 1. Take tmp/disk.img and write it to a boot drive and then boot it.")
|
||||
|
||||
else:
|
||||
if args.kernel:
|
||||
system_c.prepare(create_disk_image=True)
|
||||
system_a.prepare(create_initramfs=True)
|
||||
generator.prepare(using_kernel=True)
|
||||
|
||||
run(args.qemu_cmd,
|
||||
'-enable-kvm',
|
||||
'-m', str(args.qemu_ram) + 'M',
|
||||
'-smp', str(args.cores),
|
||||
'-no-reboot',
|
||||
'-hda', tmpdir.get_disk("sysc"),
|
||||
'-drive', 'file=' + tmpdir.get_disk("disk") + ',format=raw',
|
||||
'-drive', 'file=' + tmpdir.get_disk("external") + ',format=raw',
|
||||
'-nic', 'user,ipv6=off,model=e1000',
|
||||
'-kernel', args.kernel,
|
||||
'-initrd', system_a.initramfs_path,
|
||||
'-nographic',
|
||||
'-append', 'console=ttyS0')
|
||||
'-append', 'console=ttyS0 root=/dev/sda1 rootfstype=ext3 init=/init rw')
|
||||
else:
|
||||
system_a.prepare(create_initramfs=True, kernel_bootstrap=True)
|
||||
generator.prepare(kernel_bootstrap=True)
|
||||
run(args.qemu_cmd,
|
||||
'-enable-kvm',
|
||||
'-m', "4G",
|
||||
'-smp', str(args.cores),
|
||||
'-no-reboot',
|
||||
'-drive', 'file=' + os.path.join(system_a.tmp_dir, 'sysa.img') + ',format=raw',
|
||||
'-drive', 'file=' + os.path.join(generator.tmp_dir, 'disk.img') + ',format=raw',
|
||||
'-drive', 'file=' + tmpdir.get_disk("external") + ',format=raw',
|
||||
'-machine', 'kernel-irqchip=split',
|
||||
'-nic', 'user,ipv6=off,model=e1000',
|
||||
'-nographic')
|
||||
|
|
|
@ -1,4 +1,4 @@
|
|||
DISTFILES=/distfiles
|
||||
DISTFILES=/external/distfiles
|
||||
PREFIX=/usr
|
||||
BINDIR=${PREFIX}/bin
|
||||
LIBDIR=${PREFIX}/lib/mes
|
||||
|
|
227
sysa.py
227
sysa.py
|
@ -1,227 +0,0 @@
|
|||
#!/usr/bin/env python3
|
||||
"""System A"""
|
||||
# SPDX-License-Identifier: GPL-3.0-or-later
|
||||
# SPDX-FileCopyrightText: 2022-2023 Dor Askayo <dor.askayo@gmail.com>
|
||||
# SPDX-FileCopyrightText: 2021 Andrius Štikonas <andrius@stikonas.eu>
|
||||
# SPDX-FileCopyrightText: 2021 Melg Eight <public.melg8@gmail.com>
|
||||
# SPDX-FileCopyrightText: 2021-23 fosslinux <fosslinux@aussies.space>
|
||||
|
||||
import os
|
||||
# pylint: disable=deprecated-module
|
||||
from distutils.dir_util import copy_tree
|
||||
import shutil
|
||||
import tarfile
|
||||
|
||||
from lib.sysgeneral import SysGeneral, stage0_arch_map
|
||||
|
||||
# pylint: disable=consider-using-with
|
||||
# pylint: disable=too-many-instance-attributes
|
||||
class SysA(SysGeneral):
|
||||
"""
|
||||
Class responsible for preparing sources for System A.
|
||||
"""
|
||||
|
||||
git_dir = os.path.dirname(os.path.join(__file__))
|
||||
sys_dir = os.path.join(git_dir, 'sysa')
|
||||
sysb_dir = os.path.join(git_dir, 'sysb')
|
||||
sysc_dir = os.path.join(git_dir, 'sysc')
|
||||
cache_dir = os.path.join(sys_dir, 'distfiles')
|
||||
|
||||
# pylint: disable=too-many-arguments
|
||||
def __init__(self, tmpdir, arch, external_sources,
|
||||
early_preseed, repo_path):
|
||||
self.arch = arch
|
||||
self.early_preseed = early_preseed
|
||||
self.external_sources = external_sources
|
||||
self.repo_path = repo_path
|
||||
|
||||
self.tmp_dir = tmpdir.add_sys("sysa")
|
||||
|
||||
def prepare(self, create_initramfs, kernel_bootstrap=False):
|
||||
"""
|
||||
Prepare directory structure for System A.
|
||||
We create an empty tmp directory, unpack stage0-posix.
|
||||
Rest of the files are unpacked into more structured directory /sysa
|
||||
"""
|
||||
if self.early_preseed:
|
||||
# Extract tar containing preseed
|
||||
with tarfile.open(self.early_preseed, "r") as seed:
|
||||
seed.extractall(self.tmp_dir)
|
||||
shutil.copy2(os.path.join(self.sys_dir, 'base-preseeded.kaem'),
|
||||
os.path.join(self.tmp_dir, 'kaem.x86'))
|
||||
else:
|
||||
self.stage0_posix()
|
||||
|
||||
self.sysa()
|
||||
|
||||
# sysb must be added to sysa as it is another initramfs stage
|
||||
self.sysb()
|
||||
|
||||
self.sysc(create_initramfs)
|
||||
|
||||
if kernel_bootstrap:
|
||||
self.create_fiwix_file_list()
|
||||
self.create_builder_hex0_disk_image(os.path.join(self.tmp_dir, 'sysa.img'))
|
||||
return
|
||||
|
||||
if self.repo_path:
|
||||
repo_dir = os.path.join(self.tmp_dir, 'usr', 'src', 'repo-preseeded')
|
||||
shutil.copytree(self.repo_path, repo_dir)
|
||||
|
||||
if create_initramfs:
|
||||
self.make_initramfs()
|
||||
|
||||
def sysa(self):
|
||||
"""Copy in sysa files for sysa."""
|
||||
source_manifest = self.get_source_manifest()
|
||||
self.get_packages(source_manifest)
|
||||
|
||||
shutil.copytree(self.sys_dir, os.path.join(self.tmp_dir, 'sysa'),
|
||||
ignore=shutil.ignore_patterns('tmp'))
|
||||
|
||||
def sysb(self):
|
||||
"""Copy in sysb files for sysb."""
|
||||
shutil.copytree(self.sysb_dir, os.path.join(self.tmp_dir, 'sysb'),
|
||||
ignore=shutil.ignore_patterns('tmp'))
|
||||
|
||||
def sysc(self, create_initramfs):
|
||||
"""Copy in sysc files for sysc."""
|
||||
if create_initramfs or not self.external_sources:
|
||||
ignore = shutil.ignore_patterns('tmp', 'distfiles')
|
||||
else:
|
||||
ignore = shutil.ignore_patterns('tmp')
|
||||
shutil.copytree(self.sysc_dir, os.path.join(self.tmp_dir, 'sysc'),
|
||||
ignore=ignore)
|
||||
|
||||
def stage0_posix(self):
|
||||
"""Copy in all of the stage0-posix"""
|
||||
stage0_posix_base_dir = os.path.join(self.sys_dir, 'stage0-posix', 'src')
|
||||
copy_tree(stage0_posix_base_dir, self.tmp_dir)
|
||||
|
||||
arch = stage0_arch_map.get(self.arch, self.arch)
|
||||
kaem_optional_seed = os.path.join(self.sys_dir, 'stage0-posix', 'src', 'bootstrap-seeds',
|
||||
'POSIX', arch, 'kaem-optional-seed')
|
||||
shutil.copy2(kaem_optional_seed, os.path.join(self.tmp_dir, 'init'))
|
||||
|
||||
# stage0-posix hook to continue running live-bootstrap
|
||||
shutil.copy2(os.path.join(self.sys_dir, 'after.kaem'),
|
||||
os.path.join(self.tmp_dir, 'after.kaem'))
|
||||
|
||||
def add_fiwix_files(self, file_list_path, dirpath):
|
||||
"""Add files to the list to populate Fiwix file system"""
|
||||
for root, _, filepaths in os.walk(dirpath):
|
||||
if 'stage0-posix' in root:
|
||||
continue
|
||||
if root == os.path.join('sysc', 'distfiles'):
|
||||
continue
|
||||
with open(file_list_path, 'a', encoding="utf-8") as file_list:
|
||||
for filepath in filepaths:
|
||||
file_list.write(f"/{os.path.join(root, filepath)}\n")
|
||||
|
||||
def create_fiwix_file_list(self):
|
||||
"""Create a list of files to populate Fiwix file system"""
|
||||
file_list_path = os.path.join(self.tmp_dir, 'sysa', 'lwext4-1.0.0-lb1',
|
||||
'files', 'fiwix-file-list.txt')
|
||||
shutil.copyfile(os.path.join(self.tmp_dir, 'sysa', 'lwext4-1.0.0-lb1',
|
||||
'files', 'early-artifacts-needed-after-fiwix.txt'),
|
||||
file_list_path)
|
||||
|
||||
save_cwd = os.getcwd()
|
||||
self.add_fiwix_files(file_list_path, 'sysa')
|
||||
self.add_fiwix_files(file_list_path, 'sysb')
|
||||
self.add_fiwix_files(file_list_path, 'sysc')
|
||||
os.chdir(save_cwd)
|
||||
|
||||
@staticmethod
|
||||
def output_dir(srcfs_file, dirpath):
|
||||
"""Add a directory to srcfs file system"""
|
||||
srcline = f"src 0 {dirpath}\n"
|
||||
srcfs_file.write(srcline.encode())
|
||||
|
||||
@staticmethod
|
||||
def output_file(srcfs_file, filepath):
|
||||
"""Add a file to srcfs file system"""
|
||||
srcline = f"src {os.path.getsize(filepath)} {filepath}\n"
|
||||
srcfs_file.write(srcline.encode())
|
||||
with open(filepath, 'rb') as srcfile:
|
||||
srcfs_file.write(srcfile.read())
|
||||
|
||||
def output_tree(self, srcfs_file, treepath):
|
||||
"""Add a tree of files to srcfs file system"""
|
||||
self.output_dir(srcfs_file, treepath)
|
||||
for root, dirs, files in os.walk(treepath):
|
||||
if ".git" in root:
|
||||
continue
|
||||
for dirpath in dirs:
|
||||
if ".git" in dirpath:
|
||||
continue
|
||||
self.output_dir(srcfs_file, os.path.join(root, dirpath))
|
||||
|
||||
for filepath in files:
|
||||
if ".git" in filepath:
|
||||
continue
|
||||
self.output_file(srcfs_file, os.path.join(root, filepath))
|
||||
|
||||
def append_srcfs(self, image_file):
|
||||
"""Append srcfs file system to sysa disk image"""
|
||||
save_cwd = os.getcwd()
|
||||
|
||||
os.chdir(os.path.join(self.tmp_dir, 'sysa', 'stage0-posix', 'src'))
|
||||
self.output_tree(image_file, '.')
|
||||
|
||||
os.chdir(self.tmp_dir)
|
||||
shutil.move(os.path.join('sysa', 'stage0-posix'), '.')
|
||||
self.output_tree(image_file, 'sysa')
|
||||
self.output_tree(image_file, 'sysb')
|
||||
self.output_tree(image_file, 'sysc')
|
||||
shutil.move('stage0-posix', 'sysa')
|
||||
shutil.copyfile(os.path.join('sysa', 'after.kaem'), 'after.kaem')
|
||||
self.output_file(image_file, 'after.kaem')
|
||||
|
||||
# Add commands to kick off stage0-posix
|
||||
cmd = ' '.join(['hex0',
|
||||
'./bootstrap-seeds/POSIX/x86/hex0_x86.hex0',
|
||||
'./bootstrap-seeds/POSIX/x86/hex0-seed\n'])
|
||||
image_file.write(cmd.encode())
|
||||
cmd = ' '.join(['hex0',
|
||||
'./bootstrap-seeds/POSIX/x86/kaem-minimal.hex0',
|
||||
'./bootstrap-seeds/POSIX/x86/kaem-optional-seed\n'])
|
||||
image_file.write(cmd.encode())
|
||||
cmd = ' '.join(['./bootstrap-seeds/POSIX/x86/kaem-optional-seed', './kaem.x86\n'])
|
||||
image_file.write(cmd.encode())
|
||||
|
||||
os.chdir(save_cwd)
|
||||
|
||||
def create_builder_hex0_disk_image(self, image_file_name):
|
||||
"""Create builder-hex0 disk image"""
|
||||
shutil.copyfile(os.path.join('sysa', 'stage0-posix', 'src', 'bootstrap-seeds',
|
||||
'NATIVE', 'x86', 'builder-hex0-x86-stage1.img'),
|
||||
image_file_name)
|
||||
|
||||
with open(image_file_name, 'ab') as image_file:
|
||||
# Append stage2 hex0 source
|
||||
with open(os.path.join('kernel-bootstrap', 'builder-hex0-x86-stage2.hex0'),
|
||||
encoding="utf-8") as infile:
|
||||
image_file.write(infile.read().encode())
|
||||
# Pad to next sector
|
||||
current_size = os.stat(image_file_name).st_size
|
||||
while current_size % 512 != 0:
|
||||
image_file.write(b'\0')
|
||||
current_size += 1
|
||||
self.append_srcfs(image_file)
|
||||
|
||||
current_size = os.stat(image_file_name).st_size
|
||||
|
||||
megabyte = 1024 * 1024
|
||||
# fill file with zeros up to next megabyte
|
||||
extra = current_size % megabyte
|
||||
round_up = megabyte - extra
|
||||
with open(image_file_name, 'ab') as image_file:
|
||||
image_file.write(b'\0' * round_up)
|
||||
current_size += round_up
|
||||
|
||||
# fill file with zeros up to desired size, one megabyte at a time
|
||||
with open(image_file_name, 'ab') as image_file:
|
||||
while current_size < 16384 * megabyte:
|
||||
image_file.write(b'\0' * megabyte)
|
||||
current_size += megabyte
|
Loading…
Reference in New Issue