Rename tmpdir to target and always preserve it, dropping --preserve

There is nothing temporary about our "tmpdir" - its sole purpose is to
contain the final product of the bootstrap process. Thus, removing it
at the end of bootstrap amounts to doing the entire process for nothing.

To remedy this, --tmpdir is renamed --target, keeping the -t short form,
and defaulting to "target" instead of "tmp" to make its purpose clearer.
The --preserve option is removed, as the target is now always preserved.
This commit is contained in:
Gábor Stefanik 2023-12-26 04:28:47 +01:00
parent 9179f4bd01
commit 81e3123262
5 changed files with 79 additions and 91 deletions

View File

@ -43,15 +43,15 @@ jobs:
distfiles distfiles
key: cache-${{ hashFiles('steps/*/sources') }} key: cache-${{ hashFiles('steps/*/sources') }}
- name: Run bootstrap - name: Run bootstrap
run: ./rootfs.py --bwrap --external-sources --build-kernels --preserve --cores 2 --internal-ci pass1 run: ./rootfs.py --bwrap --external-sources --build-kernels --cores 2 --internal-ci pass1
- name: Archive created packages - name: Archive created packages
if: failure() # archive failed builds progress if: failure() # archive failed builds progress
uses: actions/upload-artifact@v3 uses: actions/upload-artifact@v3
with: with:
name: packages name: packages
path: tmp/external/repo/** path: target/external/repo/**
- name: Tar pass1 image - name: Tar pass1 image
run: tar -cf pass1_image.tar tmp run: tar -cf pass1_image.tar target
- name: Archive pass1_image - name: Archive pass1_image
uses: actions/upload-artifact@v3 uses: actions/upload-artifact@v3
with: with:
@ -96,15 +96,15 @@ jobs:
distfiles distfiles
key: cache-${{ hashFiles('steps/*/sources') }} key: cache-${{ hashFiles('steps/*/sources') }}
- name: Run bootstrap - name: Run bootstrap
run: ./rootfs.py --bwrap --external-sources --build-kernels --preserve --cores 2 --internal-ci pass2 run: ./rootfs.py --bwrap --external-sources --build-kernels --cores 2 --internal-ci pass2
- name: Archive created packages - name: Archive created packages
if: failure() # archive failed builds progress if: failure() # archive failed builds progress
uses: actions/upload-artifact@v3 uses: actions/upload-artifact@v3
with: with:
name: internal_packages_pass2 name: internal_packages_pass2
path: tmp/external/repo/** path: target/external/repo/**
- name: Tar pass2 image - name: Tar pass2 image
run: tar -cf pass2_image.tar tmp run: tar -cf pass2_image.tar target
- name: Archive pass2_image - name: Archive pass2_image
uses: actions/upload-artifact@v3 uses: actions/upload-artifact@v3
with: with:
@ -149,10 +149,10 @@ jobs:
distfiles distfiles
key: cache-${{ hashFiles('steps/*/sources') }} key: cache-${{ hashFiles('steps/*/sources') }}
- name: Run bootstrap - name: Run bootstrap
run: ./rootfs.py --bwrap --external-sources --build-kernels --preserve --cores 2 --internal-ci pass3 run: ./rootfs.py --bwrap --external-sources --build-kernels --cores 2 --internal-ci pass3
- name: Archive created packages - name: Archive created packages
if: always() # archive both failed and successful builds if: always() # archive both failed and successful builds
uses: actions/upload-artifact@v3 uses: actions/upload-artifact@v3
with: with:
name: packages name: packages
path: tmp/external/repo/** path: target/external/repo/**

View File

@ -21,7 +21,7 @@ jobs:
- name: Install pylint - name: Install pylint
run: sudo pip3 install pylint run: sudo pip3 install pylint
- name: pylint - name: pylint
run: pylint rootfs.py lib/utils.py lib/generator.py lib/tmpdir.py --disable=duplicate-code run: pylint rootfs.py lib/utils.py lib/generator.py lib/target.py --disable=duplicate-code
shellcheck: shellcheck:
name: Lint shell files name: Lint shell files

View File

@ -29,26 +29,26 @@ class Generator():
self.external_sources = external_sources self.external_sources = external_sources
self.repo_path = repo_path self.repo_path = repo_path
self.source_manifest = self.get_source_manifest(not self.external_sources) self.source_manifest = self.get_source_manifest(not self.external_sources)
self.tmp_dir = None self.target_dir = None
self.external_dir = None self.external_dir = None
def reuse(self, tmpdir): def reuse(self, target):
""" """
Reuse a previously prepared bwrap environment for further stages. Reuse a previously prepared bwrap environment for further stages.
""" """
self.tmp_dir = tmpdir.path self.target_dir = target.path
self.external_dir = os.path.join(self.tmp_dir, 'external') self.external_dir = os.path.join(self.target_dir, 'external')
self.distfiles() self.distfiles()
def prepare(self, tmpdir, using_kernel=False, kernel_bootstrap=False, target_size=0): def prepare(self, target, using_kernel=False, kernel_bootstrap=False, target_size=0):
""" """
Prepare basic media of live-bootstrap. Prepare basic media of live-bootstrap.
/steps -- contains steps to be built /steps -- contains steps to be built
/ -- contains seed to allow steps to be built, containing custom / -- contains seed to allow steps to be built, containing custom
scripts and stage0-posix scripts and stage0-posix
""" """
self.tmp_dir = tmpdir.path self.target_dir = target.path
self.external_dir = os.path.join(self.tmp_dir, 'external') self.external_dir = os.path.join(self.target_dir, 'external')
# We use ext3 here; ext4 actually has a variety of extensions that # We use ext3 here; ext4 actually has a variety of extensions that
# have been added with varying levels of recency # have been added with varying levels of recency
@ -59,33 +59,33 @@ class Generator():
# argument matrix ... or we could just use ext3 instead which # argument matrix ... or we could just use ext3 instead which
# is effectively universally the same # is effectively universally the same
if kernel_bootstrap: if kernel_bootstrap:
init_path = os.path.join(self.tmp_dir, 'init') init_path = os.path.join(self.target_dir, 'init')
os.mkdir(init_path) os.mkdir(init_path)
self.tmp_dir = init_path self.target_dir = init_path
if self.repo_path or self.external_sources: if self.repo_path or self.external_sources:
tmpdir.add_disk("external", filesystem="ext3") target.add_disk("external", filesystem="ext3")
tmpdir.mount_disk("external", "external") target.mount_disk("external", "external")
else: else:
self.external_dir = os.path.join(self.tmp_dir, 'external') self.external_dir = os.path.join(self.target_dir, 'external')
elif using_kernel: elif using_kernel:
self.tmp_dir = os.path.join(self.tmp_dir, 'disk') self.target_dir = os.path.join(self.target_dir, 'disk')
tmpdir.add_disk("disk", target.add_disk("disk",
filesystem="ext3", filesystem="ext3",
size=(target_size + "M") if target_size else "16G", size=(target_size + "M") if target_size else "16G",
bootable=True) bootable=True)
tmpdir.mount_disk("disk", "disk") target.mount_disk("disk", "disk")
self.external_dir = os.path.join(self.tmp_dir, 'external') self.external_dir = os.path.join(self.target_dir, 'external')
os.makedirs(self.external_dir, exist_ok=True) os.makedirs(self.external_dir, exist_ok=True)
if self.early_preseed: if self.early_preseed:
# Extract tar containing preseed # Extract tar containing preseed
with tarfile.open(self.early_preseed, "r") as seed: with tarfile.open(self.early_preseed, "r") as seed:
seed.extractall(self.tmp_dir) seed.extractall(self.target_dir)
shutil.copy2(os.path.join(self.git_dir, 'seed', 'preseeded.kaem'), shutil.copy2(os.path.join(self.git_dir, 'seed', 'preseeded.kaem'),
os.path.join(self.tmp_dir, 'kaem.x86')) os.path.join(self.target_dir, 'kaem.x86'))
else: else:
self.stage0_posix() self.stage0_posix()
self.seed() self.seed()
@ -101,25 +101,25 @@ class Generator():
shutil.copytree(self.repo_path, repo_dir) shutil.copytree(self.repo_path, repo_dir)
if kernel_bootstrap: if kernel_bootstrap:
self.create_builder_hex0_disk_image(self.tmp_dir + '.img', target_size) self.create_builder_hex0_disk_image(self.target_dir + '.img', target_size)
if kernel_bootstrap and (self.external_sources or self.repo_path): if kernel_bootstrap and (self.external_sources or self.repo_path):
tmpdir.umount_disk('external') target.umount_disk('external')
elif using_kernel: elif using_kernel:
tmpdir.umount_disk('disk') target.umount_disk('disk')
def steps(self): def steps(self):
"""Copy in steps.""" """Copy in steps."""
self.get_packages() self.get_packages()
shutil.copytree(os.path.join(self.git_dir, 'steps'), os.path.join(self.tmp_dir, 'steps')) shutil.copytree(os.path.join(self.git_dir, 'steps'), os.path.join(self.target_dir, 'steps'))
def stage0_posix(self): def stage0_posix(self):
"""Copy in all of the stage0-posix""" """Copy in all of the stage0-posix"""
stage0_posix_base_dir = os.path.join(self.git_dir, 'seed', 'stage0-posix') stage0_posix_base_dir = os.path.join(self.git_dir, 'seed', 'stage0-posix')
for entry in os.listdir(stage0_posix_base_dir): for entry in os.listdir(stage0_posix_base_dir):
orig = os.path.join(stage0_posix_base_dir, entry) orig = os.path.join(stage0_posix_base_dir, entry)
target = os.path.join(self.tmp_dir, entry) target = os.path.join(self.target_dir, entry)
if os.path.isfile(orig): if os.path.isfile(orig):
shutil.copy2(orig, target) shutil.copy2(orig, target)
else: else:
@ -128,14 +128,14 @@ class Generator():
arch = stage0_arch_map.get(self.arch, self.arch) arch = stage0_arch_map.get(self.arch, self.arch)
kaem_optional_seed = os.path.join(self.git_dir, 'seed', 'stage0-posix', 'bootstrap-seeds', kaem_optional_seed = os.path.join(self.git_dir, 'seed', 'stage0-posix', 'bootstrap-seeds',
'POSIX', arch, 'kaem-optional-seed') 'POSIX', arch, 'kaem-optional-seed')
shutil.copy2(kaem_optional_seed, os.path.join(self.tmp_dir, 'init')) shutil.copy2(kaem_optional_seed, os.path.join(self.target_dir, 'init'))
def seed(self): def seed(self):
"""Copy in extra seed files""" """Copy in extra seed files"""
seed_dir = os.path.join(self.git_dir, 'seed') seed_dir = os.path.join(self.git_dir, 'seed')
for entry in os.listdir(seed_dir): for entry in os.listdir(seed_dir):
if os.path.isfile(os.path.join(seed_dir, entry)): if os.path.isfile(os.path.join(seed_dir, entry)):
shutil.copy2(os.path.join(seed_dir, entry), os.path.join(self.tmp_dir, entry)) shutil.copy2(os.path.join(seed_dir, entry), os.path.join(self.target_dir, entry))
@staticmethod @staticmethod
def add_fiwix_files(file_list_path, dirpath): def add_fiwix_files(file_list_path, dirpath):
@ -149,14 +149,14 @@ class Generator():
def create_fiwix_file_list(self): def create_fiwix_file_list(self):
"""Create a list of files to populate Fiwix file system""" """Create a list of files to populate Fiwix file system"""
file_list_path = os.path.join(self.tmp_dir, 'steps', 'lwext4-1.0.0-lb1', file_list_path = os.path.join(self.target_dir, 'steps', 'lwext4-1.0.0-lb1',
'files', 'fiwix-file-list.txt') 'files', 'fiwix-file-list.txt')
shutil.copyfile(os.path.join(self.tmp_dir, 'steps', 'lwext4-1.0.0-lb1', shutil.copyfile(os.path.join(self.target_dir, 'steps', 'lwext4-1.0.0-lb1',
'files', 'early-artifacts-needed-after-fiwix.txt'), 'files', 'early-artifacts-needed-after-fiwix.txt'),
file_list_path) file_list_path)
save_cwd = os.getcwd() save_cwd = os.getcwd()
os.chdir(self.tmp_dir) os.chdir(self.target_dir)
self.add_fiwix_files(file_list_path, 'steps') self.add_fiwix_files(file_list_path, 'steps')
self.add_fiwix_files(file_list_path, 'distfiles') self.add_fiwix_files(file_list_path, 'distfiles')
os.chdir(save_cwd) os.chdir(save_cwd)
@ -170,7 +170,7 @@ class Generator():
shutil.copy2(os.path.join(self.distfiles_dir, file), shutil.copy2(os.path.join(self.distfiles_dir, file),
os.path.join(out, file)) os.path.join(out, file))
early_distfile_dir = os.path.join(self.tmp_dir, 'external', 'distfiles') early_distfile_dir = os.path.join(self.target_dir, 'external', 'distfiles')
main_distfile_dir = os.path.join(self.external_dir, 'distfiles') main_distfile_dir = os.path.join(self.external_dir, 'distfiles')
if early_distfile_dir != main_distfile_dir: if early_distfile_dir != main_distfile_dir:
@ -217,7 +217,7 @@ class Generator():
"""Append srcfs file system to disk image""" """Append srcfs file system to disk image"""
save_cwd = os.getcwd() save_cwd = os.getcwd()
os.chdir(self.tmp_dir) os.chdir(self.target_dir)
self.output_tree(image_file, '.') self.output_tree(image_file, '.')
# Add commands to kick off stage0-posix # Add commands to kick off stage0-posix

View File

@ -4,34 +4,32 @@
# SPDX-License-Identifier: GPL-3.0-or-later # SPDX-License-Identifier: GPL-3.0-or-later
""" """
Contains a class that represents a tmpdir Contains a class that represents a target directory
""" """
import enum import enum
import getpass import getpass
import os import os
import shutil
from lib.utils import mount, umount, create_disk, run_as_root from lib.utils import mount, umount, create_disk, run_as_root
class TmpType(enum.Enum): class TargetType(enum.Enum):
"""Different types of tmpdirs we can have""" """Different types of target dirs we can have"""
NONE = 0 NONE = 0
TMPFS = 1 TMPFS = 1
class Tmpdir: class Target:
""" """
Represents a tmpdir Represents a target directory
""" """
_disks = {} _disks = {}
_disk_filesystems = {} _disk_filesystems = {}
_mountpoints = {} _mountpoints = {}
def __init__(self, preserve, path="tmp"): def __init__(self, path="target"):
self.path = os.path.abspath(path) self.path = os.path.abspath(path)
self.preserve = preserve self._type = TargetType.NONE
self._type = TmpType.NONE
if not os.path.exists(self.path): if not os.path.exists(self.path):
os.mkdir(self.path) os.mkdir(self.path)
@ -41,23 +39,15 @@ class Tmpdir:
print(f"Unmounting {path}") print(f"Unmounting {path}")
umount(path) umount(path)
if not self.preserve: for disk in self._disks.values():
for disk in self._disks.values(): print(f"Detaching {disk}")
print(f"Detaching {disk}") run_as_root("losetup", "-d", disk)
run_as_root("losetup", "-d", disk)
if self._type == TmpType.TMPFS:
print(f"Unmounting tmpdir from {self.path}")
umount(self.path)
print(f"Removing {self.path}")
shutil.rmtree(self.path, ignore_errors=True)
def tmpfs(self, size="8G"): def tmpfs(self, size="8G"):
"""Mount a tmpfs""" """Mount a tmpfs"""
print(f"Mounting tmpfs on {self.path}") print(f"Mounting tmpfs on {self.path}")
mount("tmpfs", self.path, "tmpfs", f"size={size}") mount("tmpfs", self.path, "tmpfs", f"size={size}")
self._type = TmpType.TMPFS self._type = TargetType.TMPFS
# pylint: disable=too-many-arguments # pylint: disable=too-many-arguments
def add_disk(self, def add_disk(self,

View File

@ -17,7 +17,7 @@ import argparse
import os import os
from lib.utils import run, run_as_root from lib.utils import run, run_as_root
from lib.tmpdir import Tmpdir from lib.target import Target
from lib.generator import Generator, stage0_arch_map from lib.generator import Generator, stage0_arch_map
def create_configuration_file(args): def create_configuration_file(args):
@ -58,11 +58,9 @@ def main():
action="store_true") action="store_true")
parser.add_argument("-bw", "--bwrap", help="Run inside a bwrap sandbox", parser.add_argument("-bw", "--bwrap", help="Run inside a bwrap sandbox",
action="store_true") action="store_true")
parser.add_argument("-p", "--preserve", help="Do not remove temporary dir", parser.add_argument("-t", "--target", help="Target directory",
action="store_true") default="target")
parser.add_argument("-t", "--tmpdir", help="Temporary directory", parser.add_argument("--tmpfs", help="Use a tmpfs on target",
default="tmp")
parser.add_argument("--tmpfs", help="Use a tmpfs on tmpdir",
action="store_true") action="store_true")
parser.add_argument("--tmpfs-size", help="Size of the tmpfs", parser.add_argument("--tmpfs-size", help="Size of the tmpfs",
default="8G") default="8G")
@ -128,7 +126,7 @@ def main():
if args.arch != "x86": if args.arch != "x86":
print("Only x86 is supported at the moment, other arches are for development only.") print("Only x86 is supported at the moment, other arches are for development only.")
# Tmp validation # Tmpfs validation
if args.bwrap and args.tmpfs: if args.bwrap and args.tmpfs:
raise ValueError("tmpfs cannot be used with bwrap.") raise ValueError("tmpfs cannot be used with bwrap.")
@ -156,19 +154,19 @@ def main():
with open(os.path.join('steps', 'bootstrap.cfg'), 'a', encoding='UTF-8'): with open(os.path.join('steps', 'bootstrap.cfg'), 'a', encoding='UTF-8'):
pass pass
# tmpdir # target
tmpdir = Tmpdir(path=args.tmpdir, preserve=args.preserve) target = Target(path=args.target)
if args.tmpfs: if args.tmpfs:
tmpdir.tmpfs(size=args.tmpfs_size) target.tmpfs(size=args.tmpfs_size)
generator = Generator(arch=args.arch, generator = Generator(arch=args.arch,
external_sources=args.external_sources, external_sources=args.external_sources,
repo_path=args.repo, repo_path=args.repo,
early_preseed=args.early_preseed) early_preseed=args.early_preseed)
bootstrap(args, generator, tmpdir, args.target_size) bootstrap(args, generator, target, args.target_size)
def bootstrap(args, generator, tmpdir, size): def bootstrap(args, generator, target, size):
"""Kick off bootstrap process.""" """Kick off bootstrap process."""
print(f"Bootstrapping {args.arch}") print(f"Bootstrapping {args.arch}")
if args.chroot: if args.chroot:
@ -179,28 +177,28 @@ print(shutil.which('chroot'))
chroot_binary = run_as_root('python3', '-c', find_chroot, chroot_binary = run_as_root('python3', '-c', find_chroot,
capture_output=True).stdout.decode().strip() capture_output=True).stdout.decode().strip()
generator.prepare(tmpdir, using_kernel=False) generator.prepare(target, using_kernel=False)
arch = stage0_arch_map.get(args.arch, args.arch) arch = stage0_arch_map.get(args.arch, args.arch)
init = os.path.join(os.sep, 'bootstrap-seeds', 'POSIX', arch, 'kaem-optional-seed') init = os.path.join(os.sep, 'bootstrap-seeds', 'POSIX', arch, 'kaem-optional-seed')
run_as_root('env', '-i', 'PATH=/bin', chroot_binary, generator.tmp_dir, init) run_as_root('env', '-i', 'PATH=/bin', chroot_binary, generator.target_dir, init)
elif args.bwrap: elif args.bwrap:
init = '/init' init = '/init'
if not args.internal_ci or args.internal_ci == "pass1": if not args.internal_ci or args.internal_ci == "pass1":
generator.prepare(tmpdir, using_kernel=False) generator.prepare(target, using_kernel=False)
arch = stage0_arch_map.get(args.arch, args.arch) arch = stage0_arch_map.get(args.arch, args.arch)
init = os.path.join(os.sep, 'bootstrap-seeds', 'POSIX', arch, 'kaem-optional-seed') init = os.path.join(os.sep, 'bootstrap-seeds', 'POSIX', arch, 'kaem-optional-seed')
else: else:
generator.reuse(tmpdir) generator.reuse(target)
run('env', '-i', 'bwrap', '--unshare-user', run('env', '-i', 'bwrap', '--unshare-user',
'--uid', '0', '--uid', '0',
'--gid', '0', '--gid', '0',
'--unshare-net' if args.external_sources else None, '--unshare-net' if args.external_sources else None,
'--setenv', 'PATH', '/usr/bin', '--setenv', 'PATH', '/usr/bin',
'--bind', generator.tmp_dir, '/', '--bind', generator.target_dir, '/',
'--dir', '/dev', '--dir', '/dev',
'--dev-bind', '/dev/null', '/dev/null', '--dev-bind', '/dev/null', '/dev/null',
'--dev-bind', '/dev/zero', '/dev/zero', '--dev-bind', '/dev/zero', '/dev/zero',
@ -216,44 +214,44 @@ print(shutil.which('chroot'))
elif args.bare_metal: elif args.bare_metal:
if args.kernel: if args.kernel:
generator.prepare(tmpdir, using_kernel=True, target_size=size) generator.prepare(target, using_kernel=True, target_size=size)
image_path = os.path.join(args.tmpdir, os.path.relpath(generator.tmp_dir, args.tmpdir)) path = os.path.join(args.target, os.path.relpath(generator.target_dir, args.target))
print("Please:") print("Please:")
print(f" 1. Take {image_path}/initramfs and your kernel, boot using this.") print(f" 1. Take {path}/initramfs and your kernel, boot using this.")
print(f" 2. Take {image_path}/disk.img and put this on a writable storage medium.") print(f" 2. Take {path}/disk.img and put this on a writable storage medium.")
else: else:
generator.prepare(tmpdir, kernel_bootstrap=True, target_size=size) generator.prepare(target, kernel_bootstrap=True, target_size=size)
image_path = os.path.join(args.tmpdir, os.path.relpath(generator.tmp_dir, args.tmpdir)) path = os.path.join(args.target, os.path.relpath(generator.target_dir, args.target))
print("Please:") print("Please:")
print(f" 1. Take {image_path}.img and write it to a boot drive and then boot it.") print(f" 1. Take {path}.img and write it to a boot drive and then boot it.")
else: else:
if args.kernel: if args.kernel:
generator.prepare(tmpdir, using_kernel=True, target_size=size) generator.prepare(target, using_kernel=True, target_size=size)
run(args.qemu_cmd, run(args.qemu_cmd,
'-enable-kvm', '-enable-kvm',
'-m', str(args.qemu_ram) + 'M', '-m', str(args.qemu_ram) + 'M',
'-smp', str(args.cores), '-smp', str(args.cores),
'-no-reboot', '-no-reboot',
'-drive', 'file=' + tmpdir.get_disk("disk") + ',format=raw', '-drive', 'file=' + target.get_disk("disk") + ',format=raw',
'-drive', 'file=' + tmpdir.get_disk("external") + ',format=raw', '-drive', 'file=' + target.get_disk("external") + ',format=raw',
'-nic', 'user,ipv6=off,model=e1000', '-nic', 'user,ipv6=off,model=e1000',
'-kernel', args.kernel, '-kernel', args.kernel,
'-nographic', '-nographic',
'-append', 'console=ttyS0 root=/dev/sda1 rootfstype=ext3 init=/init rw') '-append', 'console=ttyS0 root=/dev/sda1 rootfstype=ext3 init=/init rw')
else: else:
generator.prepare(tmpdir, kernel_bootstrap=True, target_size=size) generator.prepare(target, kernel_bootstrap=True, target_size=size)
arg_list = [ arg_list = [
'-enable-kvm', '-enable-kvm',
'-m', str(args.qemu_ram) + 'M', '-m', str(args.qemu_ram) + 'M',
'-smp', str(args.cores), '-smp', str(args.cores),
'-no-reboot', '-no-reboot',
'-drive', 'file=' + generator.tmp_dir + '.img' + ',format=raw' '-drive', 'file=' + generator.target_dir + '.img' + ',format=raw'
] ]
if tmpdir.get_disk("external") is not None: if target.get_disk("external") is not None:
arg_list += [ arg_list += [
'-drive', 'file=' + tmpdir.get_disk("external") + ',format=raw', '-drive', 'file=' + target.get_disk("external") + ',format=raw',
] ]
arg_list += [ arg_list += [
'-machine', 'kernel-irqchip=split', '-machine', 'kernel-irqchip=split',