Merge pull request #368 from Googulator/always-preserve

Rename tmpdir to target and always preserve it, dropping --preserve
This commit is contained in:
fosslinux 2023-12-27 22:59:23 +00:00 committed by GitHub
commit 51fe3f9991
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
5 changed files with 79 additions and 91 deletions

View File

@ -43,15 +43,15 @@ jobs:
distfiles distfiles
key: cache-${{ hashFiles('steps/*/sources') }} key: cache-${{ hashFiles('steps/*/sources') }}
- name: Run bootstrap - name: Run bootstrap
run: ./rootfs.py --bwrap --external-sources --build-kernels --preserve --cores 2 --internal-ci pass1 run: ./rootfs.py --bwrap --external-sources --build-kernels --cores 2 --internal-ci pass1
- name: Archive created packages - name: Archive created packages
if: failure() # archive failed builds progress if: failure() # archive failed builds progress
uses: actions/upload-artifact@v3 uses: actions/upload-artifact@v3
with: with:
name: packages name: packages
path: tmp/external/repo/** path: target/external/repo/**
- name: Tar pass1 image - name: Tar pass1 image
run: tar -cf pass1_image.tar tmp run: tar -cf pass1_image.tar target
- name: Archive pass1_image - name: Archive pass1_image
uses: actions/upload-artifact@v3 uses: actions/upload-artifact@v3
with: with:
@ -96,15 +96,15 @@ jobs:
distfiles distfiles
key: cache-${{ hashFiles('steps/*/sources') }} key: cache-${{ hashFiles('steps/*/sources') }}
- name: Run bootstrap - name: Run bootstrap
run: ./rootfs.py --bwrap --external-sources --build-kernels --preserve --cores 2 --internal-ci pass2 run: ./rootfs.py --bwrap --external-sources --build-kernels --cores 2 --internal-ci pass2
- name: Archive created packages - name: Archive created packages
if: failure() # archive failed builds progress if: failure() # archive failed builds progress
uses: actions/upload-artifact@v3 uses: actions/upload-artifact@v3
with: with:
name: internal_packages_pass2 name: internal_packages_pass2
path: tmp/external/repo/** path: target/external/repo/**
- name: Tar pass2 image - name: Tar pass2 image
run: tar -cf pass2_image.tar tmp run: tar -cf pass2_image.tar target
- name: Archive pass2_image - name: Archive pass2_image
uses: actions/upload-artifact@v3 uses: actions/upload-artifact@v3
with: with:
@ -149,10 +149,10 @@ jobs:
distfiles distfiles
key: cache-${{ hashFiles('steps/*/sources') }} key: cache-${{ hashFiles('steps/*/sources') }}
- name: Run bootstrap - name: Run bootstrap
run: ./rootfs.py --bwrap --external-sources --build-kernels --preserve --cores 2 --internal-ci pass3 run: ./rootfs.py --bwrap --external-sources --build-kernels --cores 2 --internal-ci pass3
- name: Archive created packages - name: Archive created packages
if: always() # archive both failed and successful builds if: always() # archive both failed and successful builds
uses: actions/upload-artifact@v3 uses: actions/upload-artifact@v3
with: with:
name: packages name: packages
path: tmp/external/repo/** path: target/external/repo/**

View File

@ -21,7 +21,7 @@ jobs:
- name: Install pylint - name: Install pylint
run: sudo pip3 install pylint run: sudo pip3 install pylint
- name: pylint - name: pylint
run: pylint rootfs.py lib/utils.py lib/generator.py lib/tmpdir.py --disable=duplicate-code run: pylint rootfs.py lib/utils.py lib/generator.py lib/target.py --disable=duplicate-code
shellcheck: shellcheck:
name: Lint shell files name: Lint shell files

View File

@ -29,26 +29,26 @@ class Generator():
self.external_sources = external_sources self.external_sources = external_sources
self.repo_path = repo_path self.repo_path = repo_path
self.source_manifest = self.get_source_manifest(not self.external_sources) self.source_manifest = self.get_source_manifest(not self.external_sources)
self.tmp_dir = None self.target_dir = None
self.external_dir = None self.external_dir = None
def reuse(self, tmpdir): def reuse(self, target):
""" """
Reuse a previously prepared bwrap environment for further stages. Reuse a previously prepared bwrap environment for further stages.
""" """
self.tmp_dir = tmpdir.path self.target_dir = target.path
self.external_dir = os.path.join(self.tmp_dir, 'external') self.external_dir = os.path.join(self.target_dir, 'external')
self.distfiles() self.distfiles()
def prepare(self, tmpdir, using_kernel=False, kernel_bootstrap=False, target_size=0): def prepare(self, target, using_kernel=False, kernel_bootstrap=False, target_size=0):
""" """
Prepare basic media of live-bootstrap. Prepare basic media of live-bootstrap.
/steps -- contains steps to be built /steps -- contains steps to be built
/ -- contains seed to allow steps to be built, containing custom / -- contains seed to allow steps to be built, containing custom
scripts and stage0-posix scripts and stage0-posix
""" """
self.tmp_dir = tmpdir.path self.target_dir = target.path
self.external_dir = os.path.join(self.tmp_dir, 'external') self.external_dir = os.path.join(self.target_dir, 'external')
# We use ext3 here; ext4 actually has a variety of extensions that # We use ext3 here; ext4 actually has a variety of extensions that
# have been added with varying levels of recency # have been added with varying levels of recency
@ -59,33 +59,33 @@ class Generator():
# argument matrix ... or we could just use ext3 instead which # argument matrix ... or we could just use ext3 instead which
# is effectively universally the same # is effectively universally the same
if kernel_bootstrap: if kernel_bootstrap:
init_path = os.path.join(self.tmp_dir, 'init') init_path = os.path.join(self.target_dir, 'init')
os.mkdir(init_path) os.mkdir(init_path)
self.tmp_dir = init_path self.target_dir = init_path
if self.repo_path or self.external_sources: if self.repo_path or self.external_sources:
tmpdir.add_disk("external", filesystem="ext3") target.add_disk("external", filesystem="ext3")
tmpdir.mount_disk("external", "external") target.mount_disk("external", "external")
else: else:
self.external_dir = os.path.join(self.tmp_dir, 'external') self.external_dir = os.path.join(self.target_dir, 'external')
elif using_kernel: elif using_kernel:
self.tmp_dir = os.path.join(self.tmp_dir, 'disk') self.target_dir = os.path.join(self.target_dir, 'disk')
tmpdir.add_disk("disk", target.add_disk("disk",
filesystem="ext3", filesystem="ext3",
size=(target_size + "M") if target_size else "16G", size=(target_size + "M") if target_size else "16G",
bootable=True) bootable=True)
tmpdir.mount_disk("disk", "disk") target.mount_disk("disk", "disk")
self.external_dir = os.path.join(self.tmp_dir, 'external') self.external_dir = os.path.join(self.target_dir, 'external')
os.makedirs(self.external_dir, exist_ok=True) os.makedirs(self.external_dir, exist_ok=True)
if self.early_preseed: if self.early_preseed:
# Extract tar containing preseed # Extract tar containing preseed
with tarfile.open(self.early_preseed, "r") as seed: with tarfile.open(self.early_preseed, "r") as seed:
seed.extractall(self.tmp_dir) seed.extractall(self.target_dir)
shutil.copy2(os.path.join(self.git_dir, 'seed', 'preseeded.kaem'), shutil.copy2(os.path.join(self.git_dir, 'seed', 'preseeded.kaem'),
os.path.join(self.tmp_dir, 'kaem.x86')) os.path.join(self.target_dir, 'kaem.x86'))
else: else:
self.stage0_posix() self.stage0_posix()
self.seed() self.seed()
@ -101,25 +101,25 @@ class Generator():
shutil.copytree(self.repo_path, repo_dir) shutil.copytree(self.repo_path, repo_dir)
if kernel_bootstrap: if kernel_bootstrap:
self.create_builder_hex0_disk_image(self.tmp_dir + '.img', target_size) self.create_builder_hex0_disk_image(self.target_dir + '.img', target_size)
if kernel_bootstrap and (self.external_sources or self.repo_path): if kernel_bootstrap and (self.external_sources or self.repo_path):
tmpdir.umount_disk('external') target.umount_disk('external')
elif using_kernel: elif using_kernel:
tmpdir.umount_disk('disk') target.umount_disk('disk')
def steps(self): def steps(self):
"""Copy in steps.""" """Copy in steps."""
self.get_packages() self.get_packages()
shutil.copytree(os.path.join(self.git_dir, 'steps'), os.path.join(self.tmp_dir, 'steps')) shutil.copytree(os.path.join(self.git_dir, 'steps'), os.path.join(self.target_dir, 'steps'))
def stage0_posix(self): def stage0_posix(self):
"""Copy in all of the stage0-posix""" """Copy in all of the stage0-posix"""
stage0_posix_base_dir = os.path.join(self.git_dir, 'seed', 'stage0-posix') stage0_posix_base_dir = os.path.join(self.git_dir, 'seed', 'stage0-posix')
for entry in os.listdir(stage0_posix_base_dir): for entry in os.listdir(stage0_posix_base_dir):
orig = os.path.join(stage0_posix_base_dir, entry) orig = os.path.join(stage0_posix_base_dir, entry)
target = os.path.join(self.tmp_dir, entry) target = os.path.join(self.target_dir, entry)
if os.path.isfile(orig): if os.path.isfile(orig):
shutil.copy2(orig, target) shutil.copy2(orig, target)
else: else:
@ -128,14 +128,14 @@ class Generator():
arch = stage0_arch_map.get(self.arch, self.arch) arch = stage0_arch_map.get(self.arch, self.arch)
kaem_optional_seed = os.path.join(self.git_dir, 'seed', 'stage0-posix', 'bootstrap-seeds', kaem_optional_seed = os.path.join(self.git_dir, 'seed', 'stage0-posix', 'bootstrap-seeds',
'POSIX', arch, 'kaem-optional-seed') 'POSIX', arch, 'kaem-optional-seed')
shutil.copy2(kaem_optional_seed, os.path.join(self.tmp_dir, 'init')) shutil.copy2(kaem_optional_seed, os.path.join(self.target_dir, 'init'))
def seed(self): def seed(self):
"""Copy in extra seed files""" """Copy in extra seed files"""
seed_dir = os.path.join(self.git_dir, 'seed') seed_dir = os.path.join(self.git_dir, 'seed')
for entry in os.listdir(seed_dir): for entry in os.listdir(seed_dir):
if os.path.isfile(os.path.join(seed_dir, entry)): if os.path.isfile(os.path.join(seed_dir, entry)):
shutil.copy2(os.path.join(seed_dir, entry), os.path.join(self.tmp_dir, entry)) shutil.copy2(os.path.join(seed_dir, entry), os.path.join(self.target_dir, entry))
@staticmethod @staticmethod
def add_fiwix_files(file_list_path, dirpath): def add_fiwix_files(file_list_path, dirpath):
@ -149,14 +149,14 @@ class Generator():
def create_fiwix_file_list(self): def create_fiwix_file_list(self):
"""Create a list of files to populate Fiwix file system""" """Create a list of files to populate Fiwix file system"""
file_list_path = os.path.join(self.tmp_dir, 'steps', 'lwext4-1.0.0-lb1', file_list_path = os.path.join(self.target_dir, 'steps', 'lwext4-1.0.0-lb1',
'files', 'fiwix-file-list.txt') 'files', 'fiwix-file-list.txt')
shutil.copyfile(os.path.join(self.tmp_dir, 'steps', 'lwext4-1.0.0-lb1', shutil.copyfile(os.path.join(self.target_dir, 'steps', 'lwext4-1.0.0-lb1',
'files', 'early-artifacts-needed-after-fiwix.txt'), 'files', 'early-artifacts-needed-after-fiwix.txt'),
file_list_path) file_list_path)
save_cwd = os.getcwd() save_cwd = os.getcwd()
os.chdir(self.tmp_dir) os.chdir(self.target_dir)
self.add_fiwix_files(file_list_path, 'steps') self.add_fiwix_files(file_list_path, 'steps')
self.add_fiwix_files(file_list_path, 'distfiles') self.add_fiwix_files(file_list_path, 'distfiles')
os.chdir(save_cwd) os.chdir(save_cwd)
@ -170,7 +170,7 @@ class Generator():
shutil.copy2(os.path.join(self.distfiles_dir, file), shutil.copy2(os.path.join(self.distfiles_dir, file),
os.path.join(out, file)) os.path.join(out, file))
early_distfile_dir = os.path.join(self.tmp_dir, 'external', 'distfiles') early_distfile_dir = os.path.join(self.target_dir, 'external', 'distfiles')
main_distfile_dir = os.path.join(self.external_dir, 'distfiles') main_distfile_dir = os.path.join(self.external_dir, 'distfiles')
if early_distfile_dir != main_distfile_dir: if early_distfile_dir != main_distfile_dir:
@ -217,7 +217,7 @@ class Generator():
"""Append srcfs file system to disk image""" """Append srcfs file system to disk image"""
save_cwd = os.getcwd() save_cwd = os.getcwd()
os.chdir(self.tmp_dir) os.chdir(self.target_dir)
self.output_tree(image_file, '.') self.output_tree(image_file, '.')
# Add commands to kick off stage0-posix # Add commands to kick off stage0-posix

View File

@ -4,34 +4,32 @@
# SPDX-License-Identifier: GPL-3.0-or-later # SPDX-License-Identifier: GPL-3.0-or-later
""" """
Contains a class that represents a tmpdir Contains a class that represents a target directory
""" """
import enum import enum
import getpass import getpass
import os import os
import shutil
from lib.utils import mount, umount, create_disk, run_as_root from lib.utils import mount, umount, create_disk, run_as_root
class TmpType(enum.Enum): class TargetType(enum.Enum):
"""Different types of tmpdirs we can have""" """Different types of target dirs we can have"""
NONE = 0 NONE = 0
TMPFS = 1 TMPFS = 1
class Tmpdir: class Target:
""" """
Represents a tmpdir Represents a target directory
""" """
_disks = {} _disks = {}
_disk_filesystems = {} _disk_filesystems = {}
_mountpoints = {} _mountpoints = {}
def __init__(self, preserve, path="tmp"): def __init__(self, path="target"):
self.path = os.path.abspath(path) self.path = os.path.abspath(path)
self.preserve = preserve self._type = TargetType.NONE
self._type = TmpType.NONE
if not os.path.exists(self.path): if not os.path.exists(self.path):
os.mkdir(self.path) os.mkdir(self.path)
@ -41,23 +39,15 @@ class Tmpdir:
print(f"Unmounting {path}") print(f"Unmounting {path}")
umount(path) umount(path)
if not self.preserve: for disk in self._disks.values():
for disk in self._disks.values(): print(f"Detaching {disk}")
print(f"Detaching {disk}") run_as_root("losetup", "-d", disk)
run_as_root("losetup", "-d", disk)
if self._type == TmpType.TMPFS:
print(f"Unmounting tmpdir from {self.path}")
umount(self.path)
print(f"Removing {self.path}")
shutil.rmtree(self.path, ignore_errors=True)
def tmpfs(self, size="8G"): def tmpfs(self, size="8G"):
"""Mount a tmpfs""" """Mount a tmpfs"""
print(f"Mounting tmpfs on {self.path}") print(f"Mounting tmpfs on {self.path}")
mount("tmpfs", self.path, "tmpfs", f"size={size}") mount("tmpfs", self.path, "tmpfs", f"size={size}")
self._type = TmpType.TMPFS self._type = TargetType.TMPFS
# pylint: disable=too-many-arguments # pylint: disable=too-many-arguments
def add_disk(self, def add_disk(self,

View File

@ -17,7 +17,7 @@ import argparse
import os import os
from lib.utils import run, run_as_root from lib.utils import run, run_as_root
from lib.tmpdir import Tmpdir from lib.target import Target
from lib.generator import Generator, stage0_arch_map from lib.generator import Generator, stage0_arch_map
def create_configuration_file(args): def create_configuration_file(args):
@ -58,11 +58,9 @@ def main():
action="store_true") action="store_true")
parser.add_argument("-bw", "--bwrap", help="Run inside a bwrap sandbox", parser.add_argument("-bw", "--bwrap", help="Run inside a bwrap sandbox",
action="store_true") action="store_true")
parser.add_argument("-p", "--preserve", help="Do not remove temporary dir", parser.add_argument("-t", "--target", help="Target directory",
action="store_true") default="target")
parser.add_argument("-t", "--tmpdir", help="Temporary directory", parser.add_argument("--tmpfs", help="Use a tmpfs on target",
default="tmp")
parser.add_argument("--tmpfs", help="Use a tmpfs on tmpdir",
action="store_true") action="store_true")
parser.add_argument("--tmpfs-size", help="Size of the tmpfs", parser.add_argument("--tmpfs-size", help="Size of the tmpfs",
default="8G") default="8G")
@ -128,7 +126,7 @@ def main():
if args.arch != "x86": if args.arch != "x86":
print("Only x86 is supported at the moment, other arches are for development only.") print("Only x86 is supported at the moment, other arches are for development only.")
# Tmp validation # Tmpfs validation
if args.bwrap and args.tmpfs: if args.bwrap and args.tmpfs:
raise ValueError("tmpfs cannot be used with bwrap.") raise ValueError("tmpfs cannot be used with bwrap.")
@ -156,19 +154,19 @@ def main():
with open(os.path.join('steps', 'bootstrap.cfg'), 'a', encoding='UTF-8'): with open(os.path.join('steps', 'bootstrap.cfg'), 'a', encoding='UTF-8'):
pass pass
# tmpdir # target
tmpdir = Tmpdir(path=args.tmpdir, preserve=args.preserve) target = Target(path=args.target)
if args.tmpfs: if args.tmpfs:
tmpdir.tmpfs(size=args.tmpfs_size) target.tmpfs(size=args.tmpfs_size)
generator = Generator(arch=args.arch, generator = Generator(arch=args.arch,
external_sources=args.external_sources, external_sources=args.external_sources,
repo_path=args.repo, repo_path=args.repo,
early_preseed=args.early_preseed) early_preseed=args.early_preseed)
bootstrap(args, generator, tmpdir, args.target_size) bootstrap(args, generator, target, args.target_size)
def bootstrap(args, generator, tmpdir, size): def bootstrap(args, generator, target, size):
"""Kick off bootstrap process.""" """Kick off bootstrap process."""
print(f"Bootstrapping {args.arch}") print(f"Bootstrapping {args.arch}")
if args.chroot: if args.chroot:
@ -179,28 +177,28 @@ print(shutil.which('chroot'))
chroot_binary = run_as_root('python3', '-c', find_chroot, chroot_binary = run_as_root('python3', '-c', find_chroot,
capture_output=True).stdout.decode().strip() capture_output=True).stdout.decode().strip()
generator.prepare(tmpdir, using_kernel=False) generator.prepare(target, using_kernel=False)
arch = stage0_arch_map.get(args.arch, args.arch) arch = stage0_arch_map.get(args.arch, args.arch)
init = os.path.join(os.sep, 'bootstrap-seeds', 'POSIX', arch, 'kaem-optional-seed') init = os.path.join(os.sep, 'bootstrap-seeds', 'POSIX', arch, 'kaem-optional-seed')
run_as_root('env', '-i', 'PATH=/bin', chroot_binary, generator.tmp_dir, init) run_as_root('env', '-i', 'PATH=/bin', chroot_binary, generator.target_dir, init)
elif args.bwrap: elif args.bwrap:
init = '/init' init = '/init'
if not args.internal_ci or args.internal_ci == "pass1": if not args.internal_ci or args.internal_ci == "pass1":
generator.prepare(tmpdir, using_kernel=False) generator.prepare(target, using_kernel=False)
arch = stage0_arch_map.get(args.arch, args.arch) arch = stage0_arch_map.get(args.arch, args.arch)
init = os.path.join(os.sep, 'bootstrap-seeds', 'POSIX', arch, 'kaem-optional-seed') init = os.path.join(os.sep, 'bootstrap-seeds', 'POSIX', arch, 'kaem-optional-seed')
else: else:
generator.reuse(tmpdir) generator.reuse(target)
run('env', '-i', 'bwrap', '--unshare-user', run('env', '-i', 'bwrap', '--unshare-user',
'--uid', '0', '--uid', '0',
'--gid', '0', '--gid', '0',
'--unshare-net' if args.external_sources else None, '--unshare-net' if args.external_sources else None,
'--setenv', 'PATH', '/usr/bin', '--setenv', 'PATH', '/usr/bin',
'--bind', generator.tmp_dir, '/', '--bind', generator.target_dir, '/',
'--dir', '/dev', '--dir', '/dev',
'--dev-bind', '/dev/null', '/dev/null', '--dev-bind', '/dev/null', '/dev/null',
'--dev-bind', '/dev/zero', '/dev/zero', '--dev-bind', '/dev/zero', '/dev/zero',
@ -216,44 +214,44 @@ print(shutil.which('chroot'))
elif args.bare_metal: elif args.bare_metal:
if args.kernel: if args.kernel:
generator.prepare(tmpdir, using_kernel=True, target_size=size) generator.prepare(target, using_kernel=True, target_size=size)
image_path = os.path.join(args.tmpdir, os.path.relpath(generator.tmp_dir, args.tmpdir)) path = os.path.join(args.target, os.path.relpath(generator.target_dir, args.target))
print("Please:") print("Please:")
print(f" 1. Take {image_path}/initramfs and your kernel, boot using this.") print(f" 1. Take {path}/initramfs and your kernel, boot using this.")
print(f" 2. Take {image_path}/disk.img and put this on a writable storage medium.") print(f" 2. Take {path}/disk.img and put this on a writable storage medium.")
else: else:
generator.prepare(tmpdir, kernel_bootstrap=True, target_size=size) generator.prepare(target, kernel_bootstrap=True, target_size=size)
image_path = os.path.join(args.tmpdir, os.path.relpath(generator.tmp_dir, args.tmpdir)) path = os.path.join(args.target, os.path.relpath(generator.target_dir, args.target))
print("Please:") print("Please:")
print(f" 1. Take {image_path}.img and write it to a boot drive and then boot it.") print(f" 1. Take {path}.img and write it to a boot drive and then boot it.")
else: else:
if args.kernel: if args.kernel:
generator.prepare(tmpdir, using_kernel=True, target_size=size) generator.prepare(target, using_kernel=True, target_size=size)
run(args.qemu_cmd, run(args.qemu_cmd,
'-enable-kvm', '-enable-kvm',
'-m', str(args.qemu_ram) + 'M', '-m', str(args.qemu_ram) + 'M',
'-smp', str(args.cores), '-smp', str(args.cores),
'-no-reboot', '-no-reboot',
'-drive', 'file=' + tmpdir.get_disk("disk") + ',format=raw', '-drive', 'file=' + target.get_disk("disk") + ',format=raw',
'-drive', 'file=' + tmpdir.get_disk("external") + ',format=raw', '-drive', 'file=' + target.get_disk("external") + ',format=raw',
'-nic', 'user,ipv6=off,model=e1000', '-nic', 'user,ipv6=off,model=e1000',
'-kernel', args.kernel, '-kernel', args.kernel,
'-nographic', '-nographic',
'-append', 'console=ttyS0 root=/dev/sda1 rootfstype=ext3 init=/init rw') '-append', 'console=ttyS0 root=/dev/sda1 rootfstype=ext3 init=/init rw')
else: else:
generator.prepare(tmpdir, kernel_bootstrap=True, target_size=size) generator.prepare(target, kernel_bootstrap=True, target_size=size)
arg_list = [ arg_list = [
'-enable-kvm', '-enable-kvm',
'-m', str(args.qemu_ram) + 'M', '-m', str(args.qemu_ram) + 'M',
'-smp', str(args.cores), '-smp', str(args.cores),
'-no-reboot', '-no-reboot',
'-drive', 'file=' + generator.tmp_dir + '.img' + ',format=raw' '-drive', 'file=' + generator.target_dir + '.img' + ',format=raw'
] ]
if tmpdir.get_disk("external") is not None: if target.get_disk("external") is not None:
arg_list += [ arg_list += [
'-drive', 'file=' + tmpdir.get_disk("external") + ',format=raw', '-drive', 'file=' + target.get_disk("external") + ',format=raw',
] ]
arg_list += [ arg_list += [
'-machine', 'kernel-irqchip=split', '-machine', 'kernel-irqchip=split',