Rename tmpdir to target and always preserve it, dropping --preserve
There is nothing temporary about our "tmpdir" - its sole purpose is to contain the final product of the bootstrap process. Thus, removing it at the end of bootstrap amounts to doing the entire process for nothing. To remedy this, --tmpdir is renamed --target, keeping the -t short form, and defaulting to "target" instead of "tmp" to make its purpose clearer. The --preserve option is removed, as the target is now always preserved.
This commit is contained in:
Gábor Stefanik
parent
9179f4bd01
commit
81e3123262
|
|
@ -43,15 +43,15 @@ jobs:
|
|||
distfiles
|
||||
key: cache-${{ hashFiles('steps/*/sources') }}
|
||||
- name: Run bootstrap
|
||||
run: ./rootfs.py --bwrap --external-sources --build-kernels --preserve --cores 2 --internal-ci pass1
|
||||
run: ./rootfs.py --bwrap --external-sources --build-kernels --cores 2 --internal-ci pass1
|
||||
- name: Archive created packages
|
||||
if: failure() # archive failed builds progress
|
||||
uses: actions/upload-artifact@v3
|
||||
with:
|
||||
name: packages
|
||||
path: tmp/external/repo/**
|
||||
path: target/external/repo/**
|
||||
- name: Tar pass1 image
|
||||
run: tar -cf pass1_image.tar tmp
|
||||
run: tar -cf pass1_image.tar target
|
||||
- name: Archive pass1_image
|
||||
uses: actions/upload-artifact@v3
|
||||
with:
|
||||
|
|
@ -96,15 +96,15 @@ jobs:
|
|||
distfiles
|
||||
key: cache-${{ hashFiles('steps/*/sources') }}
|
||||
- name: Run bootstrap
|
||||
run: ./rootfs.py --bwrap --external-sources --build-kernels --preserve --cores 2 --internal-ci pass2
|
||||
run: ./rootfs.py --bwrap --external-sources --build-kernels --cores 2 --internal-ci pass2
|
||||
- name: Archive created packages
|
||||
if: failure() # archive failed builds progress
|
||||
uses: actions/upload-artifact@v3
|
||||
with:
|
||||
name: internal_packages_pass2
|
||||
path: tmp/external/repo/**
|
||||
path: target/external/repo/**
|
||||
- name: Tar pass2 image
|
||||
run: tar -cf pass2_image.tar tmp
|
||||
run: tar -cf pass2_image.tar target
|
||||
- name: Archive pass2_image
|
||||
uses: actions/upload-artifact@v3
|
||||
with:
|
||||
|
|
@ -149,10 +149,10 @@ jobs:
|
|||
distfiles
|
||||
key: cache-${{ hashFiles('steps/*/sources') }}
|
||||
- name: Run bootstrap
|
||||
run: ./rootfs.py --bwrap --external-sources --build-kernels --preserve --cores 2 --internal-ci pass3
|
||||
run: ./rootfs.py --bwrap --external-sources --build-kernels --cores 2 --internal-ci pass3
|
||||
- name: Archive created packages
|
||||
if: always() # archive both failed and successful builds
|
||||
uses: actions/upload-artifact@v3
|
||||
with:
|
||||
name: packages
|
||||
path: tmp/external/repo/**
|
||||
path: target/external/repo/**
|
||||
|
|
|
|||
|
|
@ -21,7 +21,7 @@ jobs:
|
|||
- name: Install pylint
|
||||
run: sudo pip3 install pylint
|
||||
- name: pylint
|
||||
run: pylint rootfs.py lib/utils.py lib/generator.py lib/tmpdir.py --disable=duplicate-code
|
||||
run: pylint rootfs.py lib/utils.py lib/generator.py lib/target.py --disable=duplicate-code
|
||||
|
||||
shellcheck:
|
||||
name: Lint shell files
|
||||
|
|
|
|||
|
|
@ -29,26 +29,26 @@ class Generator():
|
|||
self.external_sources = external_sources
|
||||
self.repo_path = repo_path
|
||||
self.source_manifest = self.get_source_manifest(not self.external_sources)
|
||||
self.tmp_dir = None
|
||||
self.target_dir = None
|
||||
self.external_dir = None
|
||||
|
||||
def reuse(self, tmpdir):
|
||||
def reuse(self, target):
|
||||
"""
|
||||
Reuse a previously prepared bwrap environment for further stages.
|
||||
"""
|
||||
self.tmp_dir = tmpdir.path
|
||||
self.external_dir = os.path.join(self.tmp_dir, 'external')
|
||||
self.target_dir = target.path
|
||||
self.external_dir = os.path.join(self.target_dir, 'external')
|
||||
self.distfiles()
|
||||
|
||||
def prepare(self, tmpdir, using_kernel=False, kernel_bootstrap=False, target_size=0):
|
||||
def prepare(self, target, using_kernel=False, kernel_bootstrap=False, target_size=0):
|
||||
"""
|
||||
Prepare basic media of live-bootstrap.
|
||||
/steps -- contains steps to be built
|
||||
/ -- contains seed to allow steps to be built, containing custom
|
||||
scripts and stage0-posix
|
||||
"""
|
||||
self.tmp_dir = tmpdir.path
|
||||
self.external_dir = os.path.join(self.tmp_dir, 'external')
|
||||
self.target_dir = target.path
|
||||
self.external_dir = os.path.join(self.target_dir, 'external')
|
||||
|
||||
# We use ext3 here; ext4 actually has a variety of extensions that
|
||||
# have been added with varying levels of recency
|
||||
|
|
@ -59,33 +59,33 @@ class Generator():
|
|||
# argument matrix ... or we could just use ext3 instead which
|
||||
# is effectively universally the same
|
||||
if kernel_bootstrap:
|
||||
init_path = os.path.join(self.tmp_dir, 'init')
|
||||
init_path = os.path.join(self.target_dir, 'init')
|
||||
|
||||
os.mkdir(init_path)
|
||||
self.tmp_dir = init_path
|
||||
self.target_dir = init_path
|
||||
|
||||
if self.repo_path or self.external_sources:
|
||||
tmpdir.add_disk("external", filesystem="ext3")
|
||||
tmpdir.mount_disk("external", "external")
|
||||
target.add_disk("external", filesystem="ext3")
|
||||
target.mount_disk("external", "external")
|
||||
else:
|
||||
self.external_dir = os.path.join(self.tmp_dir, 'external')
|
||||
self.external_dir = os.path.join(self.target_dir, 'external')
|
||||
elif using_kernel:
|
||||
self.tmp_dir = os.path.join(self.tmp_dir, 'disk')
|
||||
tmpdir.add_disk("disk",
|
||||
self.target_dir = os.path.join(self.target_dir, 'disk')
|
||||
target.add_disk("disk",
|
||||
filesystem="ext3",
|
||||
size=(target_size + "M") if target_size else "16G",
|
||||
bootable=True)
|
||||
tmpdir.mount_disk("disk", "disk")
|
||||
self.external_dir = os.path.join(self.tmp_dir, 'external')
|
||||
target.mount_disk("disk", "disk")
|
||||
self.external_dir = os.path.join(self.target_dir, 'external')
|
||||
|
||||
os.makedirs(self.external_dir, exist_ok=True)
|
||||
|
||||
if self.early_preseed:
|
||||
# Extract tar containing preseed
|
||||
with tarfile.open(self.early_preseed, "r") as seed:
|
||||
seed.extractall(self.tmp_dir)
|
||||
seed.extractall(self.target_dir)
|
||||
shutil.copy2(os.path.join(self.git_dir, 'seed', 'preseeded.kaem'),
|
||||
os.path.join(self.tmp_dir, 'kaem.x86'))
|
||||
os.path.join(self.target_dir, 'kaem.x86'))
|
||||
else:
|
||||
self.stage0_posix()
|
||||
self.seed()
|
||||
|
|
@ -101,25 +101,25 @@ class Generator():
|
|||
shutil.copytree(self.repo_path, repo_dir)
|
||||
|
||||
if kernel_bootstrap:
|
||||
self.create_builder_hex0_disk_image(self.tmp_dir + '.img', target_size)
|
||||
self.create_builder_hex0_disk_image(self.target_dir + '.img', target_size)
|
||||
|
||||
if kernel_bootstrap and (self.external_sources or self.repo_path):
|
||||
tmpdir.umount_disk('external')
|
||||
target.umount_disk('external')
|
||||
elif using_kernel:
|
||||
tmpdir.umount_disk('disk')
|
||||
target.umount_disk('disk')
|
||||
|
||||
def steps(self):
|
||||
"""Copy in steps."""
|
||||
self.get_packages()
|
||||
|
||||
shutil.copytree(os.path.join(self.git_dir, 'steps'), os.path.join(self.tmp_dir, 'steps'))
|
||||
shutil.copytree(os.path.join(self.git_dir, 'steps'), os.path.join(self.target_dir, 'steps'))
|
||||
|
||||
def stage0_posix(self):
|
||||
"""Copy in all of the stage0-posix"""
|
||||
stage0_posix_base_dir = os.path.join(self.git_dir, 'seed', 'stage0-posix')
|
||||
for entry in os.listdir(stage0_posix_base_dir):
|
||||
orig = os.path.join(stage0_posix_base_dir, entry)
|
||||
target = os.path.join(self.tmp_dir, entry)
|
||||
target = os.path.join(self.target_dir, entry)
|
||||
if os.path.isfile(orig):
|
||||
shutil.copy2(orig, target)
|
||||
else:
|
||||
|
|
@ -128,14 +128,14 @@ class Generator():
|
|||
arch = stage0_arch_map.get(self.arch, self.arch)
|
||||
kaem_optional_seed = os.path.join(self.git_dir, 'seed', 'stage0-posix', 'bootstrap-seeds',
|
||||
'POSIX', arch, 'kaem-optional-seed')
|
||||
shutil.copy2(kaem_optional_seed, os.path.join(self.tmp_dir, 'init'))
|
||||
shutil.copy2(kaem_optional_seed, os.path.join(self.target_dir, 'init'))
|
||||
|
||||
def seed(self):
|
||||
"""Copy in extra seed files"""
|
||||
seed_dir = os.path.join(self.git_dir, 'seed')
|
||||
for entry in os.listdir(seed_dir):
|
||||
if os.path.isfile(os.path.join(seed_dir, entry)):
|
||||
shutil.copy2(os.path.join(seed_dir, entry), os.path.join(self.tmp_dir, entry))
|
||||
shutil.copy2(os.path.join(seed_dir, entry), os.path.join(self.target_dir, entry))
|
||||
|
||||
@staticmethod
|
||||
def add_fiwix_files(file_list_path, dirpath):
|
||||
|
|
@ -149,14 +149,14 @@ class Generator():
|
|||
|
||||
def create_fiwix_file_list(self):
|
||||
"""Create a list of files to populate Fiwix file system"""
|
||||
file_list_path = os.path.join(self.tmp_dir, 'steps', 'lwext4-1.0.0-lb1',
|
||||
file_list_path = os.path.join(self.target_dir, 'steps', 'lwext4-1.0.0-lb1',
|
||||
'files', 'fiwix-file-list.txt')
|
||||
shutil.copyfile(os.path.join(self.tmp_dir, 'steps', 'lwext4-1.0.0-lb1',
|
||||
shutil.copyfile(os.path.join(self.target_dir, 'steps', 'lwext4-1.0.0-lb1',
|
||||
'files', 'early-artifacts-needed-after-fiwix.txt'),
|
||||
file_list_path)
|
||||
|
||||
save_cwd = os.getcwd()
|
||||
os.chdir(self.tmp_dir)
|
||||
os.chdir(self.target_dir)
|
||||
self.add_fiwix_files(file_list_path, 'steps')
|
||||
self.add_fiwix_files(file_list_path, 'distfiles')
|
||||
os.chdir(save_cwd)
|
||||
|
|
@ -170,7 +170,7 @@ class Generator():
|
|||
shutil.copy2(os.path.join(self.distfiles_dir, file),
|
||||
os.path.join(out, file))
|
||||
|
||||
early_distfile_dir = os.path.join(self.tmp_dir, 'external', 'distfiles')
|
||||
early_distfile_dir = os.path.join(self.target_dir, 'external', 'distfiles')
|
||||
main_distfile_dir = os.path.join(self.external_dir, 'distfiles')
|
||||
|
||||
if early_distfile_dir != main_distfile_dir:
|
||||
|
|
@ -217,7 +217,7 @@ class Generator():
|
|||
"""Append srcfs file system to disk image"""
|
||||
save_cwd = os.getcwd()
|
||||
|
||||
os.chdir(self.tmp_dir)
|
||||
os.chdir(self.target_dir)
|
||||
self.output_tree(image_file, '.')
|
||||
|
||||
# Add commands to kick off stage0-posix
|
||||
|
|
|
|||
|
|
@ -4,34 +4,32 @@
|
|||
# SPDX-License-Identifier: GPL-3.0-or-later
|
||||
|
||||
"""
|
||||
Contains a class that represents a tmpdir
|
||||
Contains a class that represents a target directory
|
||||
"""
|
||||
|
||||
import enum
|
||||
import getpass
|
||||
import os
|
||||
import shutil
|
||||
|
||||
from lib.utils import mount, umount, create_disk, run_as_root
|
||||
|
||||
class TmpType(enum.Enum):
|
||||
"""Different types of tmpdirs we can have"""
|
||||
class TargetType(enum.Enum):
|
||||
"""Different types of target dirs we can have"""
|
||||
NONE = 0
|
||||
TMPFS = 1
|
||||
|
||||
class Tmpdir:
|
||||
class Target:
|
||||
"""
|
||||
Represents a tmpdir
|
||||
Represents a target directory
|
||||
"""
|
||||
|
||||
_disks = {}
|
||||
_disk_filesystems = {}
|
||||
_mountpoints = {}
|
||||
|
||||
def __init__(self, preserve, path="tmp"):
|
||||
def __init__(self, path="target"):
|
||||
self.path = os.path.abspath(path)
|
||||
self.preserve = preserve
|
||||
self._type = TmpType.NONE
|
||||
self._type = TargetType.NONE
|
||||
|
||||
if not os.path.exists(self.path):
|
||||
os.mkdir(self.path)
|
||||
|
|
@ -41,23 +39,15 @@ class Tmpdir:
|
|||
print(f"Unmounting {path}")
|
||||
umount(path)
|
||||
|
||||
if not self.preserve:
|
||||
for disk in self._disks.values():
|
||||
print(f"Detaching {disk}")
|
||||
run_as_root("losetup", "-d", disk)
|
||||
|
||||
if self._type == TmpType.TMPFS:
|
||||
print(f"Unmounting tmpdir from {self.path}")
|
||||
umount(self.path)
|
||||
|
||||
print(f"Removing {self.path}")
|
||||
shutil.rmtree(self.path, ignore_errors=True)
|
||||
for disk in self._disks.values():
|
||||
print(f"Detaching {disk}")
|
||||
run_as_root("losetup", "-d", disk)
|
||||
|
||||
def tmpfs(self, size="8G"):
|
||||
"""Mount a tmpfs"""
|
||||
print(f"Mounting tmpfs on {self.path}")
|
||||
mount("tmpfs", self.path, "tmpfs", f"size={size}")
|
||||
self._type = TmpType.TMPFS
|
||||
self._type = TargetType.TMPFS
|
||||
|
||||
# pylint: disable=too-many-arguments
|
||||
def add_disk(self,
|
||||
60
rootfs.py
60
rootfs.py
|
|
@ -17,7 +17,7 @@ import argparse
|
|||
import os
|
||||
|
||||
from lib.utils import run, run_as_root
|
||||
from lib.tmpdir import Tmpdir
|
||||
from lib.target import Target
|
||||
from lib.generator import Generator, stage0_arch_map
|
||||
|
||||
def create_configuration_file(args):
|
||||
|
|
@ -58,11 +58,9 @@ def main():
|
|||
action="store_true")
|
||||
parser.add_argument("-bw", "--bwrap", help="Run inside a bwrap sandbox",
|
||||
action="store_true")
|
||||
parser.add_argument("-p", "--preserve", help="Do not remove temporary dir",
|
||||
action="store_true")
|
||||
parser.add_argument("-t", "--tmpdir", help="Temporary directory",
|
||||
default="tmp")
|
||||
parser.add_argument("--tmpfs", help="Use a tmpfs on tmpdir",
|
||||
parser.add_argument("-t", "--target", help="Target directory",
|
||||
default="target")
|
||||
parser.add_argument("--tmpfs", help="Use a tmpfs on target",
|
||||
action="store_true")
|
||||
parser.add_argument("--tmpfs-size", help="Size of the tmpfs",
|
||||
default="8G")
|
||||
|
|
@ -128,7 +126,7 @@ def main():
|
|||
if args.arch != "x86":
|
||||
print("Only x86 is supported at the moment, other arches are for development only.")
|
||||
|
||||
# Tmp validation
|
||||
# Tmpfs validation
|
||||
if args.bwrap and args.tmpfs:
|
||||
raise ValueError("tmpfs cannot be used with bwrap.")
|
||||
|
||||
|
|
@ -156,19 +154,19 @@ def main():
|
|||
with open(os.path.join('steps', 'bootstrap.cfg'), 'a', encoding='UTF-8'):
|
||||
pass
|
||||
|
||||
# tmpdir
|
||||
tmpdir = Tmpdir(path=args.tmpdir, preserve=args.preserve)
|
||||
# target
|
||||
target = Target(path=args.target)
|
||||
if args.tmpfs:
|
||||
tmpdir.tmpfs(size=args.tmpfs_size)
|
||||
target.tmpfs(size=args.tmpfs_size)
|
||||
|
||||
generator = Generator(arch=args.arch,
|
||||
external_sources=args.external_sources,
|
||||
repo_path=args.repo,
|
||||
early_preseed=args.early_preseed)
|
||||
|
||||
bootstrap(args, generator, tmpdir, args.target_size)
|
||||
bootstrap(args, generator, target, args.target_size)
|
||||
|
||||
def bootstrap(args, generator, tmpdir, size):
|
||||
def bootstrap(args, generator, target, size):
|
||||
"""Kick off bootstrap process."""
|
||||
print(f"Bootstrapping {args.arch}")
|
||||
if args.chroot:
|
||||
|
|
@ -179,28 +177,28 @@ print(shutil.which('chroot'))
|
|||
chroot_binary = run_as_root('python3', '-c', find_chroot,
|
||||
capture_output=True).stdout.decode().strip()
|
||||
|
||||
generator.prepare(tmpdir, using_kernel=False)
|
||||
generator.prepare(target, using_kernel=False)
|
||||
|
||||
arch = stage0_arch_map.get(args.arch, args.arch)
|
||||
init = os.path.join(os.sep, 'bootstrap-seeds', 'POSIX', arch, 'kaem-optional-seed')
|
||||
run_as_root('env', '-i', 'PATH=/bin', chroot_binary, generator.tmp_dir, init)
|
||||
run_as_root('env', '-i', 'PATH=/bin', chroot_binary, generator.target_dir, init)
|
||||
|
||||
elif args.bwrap:
|
||||
init = '/init'
|
||||
if not args.internal_ci or args.internal_ci == "pass1":
|
||||
generator.prepare(tmpdir, using_kernel=False)
|
||||
generator.prepare(target, using_kernel=False)
|
||||
|
||||
arch = stage0_arch_map.get(args.arch, args.arch)
|
||||
init = os.path.join(os.sep, 'bootstrap-seeds', 'POSIX', arch, 'kaem-optional-seed')
|
||||
else:
|
||||
generator.reuse(tmpdir)
|
||||
generator.reuse(target)
|
||||
|
||||
run('env', '-i', 'bwrap', '--unshare-user',
|
||||
'--uid', '0',
|
||||
'--gid', '0',
|
||||
'--unshare-net' if args.external_sources else None,
|
||||
'--setenv', 'PATH', '/usr/bin',
|
||||
'--bind', generator.tmp_dir, '/',
|
||||
'--bind', generator.target_dir, '/',
|
||||
'--dir', '/dev',
|
||||
'--dev-bind', '/dev/null', '/dev/null',
|
||||
'--dev-bind', '/dev/zero', '/dev/zero',
|
||||
|
|
@ -216,44 +214,44 @@ print(shutil.which('chroot'))
|
|||
|
||||
elif args.bare_metal:
|
||||
if args.kernel:
|
||||
generator.prepare(tmpdir, using_kernel=True, target_size=size)
|
||||
image_path = os.path.join(args.tmpdir, os.path.relpath(generator.tmp_dir, args.tmpdir))
|
||||
generator.prepare(target, using_kernel=True, target_size=size)
|
||||
path = os.path.join(args.target, os.path.relpath(generator.target_dir, args.target))
|
||||
print("Please:")
|
||||
print(f" 1. Take {image_path}/initramfs and your kernel, boot using this.")
|
||||
print(f" 2. Take {image_path}/disk.img and put this on a writable storage medium.")
|
||||
print(f" 1. Take {path}/initramfs and your kernel, boot using this.")
|
||||
print(f" 2. Take {path}/disk.img and put this on a writable storage medium.")
|
||||
else:
|
||||
generator.prepare(tmpdir, kernel_bootstrap=True, target_size=size)
|
||||
image_path = os.path.join(args.tmpdir, os.path.relpath(generator.tmp_dir, args.tmpdir))
|
||||
generator.prepare(target, kernel_bootstrap=True, target_size=size)
|
||||
path = os.path.join(args.target, os.path.relpath(generator.target_dir, args.target))
|
||||
print("Please:")
|
||||
print(f" 1. Take {image_path}.img and write it to a boot drive and then boot it.")
|
||||
print(f" 1. Take {path}.img and write it to a boot drive and then boot it.")
|
||||
|
||||
else:
|
||||
if args.kernel:
|
||||
generator.prepare(tmpdir, using_kernel=True, target_size=size)
|
||||
generator.prepare(target, using_kernel=True, target_size=size)
|
||||
|
||||
run(args.qemu_cmd,
|
||||
'-enable-kvm',
|
||||
'-m', str(args.qemu_ram) + 'M',
|
||||
'-smp', str(args.cores),
|
||||
'-no-reboot',
|
||||
'-drive', 'file=' + tmpdir.get_disk("disk") + ',format=raw',
|
||||
'-drive', 'file=' + tmpdir.get_disk("external") + ',format=raw',
|
||||
'-drive', 'file=' + target.get_disk("disk") + ',format=raw',
|
||||
'-drive', 'file=' + target.get_disk("external") + ',format=raw',
|
||||
'-nic', 'user,ipv6=off,model=e1000',
|
||||
'-kernel', args.kernel,
|
||||
'-nographic',
|
||||
'-append', 'console=ttyS0 root=/dev/sda1 rootfstype=ext3 init=/init rw')
|
||||
else:
|
||||
generator.prepare(tmpdir, kernel_bootstrap=True, target_size=size)
|
||||
generator.prepare(target, kernel_bootstrap=True, target_size=size)
|
||||
arg_list = [
|
||||
'-enable-kvm',
|
||||
'-m', str(args.qemu_ram) + 'M',
|
||||
'-smp', str(args.cores),
|
||||
'-no-reboot',
|
||||
'-drive', 'file=' + generator.tmp_dir + '.img' + ',format=raw'
|
||||
'-drive', 'file=' + generator.target_dir + '.img' + ',format=raw'
|
||||
]
|
||||
if tmpdir.get_disk("external") is not None:
|
||||
if target.get_disk("external") is not None:
|
||||
arg_list += [
|
||||
'-drive', 'file=' + tmpdir.get_disk("external") + ',format=raw',
|
||||
'-drive', 'file=' + target.get_disk("external") + ',format=raw',
|
||||
]
|
||||
arg_list += [
|
||||
'-machine', 'kernel-irqchip=split',
|
||||
|
|
|
|||
Loading…
Reference in New Issue