Merge pull request #368 from Googulator/always-preserve
Rename tmpdir to target and always preserve it, dropping --preserve
This commit is contained in:
fosslinux
GitHub
commit
51fe3f9991
|
|
@ -43,15 +43,15 @@ jobs:
|
|||
distfiles
|
||||
key: cache-${{ hashFiles('steps/*/sources') }}
|
||||
- name: Run bootstrap
|
||||
run: ./rootfs.py --bwrap --external-sources --build-kernels --preserve --cores 2 --internal-ci pass1
|
||||
run: ./rootfs.py --bwrap --external-sources --build-kernels --cores 2 --internal-ci pass1
|
||||
- name: Archive created packages
|
||||
if: failure() # archive failed builds progress
|
||||
uses: actions/upload-artifact@v3
|
||||
with:
|
||||
name: packages
|
||||
path: tmp/external/repo/**
|
||||
path: target/external/repo/**
|
||||
- name: Tar pass1 image
|
||||
run: tar -cf pass1_image.tar tmp
|
||||
run: tar -cf pass1_image.tar target
|
||||
- name: Archive pass1_image
|
||||
uses: actions/upload-artifact@v3
|
||||
with:
|
||||
|
|
@ -96,15 +96,15 @@ jobs:
|
|||
distfiles
|
||||
key: cache-${{ hashFiles('steps/*/sources') }}
|
||||
- name: Run bootstrap
|
||||
run: ./rootfs.py --bwrap --external-sources --build-kernels --preserve --cores 2 --internal-ci pass2
|
||||
run: ./rootfs.py --bwrap --external-sources --build-kernels --cores 2 --internal-ci pass2
|
||||
- name: Archive created packages
|
||||
if: failure() # archive failed builds progress
|
||||
uses: actions/upload-artifact@v3
|
||||
with:
|
||||
name: internal_packages_pass2
|
||||
path: tmp/external/repo/**
|
||||
path: target/external/repo/**
|
||||
- name: Tar pass2 image
|
||||
run: tar -cf pass2_image.tar tmp
|
||||
run: tar -cf pass2_image.tar target
|
||||
- name: Archive pass2_image
|
||||
uses: actions/upload-artifact@v3
|
||||
with:
|
||||
|
|
@ -149,10 +149,10 @@ jobs:
|
|||
distfiles
|
||||
key: cache-${{ hashFiles('steps/*/sources') }}
|
||||
- name: Run bootstrap
|
||||
run: ./rootfs.py --bwrap --external-sources --build-kernels --preserve --cores 2 --internal-ci pass3
|
||||
run: ./rootfs.py --bwrap --external-sources --build-kernels --cores 2 --internal-ci pass3
|
||||
- name: Archive created packages
|
||||
if: always() # archive both failed and successful builds
|
||||
uses: actions/upload-artifact@v3
|
||||
with:
|
||||
name: packages
|
||||
path: tmp/external/repo/**
|
||||
path: target/external/repo/**
|
||||
|
|
|
|||
|
|
@ -21,7 +21,7 @@ jobs:
|
|||
- name: Install pylint
|
||||
run: sudo pip3 install pylint
|
||||
- name: pylint
|
||||
run: pylint rootfs.py lib/utils.py lib/generator.py lib/tmpdir.py --disable=duplicate-code
|
||||
run: pylint rootfs.py lib/utils.py lib/generator.py lib/target.py --disable=duplicate-code
|
||||
|
||||
shellcheck:
|
||||
name: Lint shell files
|
||||
|
|
|
|||
|
|
@ -29,26 +29,26 @@ class Generator():
|
|||
self.external_sources = external_sources
|
||||
self.repo_path = repo_path
|
||||
self.source_manifest = self.get_source_manifest(not self.external_sources)
|
||||
self.tmp_dir = None
|
||||
self.target_dir = None
|
||||
self.external_dir = None
|
||||
|
||||
def reuse(self, tmpdir):
|
||||
def reuse(self, target):
|
||||
"""
|
||||
Reuse a previously prepared bwrap environment for further stages.
|
||||
"""
|
||||
self.tmp_dir = tmpdir.path
|
||||
self.external_dir = os.path.join(self.tmp_dir, 'external')
|
||||
self.target_dir = target.path
|
||||
self.external_dir = os.path.join(self.target_dir, 'external')
|
||||
self.distfiles()
|
||||
|
||||
def prepare(self, tmpdir, using_kernel=False, kernel_bootstrap=False, target_size=0):
|
||||
def prepare(self, target, using_kernel=False, kernel_bootstrap=False, target_size=0):
|
||||
"""
|
||||
Prepare basic media of live-bootstrap.
|
||||
/steps -- contains steps to be built
|
||||
/ -- contains seed to allow steps to be built, containing custom
|
||||
scripts and stage0-posix
|
||||
"""
|
||||
self.tmp_dir = tmpdir.path
|
||||
self.external_dir = os.path.join(self.tmp_dir, 'external')
|
||||
self.target_dir = target.path
|
||||
self.external_dir = os.path.join(self.target_dir, 'external')
|
||||
|
||||
# We use ext3 here; ext4 actually has a variety of extensions that
|
||||
# have been added with varying levels of recency
|
||||
|
|
@ -59,33 +59,33 @@ class Generator():
|
|||
# argument matrix ... or we could just use ext3 instead which
|
||||
# is effectively universally the same
|
||||
if kernel_bootstrap:
|
||||
init_path = os.path.join(self.tmp_dir, 'init')
|
||||
init_path = os.path.join(self.target_dir, 'init')
|
||||
|
||||
os.mkdir(init_path)
|
||||
self.tmp_dir = init_path
|
||||
self.target_dir = init_path
|
||||
|
||||
if self.repo_path or self.external_sources:
|
||||
tmpdir.add_disk("external", filesystem="ext3")
|
||||
tmpdir.mount_disk("external", "external")
|
||||
target.add_disk("external", filesystem="ext3")
|
||||
target.mount_disk("external", "external")
|
||||
else:
|
||||
self.external_dir = os.path.join(self.tmp_dir, 'external')
|
||||
self.external_dir = os.path.join(self.target_dir, 'external')
|
||||
elif using_kernel:
|
||||
self.tmp_dir = os.path.join(self.tmp_dir, 'disk')
|
||||
tmpdir.add_disk("disk",
|
||||
self.target_dir = os.path.join(self.target_dir, 'disk')
|
||||
target.add_disk("disk",
|
||||
filesystem="ext3",
|
||||
size=(target_size + "M") if target_size else "16G",
|
||||
bootable=True)
|
||||
tmpdir.mount_disk("disk", "disk")
|
||||
self.external_dir = os.path.join(self.tmp_dir, 'external')
|
||||
target.mount_disk("disk", "disk")
|
||||
self.external_dir = os.path.join(self.target_dir, 'external')
|
||||
|
||||
os.makedirs(self.external_dir, exist_ok=True)
|
||||
|
||||
if self.early_preseed:
|
||||
# Extract tar containing preseed
|
||||
with tarfile.open(self.early_preseed, "r") as seed:
|
||||
seed.extractall(self.tmp_dir)
|
||||
seed.extractall(self.target_dir)
|
||||
shutil.copy2(os.path.join(self.git_dir, 'seed', 'preseeded.kaem'),
|
||||
os.path.join(self.tmp_dir, 'kaem.x86'))
|
||||
os.path.join(self.target_dir, 'kaem.x86'))
|
||||
else:
|
||||
self.stage0_posix()
|
||||
self.seed()
|
||||
|
|
@ -101,25 +101,25 @@ class Generator():
|
|||
shutil.copytree(self.repo_path, repo_dir)
|
||||
|
||||
if kernel_bootstrap:
|
||||
self.create_builder_hex0_disk_image(self.tmp_dir + '.img', target_size)
|
||||
self.create_builder_hex0_disk_image(self.target_dir + '.img', target_size)
|
||||
|
||||
if kernel_bootstrap and (self.external_sources or self.repo_path):
|
||||
tmpdir.umount_disk('external')
|
||||
target.umount_disk('external')
|
||||
elif using_kernel:
|
||||
tmpdir.umount_disk('disk')
|
||||
target.umount_disk('disk')
|
||||
|
||||
def steps(self):
|
||||
"""Copy in steps."""
|
||||
self.get_packages()
|
||||
|
||||
shutil.copytree(os.path.join(self.git_dir, 'steps'), os.path.join(self.tmp_dir, 'steps'))
|
||||
shutil.copytree(os.path.join(self.git_dir, 'steps'), os.path.join(self.target_dir, 'steps'))
|
||||
|
||||
def stage0_posix(self):
|
||||
"""Copy in all of the stage0-posix"""
|
||||
stage0_posix_base_dir = os.path.join(self.git_dir, 'seed', 'stage0-posix')
|
||||
for entry in os.listdir(stage0_posix_base_dir):
|
||||
orig = os.path.join(stage0_posix_base_dir, entry)
|
||||
target = os.path.join(self.tmp_dir, entry)
|
||||
target = os.path.join(self.target_dir, entry)
|
||||
if os.path.isfile(orig):
|
||||
shutil.copy2(orig, target)
|
||||
else:
|
||||
|
|
@ -128,14 +128,14 @@ class Generator():
|
|||
arch = stage0_arch_map.get(self.arch, self.arch)
|
||||
kaem_optional_seed = os.path.join(self.git_dir, 'seed', 'stage0-posix', 'bootstrap-seeds',
|
||||
'POSIX', arch, 'kaem-optional-seed')
|
||||
shutil.copy2(kaem_optional_seed, os.path.join(self.tmp_dir, 'init'))
|
||||
shutil.copy2(kaem_optional_seed, os.path.join(self.target_dir, 'init'))
|
||||
|
||||
def seed(self):
|
||||
"""Copy in extra seed files"""
|
||||
seed_dir = os.path.join(self.git_dir, 'seed')
|
||||
for entry in os.listdir(seed_dir):
|
||||
if os.path.isfile(os.path.join(seed_dir, entry)):
|
||||
shutil.copy2(os.path.join(seed_dir, entry), os.path.join(self.tmp_dir, entry))
|
||||
shutil.copy2(os.path.join(seed_dir, entry), os.path.join(self.target_dir, entry))
|
||||
|
||||
@staticmethod
|
||||
def add_fiwix_files(file_list_path, dirpath):
|
||||
|
|
@ -149,14 +149,14 @@ class Generator():
|
|||
|
||||
def create_fiwix_file_list(self):
|
||||
"""Create a list of files to populate Fiwix file system"""
|
||||
file_list_path = os.path.join(self.tmp_dir, 'steps', 'lwext4-1.0.0-lb1',
|
||||
file_list_path = os.path.join(self.target_dir, 'steps', 'lwext4-1.0.0-lb1',
|
||||
'files', 'fiwix-file-list.txt')
|
||||
shutil.copyfile(os.path.join(self.tmp_dir, 'steps', 'lwext4-1.0.0-lb1',
|
||||
shutil.copyfile(os.path.join(self.target_dir, 'steps', 'lwext4-1.0.0-lb1',
|
||||
'files', 'early-artifacts-needed-after-fiwix.txt'),
|
||||
file_list_path)
|
||||
|
||||
save_cwd = os.getcwd()
|
||||
os.chdir(self.tmp_dir)
|
||||
os.chdir(self.target_dir)
|
||||
self.add_fiwix_files(file_list_path, 'steps')
|
||||
self.add_fiwix_files(file_list_path, 'distfiles')
|
||||
os.chdir(save_cwd)
|
||||
|
|
@ -170,7 +170,7 @@ class Generator():
|
|||
shutil.copy2(os.path.join(self.distfiles_dir, file),
|
||||
os.path.join(out, file))
|
||||
|
||||
early_distfile_dir = os.path.join(self.tmp_dir, 'external', 'distfiles')
|
||||
early_distfile_dir = os.path.join(self.target_dir, 'external', 'distfiles')
|
||||
main_distfile_dir = os.path.join(self.external_dir, 'distfiles')
|
||||
|
||||
if early_distfile_dir != main_distfile_dir:
|
||||
|
|
@ -217,7 +217,7 @@ class Generator():
|
|||
"""Append srcfs file system to disk image"""
|
||||
save_cwd = os.getcwd()
|
||||
|
||||
os.chdir(self.tmp_dir)
|
||||
os.chdir(self.target_dir)
|
||||
self.output_tree(image_file, '.')
|
||||
|
||||
# Add commands to kick off stage0-posix
|
||||
|
|
|
|||
|
|
@ -4,34 +4,32 @@
|
|||
# SPDX-License-Identifier: GPL-3.0-or-later
|
||||
|
||||
"""
|
||||
Contains a class that represents a tmpdir
|
||||
Contains a class that represents a target directory
|
||||
"""
|
||||
|
||||
import enum
|
||||
import getpass
|
||||
import os
|
||||
import shutil
|
||||
|
||||
from lib.utils import mount, umount, create_disk, run_as_root
|
||||
|
||||
class TmpType(enum.Enum):
|
||||
"""Different types of tmpdirs we can have"""
|
||||
class TargetType(enum.Enum):
|
||||
"""Different types of target dirs we can have"""
|
||||
NONE = 0
|
||||
TMPFS = 1
|
||||
|
||||
class Tmpdir:
|
||||
class Target:
|
||||
"""
|
||||
Represents a tmpdir
|
||||
Represents a target directory
|
||||
"""
|
||||
|
||||
_disks = {}
|
||||
_disk_filesystems = {}
|
||||
_mountpoints = {}
|
||||
|
||||
def __init__(self, preserve, path="tmp"):
|
||||
def __init__(self, path="target"):
|
||||
self.path = os.path.abspath(path)
|
||||
self.preserve = preserve
|
||||
self._type = TmpType.NONE
|
||||
self._type = TargetType.NONE
|
||||
|
||||
if not os.path.exists(self.path):
|
||||
os.mkdir(self.path)
|
||||
|
|
@ -41,23 +39,15 @@ class Tmpdir:
|
|||
print(f"Unmounting {path}")
|
||||
umount(path)
|
||||
|
||||
if not self.preserve:
|
||||
for disk in self._disks.values():
|
||||
print(f"Detaching {disk}")
|
||||
run_as_root("losetup", "-d", disk)
|
||||
|
||||
if self._type == TmpType.TMPFS:
|
||||
print(f"Unmounting tmpdir from {self.path}")
|
||||
umount(self.path)
|
||||
|
||||
print(f"Removing {self.path}")
|
||||
shutil.rmtree(self.path, ignore_errors=True)
|
||||
for disk in self._disks.values():
|
||||
print(f"Detaching {disk}")
|
||||
run_as_root("losetup", "-d", disk)
|
||||
|
||||
def tmpfs(self, size="8G"):
|
||||
"""Mount a tmpfs"""
|
||||
print(f"Mounting tmpfs on {self.path}")
|
||||
mount("tmpfs", self.path, "tmpfs", f"size={size}")
|
||||
self._type = TmpType.TMPFS
|
||||
self._type = TargetType.TMPFS
|
||||
|
||||
# pylint: disable=too-many-arguments
|
||||
def add_disk(self,
|
||||
60
rootfs.py
60
rootfs.py
|
|
@ -17,7 +17,7 @@ import argparse
|
|||
import os
|
||||
|
||||
from lib.utils import run, run_as_root
|
||||
from lib.tmpdir import Tmpdir
|
||||
from lib.target import Target
|
||||
from lib.generator import Generator, stage0_arch_map
|
||||
|
||||
def create_configuration_file(args):
|
||||
|
|
@ -58,11 +58,9 @@ def main():
|
|||
action="store_true")
|
||||
parser.add_argument("-bw", "--bwrap", help="Run inside a bwrap sandbox",
|
||||
action="store_true")
|
||||
parser.add_argument("-p", "--preserve", help="Do not remove temporary dir",
|
||||
action="store_true")
|
||||
parser.add_argument("-t", "--tmpdir", help="Temporary directory",
|
||||
default="tmp")
|
||||
parser.add_argument("--tmpfs", help="Use a tmpfs on tmpdir",
|
||||
parser.add_argument("-t", "--target", help="Target directory",
|
||||
default="target")
|
||||
parser.add_argument("--tmpfs", help="Use a tmpfs on target",
|
||||
action="store_true")
|
||||
parser.add_argument("--tmpfs-size", help="Size of the tmpfs",
|
||||
default="8G")
|
||||
|
|
@ -128,7 +126,7 @@ def main():
|
|||
if args.arch != "x86":
|
||||
print("Only x86 is supported at the moment, other arches are for development only.")
|
||||
|
||||
# Tmp validation
|
||||
# Tmpfs validation
|
||||
if args.bwrap and args.tmpfs:
|
||||
raise ValueError("tmpfs cannot be used with bwrap.")
|
||||
|
||||
|
|
@ -156,19 +154,19 @@ def main():
|
|||
with open(os.path.join('steps', 'bootstrap.cfg'), 'a', encoding='UTF-8'):
|
||||
pass
|
||||
|
||||
# tmpdir
|
||||
tmpdir = Tmpdir(path=args.tmpdir, preserve=args.preserve)
|
||||
# target
|
||||
target = Target(path=args.target)
|
||||
if args.tmpfs:
|
||||
tmpdir.tmpfs(size=args.tmpfs_size)
|
||||
target.tmpfs(size=args.tmpfs_size)
|
||||
|
||||
generator = Generator(arch=args.arch,
|
||||
external_sources=args.external_sources,
|
||||
repo_path=args.repo,
|
||||
early_preseed=args.early_preseed)
|
||||
|
||||
bootstrap(args, generator, tmpdir, args.target_size)
|
||||
bootstrap(args, generator, target, args.target_size)
|
||||
|
||||
def bootstrap(args, generator, tmpdir, size):
|
||||
def bootstrap(args, generator, target, size):
|
||||
"""Kick off bootstrap process."""
|
||||
print(f"Bootstrapping {args.arch}")
|
||||
if args.chroot:
|
||||
|
|
@ -179,28 +177,28 @@ print(shutil.which('chroot'))
|
|||
chroot_binary = run_as_root('python3', '-c', find_chroot,
|
||||
capture_output=True).stdout.decode().strip()
|
||||
|
||||
generator.prepare(tmpdir, using_kernel=False)
|
||||
generator.prepare(target, using_kernel=False)
|
||||
|
||||
arch = stage0_arch_map.get(args.arch, args.arch)
|
||||
init = os.path.join(os.sep, 'bootstrap-seeds', 'POSIX', arch, 'kaem-optional-seed')
|
||||
run_as_root('env', '-i', 'PATH=/bin', chroot_binary, generator.tmp_dir, init)
|
||||
run_as_root('env', '-i', 'PATH=/bin', chroot_binary, generator.target_dir, init)
|
||||
|
||||
elif args.bwrap:
|
||||
init = '/init'
|
||||
if not args.internal_ci or args.internal_ci == "pass1":
|
||||
generator.prepare(tmpdir, using_kernel=False)
|
||||
generator.prepare(target, using_kernel=False)
|
||||
|
||||
arch = stage0_arch_map.get(args.arch, args.arch)
|
||||
init = os.path.join(os.sep, 'bootstrap-seeds', 'POSIX', arch, 'kaem-optional-seed')
|
||||
else:
|
||||
generator.reuse(tmpdir)
|
||||
generator.reuse(target)
|
||||
|
||||
run('env', '-i', 'bwrap', '--unshare-user',
|
||||
'--uid', '0',
|
||||
'--gid', '0',
|
||||
'--unshare-net' if args.external_sources else None,
|
||||
'--setenv', 'PATH', '/usr/bin',
|
||||
'--bind', generator.tmp_dir, '/',
|
||||
'--bind', generator.target_dir, '/',
|
||||
'--dir', '/dev',
|
||||
'--dev-bind', '/dev/null', '/dev/null',
|
||||
'--dev-bind', '/dev/zero', '/dev/zero',
|
||||
|
|
@ -216,44 +214,44 @@ print(shutil.which('chroot'))
|
|||
|
||||
elif args.bare_metal:
|
||||
if args.kernel:
|
||||
generator.prepare(tmpdir, using_kernel=True, target_size=size)
|
||||
image_path = os.path.join(args.tmpdir, os.path.relpath(generator.tmp_dir, args.tmpdir))
|
||||
generator.prepare(target, using_kernel=True, target_size=size)
|
||||
path = os.path.join(args.target, os.path.relpath(generator.target_dir, args.target))
|
||||
print("Please:")
|
||||
print(f" 1. Take {image_path}/initramfs and your kernel, boot using this.")
|
||||
print(f" 2. Take {image_path}/disk.img and put this on a writable storage medium.")
|
||||
print(f" 1. Take {path}/initramfs and your kernel, boot using this.")
|
||||
print(f" 2. Take {path}/disk.img and put this on a writable storage medium.")
|
||||
else:
|
||||
generator.prepare(tmpdir, kernel_bootstrap=True, target_size=size)
|
||||
image_path = os.path.join(args.tmpdir, os.path.relpath(generator.tmp_dir, args.tmpdir))
|
||||
generator.prepare(target, kernel_bootstrap=True, target_size=size)
|
||||
path = os.path.join(args.target, os.path.relpath(generator.target_dir, args.target))
|
||||
print("Please:")
|
||||
print(f" 1. Take {image_path}.img and write it to a boot drive and then boot it.")
|
||||
print(f" 1. Take {path}.img and write it to a boot drive and then boot it.")
|
||||
|
||||
else:
|
||||
if args.kernel:
|
||||
generator.prepare(tmpdir, using_kernel=True, target_size=size)
|
||||
generator.prepare(target, using_kernel=True, target_size=size)
|
||||
|
||||
run(args.qemu_cmd,
|
||||
'-enable-kvm',
|
||||
'-m', str(args.qemu_ram) + 'M',
|
||||
'-smp', str(args.cores),
|
||||
'-no-reboot',
|
||||
'-drive', 'file=' + tmpdir.get_disk("disk") + ',format=raw',
|
||||
'-drive', 'file=' + tmpdir.get_disk("external") + ',format=raw',
|
||||
'-drive', 'file=' + target.get_disk("disk") + ',format=raw',
|
||||
'-drive', 'file=' + target.get_disk("external") + ',format=raw',
|
||||
'-nic', 'user,ipv6=off,model=e1000',
|
||||
'-kernel', args.kernel,
|
||||
'-nographic',
|
||||
'-append', 'console=ttyS0 root=/dev/sda1 rootfstype=ext3 init=/init rw')
|
||||
else:
|
||||
generator.prepare(tmpdir, kernel_bootstrap=True, target_size=size)
|
||||
generator.prepare(target, kernel_bootstrap=True, target_size=size)
|
||||
arg_list = [
|
||||
'-enable-kvm',
|
||||
'-m', str(args.qemu_ram) + 'M',
|
||||
'-smp', str(args.cores),
|
||||
'-no-reboot',
|
||||
'-drive', 'file=' + generator.tmp_dir + '.img' + ',format=raw'
|
||||
'-drive', 'file=' + generator.target_dir + '.img' + ',format=raw'
|
||||
]
|
||||
if tmpdir.get_disk("external") is not None:
|
||||
if target.get_disk("external") is not None:
|
||||
arg_list += [
|
||||
'-drive', 'file=' + tmpdir.get_disk("external") + ',format=raw',
|
||||
'-drive', 'file=' + target.get_disk("external") + ',format=raw',
|
||||
]
|
||||
arg_list += [
|
||||
'-machine', 'kernel-irqchip=split',
|
||||
|
|
|
|||
Loading…
Reference in New Issue