Rework tmpdir & associated a bit.

- Split out tmpdir logic into a separate entity & add the appropriate
  arguments and checks.
- sysb can be removed since there is now no associated logic.
- Move disk/etc logic into tmpdir.py.
This commit is contained in:
fosslinux 2023-01-28 11:11:32 +11:00
parent 4f9f56f006
commit 51b0bf8405
8 changed files with 170 additions and 195 deletions

View File

@ -13,7 +13,7 @@ pylint_task:
- apt-get -y clean
- pip3 install pylint
check_script:
- pylint rootfs.py sysa.py sysb.py sysc.py lib/utils.py lib/sysgeneral.py --disable=duplicate-code
- pylint rootfs.py sysa.py sysc.py lib/utils.py lib/sysgeneral.py lib/tmpdir.py --disable=duplicate-code
shell_lint_task:
container:

View File

@ -4,20 +4,17 @@ This file contains a few functions to be shared by all Sys* classes
"""
# SPDX-FileCopyrightText: 2022-2023 Dor Askayo <dor.askayo@gmail.com>
# SPDX-FileCopyrightText: 2021-22 fosslinux <fosslinux@aussies.space>
# SPDX-FileCopyrightText: 2021-23 fosslinux <fosslinux@aussies.space>
# SPDX-FileCopyrightText: 2021 Andrius Štikonas <andrius@stikonas.eu>
# SPDX-License-Identifier: GPL-3.0-or-later
import os
import shutil
import hashlib
import glob
import subprocess
import requests
from lib.utils import mount, umount
class SysGeneral:
"""
A class from which all Sys* class are extended.
@ -25,38 +22,12 @@ class SysGeneral:
"""
# All of these are variables defined in the individual Sys* classes
preserve_tmp = None
tmp_dir = None
cache_dir = None
base_dir = None
git_dir = None
sys_dir = None
initramfs_path = None
mounted_tmpfs = False
def __del__(self):
if not self.preserve_tmp:
self.remove_tmp()
def remove_tmp(self):
"""Remove the tmp directory"""
if self.tmp_dir is None:
return
if self.mounted_tmpfs:
print(f"Unmounting tmpfs from {self.tmp_dir}")
umount(self.tmp_dir)
print(f"Removing {self.tmp_dir}")
shutil.rmtree(self.tmp_dir, ignore_errors=True)
def mount_tmpfs(self):
"""Mount the tmpfs for this sysx"""
if not os.path.isdir(self.tmp_dir):
os.mkdir(self.tmp_dir)
print(f"Mounting tmpfs on {self.tmp_dir}")
mount('tmpfs', self.tmp_dir, 'tmpfs', 'size=8G')
self.mounted_tmpfs = True
tmp_dir = None
def check_file(self, file_name, expected_hash):
"""Check hash of downloaded source file."""

99
lib/tmpdir.py Normal file
View File

@ -0,0 +1,99 @@
#!/usr/bin/env python3
# SPDX-FileCopyrightText: 2023 fosslinux <fosslinux@aussies.space>
# SPDX-License-Identifier: GPL-3.0-or-later
"""
Contains a class that represents a tmpdir
"""
import enum
import getpass
import os
import shutil
from lib.utils import mount, umount, create_disk, run
class TmpType(enum.Enum):
"""Different types of tmpdirs we can have"""
NONE = 0
TMPFS = 1
class Tmpdir:
"""
Represents a tmpdir
"""
_syses = {}
_disks = {}
_disk_filesystems = {}
_mountpoints = {}
def __init__(self, preserve, path="tmp"):
self.path = os.path.abspath(path)
self.preserve = preserve
self._type = TmpType.NONE
if not os.path.exists(self.path):
os.mkdir(self.path)
def __del__(self):
for path in self._mountpoints:
print(f"Unmounting {path}")
umount(path)
if not self.preserve:
for disk in self._disks.values():
print(f"Detaching {disk}")
run("sudo", "losetup", "-d", disk)
if self._type == TmpType.TMPFS:
print(f"Unmounting tmpdir from {self.path}")
umount(self.path)
print(f"Removing {self.path}")
shutil.rmtree(self.path, ignore_errors=True)
def tmpfs(self, size="8G"):
"""Mount a tmpfs"""
print(f"Mounting tmpfs on {self.path}")
mount("tmpfs", self.path, "tmpfs", f"size={size}")
self._type = TmpType.TMPFS
def add_sys(self, name, subdir=None):
"""Create a subdirectory and register a sys"""
if subdir is None:
subdir = name
sys_path = os.path.join(self.path, name)
if not os.path.exists(sys_path):
os.mkdir(sys_path)
return sys_path
def add_disk(self, name, size="8G", filesystem="ext4"):
"""Add a disk"""
disk_path = os.path.join(self.path, f"{name}.img")
self._disks[name] = create_disk(disk_path, "msdos", filesystem, size)
self._disk_filesystems[name] = filesystem
# Allow executing user to access it
run("sudo", "chown", getpass.getuser(), self._disks[name])
def mount_disk(self, name, mountpoint=None):
"""Mount the disk"""
if mountpoint is None:
mountpoint = f"{name}_mnt"
mountpoint = os.path.join(self.path, mountpoint)
os.mkdir(mountpoint)
mount(self._disks[name] + "p1", mountpoint, self._disk_filesystems[name])
# Allow executing user to access it
run("sudo", "chown", getpass.getuser(), mountpoint)
self._mountpoints[name] = mountpoint
return mountpoint
def umount_disk(self, name):
"""Unmount a disk"""
umount(self._mountpoints[name])
del self._mountpoints[name]
def get_disk(self, name):
"""Get the path to a device of a disk"""
return self._disks[name]

View File

@ -5,7 +5,7 @@ This file contains a few self-contained helper functions
# SPDX-License-Identifier: GPL-3.0-or-later
# SPDX-FileCopyrightText: 2021 Andrius Štikonas <andrius@stikonas.eu>
# SPDX-FileCopyrightText: 2021-22 fosslinux <fosslinux@aussies.space>
# SPDX-FileCopyrightText: 2021-23 fosslinux <fosslinux@aussies.space>
import os
import shutil

118
rootfs.py
View File

@ -11,17 +11,16 @@ you can run bootstap inside chroot.
# SPDX-FileCopyrightText: 2021 Andrius Štikonas <andrius@stikonas.eu>
# SPDX-FileCopyrightText: 2021 Bastian Bittorf <bb@npl.de>
# SPDX-FileCopyrightText: 2021 Melg Eight <public.melg8@gmail.com>
# SPDX-FileCopyrightText: 2021-22 fosslinux <fosslinux@aussies.space>
# SPDX-FileCopyrightText: 2021-23 fosslinux <fosslinux@aussies.space>
import argparse
import os
import shutil
from sysa import SysA
from sysb import SysB
from sysc import SysC
from lib.utils import run
from lib.sysgeneral import stage0_arch_map
from lib.tmpdir import Tmpdir
def create_configuration_file(args):
"""
@ -54,7 +53,12 @@ def main():
action="store_true")
parser.add_argument("-p", "--preserve", help="Do not remove temporary dir",
action="store_true")
parser.add_argument("-t", "--tmpdir", help="Temporary directory")
parser.add_argument("-t", "--tmpdir", help="Temporary directory",
default="tmp")
parser.add_argument("--tmpfs", help="Use a tmpfs on tmpdir",
action="store_true")
parser.add_argument("--tmpfs-size", help="Size of the tmpfs",
default="8G")
parser.add_argument("--force-timestamps",
help="Force all files timestamps to be 0 unix time",
action="store_true")
@ -82,13 +86,12 @@ def main():
parser.add_argument("-qk", "--kernel", help="Kernel to use (default is ./kernel)",
default="kernel")
parser.add_argument("-m", "--minikernel", help="Use minikernel",
action="store_true")
parser.add_argument("-b", "--bare-metal", help="Build images for bare metal",
action="store_true")
args = parser.parse_args()
# Mode validation
def check_types():
count = 0
if args.qemu:
@ -97,24 +100,28 @@ def main():
count += 1
if args.bwrap:
count += 1
if args.minikernel:
count += 1
if args.bare_metal:
count += 1
return count
if check_types() > 1:
raise ValueError("No more than one of qemu, chroot, bwrap, minikernel, bare metal "
raise ValueError("No more than one of qemu, chroot, bwrap, bare metal"
"may be used.")
if check_types() == 0:
raise ValueError("One of qemu, chroot, bwrap, minikernel or bare metal must be selected.")
if args.bare_metal:
args.no_create_config = True
raise ValueError("One of qemu, chroot, bwrap, or bare metal must be selected.")
# Arch validation
if args.arch != "x86":
raise ValueError("Only x86 is supported at the moment.")
# Tmp validation
if args.bwrap and args.tmpfs:
raise ValueError("tmpfs cannot be used writh bwrap.")
# bootstrap.cfg
if args.bare_metal:
args.no_create_config = True
try:
os.remove(os.path.join('sysa', 'bootstrap.cfg'))
except FileNotFoundError:
@ -125,20 +132,21 @@ def main():
with open(os.path.join('sysa', 'bootstrap.cfg'), 'a', encoding='UTF-8'):
pass
system_c = SysC(arch=args.arch, preserve_tmp=args.preserve,
tmpdir=args.tmpdir, external_sources=args.external_sources)
system_b = SysB(arch=args.arch, preserve_tmp=args.preserve)
system_a = SysA(arch=args.arch, preserve_tmp=args.preserve,
early_preseed=args.early_preseed, tmpdir=args.tmpdir,
external_sources=args.external_sources,
sysb_dir=system_b.sys_dir, sysc_dir=system_c.sys_dir)
# tmpdir
tmpdir = Tmpdir(path=args.tmpdir, preserve=args.preserve)
if args.tmpfs:
tmpdir.tmpfs(size=args.tmpfs_size)
if args.tmpdir is not None:
os.makedirs(args.tmpdir, exist_ok=True)
# sys
system_c = SysC(arch=args.arch, tmpdir=tmpdir,
external_sources=args.external_sources)
system_a = SysA(arch=args.arch, early_preseed=args.early_preseed,
tmpdir=tmpdir, external_sources=args.external_sources,
repo_path=args.repo)
bootstrap(args, system_a, system_b, system_c)
bootstrap(args, system_a, system_c, tmpdir)
def bootstrap(args, system_a, system_b, system_c):
def bootstrap(args, system_a, system_c, tmpdir):
"""Kick off bootstrap process."""
print(f"Bootstrapping {args.arch} -- SysA")
if args.chroot:
@ -149,25 +157,17 @@ print(shutil.which('chroot'))
chroot_binary = run('sudo', 'python3', '-c', find_chroot,
capture_output=True).stdout.decode().strip()
system_c.prepare(mount_tmpfs=True,
create_disk_image=False)
system_a.prepare(mount_tmpfs=True,
create_initramfs=False,
repo_path=args.repo)
system_c.prepare(create_disk_image=False)
system_a.prepare(create_initramfs=False)
# sysa
arch = stage0_arch_map.get(args.arch, args.arch)
init = os.path.join(os.sep, 'bootstrap-seeds', 'POSIX', arch, 'kaem-optional-seed')
run('sudo', 'env', '-i', 'PATH=/bin', chroot_binary, system_a.tmp_dir, init)
elif args.bwrap:
system_c.prepare(mount_tmpfs=False,
create_disk_image=False)
system_a.prepare(mount_tmpfs=False,
create_initramfs=False,
repo_path=args.repo)
system_c.prepare(create_disk_image=False)
system_a.prepare(create_initramfs=False)
# sysa
arch = stage0_arch_map.get(args.arch, args.arch)
init = os.path.join(os.sep, 'bootstrap-seeds', 'POSIX', arch, 'kaem-optional-seed')
run('bwrap', '--unshare-user',
@ -192,55 +192,23 @@ print(shutil.which('chroot'))
'--tmpfs', '/sysc_image/tmp',
init)
elif args.minikernel:
if os.path.isdir('kritis-linux'):
shutil.rmtree('kritis-linux')
system_c.prepare(mount_tmpfs=True,
create_disk_image=True)
system_a.prepare(mount_tmpfs=True,
create_initramfs=True,
repo_path=args.repo)
run('git', 'clone',
'--depth', '1', '--branch', 'v0.7',
'https://github.com/bittorf/kritis-linux.git')
run('kritis-linux/ci_helper.sh',
'--private',
'--multi', '1',
'--repeat', '1',
'--arch', args.arch,
'--qemucpu', '486',
'--kernel', '3.18.140',
'--features', 'kflock,highrestimers',
# Hack to add -hda /dev/blah
'--ramsize', str(args.qemu_ram) + 'M -hda ' + system_b.dev_name,
'--initrd', system_a.initramfs_path,
'--log', '/tmp/bootstrap.log')
elif args.bare_metal:
system_c.prepare(mount_tmpfs=True,
create_disk_image=True)
system_a.prepare(mount_tmpfs=True,
create_initramfs=True,
repo_path=args.repo)
system_c.prepare(create_disk_image=True)
system_a.prepare(create_initramfs=True)
print("Please:")
print(" 1. Take sysa/tmp/initramfs and your kernel, boot using this.")
print(" 2. Take sysc/tmp/disk.img and put this on a writable storage medium.")
print(" 1. Take tmp/sysa/initramfs and your kernel, boot using this.")
print(" 2. Take tmp/sysc/disk.img and put this on a writable storage medium.")
else:
system_c.prepare(mount_tmpfs=True,
create_disk_image=True)
system_a.prepare(mount_tmpfs=True,
create_initramfs=True,
repo_path=args.repo)
system_c.prepare(create_disk_image=True)
system_a.prepare(create_initramfs=True)
run(args.qemu_cmd,
'-enable-kvm',
'-m', str(args.qemu_ram) + 'M',
'-no-reboot',
'-hda', system_c.dev_name,
'-hda', tmpdir.get_disk("sysc"),
'-nic', 'user,ipv6=off,model=e1000',
'-kernel', args.kernel,
'-initrd', system_a.initramfs_path,

33
sysa.py
View File

@ -4,7 +4,7 @@
# SPDX-FileCopyrightText: 2022-2023 Dor Askayo <dor.askayo@gmail.com>
# SPDX-FileCopyrightText: 2021 Andrius Štikonas <andrius@stikonas.eu>
# SPDX-FileCopyrightText: 2021 Melg Eight <public.melg8@gmail.com>
# SPDX-FileCopyrightText: 2021-22 fosslinux <fosslinux@aussies.space>
# SPDX-FileCopyrightText: 2021-23 fosslinux <fosslinux@aussies.space>
import os
from distutils.dir_util import copy_tree
@ -22,37 +22,26 @@ class SysA(SysGeneral):
git_dir = os.path.dirname(os.path.join(__file__))
sys_dir = os.path.join(git_dir, 'sysa')
sysb_dir = os.path.join(git_dir, 'sysb')
sysc_dir = os.path.join(git_dir, 'sysc')
cache_dir = os.path.join(sys_dir, 'distfiles')
# pylint: disable=too-many-arguments
def __init__(self, arch, preserve_tmp, external_sources,
early_preseed, tmpdir, sysb_dir, sysc_dir):
def __init__(self, tmpdir, arch, external_sources,
early_preseed, repo_path):
self.arch = arch
self.preserve_tmp = preserve_tmp
self.early_preseed = early_preseed
if tmpdir is None:
self.tmp_dir = os.path.join(self.git_dir, 'tmp')
else:
self.tmp_dir = os.path.join(tmpdir, 'sysa')
self.sysa_dir = os.path.join(self.tmp_dir, 'sysa')
self.base_dir = self.sysa_dir
self.sysb_dir = sysb_dir
self.sysc_dir = sysc_dir
self.external_sources = external_sources
self.repo_path = repo_path
def prepare(self, mount_tmpfs, create_initramfs, repo_path):
self.tmp_dir = tmpdir.add_sys("sysa")
def prepare(self, create_initramfs):
"""
Prepare directory structure for System A.
We create an empty tmp directory, unpack stage0-posix.
Rest of the files are unpacked into more structured directory /sysa
"""
if mount_tmpfs:
self.mount_tmpfs()
else:
os.mkdir(self.tmp_dir)
if self.early_preseed:
# Extract tar containing preseed
with tarfile.open(self.early_preseed, "r") as seed:
@ -69,9 +58,9 @@ class SysA(SysGeneral):
self.sysc(create_initramfs)
if repo_path:
if self.repo_path:
repo_dir = os.path.join(self.tmp_dir, 'usr', 'src', 'repo-preseeded')
shutil.copytree(repo_path, repo_dir)
shutil.copytree(self.repo_path, repo_dir)
if create_initramfs:
self.make_initramfs()

21
sysb.py
View File

@ -1,21 +0,0 @@
#!/usr/bin/env python3
"""System B"""
# SPDX-License-Identifier: GPL-3.0-or-later
# SPDX-FileCopyrightText: 2021 Andrius Štikonas <andrius@stikonas.eu>
# SPDX-FileCopyrightText: 2021-22 fosslinux <fosslinux@aussies.space>
import os
from lib.sysgeneral import SysGeneral
class SysB(SysGeneral):
"""
Class responsible for preparing sources for System B.
"""
git_dir = os.path.dirname(os.path.join(__file__))
sys_dir = os.path.join(git_dir, 'sysb')
def __init__(self, arch, preserve_tmp):
self.arch = arch
self.preserve_tmp = preserve_tmp

57
sysc.py
View File

@ -2,13 +2,12 @@
"""System C"""
# SPDX-License-Identifier: GPL-3.0-or-later
# SPDX-FileCopyrightText: 2022-2023 Dor Askayo <dor.askayo@gmail.com>
# SPDX-FileCopyrightText: 2021-22 fosslinux <fosslinux@aussies.space>
# SPDX-FileCopyrightText: 2021-23 fosslinux <fosslinux@aussies.space>
# SPDX-FileCopyrightText: 2021 Andrius Štikonas <andrius@stikonas.eu>
import os
import getpass
from lib.utils import mount, umount, create_disk, run, copytree
from lib.utils import copytree
from lib.sysgeneral import SysGeneral
# pylint: disable=consider-using-with
@ -21,60 +20,30 @@ class SysC(SysGeneral):
git_dir = os.path.dirname(os.path.join(__file__))
sys_dir = os.path.join(git_dir, 'sysc')
cache_dir = os.path.join(sys_dir, 'distfiles')
dev_name = None
def __init__(self, arch, preserve_tmp, tmpdir, external_sources):
def __init__(self, tmpdir, arch, external_sources):
self.arch = arch
self.preserve_tmp = preserve_tmp
self.external_sources = external_sources
self._tmpdir = tmpdir
if tmpdir is None:
self.tmp_dir = os.path.join(self.sys_dir, 'tmp')
else:
self.tmp_dir = os.path.join(tmpdir, 'sysc')
self.tmp_dir = tmpdir.add_sys("sysc")
def __del__(self):
if not self.preserve_tmp:
if self.dev_name is not None:
print(f"Detaching {self.dev_name}")
run('sudo', 'losetup', '-d', self.dev_name)
super().__del__()
def prepare(self, mount_tmpfs, create_disk_image):
def prepare(self, create_disk_image):
"""
Prepare directory structure for System C.
"""
if mount_tmpfs:
self.mount_tmpfs()
else:
os.mkdir(self.tmp_dir)
rootfs_dir = None
if create_disk_image:
# Create + mount a disk for QEMU to use
disk_path = os.path.join(self.tmp_dir, 'disk.img')
if self.external_sources:
self.dev_name = create_disk(disk_path, "msdos", "ext4", '8G')
rootfs_dir = os.path.join(self.tmp_dir, 'mnt')
os.mkdir(rootfs_dir)
mount(self.dev_name + "p1", rootfs_dir, 'ext4')
else:
self.dev_name = create_disk(disk_path, "none", "ext4", '8G')
# Use chown to allow executing user to access it
run('sudo', 'chown', getpass.getuser(), self.dev_name)
if self.external_sources:
run('sudo', 'chown', getpass.getuser(), rootfs_dir)
else:
rootfs_dir = self.tmp_dir
self._tmpdir.add_disk("sysc")
if self.external_sources:
if create_disk_image:
rootfs_dir = self._tmpdir.mount_disk("sysc")
else:
rootfs_dir = self.tmp_dir
source_manifest = self.get_source_manifest()
self.get_packages(source_manifest)
copytree(self.cache_dir, os.path.join(rootfs_dir, "distfiles"))
# Unmount tmp/mnt if it was mounted
if create_disk_image and self.external_sources:
umount(rootfs_dir)
if create_disk_image:
self._tmpdir.umount_disk("sysc")