Merge pull request #91 from stikonas/python

Port rootfs.sh to Python
master
fosslinux 2 years ago committed by GitHub
commit bd4a6a9399
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
  1. 2
      .cirrus.expect
  2. 14
      .cirrus.yml
  3. 1
      .gitignore
  4. 2
      DEVEL.md
  5. 8
      README.rst
  6. 53
      lib/utils.py
  7. 103
      rootfs.py
  8. 393
      rootfs.sh
  9. 445
      sysa.py
  10. 0
      sysa/heirloom-devtools-070527/src/.placeholder
  11. 22
      sysa/mes/checksums
  12. 22
      sysa/mes/mes.kaem
  13. 18
      sysa/pre-sha.sha256sums
  14. 1
      sysa/tcc-0.9.26/checksums
  15. 1
      sysa/tcc-0.9.27/checksums/tcc-0.9.27

@ -6,7 +6,7 @@
set timeout -1
spawn ./rootfs.sh qemu-system-x86_64 3500M
spawn python3 rootfs.py --qemu-cmd qemu-system-x86_64 --qemu-ram 3500
expect {
"not syncing: Attempted to kill init" {}
"Bootstrapping completed." {}

@ -1,7 +1,19 @@
# SPDX-FileCopyrightText: 2021 fosslinux <fosslinux@aussies.space>
# SPDX-FileCopyrightText: 2021 Andrius Štikonas <andrius@stikonas.eu>
#
# SPDX-License-Identifier: GPL-3.0-or-later
pylint_task:
container:
image: debian:bullseye
deps_script:
- apt-get -y update
- apt-get -y dist-upgrade
- apt-get -y install python3-requests pylint
- apt-get -y clean
check_script:
- pylint rootfs.py sysa.py lib/utils.py
shell_lint_task:
container:
image: debian:stable
@ -11,7 +23,7 @@ shell_lint_task:
- apt-get -y install shellcheck
- apt-get -y clean
check_script:
- shellcheck rootfs.sh sysa/run.sh sysa/helpers.sh
- shellcheck sysa/run.sh sysa/helpers.sh
reuse_lint_task:
container:

1
.gitignore vendored

@ -6,3 +6,4 @@
tmp/
kernel
sources/
__pycache__

@ -59,7 +59,7 @@ Permissable folders/files:
- After `patch`, `sha-2` is built which contains an external implementation of
`sha256sum`. We then use that currently for all remaining software.
- To extract the binaries to get their checksums, use of chroot mode is
recommended (i.e. `./rootfs.sh chroot`).
recommended (i.e. `./rootfs.py --chroot`).
Furthermore, there is a special top-level dir `dev-utils`. In here are
appropriate utilities that are often useful for development and not generally

@ -19,12 +19,12 @@ Get me started!
2. ``git submodule update --init --recursive``
3. Provide a kernel (vmlinuz file) as the name kernel in the root of the
repository.
4. ``./rootfs.sh`` - ensure your account has kvm privileges and qemu
4. ``./rootfs.py`` - ensure your account has kvm privileges and qemu
installed.
a. Alternatively, run ``./rootfs.sh chroot`` to run it in a chroot.
b. Alternatively, run ``./rootfs.sh`` but don’t run the actual
virtualization and instead copy sysa/tmp/initramfs.igz to a USB or
a. Alternatively, run ``./rootfs.py --chroot`` to run it in a chroot.
b. Alternatively, run ``./rootfs.py`` but don’t run the actual
virtualization and instead copy sysa/tmp/initramfs to a USB or
some other device and boot from bare metal.
5. Wait.

@ -0,0 +1,53 @@
#!/usr/bin/env python3
"""
This file contains a few self-contained helper functions
"""
# SPDX-License-Identifier: GPL-3.0-or-later
# SPDX-FileCopyrightText: 2021 Andrius Štikonas <andrius@stikonas.eu>
import os
import shutil
import subprocess
import sys
def run(*args, **kwargs):
"""A small wrapper around subprocess.run"""
arguments = [str(arg) for arg in args]
if kwargs.pop('verbose', False):
print(arguments)
try:
subprocess.run(arguments, check=True, **kwargs)
except subprocess.CalledProcessError:
print("Bootstrapping failed")
sys.exit(1)
def mount(source, target, fs_type, options='', **kwargs):
"""Mount filesystem"""
run('sudo', 'mount', source, target, '-t', fs_type, '-o', options, **kwargs)
def umount(target, **kwargs):
"""Unmount filesystem"""
run('sudo', 'umount', target, **kwargs)
def copytree(src, dst, ignore=shutil.ignore_patterns('*.git*')):
"""Copy directory tree into another directory"""
file_name = os.path.basename(src)
shutil.copytree(src, os.path.join(dst, file_name), ignore=ignore)
def get_target(file_name):
"""Determine package installation directory"""
bname = os.path.basename(file_name)
# Remove file extension. This is not completely trivial because
# tar archives often come with double extensions.
first_ext = os.path.splitext(os.path.basename(bname))
second_ext = os.path.splitext(first_ext[0])
if second_ext[1] == '.tar':
bname = second_ext[0]
else:
bname = first_ext[0]
return bname

@ -0,0 +1,103 @@
#!/usr/bin/env python3
"""
A helper application used to start bootstrapping process.
It has a few modes of operation, you can create initramfs with
binary seeds and sources that you can boot into or alternatively
you can run bootstap inside chroot.
"""
# SPDX-License-Identifier: GPL-3.0-or-later
# SPDX-FileCopyrightText: 2021 Andrius Štikonas <andrius@stikonas.eu>
import argparse
import glob
import os
import subprocess
from sysa import SysA
from lib.utils import run
def main():
"""
A few command line arguments to customize bootstrap.
This function also creates SysA object which prepares directory
structure with bootstrap seeds and all sources.
"""
parser = argparse.ArgumentParser()
parser.add_argument("-a", "--arch", help="Bootstrap architecture",
default="x86")
parser.add_argument("-c", "--chroot", help="Run inside chroot",
action="store_true")
parser.add_argument("-p", "--preserve", help="Do not unmount temporary dir",
action="store_true")
parser.add_argument("-t", "--tmpdir", help="Temporary directory")
# QEMU arguments
parser.add_argument("-q", "--qemu-cmd", help="QEMU command",
default="qemu-system-x86_64")
parser.add_argument("-r", "--qemu-ram", help="Memory (in megabytes) allocated to QEMU VM",
default=8000)
parser.add_argument("-k", "--kernel", help="Kernel to use (default is ./kernel)",
default="kernel")
parser.add_argument("-m", "--minikernel", help="Use minikernel",
action="store_true")
args = parser.parse_args()
if args.chroot and args.minikernel:
raise ValueError("chroot and minikernel options cannot be used simultaneously.")
if args.arch != "x86":
raise ValueError("Only x86 is supported at the moment.")
system_a = SysA(arch=args.arch, preserve_tmp=args.preserve, tmpdir=args.tmpdir)
initramfs_path = os.path.join(system_a.tmp_dir, "initramfs")
if not args.chroot:
make_initramfs(system_a.tmp_dir, initramfs_path)
bootstrap(args, system_a.tmp_dir, initramfs_path)
def make_initramfs(tmp_dir, initramfs_path):
"""Package binary bootstrap seeds and sources into initramfs."""
file_list = glob.glob(os.path.join(tmp_dir, '**'), recursive=True)
# Use built-in removeprefix once we can use Python 3.9
def remove_prefix(text, prefix):
if text.startswith(prefix):
return text[len(prefix):]
return text # or whatever
file_list = [remove_prefix(f, tmp_dir + os.sep) for f in file_list]
with open(initramfs_path, "w") as initramfs:
cpio = subprocess.Popen(["cpio", "--format", "newc", "--create", "--directory", tmp_dir],
stdin=subprocess.PIPE, stdout=initramfs)
cpio.communicate(input='\n'.join(file_list).encode())
def bootstrap(args, tmp_dir, initramfs_path):
"""Kick off bootstrap process."""
print("Bootstrapping %s" % (args.arch))
if args.chroot:
init = os.path.join(os.sep, 'bootstrap-seeds', 'POSIX', args.arch, 'kaem-optional-seed')
run('sudo', 'env', '-i', 'PATH=/bin', 'chroot', tmp_dir, init)
return
if args.minikernel:
run('git', 'clone', '--depth', '1', '--branch', 'v0.4',
'https://github.com/bittorf/kritis-linux.git')
run('kritis-linux/ci_helper.sh', '--arch', 'x86_64', '--ramsize',
'-m', str(args.qemu_ram) + 'M', '--kernel', '5.10.8', '--initrd', initramfs_path)
return
run(args.qemu_cmd,
'-enable-kvm',
'-m', str(args.qemu_ram) + 'M',
'-nographic',
'-no-reboot',
'-kernel', args.kernel,
'-initrd', initramfs_path,
'-append', "console=ttyS0")
if __name__=="__main__":
main()

@ -1,393 +0,0 @@
#!/bin/bash
# SPDX-FileCopyrightText: 2021 Andrius Štikonas <andrius@stikonas.eu>
# SPDX-FileCopyrightText: 2021 Bastian Bittorf <bb@npl.de>
# SPDX-FileCopyrightText: 2020-2021 fosslinux <fosslinux@aussies.space>
#
# SPDX-License-Identifier: GPL-3.0-or-later
set -ex
QEMU_CMD="${1:-qemu-system-x86_64}" # or 'chroot' or 'minikernel'
QEMU_RAM="${2:-8G}"
GITDIR="$PWD/$(dirname "$0")"
if [ ! -f 'rootfs.sh' ]; then
echo 'must be run from base of repo'
exit 1
fi
pushd sysa
# SYSTEM A
# Setup tmp
mkdir -p tmp/
sudo mount -t tmpfs -o size=8G tmpfs tmp
LOGFILE="$PWD/tmp/bootstrap.log"
_wget() {
local url="$1"
local dir="${CACHEDIR:-$GITDIR/sources}"
local file="${2:-$(basename "${url}")}"
mkdir -p "$dir"
test -s "$dir/$file" || command wget -O "$dir/$file" "$url"
cp -v "$dir/$file" .
checksum_do "$dir" "$file"
}
checksum_do() {
local dir="$1"
local file="$2"
local line
local store="$GITDIR/SHA256SUMS.sources"
if line=$(grep "[[:space:]][[:space:]]$file"$ "$store"); then
(cd "$dir" && echo "$line" | sha256sum -c)
else
echo 'Checksum mismatch or not found!'
exit 1
fi
}
# base: mescc-tools-seed
# copy in all the mescc-tools-seed stuff
cp -r mescc-tools-seed/src/mescc-tools-seed/x86/* tmp
cp -r mescc-tools-seed/src/mescc-tools-seed/{M2-Planet,mes-m2} tmp/
cp -r mescc-tools-seed/src/mescc-tools-patched tmp/mescc-tools
# and the kaem seed
cp bootstrap-seeds/POSIX/x86/kaem-optional-seed tmp/init
cp bootstrap-seeds/POSIX/x86/kaem-optional-seed tmp/
cp -r bootstrap-seeds tmp/
# replace the init kaem with our own custom one
mv tmp/kaem.run tmp/mescc-tools-seed.kaem.run
cp base.kaem.run tmp/kaem.run
# create directories needed
mkdir -p tmp/bin
# after mescc-tools-seed we get into our own directory because
# the mescc-tools-seed one is hella messy
mkdir -p tmp/after/bin
mkdir -p tmp/after/{lib,include}
mkdir -p tmp/after/lib/{tcc,linux}
ln -s . tmp/after/lib/x86-mes
ln -s . tmp/after/lib/linux/x86-mes
mkdir -p tmp/after/include/{mes,gnu,linux,sys,mach}
mkdir -p tmp/after/include/linux/{x86,x86_64}
mkdir -p tmp/tmp
cp after.kaem tmp/
cp after.kaem.run tmp/after/kaem.run
cp mescc-tools-seed/checksums tmp/after/mescc-tools-seed-checksums
cp helpers.sh run.sh run2.sh pre-sha.sha256sums tmp/after/
# mescc-tools-extra
cp -r mescc-tools-extra tmp/after/
# mes
cp -r mes tmp/after/
#ln -s lib/x86-mes tmp/after/mes/src/mes/x86-mes
mkdir -p tmp/after/mes/src/mes/{bin,m2}
# tcc 0.9.26
cp -r tcc-0.9.26 tmp/after/
pushd tmp/after/tcc-0.9.26/src/tcc-0.9.26
ln -s ../mes/module .
ln -s ../mes/mes .
ln -s /after/lib x86-mes
ln -s /after/lib/linux .
popd
# tcc 0.9.27
cp -r tcc-0.9.27 tmp/after/
# tar 1.12
url=https://ftp.gnu.org/gnu/tar/tar-1.12.tar.gz
cp -r tar-1.12 tmp/after
mkdir tmp/after/tar-1.12/{src,build}
pushd tmp/after/tar-1.12/src
if [ ! -f "$(basename $url)" ]; then
_wget "$url"
fi
popd
tar -C tmp/after/tar-1.12/src -xf "tmp/after/tar-1.12/src/$(basename $url)" --strip-components=1
unset url
_get_file() {
local url="$1"
local output="$2"
local target="$3"
# Get the actual file
pushd "${target}/src"
if [ ! -f "$(basename "$url")" ]; then
_wget "$url" "${output:-${url##*/}}"
fi
popd
}
_get_target() {
local url="$1"
local output="$2"
local ext="${url##*.}"
if [ "$ext" = "tar" ]; then
bname=$(basename "${output:-${url}}" ".tar")
else
bname=$(basename "${output:-${url}}" ".tar.${ext}")
fi
# this is the target
echo "tmp/after/${bname}"
}
_get_main() {
local url="$1"
local output="$2"
local make_build="$3"
local target="$4"
# Copy main files
cp -r "$(basename "${target}")" tmp/after/
# Make directories
mkdir -p "${target}/src"
if [ "${make_build}" -eq 1 ]; then
mkdir -p "${target}/build"
fi
# Also get this file
_get_file "${url}" "${output}" "${target}"
}
get_file() {
# A mapping of URL to output filenames based on index (position-dependent)
local outputs=()
local urls=()
# Argument parsing
while [ $# -gt 0 ]; do
case "$1" in
--mkbuild=*) local make_build="${1#*=}" ;;
--output=*) outputs+=("${1#*=}") ;;
# It's just another URL to download
*) urls+=("$1") ;;
esac
shift
done
if [ -z "${make_build}" ]; then
make_build=0
fi
# Actual work
# Loop over urls
local url_length="${#urls[@]}"
local target
target="$(_get_target "${urls[0]}" "${outputs[0]}")"
_get_main "${urls[0]}" "${outputs[0]}" "${make_build}" "${target}"
if [ "${url_length}" -gt 1 ]; then
url_length="$((url_length-1))"
echo "${url_length}"
for i in $(seq 1 "${url_length}"); do
_get_file "${urls[${i}]}" "${outputs[${i}]}" "${target}"
done
fi
}
# gzip 1.2.4
get_file https://ftp.gnu.org/gnu/gzip/gzip-1.2.4.tar --mkbuild=1
# sed 4.0.9
get_file https://ftp.gnu.org/gnu/sed/sed-4.0.9.tar.gz --mkbuild=1
# patch 2.5.9
get_file https://ftp.gnu.org/pub/gnu/patch/patch-2.5.9.tar.gz --mkbuild=1
# sha-2 61555d
get_file https://github.com/amosnier/sha-2/archive/61555d.tar.gz --mkbuild=1 --output=sha-2-61555d.tar.gz
# make 3.80
get_file https://ftp.gnu.org/gnu/make/make-3.80.tar.gz --mkbuild=1
# bzip2 1.0.8
get_file ftp://sourceware.org/pub/bzip2/bzip2-1.0.8.tar.gz --mkbuild=1
# coreutils 5.0
get_file https://ftp.gnu.org/gnu/coreutils/coreutils-5.0.tar.bz2 --mkbuild=1
# heirloom-devtools
get_file http://downloads.sourceforge.net/project/heirloom/heirloom-devtools/070527/heirloom-devtools-070527.tar.bz2
# bash 2.05b
get_file https://ftp.gnu.org/pub/gnu/bash/bash-2.05b.tar.gz
# flex 2.5.11
get_file http://download.nust.na/pub2/openpkg1/sources/DST/flex/flex-2.5.11.tar.gz
# musl 1.1.24
get_file https://musl.libc.org/releases/musl-1.1.24.tar.gz
# m4 1.4.7
get_file https://ftp.gnu.org/gnu/m4/m4-1.4.7.tar.gz
# flex 2.6.4
get_file https://github.com/westes/flex/releases/download/v2.6.4/flex-2.6.4.tar.gz
# bison 3.4.1
get_file https://ftp.gnu.org/gnu/bison/bison-3.4.1.tar.gz
# grep 2.4
get_file https://ftp.gnu.org/gnu/grep/grep-2.4.tar.gz
# diffutils 2.7
get_file https://ftp.gnu.org/gnu/diffutils/diffutils-2.7.tar.gz
# coreutils 6.10
get_file https://ftp.gnu.org/gnu/coreutils/coreutils-6.10.tar.gz
# gawk 3.0.4
get_file https://ftp.gnu.org/gnu/gawk/gawk-3.0.4.tar.gz
# perl 5.000
get_file https://github.com/Perl/perl5/archive/perl-5.000.tar.gz
# perl 5.003
get_file https://github.com/Perl/perl5/archive/perl-5.003.tar.gz
# perl 5.004_05
get_file https://www.cpan.org/src/5.0/perl5.004_05.tar.gz
# perl 5.005_03
get_file https://www.cpan.org/src/5.0/perl5.005_03.tar.gz
# perl 5.6.2
get_file https://www.cpan.org/src/5.0/perl-5.6.2.tar.gz
# autoconf 2.52
get_file https://ftp.gnu.org/gnu/autoconf/autoconf-2.52.tar.bz2
# automake 1.6.3
get_file https://ftp.gnu.org/gnu/automake/automake-1.6.3.tar.bz2
# automake 1.4-p6
get_file https://ftp.gnu.org/gnu/automake/automake-1.4-p6.tar.gz
# autoconf 2.13
get_file https://ftp.gnu.org/gnu/autoconf/autoconf-2.13.tar.gz
# autoconf 2.12
get_file https://ftp.gnu.org/gnu/autoconf/autoconf-2.12.tar.gz
# libtool 1.4
get_file https://ftp.gnu.org/gnu/libtool/libtool-1.4.tar.gz
# binutils 2.14
get_file https://ftp.gnu.org/gnu/binutils/binutils-2.14.tar.bz2
# autoconf 2.53
get_file https://ftp.gnu.org/gnu/autoconf/autoconf-2.53.tar.bz2
# automake 1.7
get_file https://ftp.gnu.org/gnu/automake/automake-1.7.tar.bz2
# autoconf 2.54
get_file https://ftp.gnu.org/gnu/autoconf/autoconf-2.54.tar.bz2
# autoconf 2.55
get_file https://ftp.gnu.org/gnu/autoconf/autoconf-2.55.tar.bz2
# automake 1.7.8
get_file https://ftp.gnu.org/gnu/automake/automake-1.7.8.tar.bz2
# autoconf 2.57
get_file https://ftp.gnu.org/gnu/autoconf/autoconf-2.57.tar.bz2
# autoconf 2.59
get_file https://ftp.gnu.org/gnu/autoconf/autoconf-2.59.tar.bz2
# automake 1.8.5
get_file https://ftp.gnu.org/gnu/automake/automake-1.8.5.tar.bz2
# help2man 1.36.4
get_file https://ftp.gnu.org/gnu/help2man/help2man-1.36.4.tar.gz
# autoconf 2.61
get_file https://ftp.gnu.org/gnu/autoconf/autoconf-2.61.tar.bz2
# automake 1.9.6
get_file https://ftp.gnu.org/gnu/automake/automake-1.9.6.tar.bz2
# findutils 4.2.33
get_file https://ftp.gnu.org/gnu/findutils/findutils-4.2.33.tar.gz \
https://git.savannah.gnu.org/cgit/gnulib.git/snapshot/gnulib-8e128e.tar.gz
# libtool 2.2.4
get_file https://ftp.gnu.org/gnu/libtool/libtool-2.2.4.tar.bz2
# automake 1.10.3
get_file https://ftp.gnu.org/gnu/automake/automake-1.10.3.tar.bz2
# autoconf 2.65
get_file https://ftp.gnu.org/gnu/autoconf/autoconf-2.65.tar.bz2
# gcc 4.0.4
get_file https://ftp.gnu.org/gnu/gcc/gcc-4.0.4/gcc-core-4.0.4.tar.bz2 --output=gcc-4.0.4.tar.bz2
# musl 1.2.2
get_file https://musl.libc.org/releases/musl-1.2.2.tar.gz
# bash 5.1
get_file https://ftp.gnu.org/gnu/bash/bash-5.1.tar.gz
# xz 5.0.5
get_file https://tukaani.org/xz/xz-5.0.5.tar.bz2
# automake 1.11.2
get_file https://ftp.gnu.org/gnu/automake/automake-1.11.2.tar.bz2
# autoconf 2.69
get_file https://ftp.gnu.org/gnu/autoconf/autoconf-2.69.tar.xz
# automake 1.12.6
get_file https://ftp.gnu.org/gnu/automake/automake-1.12.6.tar.xz
# automake 1.13.4
get_file https://ftp.gnu.org/gnu/automake/automake-1.13.4.tar.xz
# automake 1.15.1
get_file https://ftp.gnu.org/gnu/automake/automake-1.15.1.tar.xz
# tar 1.34
get_file https://ftp.gnu.org/gnu/tar/tar-1.34.tar.xz \
https://git.savannah.gnu.org/cgit/gnulib.git/snapshot/gnulib-30820c.tar.gz
# gmp 6.2.1
get_file https://ftp.gnu.org/gnu/gmp/gmp-6.2.1.tar.xz
# General cleanup
find tmp -name .git -exec rm -rf \;
# initramfs
cd tmp
find . | cpio -H newc -o | gzip > initramfs.igz
# Run
case "${QEMU_CMD}" in
chroot)
sudo PATH="/after/bin:${PATH}" chroot . /init 2>&1 | tee "$LOGFILE"
;;
minikernel)
git clone --depth 1 --branch v0.4 https://github.com/bittorf/kritis-linux.git
kritis-linux/ci_helper.sh \
--arch x86_64 \
--ramsize 4G \
--kernel 5.10.8 \
--initrd initramfs.igz \
--log "$LOGFILE"
;;
*)
${QEMU_CMD} -enable-kvm \
-m "${QEMU_RAM:-8G}" \
-nographic \
-no-reboot \
-kernel ../../kernel -initrd initramfs.igz -append console=ttyS0 | tee "$LOGFILE"
;;
esac
cd ../..
# Cleanup
sudo umount sysa/tmp

@ -0,0 +1,445 @@
#!/usr/bin/env python3
"""System A"""
# SPDX-License-Identifier: GPL-3.0-or-later
# SPDX-FileCopyrightText: 2021 Andrius Štikonas <andrius@stikonas.eu>
import hashlib
import os
from distutils.dir_util import copy_tree
import shutil
import requests
from lib.utils import mount, umount, copytree, run, get_target
class SysA:
"""
Class responsible for preparing sources for System A.
"""
def __init__(self, arch, preserve_tmp, tmpdir):
self.git_dir = os.path.dirname(os.path.join(__file__))
self.arch = arch
self.preserve_tmp = preserve_tmp
if tmpdir is None:
self.tmp_dir = os.path.join(self.git_dir, 'sysa', 'tmp')
else:
self.tmp_dir = tmpdir
self.sysa_dir = os.path.join(self.git_dir, 'sysa')
self.after_dir = os.path.join(self.tmp_dir, 'after')
self.prepare()
def __del__(self):
if not self.preserve_tmp:
print("Unmounting tmpfs from %s" % (self.tmp_dir))
umount(self.tmp_dir)
os.rmdir(self.tmp_dir)
def check_file(self, file_name):
"""Check hash of downloaded source file."""
checksum_store = os.path.join(self.git_dir, 'SHA256SUMS.sources')
with open(checksum_store) as checksum_file:
hashes = checksum_file.read().splitlines()
for hash_line in hashes:
if os.path.basename(file_name) in hash_line:
# Hash is in store, check it
expected_hash = hash_line.split()[0]
with open(file_name, "rb") as downloaded_file:
downloaded_content = downloaded_file.read() # read entire file as bytes
readable_hash = hashlib.sha256(downloaded_content).hexdigest()
if expected_hash == readable_hash:
return
raise Exception("Checksum mismatch")
raise Exception("File checksum is not yet recorded")
def download_file(self, url, file_name=None):
"""
Download a single source archive.
"""
cache_dir = os.path.join(self.git_dir, 'sources')
# Automatically determine file name based on URL.
if file_name is None:
file_name = os.path.basename(url)
abs_file_name = os.path.join(cache_dir, file_name)
# Create a cache directory for downloaded sources
if not os.path.isdir(cache_dir):
os.mkdir(cache_dir)
# Actually download the file
if not os.path.isfile(abs_file_name):
print("Downloading: %s" % (file_name))
request = requests.get(url, allow_redirects=True)
open(abs_file_name, 'wb').write(request.content)
# Check SHA256 hash
self.check_file(abs_file_name)
return abs_file_name
def get_file(self, url, mkbuild=False, output=None):
"""
Download and prepares source packages
url can be either:
1. a single URL
2. list of URLs to download. In this case the first URL is the primary URL
from which we derive the name of package directory
output can be used to override file name of the downloaded file(s).
mkbuild=True can be used to pre-create build directories before
mkdir is available.
"""
# Single URL
if isinstance(url, str):
assert output is None or isinstance(output, str)
file_name = url if output is None else output
urls = [url]
outputs = [output]
# Multiple URLs
elif isinstance(url, list):
assert output is None or len(output) == len(url)
file_name = url[0] if output is None else output[0]
urls = url
outputs = output if output is not None else [None] * len(url)
else:
raise TypeError("url must be either a string or a list of strings")
# Determine installation directory
target_name = get_target(file_name)
target_src_dir = os.path.join(self.after_dir, target_name, 'src')
# Install base files
src_tree = os.path.join(self.sysa_dir, target_name)
copytree(src_tree, self.after_dir)
os.mkdir(target_src_dir)
for i, _ in enumerate(urls):
# Download files into cache directory
tarball = self.download_file(urls[i], outputs[i])
# Install sources into target directory
shutil.copy2(tarball, target_src_dir)
if mkbuild:
os.mkdir(os.path.join(self.after_dir, target_name, 'build'))
def prepare(self):
"""
Prepare directory structore for System A.
We create an empty tmpfs, unpack stage0-posix.
Rest of the files are unpacked into more structured directory /after
"""
if not os.path.isdir(self.tmp_dir):
os.mkdir(self.tmp_dir)
print("Mounting tmpfs on %s" % (self.tmp_dir))
mount('tmpfs', self.tmp_dir, 'tmpfs', 'size=8G')
self.stage0_posix()
self.after()
def stage0_posix(self):
"""Copy in all the stage0-posix (formerly known as mescc-tools-seed)"""
mescc_tools_seed_base_dir = os.path.join(self.sysa_dir, 'mescc-tools-seed',
'src', 'mescc-tools-seed')
mescc_tools_seed_dir = os.path.join(mescc_tools_seed_base_dir, self.arch)
copy_tree(mescc_tools_seed_dir, self.tmp_dir)
m2_planet_dir = os.path.join(mescc_tools_seed_base_dir, 'M2-Planet')
copytree(m2_planet_dir, self.tmp_dir)
# At the moment not useful for bootstrap but easier to keep it
mes_m2_dir = os.path.join(mescc_tools_seed_base_dir, 'mes-m2')
copytree(mes_m2_dir, self.tmp_dir)
mescc_tools_patched_dir = os.path.join(self.sysa_dir, 'mescc-tools-seed',
'src', 'mescc-tools-patched')
shutil.copytree(mescc_tools_patched_dir,
os.path.join(self.tmp_dir, 'mescc-tools'), shutil.ignore_patterns('*.git*'))
# bootstrap seeds
bootstrap_seeds_dir = os.path.join(self.sysa_dir, 'bootstrap-seeds')
copytree(bootstrap_seeds_dir, self.tmp_dir)
kaem_optional_seed = os.path.join(bootstrap_seeds_dir, 'POSIX',
self.arch, 'kaem-optional-seed')
shutil.copy2(kaem_optional_seed, os.path.join(self.tmp_dir, 'init'))
# replace the init kaem with our own custom one
shutil.move(os.path.join(self.tmp_dir, 'kaem.run'),
os.path.join(self.tmp_dir, 'mescc-tools-seed.kaem.run'))
shutil.copy2(os.path.join(self.sysa_dir, 'base.kaem.run'),
os.path.join(self.tmp_dir, 'kaem.run'))
# create directories needed
os.mkdir(os.path.join(self.tmp_dir, 'bin'))
def after(self):
"""
Prepare sources in /after directory.
After mescc-tools-seed we get into our own directory because
the mescc-tools-seed one is hella messy.
"""
self.create_after_dirs()
self.mescc_tools_checksum()
self.deploy_extra_files()
self.mescc_tools_extra()
self.mes()
self.tcc_0_9_26()
self.tcc_0_9_27()
self.tar_1_12()
self.get_packages()
def create_after_dirs(self):
"""
Create some empty directories for early bootstrap
This list can be eventually reduced if we include a small
mkdir implementation written for M2-Planet.
"""
bin_dir = os.path.join(self.after_dir, 'bin')
lib_dir = os.path.join(self.after_dir, 'lib')
include_dir = os.path.join(self.after_dir, 'include')
os.mkdir(self.after_dir)
os.mkdir(bin_dir)
os.mkdir(lib_dir)
os.mkdir(include_dir)
os.mkdir(os.path.join(lib_dir, self.arch+'-mes'))
os.mkdir(os.path.join(lib_dir, 'tcc'))
os.mkdir(os.path.join(lib_dir, 'linux'))
os.mkdir(os.path.join(lib_dir, 'linux', self.arch+'-mes'))
os.mkdir(os.path.join(include_dir, 'mes'))
os.mkdir(os.path.join(include_dir, 'gnu'))
os.mkdir(os.path.join(include_dir, 'linux'))
os.mkdir(os.path.join(include_dir, 'linux', self.arch))
os.mkdir(os.path.join(include_dir, 'sys'))
os.mkdir(os.path.join(include_dir, 'mach'))
# Needed for patch to work, although can be fixed with TMPDIR
os.mkdir(os.path.join(self.tmp_dir, 'tmp'))
def mescc_tools_checksum(self):
"""Early fletcher16 checksum files"""
shutil.copy2(os.path.join(self.sysa_dir, 'mescc-tools-seed', 'checksums'),
os.path.join(self.after_dir, 'mescc-tools-seed-checksums'))
def deploy_extra_files(self):
"""Deploy misc files"""
extra_files = ['helpers.sh', 'run.sh', 'run2.sh', 'pre-sha.sha256sums']
for extra_file in extra_files:
shutil.copy2(os.path.join(self.sysa_dir, extra_file), self.after_dir)
shutil.copy2(os.path.join(self.sysa_dir, 'after.kaem'), self.tmp_dir)
shutil.copy2(os.path.join(self.sysa_dir, 'after.kaem.run'),
os.path.join(self.after_dir, 'kaem.run'))
shutil.copy2(os.path.join(self.git_dir, 'SHA256SUMS.sources'), self.after_dir)
def mescc_tools_extra(self):
"""Some additional tools such as cp and chmod (for M2-Planet)"""
copytree(os.path.join(self.sysa_dir, 'mescc-tools-extra'), self.after_dir)
def mes(self):
"""GNU Mes"""
copytree(os.path.join(self.sysa_dir, 'mes'), self.after_dir)
mes_dir = os.path.join(self.after_dir, 'mes', 'src', 'mes')
os.mkdir(os.path.join(mes_dir, 'bin'))
os.mkdir(os.path.join(mes_dir, 'm2'))
def tcc_0_9_26(self):
"""TinyCC 0.9.26 (patched by janneke)"""
copytree(os.path.join(self.sysa_dir, 'tcc-0.9.26'), self.after_dir)
def tcc_0_9_27(self):
"""TinyCC 0.9.27"""
copytree(os.path.join(self.sysa_dir, 'tcc-0.9.27'), self.after_dir)
def tar_1_12(self):
"""GNU Tar 1.12"""
# We have to pre-unpack tar sources.
# Possible alternative is to build a single C file implementation of untar.
tar_url = "https://mirrors.kernel.org/gnu/tar/tar-1.12.tar.gz"
self.get_file(tar_url)
tar_src_dir = os.path.join(self.after_dir, 'tar-1.12', 'src')
tar_tarball = os.path.join(tar_src_dir, os.path.basename(tar_url))
run('tar', '-C', tar_src_dir, '-xf', tar_tarball, '--strip-components=1')
# pylint: disable=line-too-long,too-many-statements
def get_packages(self):
"""Prepare remaining sources"""
# gzip 1.2.4
self.get_file("https://mirrors.kernel.org/gnu/gzip/gzip-1.2.4.tar", mkbuild=True)
# sed 4.0.9
self.get_file("https://mirrors.kernel.org/gnu/sed/sed-4.0.9.tar.gz", mkbuild=True)
# patch 2.5.9
self.get_file("https://ftp.gnu.org/pub/gnu/patch/patch-2.5.9.tar.gz", mkbuild=True)
# sha-2 61555d
self.get_file("https://github.com/amosnier/sha-2/archive/61555d.tar.gz", mkbuild=True,
output="sha-2-61555d.tar.gz")
# make 3.80
self.get_file("https://mirrors.kernel.org/gnu/make/make-3.80.tar.gz", mkbuild=True)
# bzip2 1.0.8
self.get_file("https://sourceware.org/pub/bzip2/bzip2-1.0.8.tar.gz", mkbuild=True)
# coreutils 5.0
self.get_file("https://mirrors.kernel.org/gnu/coreutils/coreutils-5.0.tar.bz2", mkbuild=True)
# heirloom-devtools
self.get_file("http://downloads.sourceforge.net/project/heirloom/heirloom-devtools/070527/heirloom-devtools-070527.tar.bz2")
# bash 2.05b
self.get_file("https://ftp.gnu.org/pub/gnu/bash/bash-2.05b.tar.gz")
# flex 2.5.11
self.get_file("http://download.nust.na/pub2/openpkg1/sources/DST/flex/flex-2.5.11.tar.gz")
# musl 1.1.24
self.get_file("https://musl.libc.org/releases/musl-1.1.24.tar.gz")
# m4 1.4.7
self.get_file("https://mirrors.kernel.org/gnu/m4/m4-1.4.7.tar.gz")
# flex 2.6.4
self.get_file("https://github.com/westes/flex/releases/download/v2.6.4/flex-2.6.4.tar.gz")
# bison 3.4.1
self.get_file("https://mirrors.kernel.org/gnu/bison/bison-3.4.1.tar.gz")
# grep 2.4
self.get_file("https://mirrors.kernel.org/gnu/grep/grep-2.4.tar.gz")
# diffutils 2.7
self.get_file("https://mirrors.kernel.org/gnu/diffutils/diffutils-2.7.tar.gz")
# coreutils 6.10
self.get_file("https://mirrors.kernel.org/gnu/coreutils/coreutils-6.10.tar.gz")
# gawk 3.0.4
self.get_file("https://mirrors.kernel.org/gnu/gawk/gawk-3.0.4.tar.gz")
# perl 5.000
self.get_file("https://github.com/Perl/perl5/archive/perl-5.000.tar.gz")
# perl 5.003
self.get_file("https://github.com/Perl/perl5/archive/perl-5.003.tar.gz")
# perl 5.004_05
self.get_file("https://www.cpan.org/src/5.0/perl5.004_05.tar.gz")
# perl 5.005_03
self.get_file("https://www.cpan.org/src/5.0/perl5.005_03.tar.gz")
# perl 5.6.2
self.get_file("https://www.cpan.org/src/5.0/perl-5.6.2.tar.gz")
# autoconf 2.52
self.get_file("https://mirrors.kernel.org/gnu/autoconf/autoconf-2.52.tar.bz2")
# automake 1.6.3
self.get_file("https://mirrors.kernel.org/gnu/automake/automake-1.6.3.tar.bz2")
# automake 1.4-p6
self.get_file("https://mirrors.kernel.org/gnu/automake/automake-1.4-p6.tar.gz")
# autoconf 2.13
self.get_file("https://mirrors.kernel.org/gnu/autoconf/autoconf-2.13.tar.gz")
# autoconf 2.12
self.get_file("https://mirrors.kernel.org/gnu/autoconf/autoconf-2.12.tar.gz")
# libtool 1.4
self.get_file("https://mirrors.kernel.org/gnu/libtool/libtool-1.4.tar.gz")
# binutils 2.14
self.get_file("https://mirrors.kernel.org/gnu/binutils/binutils-2.14.tar.bz2")
# autoconf 2.53
self.get_file("https://mirrors.kernel.org/gnu/autoconf/autoconf-2.53.tar.bz2")
# automake 1.7
self.get_file("https://mirrors.kernel.org/gnu/automake/automake-1.7.tar.bz2")
# autoconf 2.54
self.get_file("https://mirrors.kernel.org/gnu/autoconf/autoconf-2.54.tar.bz2")
# autoconf 2.55
self.get_file("https://mirrors.kernel.org/gnu/autoconf/autoconf-2.55.tar.bz2")
# automake 1.7.8
self.get_file("https://mirrors.kernel.org/gnu/automake/automake-1.7.8.tar.bz2")
# autoconf 2.57
self.get_file("https://mirrors.kernel.org/gnu/autoconf/autoconf-2.57.tar.bz2")
# autoconf 2.59
self.get_file("https://mirrors.kernel.org/gnu/autoconf/autoconf-2.59.tar.bz2")
# automake 1.8.5
self.get_file("https://mirrors.kernel.org/gnu/automake/automake-1.8.5.tar.bz2")
# help2man 1.36.4
self.get_file("https://mirrors.kernel.org/gnu/help2man/help2man-1.36.4.tar.gz")
# autoconf 2.61
self.get_file("https://mirrors.kernel.org/gnu/autoconf/autoconf-2.61.tar.bz2")
# automake 1.9.6
self.get_file("https://mirrors.kernel.org/gnu/automake/automake-1.9.6.tar.bz2")
# findutils 4.2.33
self.get_file(["https://mirrors.kernel.org/gnu/findutils/findutils-4.2.33.tar.gz",
"https://git.savannah.gnu.org/cgit/gnulib.git/snapshot/gnulib-8e128e.tar.gz"])
# libtool 2.2.4
self.get_file("https://mirrors.kernel.org/gnu/libtool/libtool-2.2.4.tar.bz2")
# automake 1.10.3
self.get_file("https://mirrors.kernel.org/gnu/automake/automake-1.10.3.tar.bz2")
# autoconf 2.65
self.get_file("https://mirrors.kernel.org/gnu/autoconf/autoconf-2.65.tar.bz2")
# gcc 4.0.4
self.get_file("https://mirrors.kernel.org/gnu/gcc/gcc-4.0.4/gcc-core-4.0.4.tar.bz2",
output="gcc-4.0.4.tar.bz2")
# musl 1.2.2
self.get_file("https://musl.libc.org/releases/musl-1.2.2.tar.gz")
# bash 5.1
self.get_file("https://mirrors.kernel.org/gnu/bash/bash-5.1.tar.gz")
# xz 5.0.5
self.get_file("https://tukaani.org/xz/xz-5.0.5.tar.bz2")
# automake 1.11.2
self.get_file("https://mirrors.kernel.org/gnu/automake/automake-1.11.2.tar.bz2")
# autoconf 2.69
self.get_file("https://mirrors.kernel.org/gnu/autoconf/autoconf-2.69.tar.xz")
# automake 1.12.6
self.get_file("https://mirrors.kernel.org/gnu/automake/automake-1.12.6.tar.xz")
# automake 1.13.4
self.get_file("https://mirrors.kernel.org/gnu/automake/automake-1.13.4.tar.xz")
# automake 1.15.1
self.get_file("https://mirrors.kernel.org/gnu/automake/automake-1.15.1.tar.xz")
# tar 1.34
self.get_file(["https://mirrors.kernel.org/gnu/tar/tar-1.34.tar.xz",
"https://git.savannah.gnu.org/cgit/gnulib.git/snapshot/gnulib-30820c.tar.gz"])
# gmp 6.2.1
self.get_file("https://mirrors.kernel.org/gnu/gmp/gmp-6.2.1.tar.xz")

@ -1,13 +1,13 @@
63031 /after/bin/mes
23643 /after/bin/mescc.scm
36724 /after/lib/crt1.s
42390 /after/lib/crt1.o
39638 /after/lib/x86.M1
526 /after/lib/libmescc.s
10591 /after/lib/libc+tcc.s
45027 /after/lib/libc.s
17259 /after/lib/libmescc.a
20204 /after/lib/libc+tcc.a
34645 /after/lib/libc.a
8245 /after/lib/linux/elf32-header.hex2
15474 /after/lib/linux/elf32-footer-single-main.hex2
36724 /after/lib/x86-mes/crt1.s
42390 /after/lib/x86-mes/crt1.o
39638 /after/lib/x86-mes/x86.M1
526 /after/lib/x86-mes/libmescc.s
10591 /after/lib/x86-mes/libc+tcc.s
45027 /after/lib/x86-mes/libc.s
17259 /after/lib/x86-mes/libmescc.a
20204 /after/lib/x86-mes/libc+tcc.a
34645 /after/lib/x86-mes/libc.a
8245 /after/lib/linux/x86-mes/elf32-header.hex2
15474 /after/lib/linux/x86-mes/elf32-footer-single-main.hex2

@ -212,17 +212,17 @@ ${MES} --no-auto-compile -e main ${bindir}/mescc.scm -- -L ${libdir} -nostdlib -
cp ${bindir}/mes-new ${bindir}/mes
# Install libraries
cp ${libdir}/x86-mes/libc.a ${prefix}/lib/
cp ${libdir}/x86-mes/libc+tcc.a ${prefix}/lib/
cp ${libdir}/x86-mes/libmescc.a ${prefix}/lib/
cp ${libdir}/x86-mes/libc.s ${prefix}/lib/
cp ${libdir}/x86-mes/libc+tcc.s ${prefix}/lib/
cp ${libdir}/x86-mes/libmescc.s ${prefix}/lib/
cp ${libdir}/x86-mes/x86.M1 ${prefix}/lib/
cp crt1.o ${prefix}/lib/
cp crt1.s ${prefix}/lib/
cp ${libdir}/linux/x86-mes/elf32-footer-single-main.hex2 ${prefix}/lib/linux/
cp ${libdir}/linux/x86-mes/elf32-header.hex2 ${prefix}/lib/linux/
cp ${libdir}/x86-mes/libc.a ${prefix}/lib/x86-mes/
cp ${libdir}/x86-mes/libc+tcc.a ${prefix}/lib/x86-mes/
cp ${libdir}/x86-mes/libmescc.a ${prefix}/lib/x86-mes/
cp ${libdir}/x86-mes/libc.s ${prefix}/lib/x86-mes/
cp ${libdir}/x86-mes/libc+tcc.s ${prefix}/lib/x86-mes/
cp ${libdir}/x86-mes/libmescc.s ${prefix}/lib/x86-mes/
cp ${libdir}/x86-mes/x86.M1 ${prefix}/lib/x86-mes/
cp crt1.o ${prefix}/lib/x86-mes/
cp crt1.s ${prefix}/lib/x86-mes/
cp ${libdir}/linux/x86-mes/elf32-footer-single-main.hex2 ${prefix}/lib/linux/x86-mes/
cp ${libdir}/linux/x86-mes/elf32-header.hex2 ${prefix}/lib/linux/x86-mes/
# Install header files
cp include/alloca.h ${incdir}/alloca.h

@ -20,18 +20,18 @@ bc9ddf9854bd954c71bb1cf5b0af77fd65b8fa7f290b42c75020fb8893deb53c /after/bin/sha
cacc2cda3ce6c2cc0cb761b8aa2115a003e166c94d1039974fc0221263897c52 /after/bin/tcc-0.9.26
fc2d3f6a7227836526d278d4e4b642a72c1a83c03c3363f3376100798e0eb145 /after/bin/zcat
44b5f15e1f015685fe4c3d66eda5ba52aac77b94f2edd98b764cec05ca350d49 /after/lib/crt1.o
90811dafd33ad56b8e4b0adcc04263f9329b9047b7cc337abe8151a75017172c /after/lib/crt1.s
90811dafd33ad56b8e4b0adcc04263f9329b9047b7cc337abe8151a75017172c /after/lib/x86-mes/crt1.s
09d4f9821a2566f7e56381a19259c41bd97f3c5ed83f490705acbfd1139a7736 /after/lib/crti.o
461ca1494737fab86fe1c1d3addeaf9d0ece413e353abcdea8674db3f700cda3 /after/lib/crtn.o
34f62227f8cc61d365d92a182f8f3cc91cc6c50a1bbb8f4774a4383bceaefa5f /after/lib/libc.a
34f62227f8cc61d365d92a182f8f3cc91cc6c50a1bbb8f4774a4383bceaefa5f /after/lib/libc+gnu.a
b5ce4e1288a27864156d74268090c13aea6b5a261fa81c75bfbe844d0689d03d /after/lib/libc.s
3156e619dbd85c471e2a8d053ba536eaaa8f91da657003777b8e87e7bab4266d /after/lib/libc+tcc.a
aaf89a9d6818cdb8ece73454631b1a1ae83503e5eb7777d38cdaf141cba0e530 /after/lib/libc+tcc.s
b5ce4e1288a27864156d74268090c13aea6b5a261fa81c75bfbe844d0689d03d /after/lib/x86-mes/libc.s
3156e619dbd85c471e2a8d053ba536eaaa8f91da657003777b8e87e7bab4266d /after/lib/x86-mes/libc+tcc.a
aaf89a9d6818cdb8ece73454631b1a1ae83503e5eb7777d38cdaf141cba0e530 /after/lib/x86-mes/libc+tcc.s
12c07ae103e7e3b390150a79e5c600d88de14e9bb73a066f6342582729ef5a3f /after/lib/libgetopt.a
52f697278ccdff5e457f27e10f465a91ab9858f0c6cee0683831cadb3109bbb7 /after/lib/libmescc.a
d8646707db6aa2a76fdc5dbb3521376439e357f9f1de1d67f02a1afeefd342ac /after/lib/libmescc.s
f9873d9aab12e70f24d97f8319e17a1e698ca60779ae9a6ab3ede648cd60fc61 /after/lib/linux/elf32-footer-single-main.hex2
b16ab368bc4c7b8bd896d03cba565a60e97760dea4da9f5c8a1a3d2902a79df6 /after/lib/linux/elf32-header.hex2
52f697278ccdff5e457f27e10f465a91ab9858f0c6cee0683831cadb3109bbb7 /after/lib/x86-mes/libmescc.a
d8646707db6aa2a76fdc5dbb3521376439e357f9f1de1d67f02a1afeefd342ac /after/lib/x86-mes/libmescc.s
f9873d9aab12e70f24d97f8319e17a1e698ca60779ae9a6ab3ede648cd60fc61 /after/lib/linux/x86-mes/elf32-footer-single-main.hex2
b16ab368bc4c7b8bd896d03cba565a60e97760dea4da9f5c8a1a3d2902a79df6 /after/lib/linux/x86-mes/elf32-header.hex2
a650b13efc65073fb851e9db89728089d8845c401f85faaa09801874ab058089 /after/lib/tcc/libtcc1.a
c9944a799d584abfa76f385c14ac0caf6f46d03b34bf2712493602b12826c6b2 /after/lib/x86.M1
c9944a799d584abfa76f385c14ac0caf6f46d03b34bf2712493602b12826c6b2 /after/lib/x86-mes/x86.M1

@ -9,7 +9,6 @@
35666 /after/bin/tcc-0.9.26
47832 /after/lib/libc.a
47832 /after/lib/libc+gnu.a
20204 /after/lib/libc+tcc.a
54049 /after/lib/libgetopt.a
23061 /after/lib/crt1.o
16980 /after/lib/crti.o

@ -1,7 +1,6 @@
35075 /after/bin/tcc
47832 /after/lib/libc.a
47832 /after/lib/libc+gnu.a
20204 /after/lib/libc+tcc.a
54049 /after/lib/libgetopt.a
23061 /after/lib/crt1.o
16980 /after/lib/crti.o

Loading…
Cancel
Save