Merge pull request #361 from Googulator/fix-bwrap
Fix bubblewrap support & CI workflows after simplify
This commit is contained in:
commit
81790a3271
|
@ -12,8 +12,8 @@ on:
|
||||||
- master
|
- master
|
||||||
|
|
||||||
jobs:
|
jobs:
|
||||||
sysa:
|
pass1:
|
||||||
name: Run sysa under bubblewrap
|
name: Run up to Linux build under bubblewrap
|
||||||
runs-on: ubuntu-latest
|
runs-on: ubuntu-latest
|
||||||
steps:
|
steps:
|
||||||
- name: Install bubblewrap
|
- name: Install bubblewrap
|
||||||
|
@ -30,9 +30,8 @@ jobs:
|
||||||
uses: actions/cache/restore@v3
|
uses: actions/cache/restore@v3
|
||||||
with:
|
with:
|
||||||
path: |
|
path: |
|
||||||
sysa/distfiles
|
distfiles
|
||||||
sysc/distfiles
|
key: cache-${{ hashFiles('steps/*/sources') }}
|
||||||
key: cache-${{ hashFiles('sys*/*/sources') }}
|
|
||||||
- name: Get sources
|
- name: Get sources
|
||||||
if: steps.cache.outputs.cache-hit != 'true'
|
if: steps.cache.outputs.cache-hit != 'true'
|
||||||
run: ./download-distfiles.sh
|
run: ./download-distfiles.sh
|
||||||
|
@ -41,9 +40,8 @@ jobs:
|
||||||
uses: actions/cache/save@v3
|
uses: actions/cache/save@v3
|
||||||
with:
|
with:
|
||||||
path: |
|
path: |
|
||||||
sysa/distfiles
|
distfiles
|
||||||
sysc/distfiles
|
key: cache-${{ hashFiles('steps/*/sources') }}
|
||||||
key: cache-${{ hashFiles('sys*/*/sources') }}
|
|
||||||
- name: Run bootstrap
|
- name: Run bootstrap
|
||||||
run: ./rootfs.py --bwrap --external-sources --build-kernels --preserve --cores 2 --internal-ci pass1
|
run: ./rootfs.py --bwrap --external-sources --build-kernels --preserve --cores 2 --internal-ci pass1
|
||||||
- name: Archive created packages
|
- name: Archive created packages
|
||||||
|
@ -51,18 +49,18 @@ jobs:
|
||||||
uses: actions/upload-artifact@v3
|
uses: actions/upload-artifact@v3
|
||||||
with:
|
with:
|
||||||
name: packages
|
name: packages
|
||||||
path: tmp/sysa/usr/src/repo/**
|
path: tmp/external/repo/**
|
||||||
- name: Tar sysc_image
|
- name: Tar pass1 image
|
||||||
run: tar -cf sysc_image.tar tmp/sysa/sysc_image/
|
run: tar -cf pass1_image.tar tmp
|
||||||
- name: Archive sysc_image
|
- name: Archive pass1_image
|
||||||
uses: actions/upload-artifact@v3
|
uses: actions/upload-artifact@v3
|
||||||
with:
|
with:
|
||||||
name: internal_sysc_image
|
name: internal_pass1_image
|
||||||
path: sysc_image.tar
|
path: pass1_image.tar
|
||||||
|
|
||||||
sysc1:
|
pass2:
|
||||||
name: Run sysc (part 1) under bubblewrap
|
name: Run up to Python bootstrap under bubblewrap
|
||||||
needs: sysa
|
needs: pass1
|
||||||
runs-on: ubuntu-latest
|
runs-on: ubuntu-latest
|
||||||
steps:
|
steps:
|
||||||
- name: Install bubblewrap
|
- name: Install bubblewrap
|
||||||
|
@ -74,24 +72,48 @@ jobs:
|
||||||
# There is a strange bug(?) in nongnu, when you clone a git repository
|
# There is a strange bug(?) in nongnu, when you clone a git repository
|
||||||
# against a commit != HEAD with depth=1, it errors out.
|
# against a commit != HEAD with depth=1, it errors out.
|
||||||
fetch-depth: 0
|
fetch-depth: 0
|
||||||
- name: Get sysc_image
|
- name: Get pass1_image
|
||||||
uses: actions/download-artifact@v3
|
uses: actions/download-artifact@v3
|
||||||
with:
|
with:
|
||||||
name: internal_sysc_image
|
name: internal_pass1_image
|
||||||
- name: Extract sysc_image
|
- name: Extract pass1_image
|
||||||
run: tar -xf sysc_image.tar
|
run: tar -xf pass1_image.tar
|
||||||
|
- name: Query cache for sources
|
||||||
|
id: cache
|
||||||
|
uses: actions/cache/restore@v3
|
||||||
|
with:
|
||||||
|
path: |
|
||||||
|
distfiles
|
||||||
|
key: cache-${{ hashFiles('steps/*/sources') }}
|
||||||
|
- name: Get sources
|
||||||
|
if: steps.cache.outputs.cache-hit != 'true'
|
||||||
|
run: ./download-distfiles.sh
|
||||||
|
- name: Cache sources
|
||||||
|
if: steps.cache.outputs.cache-hit != 'true'
|
||||||
|
uses: actions/cache/save@v3
|
||||||
|
with:
|
||||||
|
path: |
|
||||||
|
distfiles
|
||||||
|
key: cache-${{ hashFiles('steps/*/sources') }}
|
||||||
- name: Run bootstrap
|
- name: Run bootstrap
|
||||||
run: ./rootfs.py --bwrap --external-sources --build-kernels --preserve --cores 2 --internal-ci pass2
|
run: ./rootfs.py --bwrap --external-sources --build-kernels --preserve --cores 2 --internal-ci pass2
|
||||||
- name: Archive created packages
|
- name: Archive created packages
|
||||||
if: always() # archive failed builds progress
|
if: failure() # archive failed builds progress
|
||||||
uses: actions/upload-artifact@v3
|
uses: actions/upload-artifact@v3
|
||||||
with:
|
with:
|
||||||
name: internal_packages_sysc1
|
name: internal_packages_pass2
|
||||||
path: tmp/sysa/sysc_image/usr/src/repo/**
|
path: tmp/external/repo/**
|
||||||
|
- name: Tar pass2 image
|
||||||
|
run: tar -cf pass2_image.tar tmp
|
||||||
|
- name: Archive pass2_image
|
||||||
|
uses: actions/upload-artifact@v3
|
||||||
|
with:
|
||||||
|
name: internal_pass2_image
|
||||||
|
path: pass2_image.tar
|
||||||
|
|
||||||
sysc2:
|
pass3:
|
||||||
name: Run sysc (part 2) under bubblewrap
|
name: Run remaining builds under bubblewrap
|
||||||
needs: sysc1
|
needs: pass2
|
||||||
runs-on: ubuntu-latest
|
runs-on: ubuntu-latest
|
||||||
steps:
|
steps:
|
||||||
- name: Install bubblewrap
|
- name: Install bubblewrap
|
||||||
|
@ -103,23 +125,34 @@ jobs:
|
||||||
# There is a strange bug(?) in nongnu, when you clone a git repository
|
# There is a strange bug(?) in nongnu, when you clone a git repository
|
||||||
# against a commit != HEAD with depth=1, it errors out.
|
# against a commit != HEAD with depth=1, it errors out.
|
||||||
fetch-depth: 0
|
fetch-depth: 0
|
||||||
- name: Get sysc_image
|
- name: Get pass2_image
|
||||||
uses: actions/download-artifact@v3
|
uses: actions/download-artifact@v3
|
||||||
with:
|
with:
|
||||||
name: internal_sysc_image
|
name: internal_pass2_image
|
||||||
- name: Extract sysc_image
|
- name: Extract pass2_image
|
||||||
run: tar -xf sysc_image.tar
|
run: tar -xf pass2_image.tar
|
||||||
# By doing this, all packages that have already been compiled will come from the preseed.
|
- name: Query cache for sources
|
||||||
- name: Get packages repo progress
|
id: cache
|
||||||
uses: actions/download-artifact@v3
|
uses: actions/cache/restore@v3
|
||||||
with:
|
with:
|
||||||
name: internal_packages_sysc1
|
path: |
|
||||||
path: tmp/sysa/sysc_image/usr/src/repo-preseeded/
|
distfiles
|
||||||
|
key: cache-${{ hashFiles('steps/*/sources') }}
|
||||||
|
- name: Get sources
|
||||||
|
if: steps.cache.outputs.cache-hit != 'true'
|
||||||
|
run: ./download-distfiles.sh
|
||||||
|
- name: Cache sources
|
||||||
|
if: steps.cache.outputs.cache-hit != 'true'
|
||||||
|
uses: actions/cache/save@v3
|
||||||
|
with:
|
||||||
|
path: |
|
||||||
|
distfiles
|
||||||
|
key: cache-${{ hashFiles('steps/*/sources') }}
|
||||||
- name: Run bootstrap
|
- name: Run bootstrap
|
||||||
run: ./rootfs.py --bwrap --external-sources --build-kernels --preserve --cores 2 --internal-ci pass3
|
run: ./rootfs.py --bwrap --external-sources --build-kernels --preserve --cores 2 --internal-ci pass3
|
||||||
- name: Archive created packages
|
- name: Archive created packages
|
||||||
if: always() # archive failed builds
|
if: always() # archive both failed and successful builds
|
||||||
uses: actions/upload-artifact@v3
|
uses: actions/upload-artifact@v3
|
||||||
with:
|
with:
|
||||||
name: packages
|
name: packages
|
||||||
path: tmp/sysa/sysc_image/usr/src/repo/**
|
path: tmp/external/repo/**
|
||||||
|
|
|
@ -32,6 +32,14 @@ class Generator():
|
||||||
self.tmp_dir = None
|
self.tmp_dir = None
|
||||||
self.external_dir = None
|
self.external_dir = None
|
||||||
|
|
||||||
|
def reuse(self, tmpdir):
|
||||||
|
"""
|
||||||
|
Reuse a previously prepared bwrap environment for further stages.
|
||||||
|
"""
|
||||||
|
self.tmp_dir = tmpdir.path
|
||||||
|
self.external_dir = os.path.join(self.tmp_dir, 'external')
|
||||||
|
self.distfiles()
|
||||||
|
|
||||||
def prepare(self, tmpdir, using_kernel=False, kernel_bootstrap=False, target_size=0):
|
def prepare(self, tmpdir, using_kernel=False, kernel_bootstrap=False, target_size=0):
|
||||||
"""
|
"""
|
||||||
Prepare basic media of live-bootstrap.
|
Prepare basic media of live-bootstrap.
|
||||||
|
@ -166,11 +174,11 @@ class Generator():
|
||||||
main_distfile_dir = os.path.join(self.external_dir, 'distfiles')
|
main_distfile_dir = os.path.join(self.external_dir, 'distfiles')
|
||||||
|
|
||||||
if early_distfile_dir != main_distfile_dir:
|
if early_distfile_dir != main_distfile_dir:
|
||||||
os.makedirs(early_distfile_dir)
|
os.makedirs(early_distfile_dir, exist_ok=True)
|
||||||
copy_no_network_distfiles(early_distfile_dir)
|
copy_no_network_distfiles(early_distfile_dir)
|
||||||
|
|
||||||
if self.external_sources:
|
if self.external_sources:
|
||||||
shutil.copytree(self.distfiles_dir, main_distfile_dir)
|
shutil.copytree(self.distfiles_dir, main_distfile_dir, dirs_exist_ok=True)
|
||||||
else:
|
else:
|
||||||
os.mkdir(main_distfile_dir)
|
os.mkdir(main_distfile_dir)
|
||||||
copy_no_network_distfiles(main_distfile_dir)
|
copy_no_network_distfiles(main_distfile_dir)
|
||||||
|
|
62
rootfs.py
62
rootfs.py
|
@ -15,7 +15,6 @@ you can run bootstap inside chroot.
|
||||||
|
|
||||||
import argparse
|
import argparse
|
||||||
import os
|
import os
|
||||||
import shutil
|
|
||||||
|
|
||||||
from lib.utils import run, run_as_root
|
from lib.utils import run, run_as_root
|
||||||
from lib.tmpdir import Tmpdir
|
from lib.tmpdir import Tmpdir
|
||||||
|
@ -32,7 +31,7 @@ def create_configuration_file(args):
|
||||||
config.write(f"CHROOT={args.chroot or args.bwrap}\n")
|
config.write(f"CHROOT={args.chroot or args.bwrap}\n")
|
||||||
config.write(f"UPDATE_CHECKSUMS={args.update_checksums}\n")
|
config.write(f"UPDATE_CHECKSUMS={args.update_checksums}\n")
|
||||||
config.write(f"JOBS={args.cores}\n")
|
config.write(f"JOBS={args.cores}\n")
|
||||||
config.write(f"INTERNAL_CI={args.internal_ci}\n")
|
config.write(f"INTERNAL_CI={args.internal_ci or False}\n")
|
||||||
config.write(f"BARE_METAL={args.bare_metal}\n")
|
config.write(f"BARE_METAL={args.bare_metal}\n")
|
||||||
if (args.bare_metal or args.qemu) and not args.kernel:
|
if (args.bare_metal or args.qemu) and not args.kernel:
|
||||||
if args.repo or args.external_sources:
|
if args.repo or args.external_sources:
|
||||||
|
@ -187,50 +186,33 @@ print(shutil.which('chroot'))
|
||||||
run_as_root('env', '-i', 'PATH=/bin', chroot_binary, generator.tmp_dir, init)
|
run_as_root('env', '-i', 'PATH=/bin', chroot_binary, generator.tmp_dir, init)
|
||||||
|
|
||||||
elif args.bwrap:
|
elif args.bwrap:
|
||||||
|
init = '/init'
|
||||||
if not args.internal_ci or args.internal_ci == "pass1":
|
if not args.internal_ci or args.internal_ci == "pass1":
|
||||||
generator.prepare(tmpdir, using_kernel=False)
|
generator.prepare(tmpdir, using_kernel=False)
|
||||||
|
|
||||||
arch = stage0_arch_map.get(args.arch, args.arch)
|
arch = stage0_arch_map.get(args.arch, args.arch)
|
||||||
init = os.path.join(os.sep, 'bootstrap-seeds', 'POSIX', arch, 'kaem-optional-seed')
|
init = os.path.join(os.sep, 'bootstrap-seeds', 'POSIX', arch, 'kaem-optional-seed')
|
||||||
run('bwrap', '--unshare-user',
|
else:
|
||||||
'--uid', '0',
|
generator.reuse(tmpdir)
|
||||||
'--gid', '0',
|
|
||||||
'--unshare-net',
|
|
||||||
'--clearenv',
|
|
||||||
'--setenv', 'PATH', '/usr/bin',
|
|
||||||
'--bind', generator.tmp_dir, '/',
|
|
||||||
'--dir', '/dev',
|
|
||||||
'--dev-bind', '/dev/null', '/dev/null',
|
|
||||||
'--dev-bind', '/dev/zero', '/dev/zero',
|
|
||||||
'--dev-bind', '/dev/random', '/dev/random',
|
|
||||||
'--dev-bind', '/dev/urandom', '/dev/urandom',
|
|
||||||
'--dev-bind', '/dev/ptmx', '/dev/ptmx',
|
|
||||||
'--dev-bind', '/dev/tty', '/dev/tty',
|
|
||||||
init)
|
|
||||||
|
|
||||||
if not args.internal_ci or args.internal_ci == "pass2" or args.internal_ci == "pass3":
|
run('env', '-i', 'bwrap', '--unshare-user',
|
||||||
os.makedirs(os.path.join(generator.tmp_dir, 'stage2', 'steps'), exist_ok=True)
|
'--uid', '0',
|
||||||
shutil.copy2(os.path.join('steps', 'bootstrap.cfg'),
|
'--gid', '0',
|
||||||
os.path.join(generator.tmp_dir, 'stage2', 'steps', 'bootstrap.cfg'))
|
'--unshare-net' if args.external_sources else None,
|
||||||
run('bwrap', '--unshare-user',
|
'--setenv', 'PATH', '/usr/bin',
|
||||||
'--uid', '0',
|
'--bind', generator.tmp_dir, '/',
|
||||||
'--gid', '0',
|
'--dir', '/dev',
|
||||||
'--unshare-net' if args.external_sources else None,
|
'--dev-bind', '/dev/null', '/dev/null',
|
||||||
'--clearenv',
|
'--dev-bind', '/dev/zero', '/dev/zero',
|
||||||
'--setenv', 'PATH', '/usr/bin',
|
'--dev-bind', '/dev/random', '/dev/random',
|
||||||
'--bind', os.path.join(generator.tmp_dir, "stage2"), '/',
|
'--dev-bind', '/dev/urandom', '/dev/urandom',
|
||||||
'--dir', '/dev',
|
'--dev-bind', '/dev/ptmx', '/dev/ptmx',
|
||||||
'--dev-bind', '/dev/null', '/dev/null',
|
'--dev-bind', '/dev/tty', '/dev/tty',
|
||||||
'--dev-bind', '/dev/zero', '/dev/zero',
|
'--tmpfs', '/dev/shm',
|
||||||
'--dev-bind', '/dev/random', '/dev/random',
|
'--proc', '/proc',
|
||||||
'--dev-bind', '/dev/urandom', '/dev/urandom',
|
'--bind', '/sys', '/sys',
|
||||||
'--dev-bind', '/dev/ptmx', '/dev/ptmx',
|
'--tmpfs', '/tmp',
|
||||||
'--dev-bind', '/dev/tty', '/dev/tty',
|
init)
|
||||||
'--tmpfs', '/dev/shm',
|
|
||||||
'--proc', '/proc',
|
|
||||||
'--bind', '/sys', '/sys',
|
|
||||||
'--tmpfs', '/tmp',
|
|
||||||
'/init')
|
|
||||||
|
|
||||||
elif args.bare_metal:
|
elif args.bare_metal:
|
||||||
if args.kernel:
|
if args.kernel:
|
||||||
|
|
|
@ -15,19 +15,6 @@ test -c "/dev/urandom" || mknod -m 444 "/dev/urandom" c 1 9
|
||||||
test -c "/dev/ptmx" || mknod -m 666 "/dev/ptmx" c 5 2
|
test -c "/dev/ptmx" || mknod -m 666 "/dev/ptmx" c 5 2
|
||||||
test -c "/dev/tty" || mknod -m 666 "/dev/tty" c 5 0
|
test -c "/dev/tty" || mknod -m 666 "/dev/tty" c 5 0
|
||||||
|
|
||||||
test -b "/dev/sda" || mknod -m 600 "/dev/sda" b 8 0
|
|
||||||
test -b "/dev/sda1" || mknod -m 600 "/dev/sda1" b 8 1
|
|
||||||
test -b "/dev/sda2" || mknod -m 600 "/dev/sda2" b 8 2
|
|
||||||
test -b "/dev/sda3" || mknod -m 600 "/dev/sda3" b 8 3
|
|
||||||
test -b "/dev/sdb" || mknod -m 600 "/dev/sdb" b 8 16
|
|
||||||
test -b "/dev/sdb1" || mknod -m 600 "/dev/sdb1" b 8 17
|
|
||||||
test -b "/dev/sdb2" || mknod -m 600 "/dev/sdb2" b 8 18
|
|
||||||
test -b "/dev/sdb2" || mknod -m 600 "/dev/sdb3" b 8 19
|
|
||||||
test -b "/dev/sdc" || mknod -m 600 "/dev/sdc" b 8 32
|
|
||||||
test -b "/dev/sdc1" || mknod -m 600 "/dev/sdc1" b 8 33
|
|
||||||
test -b "/dev/sdc2" || mknod -m 600 "/dev/sdc2" b 8 34
|
|
||||||
test -b "/dev/sdc3" || mknod -m 600 "/dev/sdc3" b 8 35
|
|
||||||
|
|
||||||
test -e "/dev/stdout" || ln -s "/proc/self/fd/1" "/dev/stdout"
|
test -e "/dev/stdout" || ln -s "/proc/self/fd/1" "/dev/stdout"
|
||||||
|
|
||||||
if mount --version >/dev/null 2>&1; then
|
if mount --version >/dev/null 2>&1; then
|
||||||
|
@ -37,4 +24,16 @@ fi
|
||||||
|
|
||||||
if [ "${CHROOT}" = False ]; then
|
if [ "${CHROOT}" = False ]; then
|
||||||
test -c "/dev/console" || mknod -m 666 "/dev/console" c 5 1
|
test -c "/dev/console" || mknod -m 666 "/dev/console" c 5 1
|
||||||
|
test -b "/dev/sda" || mknod -m 600 "/dev/sda" b 8 0
|
||||||
|
test -b "/dev/sda1" || mknod -m 600 "/dev/sda1" b 8 1
|
||||||
|
test -b "/dev/sda2" || mknod -m 600 "/dev/sda2" b 8 2
|
||||||
|
test -b "/dev/sda3" || mknod -m 600 "/dev/sda3" b 8 3
|
||||||
|
test -b "/dev/sdb" || mknod -m 600 "/dev/sdb" b 8 16
|
||||||
|
test -b "/dev/sdb1" || mknod -m 600 "/dev/sdb1" b 8 17
|
||||||
|
test -b "/dev/sdb2" || mknod -m 600 "/dev/sdb2" b 8 18
|
||||||
|
test -b "/dev/sdb2" || mknod -m 600 "/dev/sdb3" b 8 19
|
||||||
|
test -b "/dev/sdc" || mknod -m 600 "/dev/sdc" b 8 32
|
||||||
|
test -b "/dev/sdc1" || mknod -m 600 "/dev/sdc1" b 8 33
|
||||||
|
test -b "/dev/sdc2" || mknod -m 600 "/dev/sdc2" b 8 34
|
||||||
|
test -b "/dev/sdc3" || mknod -m 600 "/dev/sdc3" b 8 35
|
||||||
fi
|
fi
|
||||||
|
|
|
@ -0,0 +1,7 @@
|
||||||
|
#!/bin/bash
|
||||||
|
#
|
||||||
|
# SPDX-FileCopyrightText: 2023 Gábor Stefanik <netrolller.3d@gmail.com>
|
||||||
|
#
|
||||||
|
# SPDX-License-Identifier: GPL-3.0-or-later
|
||||||
|
|
||||||
|
exit 0
|
|
@ -115,6 +115,7 @@ define: BUILD_LINUX = ( CHROOT == False || BUILD_KERNELS == True )
|
||||||
build: kexec-linux-1.0.0 ( BUILD_LINUX == True )
|
build: kexec-linux-1.0.0 ( BUILD_LINUX == True )
|
||||||
build: kexec-tools-2.0.22 ( BUILD_LINUX == True )
|
build: kexec-tools-2.0.22 ( BUILD_LINUX == True )
|
||||||
build: linux-4.9.10 ( BUILD_LINUX == True )
|
build: linux-4.9.10 ( BUILD_LINUX == True )
|
||||||
|
jump: break ( INTERNAL_CI == pass1 )
|
||||||
jump: linux ( CHROOT == False )
|
jump: linux ( CHROOT == False )
|
||||||
improve: finalize_fhs
|
improve: finalize_fhs
|
||||||
build: musl-1.2.4
|
build: musl-1.2.4
|
||||||
|
@ -174,6 +175,7 @@ build: python-3.3.7
|
||||||
build: python-3.4.10
|
build: python-3.4.10
|
||||||
build: python-3.8.16
|
build: python-3.8.16
|
||||||
build: python-3.11.1
|
build: python-3.11.1
|
||||||
|
jump: break ( INTERNAL_CI == pass1 ) # scripts are generated in pass1
|
||||||
build: gcc-10.4.0
|
build: gcc-10.4.0
|
||||||
build: binutils-2.41
|
build: binutils-2.41
|
||||||
build: gcc-13.1.0
|
build: gcc-13.1.0
|
||||||
|
|
Loading…
Reference in New Issue