diff --git a/.github/workflows/bwrap.yml b/.github/workflows/bwrap.yml index dcbe1bd..776ea20 100644 --- a/.github/workflows/bwrap.yml +++ b/.github/workflows/bwrap.yml @@ -12,8 +12,8 @@ on: - master jobs: - sysa: - name: Run sysa under bubblewrap + pass1: + name: Run up to Linux build under bubblewrap runs-on: ubuntu-latest steps: - name: Install bubblewrap @@ -30,9 +30,8 @@ jobs: uses: actions/cache/restore@v3 with: path: | - sysa/distfiles - sysc/distfiles - key: cache-${{ hashFiles('sys*/*/sources') }} + distfiles + key: cache-${{ hashFiles('steps/*/sources') }} - name: Get sources if: steps.cache.outputs.cache-hit != 'true' run: ./download-distfiles.sh @@ -41,9 +40,8 @@ jobs: uses: actions/cache/save@v3 with: path: | - sysa/distfiles - sysc/distfiles - key: cache-${{ hashFiles('sys*/*/sources') }} + distfiles + key: cache-${{ hashFiles('steps/*/sources') }} - name: Run bootstrap run: ./rootfs.py --bwrap --external-sources --build-kernels --preserve --cores 2 --internal-ci pass1 - name: Archive created packages @@ -51,18 +49,18 @@ jobs: uses: actions/upload-artifact@v3 with: name: packages - path: tmp/sysa/usr/src/repo/** - - name: Tar sysc_image - run: tar -cf sysc_image.tar tmp/sysa/sysc_image/ - - name: Archive sysc_image + path: tmp/external/repo/** + - name: Tar pass1 image + run: tar -cf pass1_image.tar tmp + - name: Archive pass1_image uses: actions/upload-artifact@v3 with: - name: internal_sysc_image - path: sysc_image.tar + name: internal_pass1_image + path: pass1_image.tar - sysc1: - name: Run sysc (part 1) under bubblewrap - needs: sysa + pass2: + name: Run up to Python bootstrap under bubblewrap + needs: pass1 runs-on: ubuntu-latest steps: - name: Install bubblewrap @@ -74,24 +72,48 @@ jobs: # There is a strange bug(?) in nongnu, when you clone a git repository # against a commit != HEAD with depth=1, it errors out. fetch-depth: 0 - - name: Get sysc_image + - name: Get pass1_image uses: actions/download-artifact@v3 with: - name: internal_sysc_image - - name: Extract sysc_image - run: tar -xf sysc_image.tar + name: internal_pass1_image + - name: Extract pass1_image + run: tar -xf pass1_image.tar + - name: Query cache for sources + id: cache + uses: actions/cache/restore@v3 + with: + path: | + distfiles + key: cache-${{ hashFiles('steps/*/sources') }} + - name: Get sources + if: steps.cache.outputs.cache-hit != 'true' + run: ./download-distfiles.sh + - name: Cache sources + if: steps.cache.outputs.cache-hit != 'true' + uses: actions/cache/save@v3 + with: + path: | + distfiles + key: cache-${{ hashFiles('steps/*/sources') }} - name: Run bootstrap run: ./rootfs.py --bwrap --external-sources --build-kernels --preserve --cores 2 --internal-ci pass2 - name: Archive created packages - if: always() # archive failed builds progress + if: failure() # archive failed builds progress uses: actions/upload-artifact@v3 with: - name: internal_packages_sysc1 - path: tmp/sysa/sysc_image/usr/src/repo/** + name: internal_packages_pass2 + path: tmp/external/repo/** + - name: Tar pass2 image + run: tar -cf pass2_image.tar tmp + - name: Archive pass2_image + uses: actions/upload-artifact@v3 + with: + name: internal_pass2_image + path: pass2_image.tar - sysc2: - name: Run sysc (part 2) under bubblewrap - needs: sysc1 + pass3: + name: Run remaining builds under bubblewrap + needs: pass2 runs-on: ubuntu-latest steps: - name: Install bubblewrap @@ -103,23 +125,34 @@ jobs: # There is a strange bug(?) in nongnu, when you clone a git repository # against a commit != HEAD with depth=1, it errors out. fetch-depth: 0 - - name: Get sysc_image + - name: Get pass2_image uses: actions/download-artifact@v3 with: - name: internal_sysc_image - - name: Extract sysc_image - run: tar -xf sysc_image.tar - # By doing this, all packages that have already been compiled will come from the preseed. - - name: Get packages repo progress - uses: actions/download-artifact@v3 + name: internal_pass2_image + - name: Extract pass2_image + run: tar -xf pass2_image.tar + - name: Query cache for sources + id: cache + uses: actions/cache/restore@v3 with: - name: internal_packages_sysc1 - path: tmp/sysa/sysc_image/usr/src/repo-preseeded/ + path: | + distfiles + key: cache-${{ hashFiles('steps/*/sources') }} + - name: Get sources + if: steps.cache.outputs.cache-hit != 'true' + run: ./download-distfiles.sh + - name: Cache sources + if: steps.cache.outputs.cache-hit != 'true' + uses: actions/cache/save@v3 + with: + path: | + distfiles + key: cache-${{ hashFiles('steps/*/sources') }} - name: Run bootstrap run: ./rootfs.py --bwrap --external-sources --build-kernels --preserve --cores 2 --internal-ci pass3 - name: Archive created packages - if: always() # archive failed builds + if: always() # archive both failed and successful builds uses: actions/upload-artifact@v3 with: name: packages - path: tmp/sysa/sysc_image/usr/src/repo/** + path: tmp/external/repo/** diff --git a/lib/generator.py b/lib/generator.py index 82ecd1e..326101a 100755 --- a/lib/generator.py +++ b/lib/generator.py @@ -32,6 +32,14 @@ class Generator(): self.tmp_dir = None self.external_dir = None + def reuse(self, tmpdir): + """ + Reuse a previously prepared bwrap environment for further stages. + """ + self.tmp_dir = tmpdir.path + self.external_dir = os.path.join(self.tmp_dir, 'external') + self.distfiles() + def prepare(self, tmpdir, using_kernel=False, kernel_bootstrap=False, target_size=0): """ Prepare basic media of live-bootstrap. @@ -166,11 +174,11 @@ class Generator(): main_distfile_dir = os.path.join(self.external_dir, 'distfiles') if early_distfile_dir != main_distfile_dir: - os.makedirs(early_distfile_dir) + os.makedirs(early_distfile_dir, exist_ok=True) copy_no_network_distfiles(early_distfile_dir) if self.external_sources: - shutil.copytree(self.distfiles_dir, main_distfile_dir) + shutil.copytree(self.distfiles_dir, main_distfile_dir, dirs_exist_ok=True) else: os.mkdir(main_distfile_dir) copy_no_network_distfiles(main_distfile_dir) diff --git a/rootfs.py b/rootfs.py index 8641d02..18834ab 100755 --- a/rootfs.py +++ b/rootfs.py @@ -15,7 +15,6 @@ you can run bootstap inside chroot. import argparse import os -import shutil from lib.utils import run, run_as_root from lib.tmpdir import Tmpdir @@ -32,7 +31,7 @@ def create_configuration_file(args): config.write(f"CHROOT={args.chroot or args.bwrap}\n") config.write(f"UPDATE_CHECKSUMS={args.update_checksums}\n") config.write(f"JOBS={args.cores}\n") - config.write(f"INTERNAL_CI={args.internal_ci}\n") + config.write(f"INTERNAL_CI={args.internal_ci or False}\n") config.write(f"BARE_METAL={args.bare_metal}\n") if (args.bare_metal or args.qemu) and not args.kernel: if args.repo or args.external_sources: @@ -187,50 +186,33 @@ print(shutil.which('chroot')) run_as_root('env', '-i', 'PATH=/bin', chroot_binary, generator.tmp_dir, init) elif args.bwrap: + init = '/init' if not args.internal_ci or args.internal_ci == "pass1": generator.prepare(tmpdir, using_kernel=False) arch = stage0_arch_map.get(args.arch, args.arch) init = os.path.join(os.sep, 'bootstrap-seeds', 'POSIX', arch, 'kaem-optional-seed') - run('bwrap', '--unshare-user', - '--uid', '0', - '--gid', '0', - '--unshare-net', - '--clearenv', - '--setenv', 'PATH', '/usr/bin', - '--bind', generator.tmp_dir, '/', - '--dir', '/dev', - '--dev-bind', '/dev/null', '/dev/null', - '--dev-bind', '/dev/zero', '/dev/zero', - '--dev-bind', '/dev/random', '/dev/random', - '--dev-bind', '/dev/urandom', '/dev/urandom', - '--dev-bind', '/dev/ptmx', '/dev/ptmx', - '--dev-bind', '/dev/tty', '/dev/tty', - init) + else: + generator.reuse(tmpdir) - if not args.internal_ci or args.internal_ci == "pass2" or args.internal_ci == "pass3": - os.makedirs(os.path.join(generator.tmp_dir, 'stage2', 'steps'), exist_ok=True) - shutil.copy2(os.path.join('steps', 'bootstrap.cfg'), - os.path.join(generator.tmp_dir, 'stage2', 'steps', 'bootstrap.cfg')) - run('bwrap', '--unshare-user', - '--uid', '0', - '--gid', '0', - '--unshare-net' if args.external_sources else None, - '--clearenv', - '--setenv', 'PATH', '/usr/bin', - '--bind', os.path.join(generator.tmp_dir, "stage2"), '/', - '--dir', '/dev', - '--dev-bind', '/dev/null', '/dev/null', - '--dev-bind', '/dev/zero', '/dev/zero', - '--dev-bind', '/dev/random', '/dev/random', - '--dev-bind', '/dev/urandom', '/dev/urandom', - '--dev-bind', '/dev/ptmx', '/dev/ptmx', - '--dev-bind', '/dev/tty', '/dev/tty', - '--tmpfs', '/dev/shm', - '--proc', '/proc', - '--bind', '/sys', '/sys', - '--tmpfs', '/tmp', - '/init') + run('env', '-i', 'bwrap', '--unshare-user', + '--uid', '0', + '--gid', '0', + '--unshare-net' if args.external_sources else None, + '--setenv', 'PATH', '/usr/bin', + '--bind', generator.tmp_dir, '/', + '--dir', '/dev', + '--dev-bind', '/dev/null', '/dev/null', + '--dev-bind', '/dev/zero', '/dev/zero', + '--dev-bind', '/dev/random', '/dev/random', + '--dev-bind', '/dev/urandom', '/dev/urandom', + '--dev-bind', '/dev/ptmx', '/dev/ptmx', + '--dev-bind', '/dev/tty', '/dev/tty', + '--tmpfs', '/dev/shm', + '--proc', '/proc', + '--bind', '/sys', '/sys', + '--tmpfs', '/tmp', + init) elif args.bare_metal: if args.kernel: diff --git a/steps/improve/populate_device_nodes.sh b/steps/improve/populate_device_nodes.sh index 35856e9..98ffc8d 100755 --- a/steps/improve/populate_device_nodes.sh +++ b/steps/improve/populate_device_nodes.sh @@ -15,19 +15,6 @@ test -c "/dev/urandom" || mknod -m 444 "/dev/urandom" c 1 9 test -c "/dev/ptmx" || mknod -m 666 "/dev/ptmx" c 5 2 test -c "/dev/tty" || mknod -m 666 "/dev/tty" c 5 0 -test -b "/dev/sda" || mknod -m 600 "/dev/sda" b 8 0 -test -b "/dev/sda1" || mknod -m 600 "/dev/sda1" b 8 1 -test -b "/dev/sda2" || mknod -m 600 "/dev/sda2" b 8 2 -test -b "/dev/sda3" || mknod -m 600 "/dev/sda3" b 8 3 -test -b "/dev/sdb" || mknod -m 600 "/dev/sdb" b 8 16 -test -b "/dev/sdb1" || mknod -m 600 "/dev/sdb1" b 8 17 -test -b "/dev/sdb2" || mknod -m 600 "/dev/sdb2" b 8 18 -test -b "/dev/sdb2" || mknod -m 600 "/dev/sdb3" b 8 19 -test -b "/dev/sdc" || mknod -m 600 "/dev/sdc" b 8 32 -test -b "/dev/sdc1" || mknod -m 600 "/dev/sdc1" b 8 33 -test -b "/dev/sdc2" || mknod -m 600 "/dev/sdc2" b 8 34 -test -b "/dev/sdc3" || mknod -m 600 "/dev/sdc3" b 8 35 - test -e "/dev/stdout" || ln -s "/proc/self/fd/1" "/dev/stdout" if mount --version >/dev/null 2>&1; then @@ -37,4 +24,16 @@ fi if [ "${CHROOT}" = False ]; then test -c "/dev/console" || mknod -m 666 "/dev/console" c 5 1 + test -b "/dev/sda" || mknod -m 600 "/dev/sda" b 8 0 + test -b "/dev/sda1" || mknod -m 600 "/dev/sda1" b 8 1 + test -b "/dev/sda2" || mknod -m 600 "/dev/sda2" b 8 2 + test -b "/dev/sda3" || mknod -m 600 "/dev/sda3" b 8 3 + test -b "/dev/sdb" || mknod -m 600 "/dev/sdb" b 8 16 + test -b "/dev/sdb1" || mknod -m 600 "/dev/sdb1" b 8 17 + test -b "/dev/sdb2" || mknod -m 600 "/dev/sdb2" b 8 18 + test -b "/dev/sdb2" || mknod -m 600 "/dev/sdb3" b 8 19 + test -b "/dev/sdc" || mknod -m 600 "/dev/sdc" b 8 32 + test -b "/dev/sdc1" || mknod -m 600 "/dev/sdc1" b 8 33 + test -b "/dev/sdc2" || mknod -m 600 "/dev/sdc2" b 8 34 + test -b "/dev/sdc3" || mknod -m 600 "/dev/sdc3" b 8 35 fi diff --git a/steps/jump/break.sh b/steps/jump/break.sh new file mode 100644 index 0000000..e6788ce --- /dev/null +++ b/steps/jump/break.sh @@ -0,0 +1,7 @@ +#!/bin/bash +# +# SPDX-FileCopyrightText: 2023 Gábor Stefanik +# +# SPDX-License-Identifier: GPL-3.0-or-later + +exit 0 diff --git a/steps/manifest b/steps/manifest index 164ce2c..b5aed41 100644 --- a/steps/manifest +++ b/steps/manifest @@ -115,6 +115,7 @@ define: BUILD_LINUX = ( CHROOT == False || BUILD_KERNELS == True ) build: kexec-linux-1.0.0 ( BUILD_LINUX == True ) build: kexec-tools-2.0.22 ( BUILD_LINUX == True ) build: linux-4.9.10 ( BUILD_LINUX == True ) +jump: break ( INTERNAL_CI == pass1 ) jump: linux ( CHROOT == False ) improve: finalize_fhs build: musl-1.2.4 @@ -174,6 +175,7 @@ build: python-3.3.7 build: python-3.4.10 build: python-3.8.16 build: python-3.11.1 +jump: break ( INTERNAL_CI == pass1 ) # scripts are generated in pass1 build: gcc-10.4.0 build: binutils-2.41 build: gcc-13.1.0