From 372374756272db4bd1621b38ae2e6faf68922847 Mon Sep 17 00:00:00 2001 From: fosslinux Date: Mon, 1 May 2023 20:49:41 +1000 Subject: [PATCH 1/3] Add internal-ci argument for ci passes to rootfs.py --- rootfs.py | 71 +++++++++++++++++++++++++++++-------------------------- 1 file changed, 37 insertions(+), 34 deletions(-) diff --git a/rootfs.py b/rootfs.py index 20efeba..48d713b 100755 --- a/rootfs.py +++ b/rootfs.py @@ -84,6 +84,7 @@ def main(): help="Path to prebuilt binary packages.", nargs=None) parser.add_argument("--early-preseed", help="Skip early stages of live-bootstrap.", nargs=None) + parser.add_argument("--internal-ci", help="INTERNAL for github CI") # QEMU arguments parser.add_argument("-q", "--qemu", help="Use QEMU", @@ -177,42 +178,44 @@ print(shutil.which('chroot')) run('sudo', 'env', '-i', 'PATH=/bin', chroot_binary, system_a.tmp_dir, init) elif args.bwrap: - system_c.prepare(create_disk_image=False) - system_a.prepare(create_initramfs=False) + if not args.internal_ci or args.internal_ci == "pass1": + system_c.prepare(create_disk_image=False) + system_a.prepare(create_initramfs=False) - arch = stage0_arch_map.get(args.arch, args.arch) - init = os.path.join(os.sep, 'bootstrap-seeds', 'POSIX', arch, 'kaem-optional-seed') - run('bwrap', '--unshare-user', - '--uid', '0', - '--gid', '0', - '--unshare-net', - '--clearenv', - '--setenv', 'PATH', '/usr/bin', - '--bind', system_a.tmp_dir, '/', - '--dir', '/dev', - '--dev-bind', '/dev/null', '/dev/null', - '--dev-bind', '/dev/zero', '/dev/zero', - '--dev-bind', '/dev/random', '/dev/random', - '--dev-bind', '/dev/urandom', '/dev/urandom', - init) + arch = stage0_arch_map.get(args.arch, args.arch) + init = os.path.join(os.sep, 'bootstrap-seeds', 'POSIX', arch, 'kaem-optional-seed') + run('bwrap', '--unshare-user', + '--uid', '0', + '--gid', '0', + '--unshare-net', + '--clearenv', + '--setenv', 'PATH', '/usr/bin', + '--bind', system_a.tmp_dir, '/', + '--dir', '/dev', + '--dev-bind', '/dev/null', '/dev/null', + '--dev-bind', '/dev/zero', '/dev/zero', + '--dev-bind', '/dev/random', '/dev/random', + '--dev-bind', '/dev/urandom', '/dev/urandom', + init) - run('bwrap', '--unshare-user', - '--uid', '0', - '--gid', '0', - '--unshare-net' if args.external_sources else None, - '--clearenv', - '--setenv', 'PATH', '/usr/bin', - '--bind', system_a.tmp_dir + "/sysc_image", '/', - '--dir', '/dev', - '--dev-bind', '/dev/null', '/dev/null', - '--dev-bind', '/dev/zero', '/dev/zero', - '--dev-bind', '/dev/random', '/dev/random', - '--dev-bind', '/dev/urandom', '/dev/urandom', - '--tmpfs', '/dev/shm', - '--proc', '/proc', - '--bind', '/sys', '/sys', - '--tmpfs', '/tmp', - '/init') + if not args.internal_ci or args.internal_ci == "pass2": + run('bwrap', '--unshare-user', + '--uid', '0', + '--gid', '0', + '--unshare-net' if args.external_sources else None, + '--clearenv', + '--setenv', 'PATH', '/usr/bin', + '--bind', system_a.tmp_dir + "/sysc_image", '/', + '--dir', '/dev', + '--dev-bind', '/dev/null', '/dev/null', + '--dev-bind', '/dev/zero', '/dev/zero', + '--dev-bind', '/dev/random', '/dev/random', + '--dev-bind', '/dev/urandom', '/dev/urandom', + '--tmpfs', '/dev/shm', + '--proc', '/proc', + '--bind', '/sys', '/sys', + '--tmpfs', '/tmp', + '/init') elif args.bare_metal: if args.kernel: From f124bc86bef4da3a20fb60e337ae66cf99d56381 Mon Sep 17 00:00:00 2001 From: fosslinux Date: Mon, 1 May 2023 20:49:46 +1000 Subject: [PATCH 2/3] Work around GH actions time limit by splitting sysa and sysc into 2 --- .github/workflows/bwrap.yml | 43 +++++++++++++++++++++++++++++++++---- 1 file changed, 39 insertions(+), 4 deletions(-) diff --git a/.github/workflows/bwrap.yml b/.github/workflows/bwrap.yml index 1f4fcbd..aac301b 100644 --- a/.github/workflows/bwrap.yml +++ b/.github/workflows/bwrap.yml @@ -12,8 +12,8 @@ on: - master jobs: - run: - name: Run under bubblewrap + sysa: + name: Run sysa under bubblewrap runs-on: ubuntu-latest steps: - name: Install bubblewrap @@ -45,8 +45,43 @@ jobs: sysc/distfiles key: cache-${{ hashFiles('sys*/*/sources') }} - name: Run bootstrap - id: bootstrap - run: ./rootfs.py --bwrap --external-sources --build-kernels --preserve --cores 2 + run: ./rootfs.py --bwrap --external-sources --build-kernels --preserve --cores 2 --internal-ci pass1 + - name: Archive created packages + if: failure() # archive failed builds progress + uses: actions/upload-artifact@v3 + with: + name: packages + path: tmp/sysa/usr/src/repo/** + - name: Tar sysc_image + run: tar -cf sysc_image.tar tmp/sysa/sysc_image/ + - name: Archive sysc_image + uses: actions/upload-artifact@v3 + with: + name: sysc_image + path: sysc_image.tar + + sysc: + name: Run sysc under bubblewrap + needs: sysa + runs-on: ubuntu-latest + steps: + - name: Install bubblewrap + run: sudo apt install bubblewrap + - name: Checkout repo + uses: actions/checkout@v3 + with: + submodules: recursive + # There is a strange bug(?) in nongnu, when you clone a git repository + # against a commit != HEAD with depth=1, it errors out. + fetch-depth: 0 + - name: Get sysc_image + uses: actions/download-artifact@v3 + with: + name: sysc_image + - name: Extract sysc_image + run: tar -xf sysc_image.tar + - name: Run bootstrap + run: ./rootfs.py --bwrap --external-sources --build-kernels --preserve --cores 2 --internal-ci pass2 - name: Archive created packages if: always() # archive failed builds uses: actions/upload-artifact@v3 From ce33a6e9f9c147416fbfe14d412831818e6c359a Mon Sep 17 00:00:00 2001 From: fosslinux Date: Wed, 3 May 2023 08:48:40 +1000 Subject: [PATCH 3/3] Actually, split into 3, sysc is too long --- .github/workflows/bwrap.yml | 43 +++++++++++++++++++++++++++++++++---- rootfs.py | 6 +++++- sysc/init | 1 + sysc/run2.sh | 2 ++ 4 files changed, 47 insertions(+), 5 deletions(-) diff --git a/.github/workflows/bwrap.yml b/.github/workflows/bwrap.yml index aac301b..dcbe1bd 100644 --- a/.github/workflows/bwrap.yml +++ b/.github/workflows/bwrap.yml @@ -57,11 +57,11 @@ jobs: - name: Archive sysc_image uses: actions/upload-artifact@v3 with: - name: sysc_image + name: internal_sysc_image path: sysc_image.tar - sysc: - name: Run sysc under bubblewrap + sysc1: + name: Run sysc (part 1) under bubblewrap needs: sysa runs-on: ubuntu-latest steps: @@ -77,11 +77,46 @@ jobs: - name: Get sysc_image uses: actions/download-artifact@v3 with: - name: sysc_image + name: internal_sysc_image - name: Extract sysc_image run: tar -xf sysc_image.tar - name: Run bootstrap run: ./rootfs.py --bwrap --external-sources --build-kernels --preserve --cores 2 --internal-ci pass2 + - name: Archive created packages + if: always() # archive failed builds progress + uses: actions/upload-artifact@v3 + with: + name: internal_packages_sysc1 + path: tmp/sysa/sysc_image/usr/src/repo/** + + sysc2: + name: Run sysc (part 2) under bubblewrap + needs: sysc1 + runs-on: ubuntu-latest + steps: + - name: Install bubblewrap + run: sudo apt install bubblewrap + - name: Checkout repo + uses: actions/checkout@v3 + with: + submodules: recursive + # There is a strange bug(?) in nongnu, when you clone a git repository + # against a commit != HEAD with depth=1, it errors out. + fetch-depth: 0 + - name: Get sysc_image + uses: actions/download-artifact@v3 + with: + name: internal_sysc_image + - name: Extract sysc_image + run: tar -xf sysc_image.tar + # By doing this, all packages that have already been compiled will come from the preseed. + - name: Get packages repo progress + uses: actions/download-artifact@v3 + with: + name: internal_packages_sysc1 + path: tmp/sysa/sysc_image/usr/src/repo-preseeded/ + - name: Run bootstrap + run: ./rootfs.py --bwrap --external-sources --build-kernels --preserve --cores 2 --internal-ci pass3 - name: Archive created packages if: always() # archive failed builds uses: actions/upload-artifact@v3 diff --git a/rootfs.py b/rootfs.py index 48d713b..4e96dbc 100755 --- a/rootfs.py +++ b/rootfs.py @@ -15,6 +15,7 @@ you can run bootstap inside chroot. import argparse import os +import shutil from sysa import SysA from sysc import SysC @@ -35,6 +36,7 @@ def create_configuration_file(args): config.write(f"UPDATE_CHECKSUMS={args.update_checksums}\n") config.write(f"JOBS={args.cores}\n") config.write("DISK=sda1\n") + config.write(f"INTERNAL_CI={args.internal_ci}\n") if (args.bare_metal or args.qemu) and not args.kernel: config.write("KERNEL_BOOTSTRAP=True\n") else: @@ -198,7 +200,9 @@ print(shutil.which('chroot')) '--dev-bind', '/dev/urandom', '/dev/urandom', init) - if not args.internal_ci or args.internal_ci == "pass2": + if not args.internal_ci or args.internal_ci == "pass2" or args.internal_ci == "pass3": + shutil.copy2(os.path.join('sysa', 'bootstrap.cfg'), + os.path.join('tmp', 'sysa', 'sysc_image', 'usr', 'src', 'bootstrap.cfg')) run('bwrap', '--unshare-user', '--uid', '0', '--gid', '0', diff --git a/sysc/init b/sysc/init index 6a6ef41..f0f1b07 100755 --- a/sysc/init +++ b/sysc/init @@ -78,6 +78,7 @@ DESTDIR=${DESTDIR} DISTFILES=${DISTFILES} SRCDIR=${SRCDIR} MAKEJOBS=${MAKEJOBS} +INTERNAL_CI=${INTERNAL_CI} EOF exec ./run.sh diff --git a/sysc/run2.sh b/sysc/run2.sh index c56aba7..394ca34 100755 --- a/sysc/run2.sh +++ b/sysc/run2.sh @@ -120,6 +120,8 @@ build python-3.8.16 build python-3.11.1 +[ "${INTERNAL_CI}" = "pass2" ] && exit 0 + build gcc-10.4.0 build binutils-2.38 pass2.sh