Merge pull request #286 from fosslinux/ci-length

split CI into three (as a temporary solution)

.github/workflows/bwrap.yml

@@ -12,8 +12,8 @@ on:
       - master
 
 jobs:
-  run:
+  sysa:
-    name: Run under bubblewrap
+    name: Run sysa under bubblewrap
     runs-on: ubuntu-latest
     steps:
       - name: Install bubblewrap
@@ -45,8 +45,78 @@ jobs:
             sysc/distfiles
           key: cache-${{ hashFiles('sys*/*/sources') }}
       - name: Run bootstrap
-        id: bootstrap
+        run: ./rootfs.py --bwrap --external-sources --build-kernels --preserve --cores 2 --internal-ci pass1
-        run: ./rootfs.py --bwrap --external-sources --build-kernels --preserve --cores 2
+      - name: Archive created packages
+        if: failure() # archive failed builds progress
+        uses: actions/upload-artifact@v3
+        with:
+          name: packages
+          path: tmp/sysa/usr/src/repo/**
+      - name: Tar sysc_image
+        run: tar -cf sysc_image.tar tmp/sysa/sysc_image/
+      - name: Archive sysc_image
+        uses: actions/upload-artifact@v3
+        with:
+          name: internal_sysc_image
+          path: sysc_image.tar
+
+  sysc1:
+    name: Run sysc (part 1) under bubblewrap
+    needs: sysa
+    runs-on: ubuntu-latest
+    steps:
+      - name: Install bubblewrap
+        run: sudo apt install bubblewrap
+      - name: Checkout repo
+        uses: actions/checkout@v3
+        with:
+          submodules: recursive
+          # There is a strange bug(?) in nongnu, when you clone a git repository
+          # against a commit != HEAD with depth=1, it errors out.
+          fetch-depth: 0
+      - name: Get sysc_image
+        uses: actions/download-artifact@v3
+        with:
+          name: internal_sysc_image
+      - name: Extract sysc_image
+        run: tar -xf sysc_image.tar
+      - name: Run bootstrap
+        run: ./rootfs.py --bwrap --external-sources --build-kernels --preserve --cores 2 --internal-ci pass2
+      - name: Archive created packages
+        if: always() # archive failed builds progress
+        uses: actions/upload-artifact@v3
+        with:
+          name: internal_packages_sysc1
+          path: tmp/sysa/sysc_image/usr/src/repo/**
+
+  sysc2:
+    name: Run sysc (part 2) under bubblewrap
+    needs: sysc1
+    runs-on: ubuntu-latest
+    steps:
+      - name: Install bubblewrap
+        run: sudo apt install bubblewrap
+      - name: Checkout repo
+        uses: actions/checkout@v3
+        with:
+          submodules: recursive
+          # There is a strange bug(?) in nongnu, when you clone a git repository
+          # against a commit != HEAD with depth=1, it errors out.
+          fetch-depth: 0
+      - name: Get sysc_image
+        uses: actions/download-artifact@v3
+        with:
+          name: internal_sysc_image
+      - name: Extract sysc_image
+        run: tar -xf sysc_image.tar
+      # By doing this, all packages that have already been compiled will come from the preseed.
+      - name: Get packages repo progress
+        uses: actions/download-artifact@v3
+        with:
+          name: internal_packages_sysc1
+          path: tmp/sysa/sysc_image/usr/src/repo-preseeded/
+      - name: Run bootstrap
+        run: ./rootfs.py --bwrap --external-sources --build-kernels --preserve --cores 2 --internal-ci pass3
       - name: Archive created packages
         if: always() # archive failed builds
         uses: actions/upload-artifact@v3

rootfs.py

@@ -15,6 +15,7 @@ you can run bootstap inside chroot.
 
 import argparse
 import os
+import shutil
 
 from sysa import SysA
 from sysc import SysC
@@ -35,6 +36,7 @@ def create_configuration_file(args):
         config.write(f"UPDATE_CHECKSUMS={args.update_checksums}\n")
         config.write(f"JOBS={args.cores}\n")
         config.write("DISK=sda1\n")
+        config.write(f"INTERNAL_CI={args.internal_ci}\n")
         if (args.bare_metal or args.qemu) and not args.kernel:
             config.write("KERNEL_BOOTSTRAP=True\n")
         else:
@@ -84,6 +86,7 @@ def main():
                         help="Path to prebuilt binary packages.", nargs=None)
     parser.add_argument("--early-preseed",
                         help="Skip early stages of live-bootstrap.", nargs=None)
+    parser.add_argument("--internal-ci", help="INTERNAL for github CI")
 
     # QEMU arguments
     parser.add_argument("-q", "--qemu", help="Use QEMU",
@@ -177,42 +180,46 @@ print(shutil.which('chroot'))
         run('sudo', 'env', '-i', 'PATH=/bin', chroot_binary, system_a.tmp_dir, init)
 
     elif args.bwrap:
-        system_c.prepare(create_disk_image=False)
+        if not args.internal_ci or args.internal_ci == "pass1":
-        system_a.prepare(create_initramfs=False)
+            system_c.prepare(create_disk_image=False)
+            system_a.prepare(create_initramfs=False)
 
-        arch = stage0_arch_map.get(args.arch, args.arch)
+            arch = stage0_arch_map.get(args.arch, args.arch)
-        init = os.path.join(os.sep, 'bootstrap-seeds', 'POSIX', arch, 'kaem-optional-seed')
+            init = os.path.join(os.sep, 'bootstrap-seeds', 'POSIX', arch, 'kaem-optional-seed')
-        run('bwrap', '--unshare-user',
+            run('bwrap', '--unshare-user',
-                     '--uid', '0',
+                         '--uid', '0',
-                     '--gid', '0',
+                         '--gid', '0',
-                     '--unshare-net',
+                         '--unshare-net',
-                     '--clearenv',
+                         '--clearenv',
-                     '--setenv', 'PATH', '/usr/bin',
+                         '--setenv', 'PATH', '/usr/bin',
-                     '--bind', system_a.tmp_dir, '/',
+                         '--bind', system_a.tmp_dir, '/',
-                     '--dir', '/dev',
+                         '--dir', '/dev',
-                     '--dev-bind', '/dev/null', '/dev/null',
+                         '--dev-bind', '/dev/null', '/dev/null',
-                     '--dev-bind', '/dev/zero', '/dev/zero',
+                         '--dev-bind', '/dev/zero', '/dev/zero',
-                     '--dev-bind', '/dev/random', '/dev/random',
+                         '--dev-bind', '/dev/random', '/dev/random',
-                     '--dev-bind', '/dev/urandom', '/dev/urandom',
+                         '--dev-bind', '/dev/urandom', '/dev/urandom',
-                     init)
+                         init)
 
-        run('bwrap', '--unshare-user',
+        if not args.internal_ci or args.internal_ci == "pass2" or args.internal_ci == "pass3":
-                     '--uid', '0',
+            shutil.copy2(os.path.join('sysa', 'bootstrap.cfg'),
-                     '--gid', '0',
+                         os.path.join('tmp', 'sysa', 'sysc_image', 'usr', 'src', 'bootstrap.cfg'))
-                     '--unshare-net' if args.external_sources else None,
+            run('bwrap', '--unshare-user',
-                     '--clearenv',
+                         '--uid', '0',
-                     '--setenv', 'PATH', '/usr/bin',
+                         '--gid', '0',
-                     '--bind', system_a.tmp_dir + "/sysc_image", '/',
+                         '--unshare-net' if args.external_sources else None,
-                     '--dir', '/dev',
+                         '--clearenv',
-                     '--dev-bind', '/dev/null', '/dev/null',
+                         '--setenv', 'PATH', '/usr/bin',
-                     '--dev-bind', '/dev/zero', '/dev/zero',
+                         '--bind', system_a.tmp_dir + "/sysc_image", '/',
-                     '--dev-bind', '/dev/random', '/dev/random',
+                         '--dir', '/dev',
-                     '--dev-bind', '/dev/urandom', '/dev/urandom',
+                         '--dev-bind', '/dev/null', '/dev/null',
-                     '--tmpfs', '/dev/shm',
+                         '--dev-bind', '/dev/zero', '/dev/zero',
-                     '--proc', '/proc',
+                         '--dev-bind', '/dev/random', '/dev/random',
-                     '--bind', '/sys', '/sys',
+                         '--dev-bind', '/dev/urandom', '/dev/urandom',
-                     '--tmpfs', '/tmp',
+                         '--tmpfs', '/dev/shm',
-                     '/init')
+                         '--proc', '/proc',
+                         '--bind', '/sys', '/sys',
+                         '--tmpfs', '/tmp',
+                         '/init')
 
     elif args.bare_metal:
         if args.kernel:

sysc/init

@@ -78,6 +78,7 @@ DESTDIR=${DESTDIR}
 DISTFILES=${DISTFILES}
 SRCDIR=${SRCDIR}
 MAKEJOBS=${MAKEJOBS}
+INTERNAL_CI=${INTERNAL_CI}
 EOF
 
 exec ./run.sh

sysc/run2.sh

@@ -120,6 +120,8 @@ build python-3.8.16
 
 build python-3.11.1
 
+[ "${INTERNAL_CI}" = "pass2" ] && exit 0
+
 build gcc-10.4.0
 
 build binutils-2.38 pass2.sh