Fix a typo in CFLAGS variable name.

Drop non-reproducible manpage from Grub package

.github/workflows/bwrap.yml

@@ -12,8 +12,8 @@ on:
       - master
 
 jobs:
-  run:
-    name: Run under bubblewrap
+  pass1:
+    name: Run up to Linux build under bubblewrap
     runs-on: ubuntu-latest
     steps:
       - name: Install bubblewrap
@@ -30,9 +30,8 @@ jobs:
         uses: actions/cache/restore@v3
         with:
           path: |
-            sysa/distfiles
-            sysc/distfiles
-          key: cache-${{ hashFiles('sys*/*/sources') }}
+            distfiles
+          key: cache-${{ hashFiles('steps/*/sources') }}
       - name: Get sources
         if: steps.cache.outputs.cache-hit != 'true'
         run: ./download-distfiles.sh
@@ -41,15 +40,119 @@ jobs:
         uses: actions/cache/save@v3
         with:
           path: |
-            sysa/distfiles
-            sysc/distfiles
-          key: cache-${{ hashFiles('sys*/*/sources') }}
+            distfiles
+          key: cache-${{ hashFiles('steps/*/sources') }}
       - name: Run bootstrap
-        id: bootstrap
-        run: ./rootfs.py --bwrap --external-sources --build-kernels --preserve --cores 2
+        run: ./rootfs.py --bwrap --external-sources --build-kernels --cores 2 --internal-ci pass1
       - name: Archive created packages
-        if: always() # archive failed builds
+        if: failure() # archive failed builds progress
         uses: actions/upload-artifact@v3
         with:
           name: packages
-          path: tmp/sysa/sysc_image/usr/src/repo/**
+          path: target/external/repo/**
+      - name: Tar pass1 image
+        run: tar -cf pass1_image.tar target
+      - name: Archive pass1_image
+        uses: actions/upload-artifact@v3
+        with:
+          name: internal_pass1_image
+          path: pass1_image.tar
+
+  pass2:
+    name: Run up to Python bootstrap under bubblewrap
+    needs: pass1
+    runs-on: ubuntu-latest
+    steps:
+      - name: Install bubblewrap
+        run: sudo apt install bubblewrap
+      - name: Checkout repo
+        uses: actions/checkout@v3
+        with:
+          submodules: recursive
+          # There is a strange bug(?) in nongnu, when you clone a git repository
+          # against a commit != HEAD with depth=1, it errors out.
+          fetch-depth: 0
+      - name: Get pass1_image
+        uses: actions/download-artifact@v3
+        with:
+          name: internal_pass1_image
+      - name: Extract pass1_image
+        run: tar -xf pass1_image.tar
+      - name: Query cache for sources
+        id: cache
+        uses: actions/cache/restore@v3
+        with:
+          path: |
+            distfiles
+          key: cache-${{ hashFiles('steps/*/sources') }}
+      - name: Get sources
+        if: steps.cache.outputs.cache-hit != 'true'
+        run: ./download-distfiles.sh
+      - name: Cache sources
+        if: steps.cache.outputs.cache-hit != 'true'
+        uses: actions/cache/save@v3
+        with:
+          path: |
+            distfiles
+          key: cache-${{ hashFiles('steps/*/sources') }}
+      - name: Run bootstrap
+        run: ./rootfs.py --bwrap --external-sources --build-kernels --cores 2 --internal-ci pass2
+      - name: Archive created packages
+        if: failure() # archive failed builds progress
+        uses: actions/upload-artifact@v3
+        with:
+          name: internal_packages_pass2
+          path: target/external/repo/**
+      - name: Tar pass2 image
+        run: tar -cf pass2_image.tar target
+      - name: Archive pass2_image
+        uses: actions/upload-artifact@v3
+        with:
+          name: internal_pass2_image
+          path: pass2_image.tar
+
+  pass3:
+    name: Run remaining builds under bubblewrap
+    needs: pass2
+    runs-on: ubuntu-latest
+    steps:
+      - name: Install bubblewrap
+        run: sudo apt install bubblewrap
+      - name: Checkout repo
+        uses: actions/checkout@v3
+        with:
+          submodules: recursive
+          # There is a strange bug(?) in nongnu, when you clone a git repository
+          # against a commit != HEAD with depth=1, it errors out.
+          fetch-depth: 0
+      - name: Get pass2_image
+        uses: actions/download-artifact@v3
+        with:
+          name: internal_pass2_image
+      - name: Extract pass2_image
+        run: tar -xf pass2_image.tar
+      - name: Query cache for sources
+        id: cache
+        uses: actions/cache/restore@v3
+        with:
+          path: |
+            distfiles
+          key: cache-${{ hashFiles('steps/*/sources') }}
+      - name: Get sources
+        if: steps.cache.outputs.cache-hit != 'true'
+        run: ./download-distfiles.sh
+      - name: Cache sources
+        if: steps.cache.outputs.cache-hit != 'true'
+        uses: actions/cache/save@v3
+        with:
+          path: |
+            distfiles
+          key: cache-${{ hashFiles('steps/*/sources') }}
+      - name: Run bootstrap
+        run: ./rootfs.py --bwrap --external-sources --build-kernels --cores 2 --internal-ci pass3
+      - name: Archive created packages
+        if: always() # archive both failed and successful builds
+        uses: actions/upload-artifact@v3
+        with:
+          name: packages
+          path: target/external/repo/**

.github/workflows/lint.yml

@@ -21,7 +21,7 @@ jobs:
       - name: Install pylint
         run: sudo pip3 install pylint
       - name: pylint
-        run: pylint rootfs.py sysa.py sysc.py lib/utils.py lib/sysgeneral.py lib/tmpdir.py --disable=duplicate-code
+        run: pylint rootfs.py lib/utils.py lib/generator.py lib/target.py check_substitutes.py --disable=duplicate-code
 
   shellcheck:
     name: Lint shell files
@@ -30,7 +30,7 @@ jobs:
       - name: Checkout repo
         uses: actions/checkout@v3
       - name: shellcheck
-        run: shellcheck sysa/run.sh sysa/run2.sh sysb/init sysb/run.sh sysc/init sysc/run.sh sysa/helpers.sh download-distfiles.sh
+        run: shellcheck steps/helpers.sh download-distfiles.sh
 
   reuse:
     name: Lint reuse information
@@ -42,3 +42,12 @@ jobs:
         run: sudo pip3 install reuse
       - name: reuse
         run: reuse lint
+
+  substitutes:
+    name: Check validity of substituted tarballs
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout repo
+        uses: actions/checkout@v3
+      - name: Check validity of substitutes
+        run: ./check_substitutes.py

.gitignore

@@ -3,9 +3,8 @@
 #
 # SPDX-License-Identifier: MIT 
 
-tmp/
+target/
 kernel
-sysa/distfiles/
-sysc/distfiles/
+distfiles/
 __pycache__
-sysa/bootstrap.cfg
+steps/bootstrap.cfg

.gitmodules

@@ -2,7 +2,10 @@
 # SPDX-FileCopyrightText: 2021 fosslinux <fosslinux@aussies.space>
 #
 # SPDX-License-Identifier: MIT 
+[submodule "seed/stage0-posix"]
+	path = seed/stage0-posix
+	url = https://github.com/oriansj/stage0-posix
 
-[submodule "sysa/stage0-posix/src"]
-	path = sysa/stage0-posix/src
-	url = https://github.com/oriansj/stage0-posix/
+[submodule "builder-hex0"]
+	path = builder-hex0
+	url = https://github.com/ironmeld/builder-hex0

.reuse/dep5

@@ -9,6 +9,6 @@ Source: https://github.com/fosslinux/live-bootstrap
 # Copyright: $YEAR $NAME <$CONTACT>
 # License: ...
 
-Files: sys*/*/sources sys*/SHA256SUMS.sources sysa/checksum-transcriber.*.SHA256SUM sysa/pre-sha.sha256sums sysa/*/*.checksums sysa/SHA256SUMS.pkgs sysc/musl-1.2.3/ld-musl-i386.path sysa/*/simple-patches/*
+Files: steps/*/sources steps/*/*.checksums steps/SHA256SUMS.pkgs steps/*/simple-patches/* seed/*.checksums
 Copyright: none
 License: MIT

DEVEL.md

@@ -14,31 +14,40 @@ and that a full build completes.
 
 ## Structure
 
-Each system corresponds to a reboot of the live environment. There is only one
-appropriate structure as shown below (eg for sysa):
-
 ```
-sysa
-├── any-global-files.sh
+seed
+├── seed.kaem
+├── script-generator.c
+├── ...
+└── stage0-posix
+
+steps
+├── manifest
+├── any-global-files
+├── jump
+│   └── linux.sh
+├── improve
+│   └── x.sh
 ├── somepackage-version
-│   ├── somepackage-version.kaem (or .sh)
+│   ├── pass1.kaem
+│   ├── pass2.sh
 │   ├── files
 │   ├── simple-patches
 │   ├── mk
 │   └── patches
-└── tmp
 ```
 
-Global scripts that drive the entire system go directly under `sysx`. `tmp`
-contains the temporary system used for QEMU or a chroot.
+The `seed` directory contains everything required for `script-generator` to be
+run.
 
-Then, each package is in its own specific directory, named `package-version`.
-It then diverges based upon which driver is being used:
-
-- `kaem`: A file named `package-version.kaem` is called by the master script.
-- `bash`: The `build` function from helper.sh is called from the master script.
-  There are default functions run which can be overridden by an optional script
-  `package-version.sh` within the package-specific directory.
+In the `steps` directory, the bootstrap process is defined in `manifest`.
+Each package to be built is named `package-version`.
+Each subsequent build of a package is the nth pass. Scripts are named
+accordingly; eg, the first build would be called `pass1.sh`, the second would be
+`pass2.sh`, etc.
+Scripts run in kaem era should be denoted as such in their filename;
+`pass1.kaem`, for example. Pass numbers do not reset after kaem, ie, you cannot
+have both `pass1.kaem` and `pass1.sh`.
 
 In this folder, there are other folders/files. `*.checksums` are
 required for early packages that are build with kaem, others are optional.
@@ -51,21 +60,16 @@ Permissible folders/files:
 - `simple-patches`: patches for the source that use the before/after convention of simple-patch.c
 - `*.checksums`: files containing the checksums for the resulting binaries and
 libraries that are compiled and installed.
-  - Up to and including `coreutils-6.10`, `sha256sum` from `stage0-posix`
-    is used for the checksumming. Later we switch to GNU version.
-  - To extract checksums of the binaries, use of qemu mode is recommended
-    (i.e. `./rootfs.py -q -qk $kernel --update-checksums`).
-- compilation script
-
-The directory m2-functions is used for M2-Planet functions (currently).
+  - Otherwise, the package's checksum is in SHA256SUMS.pkgs.
+- compilation script(s)
 
 ## Conventions
 
 - **Patches:**
   - all patches are `-p0`
   - all patches begin with a patch header
-- **README:**
-  - all stages are explained in README
+- **parts.rst:**
+  - all packages are explained in `parts.rst`
 - **General:**
   - Where possible, all blocks of text should be limited to a length of 80
     characters.
@@ -79,9 +83,3 @@ The directory m2-functions is used for M2-Planet functions (currently).
   - Patches are licensed under the license of the project which they are
     patching.
   - All files (excluding files within submodules) must comply with REUSE v3.0.
-
-## git
-
-All changes must be submitted as PRs. Pushing to master is disallowed, even if
-push access is granted to a user. Only pushes to master should be merging of
-patches into master.

LICENSES/CC0-1.0.txt

@@ -0,0 +1,121 @@
+Creative Commons Legal Code
+
+CC0 1.0 Universal
+
+    CREATIVE COMMONS CORPORATION IS NOT A LAW FIRM AND DOES NOT PROVIDE
+    LEGAL SERVICES. DISTRIBUTION OF THIS DOCUMENT DOES NOT CREATE AN
+    ATTORNEY-CLIENT RELATIONSHIP. CREATIVE COMMONS PROVIDES THIS
+    INFORMATION ON AN "AS-IS" BASIS. CREATIVE COMMONS MAKES NO WARRANTIES
+    REGARDING THE USE OF THIS DOCUMENT OR THE INFORMATION OR WORKS
+    PROVIDED HEREUNDER, AND DISCLAIMS LIABILITY FOR DAMAGES RESULTING FROM
+    THE USE OF THIS DOCUMENT OR THE INFORMATION OR WORKS PROVIDED
+    HEREUNDER.
+
+Statement of Purpose
+
+The laws of most jurisdictions throughout the world automatically confer
+exclusive Copyright and Related Rights (defined below) upon the creator
+and subsequent owner(s) (each and all, an "owner") of an original work of
+authorship and/or a database (each, a "Work").
+
+Certain owners wish to permanently relinquish those rights to a Work for
+the purpose of contributing to a commons of creative, cultural and
+scientific works ("Commons") that the public can reliably and without fear
+of later claims of infringement build upon, modify, incorporate in other
+works, reuse and redistribute as freely as possible in any form whatsoever
+and for any purposes, including without limitation commercial purposes.
+These owners may contribute to the Commons to promote the ideal of a free
+culture and the further production of creative, cultural and scientific
+works, or to gain reputation or greater distribution for their Work in
+part through the use and efforts of others.
+
+For these and/or other purposes and motivations, and without any
+expectation of additional consideration or compensation, the person
+associating CC0 with a Work (the "Affirmer"), to the extent that he or she
+is an owner of Copyright and Related Rights in the Work, voluntarily
+elects to apply CC0 to the Work and publicly distribute the Work under its
+terms, with knowledge of his or her Copyright and Related Rights in the
+Work and the meaning and intended legal effect of CC0 on those rights.
+
+1. Copyright and Related Rights. A Work made available under CC0 may be
+protected by copyright and related or neighboring rights ("Copyright and
+Related Rights"). Copyright and Related Rights include, but are not
+limited to, the following:
+
+  i. the right to reproduce, adapt, distribute, perform, display,
+     communicate, and translate a Work;
+ ii. moral rights retained by the original author(s) and/or performer(s);
+iii. publicity and privacy rights pertaining to a person's image or
+     likeness depicted in a Work;
+ iv. rights protecting against unfair competition in regards to a Work,
+     subject to the limitations in paragraph 4(a), below;
+  v. rights protecting the extraction, dissemination, use and reuse of data
+     in a Work;
+ vi. database rights (such as those arising under Directive 96/9/EC of the
+     European Parliament and of the Council of 11 March 1996 on the legal
+     protection of databases, and under any national implementation
+     thereof, including any amended or successor version of such
+     directive); and
+vii. other similar, equivalent or corresponding rights throughout the
+     world based on applicable law or treaty, and any national
+     implementations thereof.
+
+2. Waiver. To the greatest extent permitted by, but not in contravention
+of, applicable law, Affirmer hereby overtly, fully, permanently,
+irrevocably and unconditionally waives, abandons, and surrenders all of
+Affirmer's Copyright and Related Rights and associated claims and causes
+of action, whether now known or unknown (including existing as well as
+future claims and causes of action), in the Work (i) in all territories
+worldwide, (ii) for the maximum duration provided by applicable law or
+treaty (including future time extensions), (iii) in any current or future
+medium and for any number of copies, and (iv) for any purpose whatsoever,
+including without limitation commercial, advertising or promotional
+purposes (the "Waiver"). Affirmer makes the Waiver for the benefit of each
+member of the public at large and to the detriment of Affirmer's heirs and
+successors, fully intending that such Waiver shall not be subject to
+revocation, rescission, cancellation, termination, or any other legal or
+equitable action to disrupt the quiet enjoyment of the Work by the public
+as contemplated by Affirmer's express Statement of Purpose.
+
+3. Public License Fallback. Should any part of the Waiver for any reason
+be judged legally invalid or ineffective under applicable law, then the
+Waiver shall be preserved to the maximum extent permitted taking into
+account Affirmer's express Statement of Purpose. In addition, to the
+extent the Waiver is so judged Affirmer hereby grants to each affected
+person a royalty-free, non transferable, non sublicensable, non exclusive,
+irrevocable and unconditional license to exercise Affirmer's Copyright and
+Related Rights in the Work (i) in all territories worldwide, (ii) for the
+maximum duration provided by applicable law or treaty (including future
+time extensions), (iii) in any current or future medium and for any number
+of copies, and (iv) for any purpose whatsoever, including without
+limitation commercial, advertising or promotional purposes (the
+"License"). The License shall be deemed effective as of the date CC0 was
+applied by Affirmer to the Work. Should any part of the License for any
+reason be judged legally invalid or ineffective under applicable law, such
+partial invalidity or ineffectiveness shall not invalidate the remainder
+of the License, and in such case Affirmer hereby affirms that he or she
+will not (i) exercise any of his or her remaining Copyright and Related
+Rights in the Work or (ii) assert any associated claims and causes of
+action with respect to the Work, in either case contrary to Affirmer's
+express Statement of Purpose.
+
+4. Limitations and Disclaimers.
+
+ a. No trademark or patent rights held by Affirmer are waived, abandoned,
+    surrendered, licensed or otherwise affected by this document.
+ b. Affirmer offers the Work as-is and makes no representations or
+    warranties of any kind concerning the Work, express, implied,
+    statutory or otherwise, including without limitation warranties of
+    title, merchantability, fitness for a particular purpose, non
+    infringement, or the absence of latent or other defects, accuracy, or
+    the present or absence of errors, whether or not discoverable, all to
+    the greatest extent permissible under applicable law.
+ c. Affirmer disclaims responsibility for clearing rights of other persons
+    that may apply to the Work or any use thereof, including without
+    limitation any person's Copyright and Related Rights in the Work.
+    Further, Affirmer disclaims responsibility for obtaining any necessary
+    consents, permissions or other rights required for any use of the
+    Work.
+ d. Affirmer understands and acknowledges that Creative Commons is not a
+    party to this document and has no duty or obligation with respect to
+    this CC0 or use of the Work.

LICENSES/CDDL-1.0.txt

@@ -1,298 +0,0 @@
-COMMON DEVELOPMENT AND DISTRIBUTION LICENSE (CDDL)
-Version 1.0
-
-1. Definitions.
-
-1.1. “Contributor” means each individual or entity that creates or contributes
-to the creation of Modifications.
-
-1.2. “Contributor Version” means the combination of the Original Software,
-prior Modifications used by a Contributor (if any), and the Modifications
-made by that particular Contributor.
-
-1.3. “Covered Software” means (a) the Original Software, or (b) Modifications,
-or (c) the combination of files containing Original Software with files containing
-Modifications, in each case including portions thereof.
-
-1.4. “Executable” means the Covered Software in any form other than Source
-Code.
-
-1.5. “Initial Developer” means the individual or entity that first makes Original
-Software available under this License.
-
-1.6. “Larger Work” means a work which combines Covered Software or portions
-thereof with code not governed by the terms of this License.
-
-1.7. “License” means this document.
-
-1.8. “Licensable” means having the right to grant, to the maximum extent possible,
-whether at the time of the initial grant or subsequently acquired, any and
-all of the rights conveyed herein.
-
-1.9. “Modifications” means the Source Code and Executable form of any of the
-following:
-
-A. Any file that results from an addition to, deletion from or modification
-of the contents of a file containing Original Software or previous Modifications;
-
-B. Any new file that contains any part of the Original Software or previous
-Modification; or
-
-C. Any new file that is contributed or otherwise made available under the
-terms of this License.
-
-1.10. “Original Software” means the Source Code and Executable form of computer
-software code that is originally released under this License.
-
-1.11. “Patent Claims” means any patent claim(s), now owned or hereafter acquired,
-including without limitation, method, process, and apparatus claims, in any
-patent Licensable by grantor.
-
-1.12. “Source Code” means (a) the common form of computer software code in
-which modifications are made and (b) associated documentation included in
-or with such code.
-
-1.13. “You” (or “Your”) means an individual or a legal entity exercising rights
-under, and complying with all of the terms of, this License. For legal entities,
-“You” includes any entity which controls, is controlled by, or is under common
-control with You. For purposes of this definition, “control” means (a) the
-power, direct or indirect, to cause the direction or management of such entity,
-whether by contract or otherwise, or (b) ownership of more than fifty percent
-(50%) of the outstanding shares or beneficial ownership of such entity.
-
-2. License Grants.
-
-2.1. The Initial Developer Grant.
-Conditioned upon Your compliance with Section 3.1 below and subject to third
-party intellectual property claims, the Initial Developer hereby grants You
-a world-wide, royalty-free, non-exclusive license:
-
-(a) under intellectual property rights (other than patent or trademark) Licensable
-by Initial Developer, to use, reproduce, modify, display, perform, sublicense
-and distribute the Original Software (or portions thereof), with or without
-Modifications, and/or as part of a Larger Work; and
-
-(b) under Patent Claims infringed by the making, using or selling of Original
-Software, to make, have made, use, practice, sell, and offer for sale, and/or
-otherwise dispose of the Original Software (or portions thereof).
-
-(c) The licenses granted in Sections 2.1(a) and (b) are effective on the date
-Initial Developer first distributes or otherwise makes the Original Software
-available to a third party under the terms of this License.
-
-(d) Notwithstanding Section 2.1(b) above, no patent license is granted: (1)
-for code that You delete from the Original Software, or (2) for infringements
-caused by: (i) the modification of the Original Software, or (ii) the combination
-of the Original Software with other software or devices.
-
-2.2. Contributor Grant.
-Conditioned upon Your compliance with Section 3.1 below and subject to third
-party intellectual property claims, each Contributor hereby grants You a world-wide,
-royalty-free, non-exclusive license:
-
-(a) under intellectual property rights (other than patent or trademark) Licensable
-by Contributor to use, reproduce, modify, display, perform, sublicense and
-distribute the Modifications created by such Contributor (or portions thereof),
-either on an unmodified basis, with other Modifications, as Covered Software
-and/or as part of a Larger Work; and
-
-(b) under Patent Claims infringed by the making, using, or selling of Modifications
-made by that Contributor either alone and/or in combination with its Contributor
-Version (or portions of such combination), to make, use, sell, offer for sale,
-have made, and/or otherwise dispose of: (1) Modifications made by that Contributor
-(or portions thereof); and (2) the combination of Modifications made by that
-Contributor with its Contributor Version (or portions of such combination).
-
-(c) The licenses granted in Sections 2.2(a) and 2.2(b) are effective on the
-date Contributor first distributes or otherwise makes the Modifications available
-to a third party.
-
-(d) Notwithstanding Section 2.2(b) above, no patent license is granted: (1)
-for any code that Contributor has deleted from the Contributor Version; (2)
-for infringements caused by: (i) third party modifications of Contributor
-Version, or (ii) the combination of Modifications made by that Contributor
-with other software (except as part of the Contributor Version) or other devices;
-or (3) under Patent Claims infringed by Covered Software in the absence of
-Modifications made by that Contributor.
-
-3. Distribution Obligations.
-
-3.1. Availability of Source Code.
-Any Covered Software that You distribute or otherwise make available in Executable
-form must also be made available in Source Code form and that Source Code
-form must be distributed only under the terms of this License. You must include
-a copy of this License with every copy of the Source Code form of the Covered
-Software You distribute or otherwise make available. You must inform recipients
-of any such Covered Software in Executable form as to how they can obtain
-such Covered Software in Source Code form in a reasonable manner on or through
-a medium customarily used for software exchange.
-
-3.2. Modifications.
-The Modifications that You create or to which You contribute are governed
-by the terms of this License. You represent that You believe Your Modifications
-are Your original creation(s) and/or You have sufficient rights to grant the
-rights conveyed by this License.
-
-3.3. Required Notices.
-You must include a notice in each of Your Modifications that identifies You
-as the Contributor of the Modification. You may not remove or alter any copyright,
-patent or trademark notices contained within the Covered Software, or any
-notices of licensing or any descriptive text giving attribution to any Contributor
-or the Initial Developer.
-
-3.4. Application of Additional Terms.
-You may not offer or impose any terms on any Covered Software in Source Code
-form that alters or restricts the applicable version of this License or the
-recipients’ rights hereunder. You may choose to offer, and to charge a fee
-for, warranty, support, indemnity or liability obligations to one or more
-recipients of Covered Software. However, you may do so only on Your own behalf,
-and not on behalf of the Initial Developer or any Contributor. You must make
-it absolutely clear that any such warranty, support, indemnity or liability
-obligation is offered by You alone, and You hereby agree to indemnify the
-Initial Developer and every Contributor for any liability incurred by the
-Initial Developer or such Contributor as a result of warranty, support, indemnity
-or liability terms You offer.
-
-3.5. Distribution of Executable Versions.
-You may distribute the Executable form of the Covered Software under the terms
-of this License or under the terms of a license of Your choice, which may
-contain terms different from this License, provided that You are in compliance
-with the terms of this License and that the license for the Executable form
-does not attempt to limit or alter the recipient’s rights in the Source Code
-form from the rights set forth in this License. If You distribute the Covered
-Software in Executable form under a different license, You must make it absolutely
-clear that any terms which differ from this License are offered by You alone,
-not by the Initial Developer or Contributor. You hereby agree to indemnify
-the Initial Developer and every Contributor for any liability incurred by
-the Initial Developer or such Contributor as a result of any such terms You
-offer.
-
-3.6. Larger Works.
-You may create a Larger Work by combining Covered Software with other code
-not governed by the terms of this License and distribute the Larger Work as
-a single product. In such a case, You must make sure the requirements of this
-License are fulfilled for the Covered Software.
-
-4. Versions of the License.
-
-4.1. New Versions.
-Sun Microsystems, Inc. is the initial license steward and may publish revised
-and/or new versions of this License from time to time. Each version will be
-given a distinguishing version number. Except as provided in Section 4.3,
-no one other than the license steward has the right to modify this License.
-
-4.2. Effect of New Versions.
-You may always continue to use, distribute or otherwise make the Covered Software
-available under the terms of the version of the License under which You originally
-received the Covered Software. If the Initial Developer includes a notice
-in the Original Software prohibiting it from being distributed or otherwise
-made available under any subsequent version of the License, You must distribute
-and make the Covered Software available under the terms of the version of
-the License under which You originally received the Covered Software. Otherwise,
-You may also choose to use, distribute or otherwise make the Covered Software
-available under the terms of any subsequent version of the License published
-by the license steward.
-
-4.3. Modified Versions.
-When You are an Initial Developer and You want to create a new license for
-Your Original Software, You may create and use a modified version of this
-License if You: (a) rename the license and remove any references to the name
-of the license steward (except to note that the license differs from this
-License); and (b) otherwise make it clear that the license contains terms
-which differ from this License.
-
-5. DISCLAIMER OF WARRANTY.
-
-COVERED SOFTWARE IS PROVIDED UNDER THIS LICENSE ON AN “AS IS” BASIS, WITHOUT
-WARRANTY OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, WITHOUT LIMITATION,
-WARRANTIES THAT THE COVERED SOFTWARE IS FREE OF DEFECTS, MERCHANTABLE, FIT
-FOR A PARTICULAR PURPOSE OR NON-INFRINGING. THE ENTIRE RISK AS TO THE QUALITY
-AND PERFORMANCE OF THE COVERED SOFTWARE IS WITH YOU. SHOULD ANY COVERED SOFTWARE
-PROVE DEFECTIVE IN ANY RESPECT, YOU (NOT THE INITIAL DEVELOPER OR ANY OTHER
-CONTRIBUTOR) ASSUME THE COST OF ANY NECESSARY SERVICING, REPAIR OR CORRECTION.
-THIS DISCLAIMER OF WARRANTY CONSTITUTES AN ESSENTIAL PART OF THIS LICENSE.
-NO USE OF ANY COVERED SOFTWARE IS AUTHORIZED HEREUNDER EXCEPT UNDER THIS DISCLAIMER.
-
-6. TERMINATION.
-
-6.1. This License and the rights granted hereunder will terminate automatically
-if You fail to comply with terms herein and fail to cure such breach within
-30 days of becoming aware of the breach. Provisions which, by their nature,
-must remain in effect beyond the termination of this License shall survive.
-
-6.2. If You assert a patent infringement claim (excluding declaratory judgment
-actions) against Initial Developer or a Contributor (the Initial Developer
-or Contributor against whom You assert such claim is referred to as “Participant”)
-alleging that the Participant Software (meaning the Contributor Version where
-the Participant is a Contributor or the Original Software where the Participant
-is the Initial Developer) directly or indirectly infringes any patent, then
-any and all rights granted directly or indirectly to You by such Participant,
-the Initial Developer (if the Initial Developer is not the Participant) and
-all Contributors under Sections 2.1 and/or 2.2 of this License shall, upon
-60 days notice from Participant terminate prospectively and automatically
-at the expiration of such 60 day notice period, unless if within such 60 day
-period You withdraw Your claim with respect to the Participant Software against
-such Participant either unilaterally or pursuant to a written agreement with
-Participant.
-
-6.3. In the event of termination under Sections 6.1 or 6.2 above, all end
-user licenses that have been validly granted by You or any distributor hereunder
-prior to termination (excluding licenses granted to You by any distributor)
-shall survive termination.
-
-7. LIMITATION OF LIABILITY.
-
-UNDER NO CIRCUMSTANCES AND UNDER NO LEGAL THEORY, WHETHER TORT (INCLUDING
-NEGLIGENCE), CONTRACT, OR OTHERWISE, SHALL YOU, THE INITIAL DEVELOPER, ANY
-OTHER CONTRIBUTOR, OR ANY DISTRIBUTOR OF COVERED SOFTWARE, OR ANY SUPPLIER
-OF ANY OF SUCH PARTIES, BE LIABLE TO ANY PERSON FOR ANY INDIRECT, SPECIAL,
-INCIDENTAL, OR CONSEQUENTIAL DAMAGES OF ANY CHARACTER INCLUDING, WITHOUT LIMITATION,
-DAMAGES FOR LOST PROFITS, LOSS OF GOODWILL, WORK STOPPAGE, COMPUTER FAILURE
-OR MALFUNCTION, OR ANY AND ALL OTHER COMMERCIAL DAMAGES OR LOSSES, EVEN IF
-SUCH PARTY SHALL HAVE BEEN INFORMED OF THE POSSIBILITY OF SUCH DAMAGES. THIS
-LIMITATION OF LIABILITY SHALL NOT APPLY TO LIABILITY FOR DEATH OR PERSONAL
-INJURY RESULTING FROM SUCH PARTY’S NEGLIGENCE TO THE EXTENT APPLICABLE LAW
-PROHIBITS SUCH LIMITATION. SOME JURISDICTIONS DO NOT ALLOW THE EXCLUSION OR
-LIMITATION OF INCIDENTAL OR CONSEQUENTIAL DAMAGES, SO THIS EXCLUSION AND LIMITATION
-MAY NOT APPLY TO YOU.
-
-8. U.S. GOVERNMENT END USERS.
-
-The Covered Software is a “commercial item,” as that term is defined in 48
-C.F.R. 2.101 (Oct. 1995), consisting of “commercial computer software” (as
-that term is defined at 48 C.F.R. § 252.227-7014(a)(1)) and “commercial computer
-software documentation” as such terms are used in 48 C.F.R. 12.212 (Sept.
-1995). Consistent with 48 C.F.R. 12.212 and 48 C.F.R. 227.7202-1 through 227.7202-4
-(June 1995), all U.S. Government End Users acquire Covered Software with only
-those rights set forth herein. This U.S. Government Rights clause is in lieu
-of, and supersedes, any other FAR, DFAR, or other clause or provision that
-addresses Government rights in computer software under this License.
-
-9. MISCELLANEOUS.
-
-This License represents the complete agreement concerning subject matter hereof.
-If any provision of this License is held to be unenforceable, such provision
-shall be reformed only to the extent necessary to make it enforceable. This
-License shall be governed by the law of the jurisdiction specified in a notice
-contained within the Original Software (except to the extent applicable law,
-if any, provides otherwise), excluding such jurisdiction’s conflict-of-law
-provisions. Any litigation relating to this License shall be subject to the
-jurisdiction of the courts located in the jurisdiction and venue specified
-in a notice contained within the Original Software, with the losing party
-responsible for costs, including, without limitation, court costs and reasonable
-attorneys’ fees and expenses. The application of the United Nations Convention
-on Contracts for the International Sale of Goods is expressly excluded. Any
-law or regulation which provides that the language of a contract shall be
-construed against the drafter shall not apply to this License. You agree that
-You alone are responsible for compliance with the United States export administration
-regulations (and the export control laws and regulation of any other countries)
-when You use, distribute or otherwise make available any Covered Software.
-
-10. RESPONSIBILITY FOR CLAIMS.
-
-As between Initial Developer and the Contributors, each party is responsible
-for claims and damages arising, directly or indirectly, out of its utilization
-of rights under this License and You agree to work with Initial Developer
-and Contributors to distribute such responsibility on an equitable basis.
-Nothing herein is intended or shall be deemed to constitute any admission
-of liability.

README.rst

@@ -12,94 +12,91 @@ An attempt to provide a reproducible, automatic, complete end-to-end
 bootstrap from a minimal number of binary seeds to a supported fully
 functioning operating system.
 
-Get me started!
----------------
+How do I use this?
+------------------
+
+Quick start:
+
+See ``./rootfs.py --help`` and follow the instructions given there.
+This uses a variety of userland tools to prepare the bootstrap.
+
+(*Currently, there is no way to perform the bootstrap without external
+preparations! This is a currently unsolved problem.*)
+
+Without using Python:
 
 1. ``git clone https://github.com/fosslinux/live-bootstrap``
 2. ``git submodule update --init --recursive``
-3. Provide a kernel (vmlinuz file) as the name ``kernel`` in the root of the
-   repository. **This must be a 32-bit kernel.**
-4. ``./rootfs.py --qemu`` - ensure your account has kvm privileges and qemu
-   installed.
-
-   a. Alternatively, run ``./rootfs.py --chroot`` to run it in a chroot.
-   b. Alternatively, run ``./rootfs.py --bwrap`` to run it in a bubblewrap
-      sandbox. When user namespaces are supported, this mode is rootless.
-   c. Alternatively, run ``./rootfs.py`` but don’t run the actual
-      virtualization and instead copy sysa/tmp/initramfs to a USB or
-      some other device and boot from bare metal. NOTE: we now require
-      a hard drive. This is currently hardcoded as sda. You also need
-      to put ``sysc/tmp/disk.img`` onto your sda on the bootstrapping
-      machine.
-   d. Alternatively, do not use python at all, see "Python-less build"
-      below.
-
-5. Wait.
-6. If you can, observe the many binaries in ``/usr/bin``! When the
-   bootstrap is completed ``bash`` is launched providing a shell to
-   explore the system.
-
+3. Consider whether you are going to run this in a chroot, in QEMU, or on bare
+   metal. (All of this *can* be automated, but not in a trustable way. See
+   further below.)
+   a. **chroot:** Create a directory where the chroot will reside, run
+      ``./download-distfiles.sh``, and copy:
+      * The entire contents of ``seed/stage0-posix`` into that directory.
+      * All other files in ``seed`` into that directory.
+      * ``steps/`` and ``distfiles/`` into that directory.
+        * At least all files listed in ``steps/pre-network-sources`` must be
+          copied in. All other files will be obtained from the network.
+      * Run ``/bootstrap-seeds/POSIX/x86/kaem-optional-seed`` in the chroot.
+        (Eg, ``chroot rootfs /bootstrap-seeds/POSIX/x86/kaem-optional-seed``).
+   b. **QEMU:** Create two blank disk images.
+      * Generate ``builder-hex0-x86-stage1.img`` from hex0 source:
+        ``sed 's/[;#].*$//g' builder-hex0/builder-hex0-x86-stage1-hex0 | xxd -r -p``
+      * On the first image, write ``builder-hex0-x86-stage1.img`` to it, followed
+        by ``kernel-bootstrap/builder-hex0-x86-stage2.hex0``, followed by zeros
+        padding the disk to the next sector.
+      * distfiles can be obtained using ``./download-distfiles.sh``.
+      * See the list in part a. For every file within that list, write a line to
+        the disk ``src <size-of-file> <path-to-file>``, followed by the contents
+        of the file.
+        * *Only* copy distfiles listed in ``sources`` files for ``build:`` steps
+          manifested before ``improve: get_network`` into this disk.
+      * Optionally (if you don't do this, distfiles will be network downloaded):
+        * On the second image, create an MSDOS partition table and one ext3
+          partition.
+        * Copy ``distfiles/`` into this disk.
+      * Run QEMU, with 4+G RAM, optionally SMP (multicore), both drives (in the
+        order introduced above), a NIC with model E1000 (``-nic
+        user,model=e1000``), and ``-machine kernel-irqchip=split``.
+   c. **Bare metal:** Follow the same steps as QEMU, but the disks need to be
+   two different *physical* disks, and boot from the first disk.
 
 Background
 ----------
 
-This project is a part of the bootstrappable project, a project that
-aims to be able to build complete computing platforms through the use of
-source code. When you build a compiler like GCC, you need another C
-compiler to compile the compiler - turtles all the way down. Even the
-first GCC compiler was written in C. There has to be a way to break the
-chain…
+Problem statement
+=================
 
-There has been significant work on this over the last 5 years, from
-Jeremiah Orians’ stage0, hex2 and M2-Planet to janneke’s Mes. We have a
-currently, fully-functioning chain of bootstrapping from the 357-byte
-hex0 seed to a complete GCC compiler and hence a full Linux operating
-system. From there, it is trivial to move to other UNIXes. However,
-there is only currently one vector through which this can be
-automatically done, GNU Guix.
+live-bootstrap's overarching problem statement is;
 
-While the primary author of this project does not believe Guix is a bad
-project, the great reliance on Guile, the complexity of many of the
-scripts and the rather steep learning curve to install and run Guix make
-it a very non plug-and-play solution. Furthermore, there is currently
-(Jan 2021) no possible way to run the bootstrap from outside of a
-pre-existing Linux environment. Additionally, Guix uses many scripts and
-distributed files that cannot be considered source code.
+> How can a usable Linux system be created with only human-auditable, and
+wherever possible, human-written, source code?
 
-(NOTE: Guix is working on a Full Source Bootstrap, but I’m not
-completely sure what that entails).
+Clarifications:
 
-Furthermore, having an alternative bootstrap automation tool allows
-people to have greater trust in the bootstrap procedure.
+* "usable" means a modern toolchain, with appropriate utilities, that can be
+  used to expand the amount of software on the system, interactively, or
+  non-interactively.
+* "human-auditable" is discretionary, but is usually fairly strict. See
+  "Specific things to be bootstrapped" below.
 
-Comparison between GNU Guix and live-bootstrap
-----------------------------------------------
+Why is this difficult?
+======================
 
-+----------------------+----------------------+----------------------+
-| Item                 | Guix                 | live-bootstrap       |
-+======================+======================+======================+
-| Total size of seeds  | ~30MB (Reduced       | ~1KB                 |
-| [1]                  | Source Bootstrap)    |                      |
-|                      | [2]                  |                      |
-+----------------------+----------------------+----------------------+
-| Use of kernel        | Linux-Libre Kernel   | Any Linux Kernel     |
-|                      |                      | (2.6+) [3]           |
-+----------------------+----------------------+----------------------+
-| Implementation       | Yes                  | No (in development)  |
-| complete             |                      |                      |
-+----------------------+----------------------+----------------------+
-| Automation           | Almost fully         | Optional user        |
-|                      | automatic            | customization        |
-+----------------------+----------------------+----------------------+
+The core of a modern Linux system is primarily written in C and C++. C and C++
+are **self-hosting**, ie, nearly every single C compiler is written in C.
 
-[1]: Both projects only use software licensed under a FSF-approved
-free software license. Kernel is excluded from seed.
-[2]: Reiterating that Guix is working on a full source bootstrap,
-although that still uses guile (~12 MB). [3]: Work is ongoing to use
-other, smaller POSIX kernels.
+Every single version of GCC was written in C. To avoid using an existing
+toolchain, we need some way to be able to compile a GCC version without C. We
+can use a less well-featured compiler, TCC, to do this. And so forth, until we
+get to a fairly primitive C compiler written in assembly, ``cc_x86``.
 
-Why would I want bootstrapping?
--------------------------------
+Going up through this process requires a bunch of other utilities as well; the
+autotools suite, guile and autogen, etc. These also have to be matched
+appropriately to the toolchain available.
+
+Why should I care?
+------------------
 
 That is outside of the scope of this README. Here’s a few things you can
 look at:
@@ -117,7 +114,7 @@ bootstrapping. However, there are a number of non-auditable files used
 in many of their packages. Here is a list of file types that we deem
 unsuitable for bootstrapping.
 
-1. Binaries (apart from seed hex0, kaem, kernel).
+1. Binaries (apart from seed hex0, kaem, builder-hex0).
 2. Any pre-generated configure scripts, or Makefile.in’s from autotools.
 3. Pre-generated bison/flex parsers (identifiable through a ``.y``
    file).
@@ -131,56 +128,18 @@ How does this work?
 
 **For a more in-depth discussion, see parts.rst.**
 
-sysa
-~~~~
+Firstly, ``builder-hex0`` is launched. ``builder-hex0`` is a minimal kernel that is
+written in ``hex0``, existing in 3 self-bootstrapping stages.
 
-sysa is the first ‘system’ used in live-bootstrap. We move to a new
-system after a reboot, which often occurs after the movement to a new
-kernel. It is run by the seed Linux kernel provided by the user. It
-compiles everything we need to be able to compile our own Linux kernel.
-It runs fully in an initramfs and does not rely on disk support in the
-seed Linux kernel.
+This is capable of executing the entirety of ``stage0-posix``, (see
+``seed/stage0-posix``), which produces a variety of useful utilities and a basic
+C language, ``M2-Planet``.
 
-sysb
-~~~~
+``stage0-posix`` runs a file called ``after.kaem``. This is a shell script that
+builds and runs a small program called ``script-generator``. This program reads
+``steps/manifest`` and converts it into a series of shell scripts that can be
+executed in sequence to complete the bootstrap.
 
-sysb is the second 'system' of live-bootstrap. This uses the Linux 4.9.10
-kernel compiled within sysa. As we do not rely on disk support in sysa, we
-need this intermediate system to be able to add the missing binaries to sysc
-before moving into it. This is executed through kexec from sysa. At this point,
-a SATA disk IS required.
-
-sysc
-~~~~
-
-sysc is the (current) last 'system' of live-bootstrap. This is a continuation
-from sysb, executed through util-linux's ``switch_root`` command which moves
-the entire rootfs without a reboot. Every package from here on out is compiled
-under this system, taking binaries from sysa. Chroot and bubblewrap modes skip
-sysb, as it is obviously irrelevant to them.
-
-Python-less build
------------------
-
-Python is no longer a requirement to set up the build system. The
-repository is almost completely in a form where it can be used as the
-source of a build.
-
-1. Download required tarballs into ``sysa/distfiles`` and ``sysc/distfiles``.
-   You can use the ``download-distfiles.sh`` script.
-2. Copy sysa/stage0-posix/src/* to the root of the repository.
-3. Copy sysa/stage0-posix/src/bootstrap-seeds/POSIX/x86/kaem-optional-seed
-   to init in the root of the repository.
-4. Copy sysa/after.kaem to after.kaem
-5. Create a CPIO archive (eg, ``cpio --format newc --create --directory . > ../initramfs``).
-6. Boot your initramfs and kernel.
-
-chroot builds
-~~~~~~~~~~~~~
-
-For chroot  based bootstraps you can skip creation of initramfs and instead start bootstrap with
-
-``sudo chroot . bootstrap-seeds/POSIX/x86/kaem-optional-seed``
-
-It is also recommended to copy everything to a new directory as bootstrapping messes up with files
-in git repository and cannot be re-run again.
+From this point forward, ``steps/manifest`` is effectively self documenting.
+Each package built exists in ``steps/<pkg>``, and the build scripts can be seen
+there.

builder-hex0

@@ -0,0 +1 @@
+Subproject commit a2781242d19e6be891b453d8fa827137ab5db31a

check_substitutes.py

@@ -0,0 +1,162 @@
+#!/usr/bin/env python3
+# SPDX-License-Identifier: GPL-3.0-or-later
+#
+# SPDX-FileCopyrightText: 2024 fosslinux <fosslinux@aussies.space>
+
+"""Check that substituted files are the same."""
+import bz2
+import filecmp
+import gzip
+import itertools
+import lzma
+import shutil
+import tarfile
+import tempfile
+import sys
+import os
+
+from lib.generator import Generator
+
+# Get a temporary directory to work in
+working = tempfile.mkdtemp()
+
+# Colour constants
+# pylint: disable=too-few-public-methods
+class Colors():
+    """ANSI Color Codes"""
+    GREY = "\033[90m"
+    RED = "\033[91m"
+    GREEN = "\033[92m"
+    ORANGE = "\033[91m\033[93m"
+    YELLOW = "\033[93m"
+    END = "\033[0m"
+
+def traverse_path(base_root):
+    """Takes a path and returns a set of all directories and files in that path."""
+    all_dirs = set()
+    all_files = set()
+    for root, directories, files in os.walk(base_root, topdown=True):
+        for d in directories:
+            all_dirs.add(os.path.join(root, d).lstrip(base_root))
+        for f in files:
+            all_files.add(os.path.join(root, f).lstrip(base_root))
+    return (all_dirs, all_files)
+
+class Distfile():
+    """Represents one distfile and operations performed on it."""
+    def __init__(self, i, url):
+        self.i = i
+        self.url = url
+        self.out_file = f"{i}-{os.path.basename(url)}"
+        self.filepath = ""
+
+    def download(self):
+        """Downloads the distfile."""
+        Generator.download_file(self.url, working, self.out_file, silent=True)
+        self.filepath = os.path.join(working, self.out_file)
+
+    def decompress(self):
+        """Decompresses the distfile."""
+        compression = self.out_file.split('.')[-1]
+        decompress_func = {
+            "gz": gzip.open,
+            "tgz": gzip.open,
+            "bz2": bz2.open,
+            "xz": lzma.open,
+            "lzma": lzma.open
+        }
+        if compression not in decompress_func:
+            # No decompression needed
+            return
+        # Remove the compression extension
+        new_path = '.'.join(self.filepath.split('.')[:-1])
+        # tgz -> .tar
+        if compression == "tgz":
+            new_path = f"{new_path}.tar"
+        # Move the decompressed binary stream to a new file
+        with decompress_func[compression](self.filepath, 'rb') as fin:
+            with open(new_path, 'wb') as fout:
+                shutil.copyfileobj(fin, fout)
+        self.filepath = new_path
+
+    def extract(self):
+        """Extracts the distfile."""
+        # Sanity check
+        if not tarfile.is_tarfile(self.filepath):
+            return
+        out_dir = os.path.join(working, f"{self.i}")
+        os.mkdir(out_dir)
+        with tarfile.open(self.filepath, 'r') as f:
+            f.extractall(path=out_dir)
+        self.filepath = out_dir
+
+    # It makes more sense here to label them d1 and d2 rather than have one be self.
+    # pylint: disable=no-self-argument
+    def compare(d1, d2):
+        """Compares the distfile to another distfile."""
+        if not os.path.isdir(d1.filepath):
+            # Compare files
+            return filecmp.cmp(d1.filepath, d2.filepath, shallow=False)
+        if not os.path.isdir(d2.filepath):
+            # Then, d2 is a file and d1 is a directory
+            return False
+        # Otherwise it's two directories
+        dirnames1, filenames1 = traverse_path(d1.filepath)
+        dirnames2, filenames2 = traverse_path(d2.filepath)
+        if dirnames1 != dirnames2:
+            return False
+        if filenames1 != filenames2:
+            return False
+        return filecmp.cmpfiles(d1.filepath, d2.filepath, filenames1, shallow=False)
+
+def check(*args):
+    """Check if a list of distfiles are equivalent."""
+    notequiv = []
+    # Find all pairs that are not equivalent
+    for pair in itertools.combinations(args, 2):
+        if pair[0].compare(pair[1]):
+            print(f"{Colors.GREY}DEBUG: {pair[0].url} is equivalent to {pair[1].url}{Colors.END}")
+        else:
+            notequiv.append(pair)
+
+    # Decompress all, and check again
+    for d in {y for x in notequiv for y in x}:
+        d.decompress()
+    for pair in notequiv.copy():
+        if pair[0].compare(pair[1]):
+            # pylint: disable=line-too-long
+            print(f"{Colors.YELLOW}NOTE: {pair[0].url} is equivalent to {pair[1].url} when decompressed{Colors.END}")
+            notequiv.remove(pair)
+
+    # Extract all, and check again
+    for d in {y for x in notequiv for y in x}:
+        d.extract()
+    has_error = False
+    for pair in notequiv:
+        if pair[0].compare(pair[1]):
+            # pylint: disable=line-too-long
+            print(f"{Colors.ORANGE}WARN: {pair[0].url} is equivalent to {pair[1].url} when extracted{Colors.END}")
+        else:
+            has_error = True
+            # pylint: disable=line-too-long
+            print(f"{Colors.RED}ERROR: {pair[0].url} is not equivalent to {pair[1].url}!{Colors.END}")
+
+    return has_error
+
+def main():
+    """Main function."""
+    has_error = False
+    with open("substitutes", 'r', encoding="utf-8") as f:
+        for line in f.readlines():
+            urls = line.strip().split(' ')
+            distfiles = []
+            for i, url in enumerate(urls):
+                distfiles.append(Distfile(i, url))
+            for distfile in distfiles:
+                distfile.download()
+            if check(*distfiles):
+                has_error = True
+    sys.exit(has_error)
+
+if __name__ == "__main__":
+    main()

download-distfiles.sh

@@ -15,32 +15,47 @@ download_source() {
     local dest_path="${distfiles}/${fname}"
     if ! [ -e "${dest_path}" ]; then
         echo "Downloading ${fname}"
-        curl --fail --location "${url}" --output "${dest_path}"
+        curl --fail --location "${url}" --output "${dest_path}" || true
     fi
+}
+
+check_source() {
+    local distfiles="${1}"
+    local url="${2}"
+    local checksum="${3}"
+    local fname="${4}"
+    # Default to basename of url if not given
+    fname="${fname:-$(basename "${url}")}"
+
+    local dest_path="${distfiles}/${fname}"
     echo "${checksum}  ${dest_path}" | sha256sum -c
 }
 
-download_for_sys() {
-    local sysdir="${1}"
-    local distfiles="${sysdir}/distfiles"
-
-    mkdir -p "${distfiles}"
-
-    local entry
-    for entry in "${sysdir}"/*; do
-        [ -e "${entry}/sources" ] || continue
-
-        local line
-        # shellcheck disable=SC2162
-        while read line; do
-            # This is intentional - we want to split out ${line} into separate arguments.
-            # shellcheck disable=SC2086
-            download_source "${distfiles}" ${line}
-        done < "${entry}/sources"
-    done
-}
-
 set -e
+
 cd "$(dirname "$(readlink -f "$0")")"
-download_for_sys sysa
-download_for_sys sysc
+mkdir -p distfiles
+
+# First, try to download anything missing - ignore failing mirrors
+for entry in steps/*; do
+    [ -e "${entry}/sources" ] || continue
+
+    # shellcheck disable=SC2162
+    while read line; do
+        # This is intentional - we want to split out ${line} into separate arguments.
+        # shellcheck disable=SC2086
+        download_source distfiles ${line}
+    done < "${entry}/sources"
+done
+
+# Then, check if everything has been obtained at least once
+for entry in steps/*; do
+    [ -e "${entry}/sources" ] || continue
+
+    # shellcheck disable=SC2162
+    while read line; do
+        # This is intentional - we want to split out ${line} into separate arguments.
+        # shellcheck disable=SC2086
+        check_source distfiles ${line}
+    done < "${entry}/sources"
+done

lib/generator.py

@@ -0,0 +1,357 @@
+#!/usr/bin/env python3
+"""
+This file contains all code required to generate the boot image for live-bootstrap
+"""
+# SPDX-License-Identifier: GPL-3.0-or-later
+# SPDX-FileCopyrightText: 2022-2023 Dor Askayo <dor.askayo@gmail.com>
+# SPDX-FileCopyrightText: 2021 Andrius Štikonas <andrius@stikonas.eu>
+# SPDX-FileCopyrightText: 2021 Melg Eight <public.melg8@gmail.com>
+# SPDX-FileCopyrightText: 2021-23 fosslinux <fosslinux@aussies.space>
+
+import hashlib
+import os
+import shutil
+import tarfile
+import traceback
+import requests
+
+# pylint: disable=too-many-instance-attributes
+class Generator():
+    """
+    Class responsible for generating the basic media to be consumed.
+    """
+
+    git_dir = os.path.join(os.path.dirname(os.path.join(__file__)), '..')
+    distfiles_dir = os.path.join(git_dir, 'distfiles')
+
+    def __init__(self, arch, external_sources, early_preseed, repo_path):
+        self.arch = arch
+        self.early_preseed = early_preseed
+        self.external_sources = external_sources
+        self.repo_path = repo_path
+        self.source_manifest = self.get_source_manifest(not self.external_sources)
+        self.early_source_manifest = self.get_source_manifest(True)
+        self.target_dir = None
+        self.external_dir = None
+
+    def reuse(self, target):
+        """
+        Reuse a previously prepared bwrap environment for further stages.
+        """
+        self.target_dir = target.path
+        self.external_dir = os.path.join(self.target_dir, 'external')
+        self.distfiles()
+
+    def prepare(self, target, using_kernel=False, kernel_bootstrap=False, target_size=0):
+        """
+        Prepare basic media of live-bootstrap.
+        /steps -- contains steps to be built
+        / -- contains seed to allow steps to be built, containing custom
+             scripts and stage0-posix
+        """
+        self.target_dir = target.path
+        self.external_dir = os.path.join(self.target_dir, 'external')
+
+        # We use ext3 here; ext4 actually has a variety of extensions that
+        # have been added with varying levels of recency
+        # Linux 4.9.10 does not support a bunch of them
+        # Attempting to disable extensions that a particular e2fsprogs
+        # is *unaware* of causes the filesystem creation to fail
+        # We could hypothetically detect e2fsprogs version and create an
+        # argument matrix ... or we could just use ext3 instead which
+        # is effectively universally the same
+        if kernel_bootstrap:
+            self.target_dir = os.path.join(self.target_dir, 'init')
+            os.mkdir(self.target_dir)
+
+            if not self.repo_path and not self.external_sources:
+                self.external_dir = os.path.join(self.target_dir, 'external')
+        elif using_kernel:
+            self.target_dir = os.path.join(self.target_dir, 'disk')
+            self.external_dir = os.path.join(self.target_dir, 'external')
+
+        os.makedirs(self.external_dir)
+
+        if self.early_preseed:
+            # Extract tar containing preseed
+            with tarfile.open(self.early_preseed, "r") as seed:
+                seed.extractall(self.target_dir)
+            shutil.copy2(os.path.join(self.git_dir, 'seed', 'preseeded.kaem'),
+                         os.path.join(self.target_dir, 'kaem.x86'))
+        else:
+            self.stage0_posix(kernel_bootstrap)
+            self.seed()
+
+        self.steps()
+
+        self.distfiles()
+
+        if self.repo_path:
+            repo_dir = os.path.join(self.external_dir, 'repo-preseeded')
+            shutil.copytree(self.repo_path, repo_dir)
+
+        if kernel_bootstrap:
+            self.create_builder_hex0_disk_image(self.target_dir + '.img', target_size)
+
+            if self.repo_path or self.external_sources:
+                mkfs_args = ['-d', os.path.join(target.path, 'external')]
+                target.add_disk("external", filesystem="ext3", mkfs_args=mkfs_args)
+        elif using_kernel:
+            mkfs_args = ['-F', '-d', os.path.join(target.path, 'disk')]
+            target.add_disk("disk",
+                            filesystem="ext3",
+                            size=(str(target_size) + "M") if target_size else "16G",
+                            bootable=True,
+                            mkfs_args=mkfs_args)
+
+    def steps(self):
+        """Copy in steps."""
+        self.get_packages()
+
+        shutil.copytree(os.path.join(self.git_dir, 'steps'), os.path.join(self.target_dir, 'steps'))
+
+    def stage0_posix(self, kernel_bootstrap=False):
+        """Copy in all of the stage0-posix"""
+        stage0_posix_base_dir = os.path.join(self.git_dir, 'seed', 'stage0-posix')
+        for entry in os.listdir(stage0_posix_base_dir):
+            if kernel_bootstrap and entry == 'bootstrap-seeds':
+                continue
+            orig = os.path.join(stage0_posix_base_dir, entry)
+            target = os.path.join(self.target_dir, entry)
+            if os.path.isfile(orig):
+                shutil.copy2(orig, target)
+            else:
+                shutil.copytree(orig, target)
+
+        if not kernel_bootstrap:
+            arch = stage0_arch_map.get(self.arch, self.arch)
+            kaem_optional_seed = os.path.join(self.git_dir, 'seed', 'stage0-posix',
+                                              'bootstrap-seeds', 'POSIX', arch,
+                                              'kaem-optional-seed')
+            shutil.copy2(kaem_optional_seed, os.path.join(self.target_dir, 'init'))
+
+    def seed(self):
+        """Copy in extra seed files"""
+        seed_dir = os.path.join(self.git_dir, 'seed')
+        for entry in os.listdir(seed_dir):
+            if os.path.isfile(os.path.join(seed_dir, entry)):
+                shutil.copy2(os.path.join(seed_dir, entry), os.path.join(self.target_dir, entry))
+
+    def distfiles(self):
+        """Copy in distfiles"""
+        def copy_no_network_distfiles(out, early):
+            # Note that "no disk" implies "no network" for kernel bootstrap mode
+            manifest = self.early_source_manifest if early else self.source_manifest
+            for file in manifest:
+                file = file[3].strip()
+                shutil.copy2(os.path.join(self.distfiles_dir, file),
+                             os.path.join(out, file))
+
+        early_distfile_dir = os.path.join(self.target_dir, 'external', 'distfiles')
+        main_distfile_dir = os.path.join(self.external_dir, 'distfiles')
+
+        if early_distfile_dir != main_distfile_dir:
+            os.makedirs(early_distfile_dir, exist_ok=True)
+            copy_no_network_distfiles(early_distfile_dir, True)
+
+        if self.external_sources:
+            shutil.copytree(self.distfiles_dir, main_distfile_dir, dirs_exist_ok=True)
+        else:
+            os.mkdir(main_distfile_dir)
+            copy_no_network_distfiles(main_distfile_dir, False)
+
+    @staticmethod
+    def output_dir(srcfs_file, dirpath):
+        """Add a directory to srcfs file system"""
+        srcline = f"src 0 {dirpath}\n"
+        srcfs_file.write(srcline.encode())
+
+    @staticmethod
+    def output_file(srcfs_file, filepath):
+        """Add a file to srcfs file system"""
+        srcline = f"src {os.path.getsize(filepath)} {filepath}\n"
+        srcfs_file.write(srcline.encode())
+        with open(filepath, 'rb') as srcfile:
+            srcfs_file.write(srcfile.read())
+
+    def output_tree(self, srcfs_file, treepath):
+        """Add a tree of files to srcfs file system"""
+        self.output_dir(srcfs_file, treepath)
+        for root, dirs, files in os.walk(treepath):
+            if ".git" in root:
+                continue
+            for dirpath in dirs:
+                if ".git" in dirpath:
+                    continue
+                self.output_dir(srcfs_file, os.path.join(root, dirpath))
+
+            for filepath in files:
+                if ".git" in filepath:
+                    continue
+                self.output_file(srcfs_file, os.path.join(root, filepath))
+
+    def append_srcfs(self, image_file):
+        """Append srcfs file system to disk image"""
+        save_cwd = os.getcwd()
+
+        os.chdir(self.target_dir)
+        self.output_tree(image_file, '.')
+
+        # Add commands to kick off stage0-posix
+        cmd = ' '.join(['src',
+                        '0',
+                        '/bootstrap-seeds\n'])
+        image_file.write(cmd.encode())
+        cmd = ' '.join(['src',
+                        '0',
+                        '/bootstrap-seeds/POSIX\n'])
+        image_file.write(cmd.encode())
+        cmd = ' '.join(['src',
+                        '0',
+                        '/bootstrap-seeds/POSIX/x86\n'])
+        image_file.write(cmd.encode())
+        cmd = ' '.join(['hex0',
+                        '/x86/hex0_x86.hex0',
+                        '/bootstrap-seeds/POSIX/x86/hex0-seed\n'])
+        image_file.write(cmd.encode())
+        cmd = ' '.join(['hex0',
+                        '/x86/kaem-minimal.hex0',
+                        '/bootstrap-seeds/POSIX/x86/kaem-optional-seed\n'])
+        image_file.write(cmd.encode())
+        cmd = ' '.join(['hex0',
+                        '/x86/kaem-minimal.hex0',
+                        '/init\n'])
+        image_file.write(cmd.encode())
+        cmd = ' '.join(['/bootstrap-seeds/POSIX/x86/kaem-optional-seed', '/kaem.x86\n'])
+        image_file.write(cmd.encode())
+        os.chdir(save_cwd)
+
+    def create_builder_hex0_disk_image(self, image_file_name, size):
+        """Create builder-hex0 disk image"""
+        with open(image_file_name, 'ab') as image_file:
+            # Compile and write stage1 binary seed
+            with open(os.path.join('builder-hex0', 'builder-hex0-x86-stage1.hex0'),
+                      encoding="utf-8") as infile:
+                for line in infile:
+                    image_file.write(bytes.fromhex(line.split('#')[0].split(';')[0].strip()))
+            # Append stage2 hex0 source
+            with open(os.path.join('builder-hex0', 'builder-hex0-x86-stage2.hex0'),
+                      encoding="utf-8") as infile:
+                image_file.write(infile.read().encode())
+            # Pad to next sector
+            current_size = os.stat(image_file_name).st_size
+            while current_size % 512 != 0:
+                image_file.write(b'\0')
+                current_size += 1
+            self.append_srcfs(image_file)
+
+        current_size = os.stat(image_file_name).st_size
+
+        megabyte = 1024 * 1024
+        # fill file with zeros up to next megabyte
+        extra = current_size % megabyte
+        round_up = megabyte - extra
+        with open(image_file_name, 'ab') as image_file:
+            image_file.write(b'\0' * round_up)
+        current_size += round_up
+
+        # extend file up to desired size
+        if current_size < size * megabyte:
+            with open(image_file_name, 'ab') as image_file:
+                image_file.truncate(size * megabyte)
+
+    @staticmethod
+    def check_file(file_name, expected_hash):
+        """Check hash of downloaded source file."""
+        with open(file_name, "rb") as downloaded_file:
+            downloaded_content = downloaded_file.read() # read entire file as bytes
+        readable_hash = hashlib.sha256(downloaded_content).hexdigest()
+        if expected_hash == readable_hash:
+            return
+        raise ValueError(f"Checksum mismatch for file {os.path.basename(file_name)}:\n\
+expected: {expected_hash}\n\
+actual:   {readable_hash}\n\
+When in doubt, try deleting the file in question -- it will be downloaded again when running \
+this script the next time")
+
+    @staticmethod
+    def download_file(url, directory, file_name, silent=False):
+        """
+        Download a single source archive.
+        """
+        abs_file_name = os.path.join(directory, file_name)
+
+        # Create a directory for downloaded file
+        if not os.path.isdir(directory):
+            os.mkdir(directory)
+
+        # Actually download the file
+        headers = {
+                "Accept-Encoding": "identity",
+                "User-Agent": "curl/7.88.1"
+        }
+        if not os.path.isfile(abs_file_name):
+            if not silent:
+                print(f"Downloading: {file_name}")
+            response = requests.get(url, allow_redirects=True, stream=True,
+                    headers=headers, timeout=20)
+            if response.status_code == 200:
+                with open(abs_file_name, 'wb') as target_file:
+                    target_file.write(response.raw.read())
+            else:
+                raise requests.HTTPError("Download failed: HTTP " +
+                    str(response.status_code) + " " + response.reason)
+        return abs_file_name
+
+    def get_packages(self):
+        """Prepare remaining sources"""
+        for line in self.source_manifest:
+            try:
+                path = self.download_file(line[2], line[1], line[3])
+            except requests.HTTPError:
+                print(traceback.format_exc())
+        for line in self.source_manifest:
+            path = os.path.join(line[1], line[3])
+            self.check_file(path, line[0])
+
+    @classmethod
+    def get_source_manifest(cls, pre_network=False):
+        """
+        Generate a source manifest for the system.
+        """
+        entries = []
+        directory = os.path.relpath(cls.distfiles_dir, cls.git_dir)
+
+        # Find all source files
+        steps_dir = os.path.join(cls.git_dir, 'steps')
+        with open(os.path.join(steps_dir, 'manifest'), 'r', encoding="utf_8") as file:
+            for line in file:
+                if pre_network and line.strip().startswith("improve: ") and "network" in line:
+                    break
+
+                if not line.strip().startswith("build: "):
+                    continue
+
+                step = line.split(" ")[1].split("#")[0].strip()
+                sourcef = os.path.join(steps_dir, step, "sources")
+                if os.path.exists(sourcef):
+                    # Read sources from the source file
+                    with open(sourcef, "r", encoding="utf_8") as sources:
+                        for source in sources.readlines():
+                            source = source.strip().split(" ")
+
+                            if len(source) > 2:
+                                file_name = source[2]
+                            else:
+                                # Automatically determine file name based on URL.
+                                file_name = os.path.basename(source[0])
+
+                            entry = (source[1], directory, source[0], file_name)
+                            if entry not in entries:
+                                entries.append(entry)
+
+        return entries
+
+stage0_arch_map = {
+    "amd64": "AMD64",
+}

lib/sysgeneral.py

@@ -1,132 +0,0 @@
-#!/usr/bin/env python3
-"""
-This file contains a few functions to be shared by all Sys* classes
-"""
-
-# SPDX-FileCopyrightText: 2022-2023 Dor Askayo <dor.askayo@gmail.com>
-# SPDX-FileCopyrightText: 2021-23 fosslinux <fosslinux@aussies.space>
-# SPDX-FileCopyrightText: 2021 Andrius Štikonas <andrius@stikonas.eu>
-# SPDX-License-Identifier: GPL-3.0-or-later
-
-import os
-import hashlib
-import glob
-import subprocess
-
-import requests
-
-class SysGeneral:
-    """
-    A class from which all Sys* class are extended.
-    Contains functions used in all Sys*
-    """
-
-    # All of these are variables defined in the individual Sys* classes
-    cache_dir = None
-    base_dir = None
-    git_dir = None
-    sys_dir = None
-    initramfs_path = None
-    tmp_dir = None
-
-    def check_file(self, file_name, expected_hash):
-        """Check hash of downloaded source file."""
-        with open(file_name, "rb") as downloaded_file:
-            downloaded_content = downloaded_file.read() # read entire file as bytes
-        readable_hash = hashlib.sha256(downloaded_content).hexdigest()
-        if expected_hash == readable_hash:
-            return
-        raise ValueError(f"Checksum mismatch for file {os.path.basename(file_name)}:\n\
-expected: {expected_hash}\n\
-actual:   {readable_hash}\n\
-When in doubt, try deleting the file in question -- it will be downloaded again when running \
-this script the next time")
-
-    def download_file(self, url, directory, file_name):
-        """
-        Download a single source archive.
-        """
-        abs_file_name = os.path.join(directory, file_name)
-
-        # Create a directory for downloaded file
-        if not os.path.isdir(directory):
-            os.mkdir(directory)
-
-        # Actually download the file
-        headers = {
-                "Accept-Encoding": "identity"
-        }
-        if not os.path.isfile(abs_file_name):
-            print(f"Downloading: {file_name}")
-            response = requests.get(url, allow_redirects=True, stream=True,
-                    headers=headers, timeout=20)
-            if response.status_code == 200:
-                with open(abs_file_name, 'wb') as target_file:
-                    target_file.write(response.raw.read())
-            else:
-                raise requests.HTTPError("Download failed.")
-        return abs_file_name
-
-    def get_packages(self, source_manifest):
-        """Prepare remaining sources"""
-        for line in source_manifest.split("\n"):
-            line = line.strip().split(" ")
-
-            path = self.download_file(line[2], line[1], line[3])
-            self.check_file(path, line[0])
-
-    @classmethod
-    def get_source_manifest(cls):
-        """
-        Generate a source manifest for the system.
-        """
-        manifest_lines = []
-        directory = os.path.relpath(cls.cache_dir, cls.git_dir)
-
-        # Find all source files
-        for file in os.listdir(cls.sys_dir):
-            if os.path.isdir(os.path.join(cls.sys_dir, file)):
-                sourcef = os.path.join(cls.sys_dir, file, "sources")
-                if os.path.exists(sourcef):
-                    # Read sources from the source file
-                    with open(sourcef, "r", encoding="utf_8") as sources:
-                        for line in sources.readlines():
-                            line = line.strip().split(" ")
-
-                            if len(line) > 2:
-                                file_name = line[2]
-                            else:
-                                # Automatically determine file name based on URL.
-                                file_name = os.path.basename(line[0])
-
-                            manifest_lines.append(f"{line[1]} {directory} {line[0]} {file_name}")
-
-        return "\n".join(manifest_lines)
-
-    def make_initramfs(self):
-        """Package binary bootstrap seeds and sources into initramfs."""
-        self.initramfs_path = os.path.join(self.tmp_dir, 'initramfs')
-
-        # Create a list of files to go within the initramfs
-        file_list = glob.glob(os.path.join(self.tmp_dir, '**'), recursive=True)
-
-        # Use built-in removeprefix once we can use Python 3.9
-        def remove_prefix(text, prefix):
-            if text.startswith(prefix):
-                return text[len(prefix):]
-            return text  # or whatever
-
-        file_list = [remove_prefix(f, self.tmp_dir + os.sep) for f in file_list]
-
-        # Create the initramfs
-        with open(self.initramfs_path, "w", encoding="utf_8") as initramfs:
-            # pylint: disable=consider-using-with
-            cpio = subprocess.Popen(
-                    ["cpio", "--format", "newc", "--create",
-                        "--directory", self.tmp_dir],
-                     stdin=subprocess.PIPE, stdout=initramfs)
-            cpio.communicate(input='\n'.join(file_list).encode())
-
-stage0_arch_map = {
-    "amd64": "AMD64",
-}

lib/target.py

@@ -0,0 +1,61 @@
+#!/usr/bin/env python3
+
+# SPDX-FileCopyrightText: 2023 fosslinux <fosslinux@aussies.space>
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+"""
+Contains a class that represents a target directory
+"""
+
+import enum
+import os
+
+from lib.utils import mount, create_disk
+
+class TargetType(enum.Enum):
+    """Different types of target dirs we can have"""
+    NONE = 0
+    TMPFS = 1
+
+class Target:
+    """
+    Represents a target directory
+    """
+
+    _disks = {}
+    _mountpoints = {}
+
+    def __init__(self, path="target"):
+        self.path = os.path.abspath(path)
+        self._type = TargetType.NONE
+
+        if not os.path.exists(self.path):
+            os.mkdir(self.path)
+
+    def tmpfs(self, size="8G"):
+        """Mount a tmpfs"""
+        print(f"Mounting tmpfs on {self.path}")
+        mount("tmpfs", self.path, "tmpfs", f"size={size}")
+        self._type = TargetType.TMPFS
+
+    # pylint: disable=too-many-arguments
+    def add_disk(self,
+                 name,
+                 size="16G",
+                 filesystem="ext4",
+                 tabletype="msdos",
+                 bootable=False,
+                 mkfs_args=None):
+        """Add a disk"""
+        disk_path = os.path.join(self.path, f"{name}.img")
+        create_disk(disk_path,
+                    tabletype,
+                    filesystem,
+                    size,
+                    bootable,
+                    mkfs_args)
+        self._disks[name] = disk_path
+
+    def get_disk(self, name):
+        """Get the path to a device of a disk"""
+        return self._disks.get(name)

lib/tmpdir.py

@@ -1,99 +0,0 @@
-#!/usr/bin/env python3
-
-# SPDX-FileCopyrightText: 2023 fosslinux <fosslinux@aussies.space>
-# SPDX-License-Identifier: GPL-3.0-or-later
-
-"""
-Contains a class that represents a tmpdir
-"""
-
-import enum
-import getpass
-import os
-import shutil
-
-from lib.utils import mount, umount, create_disk, run
-
-class TmpType(enum.Enum):
-    """Different types of tmpdirs we can have"""
-    NONE = 0
-    TMPFS = 1
-
-class Tmpdir:
-    """
-    Represents a tmpdir
-    """
-
-    _syses = {}
-    _disks = {}
-    _disk_filesystems = {}
-    _mountpoints = {}
-
-    def __init__(self, preserve, path="tmp"):
-        self.path = os.path.abspath(path)
-        self.preserve = preserve
-        self._type = TmpType.NONE
-
-        if not os.path.exists(self.path):
-            os.mkdir(self.path)
-
-    def __del__(self):
-        for path in self._mountpoints:
-            print(f"Unmounting {path}")
-            umount(path)
-
-        if not self.preserve:
-            for disk in self._disks.values():
-                print(f"Detaching {disk}")
-                run("sudo", "losetup", "-d", disk)
-
-            if self._type == TmpType.TMPFS:
-                print(f"Unmounting tmpdir from {self.path}")
-                umount(self.path)
-
-            print(f"Removing {self.path}")
-            shutil.rmtree(self.path, ignore_errors=True)
-
-    def tmpfs(self, size="8G"):
-        """Mount a tmpfs"""
-        print(f"Mounting tmpfs on {self.path}")
-        mount("tmpfs", self.path, "tmpfs", f"size={size}")
-        self._type = TmpType.TMPFS
-
-    def add_sys(self, name, subdir=None):
-        """Create a subdirectory and register a sys"""
-        if subdir is None:
-            subdir = name
-        sys_path = os.path.join(self.path, name)
-        if not os.path.exists(sys_path):
-            os.mkdir(sys_path)
-        return sys_path
-
-    def add_disk(self, name, size="8G", filesystem="ext4"):
-        """Add a disk"""
-        disk_path = os.path.join(self.path, f"{name}.img")
-        self._disks[name] = create_disk(disk_path, "msdos", filesystem, size)
-        self._disk_filesystems[name] = filesystem
-        # Allow executing user to access it
-        run("sudo", "chown", getpass.getuser(), self._disks[name])
-
-    def mount_disk(self, name, mountpoint=None):
-        """Mount the disk"""
-        if mountpoint is None:
-            mountpoint = f"{name}_mnt"
-        mountpoint = os.path.join(self.path, mountpoint)
-        os.mkdir(mountpoint)
-        mount(self._disks[name] + "p1", mountpoint, self._disk_filesystems[name])
-        # Allow executing user to access it
-        run("sudo", "chown", getpass.getuser(), mountpoint)
-        self._mountpoints[name] = mountpoint
-        return mountpoint
-
-    def umount_disk(self, name):
-        """Unmount a disk"""
-        umount(self._mountpoints[name])
-        del self._mountpoints[name]
-
-    def get_disk(self, name):
-        """Get the path to a device of a disk"""
-        return self._disks[name]

lib/utils.py

@@ -25,27 +25,33 @@ def run(*args, **kwargs):
         print("Bootstrapping failed")
         sys.exit(1)
 
-def create_disk(image, disk_type, fs_type, size):
+def run_as_root(*args, **kwargs):
+    """A helper for run that invokes sudo when unprivileged"""
+    if os.geteuid() != 0:
+        return run("sudo", *args, **kwargs)
+    return run(*args, **kwargs)
+
+# pylint: disable=too-many-arguments
+def create_disk(image, disk_type, fs_type, size, bootable=False, mkfs_args=None):
     """Create a disk image, with a filesystem on it"""
+    if mkfs_args is None:
+        mkfs_args = []
     run('truncate', '-s', size, image)
-    # First find the device we will use, then actually use it
-    loop_dev = run('sudo', 'losetup', '-f', capture_output=True).stdout.decode().strip()
-    run('sudo', 'losetup', '-P', loop_dev, image)
     # Create the partition
     if disk_type != "none":
-        run('sudo', 'parted', '--script', image, 'mklabel', disk_type, 'mkpart',
-                'primary', 'ext4', '0%', '100%')
-        run('sudo', 'partprobe', loop_dev)
-        run('sudo', 'mkfs.' + fs_type, loop_dev + "p1")
-    return loop_dev
+        # 1 GiB if bootable, 1 MiB otherwise
+        offset = str(1024 * 1024 * (1024 if bootable else 1))
+        run('parted', '--script', image, 'mklabel', disk_type, 'mkpart',
+            'primary', fs_type, offset + 'B', '100%')
+        run('mkfs.' + fs_type, image, '-E', 'offset=' + offset, *mkfs_args)
 
 def mount(source, target, fs_type, options='', **kwargs):
     """Mount filesystem"""
-    run('sudo', 'mount', source, target, '-t', fs_type, '-o', options, **kwargs)
+    run_as_root('mount', source, target, '-t', fs_type, '-o', options, **kwargs)
 
 def umount(target, **kwargs):
     """Unmount filesystem"""
-    run('sudo', 'umount', '--recursive', target, **kwargs)
+    run_as_root('umount', '--recursive', target, **kwargs)
 
 def copytree(src, dst, ignore=shutil.ignore_patterns('*.git*')):
     """Copy directory tree into another directory"""

rootfs.py

@@ -12,40 +12,49 @@ you can run bootstap inside chroot.
 # SPDX-FileCopyrightText: 2021 Bastian Bittorf <bb@npl.de>
 # SPDX-FileCopyrightText: 2021 Melg Eight <public.melg8@gmail.com>
 # SPDX-FileCopyrightText: 2021-23 fosslinux <fosslinux@aussies.space>
+# SPDX-FileCopyrightText: 2023-24 Gábor Stefanik <netrolller.3d@gmail.com>
 
 import argparse
 import os
 
-from sysa import SysA
-from sysc import SysC
-from lib.utils import run
-from lib.sysgeneral import stage0_arch_map
-from lib.tmpdir import Tmpdir
+from lib.utils import run, run_as_root
+from lib.target import Target
+from lib.generator import Generator, stage0_arch_map
 
 def create_configuration_file(args):
     """
     Creates bootstrap.cfg file which would contain options used to
     customize bootstrap.
     """
-    config_path = os.path.join('sysa', 'bootstrap.cfg')
+    config_path = os.path.join('steps', 'bootstrap.cfg')
     with open(config_path, "w", encoding="utf_8") as config:
+        config.write(f"ARCH={args.arch}\n")
+        config.write(f"ARCH_DIR={stage0_arch_map.get(args.arch, args.arch)}\n")
         config.write(f"FORCE_TIMESTAMPS={args.force_timestamps}\n")
         config.write(f"CHROOT={args.chroot or args.bwrap}\n")
-        config.write(f"CHROOT_ONLY_SYSA={args.bwrap}\n")
         config.write(f"UPDATE_CHECKSUMS={args.update_checksums}\n")
         config.write(f"JOBS={args.cores}\n")
-        config.write("DISK=sda1\n")
+        config.write(f"SWAP_SIZE={args.swap}\n")
+        config.write(f"FINAL_JOBS={args.cores}\n")
+        config.write(f"INTERNAL_CI={args.internal_ci or False}\n")
+        config.write(f"INTERACTIVE={args.interactive}\n")
+        config.write(f"BARE_METAL={args.bare_metal or (args.qemu and args.interactive)}\n")
         if (args.bare_metal or args.qemu) and not args.kernel:
+            if args.repo or args.external_sources:
+                config.write("DISK=sdb1\n")
+            else:
+                config.write("DISK=sda\n")
             config.write("KERNEL_BOOTSTRAP=True\n")
         else:
+            config.write("DISK=sda1\n")
             config.write("KERNEL_BOOTSTRAP=False\n")
         config.write(f"BUILD_KERNELS={args.update_checksums or args.build_kernels}\n")
 
-# pylint: disable=too-many-statements
+# pylint: disable=too-many-statements,too-many-branches
 def main():
     """
     A few command line arguments to customize bootstrap.
-    This function also creates SysA object which prepares directory
+    This function also creates object which prepares directory
     structure with bootstrap seeds and all sources.
     """
     parser = argparse.ArgumentParser()
@@ -55,11 +64,9 @@ def main():
                         action="store_true")
     parser.add_argument("-bw", "--bwrap", help="Run inside a bwrap sandbox",
                         action="store_true")
-    parser.add_argument("-p", "--preserve", help="Do not remove temporary dir",
-                        action="store_true")
-    parser.add_argument("-t", "--tmpdir", help="Temporary directory",
-                        default="tmp")
-    parser.add_argument("--tmpfs", help="Use a tmpfs on tmpdir",
+    parser.add_argument("-t", "--target", help="Target directory",
+                        default="target")
+    parser.add_argument("--tmpfs", help="Use a tmpfs on target",
                         action="store_true")
     parser.add_argument("--tmpfs-size", help="Size of the tmpfs",
                         default="8G")
@@ -69,21 +76,27 @@ def main():
                         help="Force all files timestamps to be 0 unix time",
                         action="store_true")
     parser.add_argument("--update-checksums",
-                        help="Update checksum files.",
+                        help="Update checksum files",
                         action="store_true")
     parser.add_argument("--external-sources",
-                        help="Download sources externally from live-bootstrap.",
+                        help="Download sources externally from live-bootstrap",
                         action="store_true")
     parser.add_argument("--build-kernels",
-                        help="Also build kernels in chroot and bwrap builds.",
+                        help="Also build kernels in chroot and bwrap builds",
                         action="store_true")
     parser.add_argument("--no-create-config",
                         help="Do not automatically create config file",
                         action="store_true")
+    parser.add_argument("-i", "--interactive",
+                        help="Use interactive prompts to resolve issues during bootstrap",
+                        action="store_true")
     parser.add_argument("-r", "--repo",
-                        help="Path to prebuilt binary packages.", nargs=None)
+                        help="Path to prebuilt binary packages", nargs=None)
     parser.add_argument("--early-preseed",
-                        help="Skip early stages of live-bootstrap.", nargs=None)
+                        help="Skip early stages of live-bootstrap", nargs=None)
+    parser.add_argument("--internal-ci", help="INTERNAL for github CI")
+    parser.add_argument("-s", "--swap", help="Swap space to allocate in Linux",
+                        default=0)
 
     # QEMU arguments
     parser.add_argument("-q", "--qemu", help="Use QEMU",
@@ -92,7 +105,9 @@ def main():
                         default="qemu-system-x86_64")
     parser.add_argument("-qr", "--qemu-ram", help="Memory (in megabytes) allocated to QEMU VM",
                         default=4096)
-    parser.add_argument("-qk", "--kernel", help="Custom sysa kernel to use")
+    parser.add_argument("-qs", "--target-size", help="Size of the target image (for QEMU only)",
+                        default="16G")
+    parser.add_argument("-qk", "--kernel", help="Custom early kernel to use")
 
     parser.add_argument("-b", "--bare-metal", help="Build images for bare metal",
                         action="store_true")
@@ -120,9 +135,9 @@ def main():
 
     # Arch validation
     if args.arch != "x86":
-        raise ValueError("Only x86 is supported at the moment.")
+        print("Only x86 is supported at the moment, other arches are for development only.")
 
-    # Tmp validation
+    # Tmpfs validation
     if args.bwrap and args.tmpfs:
         raise ValueError("tmpfs cannot be used with bwrap.")
 
@@ -130,129 +145,151 @@ def main():
     if int(args.cores) < 1:
         raise ValueError("Must use one or more cores.")
 
-    # bootstrap.cfg
-    if args.bare_metal:
-        args.no_create_config = True
+    # Target image size validation
+    if args.qemu:
+        if int(str(args.target_size).rstrip('gGmM')) < 1:
+            raise ValueError("Please specify a positive target size for qemu.")
+        args.target_size = (int(str(args.target_size).rstrip('gGmM')) *
+            (1024 if str(args.target_size).lower().endswith('g') else 1))
+    else:
+        args.target_size = 0
 
+    # Swap file size validation
+    if args.qemu or args.bare_metal:
+        args.swap = (int(str(args.swap).rstrip('gGmM')) *
+            (1024 if str(args.swap).lower().endswith('g') else 1))
+    else:
+        args.swap = 0
+
+    # Set constant umask
+    os.umask(0o022)
+
+    # bootstrap.cfg
     try:
-        os.remove(os.path.join('sysa', 'bootstrap.cfg'))
+        os.remove(os.path.join('steps', 'bootstrap.cfg'))
     except FileNotFoundError:
         pass
     if not args.no_create_config:
         create_configuration_file(args)
     else:
-        with open(os.path.join('sysa', 'bootstrap.cfg'), 'a', encoding='UTF-8'):
+        with open(os.path.join('steps', 'bootstrap.cfg'), 'a', encoding='UTF-8'):
             pass
 
-    # tmpdir
-    tmpdir = Tmpdir(path=args.tmpdir, preserve=args.preserve)
+    # target
+    target = Target(path=args.target)
     if args.tmpfs:
-        tmpdir.tmpfs(size=args.tmpfs_size)
+        target.tmpfs(size=args.tmpfs_size)
 
-    # sys
-    system_c = SysC(arch=args.arch, tmpdir=tmpdir,
-                    external_sources=args.external_sources)
-    system_a = SysA(arch=args.arch, early_preseed=args.early_preseed,
-                    tmpdir=tmpdir, external_sources=args.external_sources,
-                    repo_path=args.repo)
+    generator = Generator(arch=args.arch,
+                          external_sources=args.external_sources,
+                          repo_path=args.repo,
+                          early_preseed=args.early_preseed)
 
-    bootstrap(args, system_a, system_c, tmpdir)
+    bootstrap(args, generator, target, args.target_size)
 
-def bootstrap(args, system_a, system_c, tmpdir):
+def bootstrap(args, generator, target, size):
     """Kick off bootstrap process."""
-    print(f"Bootstrapping {args.arch} -- SysA")
+    print(f"Bootstrapping {args.arch}", flush=True)
     if args.chroot:
         find_chroot = """
 import shutil
 print(shutil.which('chroot'))
 """
-        chroot_binary = run('sudo', 'python3', '-c', find_chroot,
-                            capture_output=True).stdout.decode().strip()
+        chroot_binary = run_as_root('python3', '-c', find_chroot,
+                                    capture_output=True).stdout.decode().strip()
 
-        system_c.prepare(create_disk_image=False)
-        system_a.prepare(create_initramfs=False)
+        generator.prepare(target, using_kernel=False)
 
         arch = stage0_arch_map.get(args.arch, args.arch)
         init = os.path.join(os.sep, 'bootstrap-seeds', 'POSIX', arch, 'kaem-optional-seed')
-        run('sudo', 'env', '-i', 'PATH=/bin', chroot_binary, system_a.tmp_dir, init)
+        run_as_root('env', '-i', 'PATH=/bin', chroot_binary, generator.target_dir, init)
 
     elif args.bwrap:
-        system_c.prepare(create_disk_image=False)
-        system_a.prepare(create_initramfs=False)
+        init = '/init'
+        if not args.internal_ci or args.internal_ci == "pass1":
+            generator.prepare(target, using_kernel=False)
 
-        arch = stage0_arch_map.get(args.arch, args.arch)
-        init = os.path.join(os.sep, 'bootstrap-seeds', 'POSIX', arch, 'kaem-optional-seed')
-        run('bwrap', '--unshare-user',
-                     '--uid', '0',
-                     '--gid', '0',
-                     '--unshare-net',
-                     '--clearenv',
-                     '--setenv', 'PATH', '/usr/bin',
-                     '--bind', system_a.tmp_dir, '/',
-                     '--dir', '/dev',
-                     '--dev-bind', '/dev/null', '/dev/null',
-                     '--dev-bind', '/dev/zero', '/dev/zero',
-                     '--dev-bind', '/dev/random', '/dev/random',
-                     '--dev-bind', '/dev/urandom', '/dev/urandom',
-                     init)
+            arch = stage0_arch_map.get(args.arch, args.arch)
+            init = os.path.join(os.sep, 'bootstrap-seeds', 'POSIX', arch, 'kaem-optional-seed')
+        else:
+            generator.reuse(target)
 
-        run('bwrap', '--unshare-user',
-                     '--uid', '0',
-                     '--gid', '0',
-                     '--unshare-net' if args.external_sources else None,
-                     '--clearenv',
-                     '--setenv', 'PATH', '/usr/bin',
-                     '--bind', system_a.tmp_dir + "/sysc_image", '/',
-                     '--dir', '/dev',
-                     '--dev-bind', '/dev/null', '/dev/null',
-                     '--dev-bind', '/dev/zero', '/dev/zero',
-                     '--dev-bind', '/dev/random', '/dev/random',
-                     '--dev-bind', '/dev/urandom', '/dev/urandom',
-                     '--tmpfs', '/dev/shm',
-                     '--proc', '/proc',
-                     '--bind', '/sys', '/sys',
-                     '--tmpfs', '/tmp',
-                     '/init')
+        run('env', '-i', 'bwrap', '--unshare-user',
+                                  '--uid', '0',
+                                  '--gid', '0',
+                                  '--unshare-net' if args.external_sources else None,
+                                  '--setenv', 'PATH', '/usr/bin',
+                                  '--bind', generator.target_dir, '/',
+                                  '--dir', '/dev',
+                                  '--dev-bind', '/dev/null', '/dev/null',
+                                  '--dev-bind', '/dev/zero', '/dev/zero',
+                                  '--dev-bind', '/dev/random', '/dev/random',
+                                  '--dev-bind', '/dev/urandom', '/dev/urandom',
+                                  '--dev-bind', '/dev/ptmx', '/dev/ptmx',
+                                  '--dev-bind', '/dev/tty', '/dev/tty',
+                                  '--tmpfs', '/dev/shm',
+                                  '--proc', '/proc',
+                                  '--bind', '/sys', '/sys',
+                                  '--tmpfs', '/tmp',
+                                  init)
 
     elif args.bare_metal:
         if args.kernel:
-            system_c.prepare(create_disk_image=True)
-            system_a.prepare(create_initramfs=True)
+            generator.prepare(target, using_kernel=True, target_size=size)
+            path = os.path.join(args.target, os.path.relpath(generator.target_dir, args.target))
             print("Please:")
-            print("  1. Take tmp/sysa/initramfs and your kernel, boot using this.")
-            print("  2. Take tmp/sysc/disk.img and put this on a writable storage medium.")
+            print(f"  1. Take {path}/initramfs and your kernel, boot using this.")
+            print(f"  2. Take {path}/disk.img and put this on a writable storage medium.")
         else:
-            system_a.prepare(create_initramfs=True, kernel_bootstrap=True)
+            generator.prepare(target, kernel_bootstrap=True, target_size=size)
+            path = os.path.join(args.target, os.path.relpath(generator.target_dir, args.target))
             print("Please:")
-            print("  1. Take tmp/sysa/sysa.img and write it to a boot drive and then boot it.")
+            print(f"  1. Take {path}.img and write it to a boot drive and then boot it.")
 
     else:
         if args.kernel:
-            system_c.prepare(create_disk_image=True)
-            system_a.prepare(create_initramfs=True)
+            generator.prepare(target, using_kernel=True, target_size=size)
 
-            run(args.qemu_cmd,
+            arg_list = [
                 '-enable-kvm',
                 '-m', str(args.qemu_ram) + 'M',
                 '-smp', str(args.cores),
-                '-no-reboot',
-                '-hda', tmpdir.get_disk("sysc"),
+                '-drive', 'file=' + target.get_disk("disk") + ',format=raw'
+            ]
+            if target.get_disk("external") is not None:
+                arg_list += [
+                    '-drive', 'file=' + target.get_disk("external") + ',format=raw',
+                ]
+            arg_list += [
                 '-nic', 'user,ipv6=off,model=e1000',
                 '-kernel', args.kernel,
-                '-initrd', system_a.initramfs_path,
-                '-nographic',
-                '-append', 'console=ttyS0')
+                '-append',
+            ]
+            if args.interactive:
+                arg_list += ['consoleblank=0 earlyprintk=vga root=/dev/sda1 '
+                             'rootfstype=ext3 init=/init rw']
+            else:
+                arg_list += ['console=ttyS0 earlycon=uart8250,io,0x3f8,115200n8 '
+                             'root=/dev/sda1 rootfstype=ext3 init=/init rw']
         else:
-            system_a.prepare(create_initramfs=True, kernel_bootstrap=True)
-            run(args.qemu_cmd,
+            generator.prepare(target, kernel_bootstrap=True, target_size=size)
+            arg_list = [
                 '-enable-kvm',
-                '-m', "4G",
+                '-m', str(args.qemu_ram) + 'M',
                 '-smp', str(args.cores),
-                '-no-reboot',
-                '-drive', 'file=' + os.path.join(system_a.tmp_dir, 'sysa.img') + ',format=raw',
+                '-drive', 'file=' + generator.target_dir + '.img' + ',format=raw'
+            ]
+            if target.get_disk("external") is not None:
+                arg_list += [
+                    '-drive', 'file=' + target.get_disk("external") + ',format=raw',
+                ]
+            arg_list += [
                 '-machine', 'kernel-irqchip=split',
-                '-nic', 'user,ipv6=off,model=e1000',
-                '-nographic')
+                '-nic', 'user,ipv6=off,model=e1000'
+            ]
+        if not args.interactive:
+            arg_list += ['-no-reboot', '-nographic']
+        run(args.qemu_cmd, *arg_list)
 
 if __name__ == "__main__":
     main()

seed/after.kaem

@@ -9,17 +9,7 @@
 
 set -ex
 
-# Set commonly used variables
-sysa=/sysa
-distfiles=/sysa/distfiles
-prefix=/usr
-bindir=${prefix}/bin
-libdir=${libdir}/mes
-incdir=${prefix}/include
-srcdir=${prefix}/src
-TMPDIR=/tmp # tmpdir is needed for patch to work
-PATH=${bindir}
+PATH=/${ARCH_DIR}/bin
 
-cd ${sysa}
-
-exec bash run.sh
+catm seed-full.kaem /steps/bootstrap.cfg /steps/env seed.kaem
+kaem --file seed-full.kaem

seed/preseeded.kaem

@@ -0,0 +1,8 @@
+#!/bin/sh
+#
+# SPDX-FileCopyrightText: 2023 fosslinux <fosslinux@aussies.space>
+#
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+/script-generator /steps/manifest
+/usr/bin/kaem --file /preseed-jump.kaem

seed/script-generator.amd64.checksums

@@ -0,0 +1 @@
+1927a357c893d60952616146e788af25e637635b2862650914bf895d27f556a6  script-generator

seed/script-generator.c

@@ -0,0 +1,641 @@
+/*
+ * SPDX-FileCopyrightText: 2023 fosslinux <fosslinux@aussies.space>
+ *
+ * SPDX-License-Identifier: GPL-3.0-or-later
+ */
+
+#define MAX_TOKEN 64
+#define MAX_STRING 2048
+
+#include <bootstrappable.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+
+struct Token {
+	char *val;
+	struct Token *next;
+};
+typedef struct Token Token;
+
+#define TYPE_BUILD 1
+#define TYPE_IMPROVE 2
+#define TYPE_DEFINE 3
+#define TYPE_JUMP 4
+#define TYPE_UNINSTALL 5
+
+struct Directive {
+	Token *tok;
+	struct Directive *next;
+	int type;
+	char *arg; /* The primary argument */
+};
+typedef struct Directive Directive;
+
+/* Tokenizer. */
+
+/* Skip over a comment. */
+char consume_comment(FILE *in) {
+	/* Discard the rest of the line. */
+	char c = fgetc(in);
+	while (c != -1 && c != '\n')
+		c = fgetc(in);
+	return c;
+}
+
+char consume_line(FILE *in, Directive *directive) {
+	char c = fgetc(in);
+
+	/* Short-circuit if whole line is comment or blank line. */
+	if (c == '#') {
+		c = consume_comment(in);
+		return c;
+	} else if (c == '\n' || c == -1) {
+		return c;
+	}
+
+	/* Ok, we will have something to put here. */
+	directive->next = calloc(1, sizeof(Directive));
+	directive = directive->next;
+
+	Token *head = calloc(1, sizeof(Token));
+	Token *cur = head;
+	char *out;
+	int i = 0;
+	while (c != -1 && c != '\n') {
+		/* Initialize next token. */
+		cur->next = calloc(1, sizeof(Token));
+		cur = cur->next;
+		cur->val = calloc(MAX_TOKEN, sizeof(char));
+		out = cur->val;
+		/* Copy line to token until a space (or EOL/EOF) or comment is found. */
+		while (c != -1 && c != '\n' && c != ' ' && c != '#') {
+			out[0] = c;
+			out += 1;
+			c = fgetc(in);
+		}
+		/* Go to start of next token. */
+		if (c == ' ') {
+			c = fgetc(in);
+		}
+		/* Handle comment. */
+		if (c == '#') {
+			c = consume_comment(in);
+		}
+	}
+
+	/* Add information to directive. */
+	directive->tok = head->next;
+
+	return c;
+}
+
+Directive *tokenizer(FILE *in) {
+	Directive *head = calloc(1, sizeof(Directive));
+	Directive *cur = head;
+
+	char c;
+	while (c != -1) {
+		/*
+		 * Note that consume_line fills cur->next, not cur.
+		 * This avoids having an empty last Directive.
+		 */
+		c = consume_line(in, cur);
+		if (cur->next != NULL) {
+			cur = cur->next;
+		}
+	}
+	return head->next;
+}
+
+/* Config variables. */
+
+struct Variable {
+	char *name;
+	char *val;
+	struct Variable *next;
+};
+typedef struct Variable Variable;
+
+Variable *variables;
+
+Variable *load_config() {
+	FILE *config = fopen("/steps/bootstrap.cfg", "r");
+	/* File does not exist check. */
+	if (config == NULL) {
+		return NULL;
+	}
+
+	char *line = calloc(MAX_STRING, sizeof(char));
+	Variable *head = calloc(1, sizeof(Variable));
+	Variable *cur = head;
+	/* For each line... */
+	char *equals;
+	while (fgets(line, MAX_STRING, config) != 0) {
+		/* Weird M2-Planet behaviour. */
+		if (*line == 0) {
+			break;
+		}
+		cur->next = calloc(1, sizeof(Variable));
+		cur = cur->next;
+		/* Split on the equals. First half is name, second half is value. */
+		equals = strchr(line, '=');
+		if (equals == 0) {
+			fputs("bootstrap.cfg should have the format var=val on each line.", stderr);
+			exit(1);
+		}
+		cur->name = calloc(equals - line + 1, sizeof(char));
+		strncpy(cur->name, line, equals - line);
+		equals += 1;
+		cur->val = calloc(strlen(equals), sizeof(char));
+		strncpy(cur->val, equals, strlen(equals) - 1);
+		line = calloc(MAX_STRING, sizeof(char));
+	}
+	variables = head->next;
+	fclose(config);
+}
+
+void output_config(FILE *out) {
+	Variable *variable;
+	for (variable = variables; variable != NULL; variable = variable->next) {
+		fputs(variable->name, out);
+		fputs("=", out);
+		fputs(variable->val, out);
+		fputs("\n", out);
+	}
+}
+
+char *get_var(char *name) {
+	/* Search through existing variables. */
+	Variable *var;
+	Variable *last = NULL;
+	for (var = variables; var != NULL; var = var->next) {
+		if (strcmp(name, var->name) == 0) {
+			return var->val;
+		}
+		last = var;
+	}
+
+	/* If the variable is unset, prompt the user. */
+	if (variables == NULL) {
+		variables = calloc(1, sizeof(Variable));
+		var = variables;
+	} else {
+		last->next = calloc(1, sizeof(Variable));
+		var = last->next;
+	}
+	var->name = calloc(strlen(name) + 1, sizeof(char));
+	strcpy(var->name, name);
+	var->val = calloc(MAX_STRING, sizeof(char));
+	fputs("You have not set a value for ", stdout);
+	fputs(name, stdout);
+	fputs(" in bootstrap.cfg. Please set it now:\n", stdout);
+	while (fgets(var->val, MAX_STRING, stdin) == 0 || var->val[0] == '\n') {
+		fputs("Error inputting, try again:\n", stdout);
+	}
+	if (var->val[0] == 0) {
+		fputs("You put in an EOF!\n", stderr);
+		exit(1);
+	}
+	/* Trim the newline. */
+	var->val[strlen(var->val)] = 0;
+	return var->val;
+}
+
+/* Recursive descent interpreter. */
+
+Token *fill(Token *tok, Directive *directive, int type) {
+	directive->type = type;
+	directive->arg = tok->val;
+	return tok->next;
+}
+
+Token *logic(Token *tok, char **val) {
+	/* logic = "("
+	 *	 (name |
+	 *	 (name "==" value) |
+	 *	 (name "!=" value) |
+	 *	 (logic "||" logic) |
+	 *	 (logic "&&" logic))
+	 *	 ")"
+	 */
+
+	char *lhs = tok->val;
+	char *rhs;
+	tok = tok->next;
+	if (strcmp(tok->val, ")") == 0) {
+		/* Case where it's just a constant. */
+		*val = lhs;
+		return tok;
+	} else if (strcmp(tok->val, "==") == 0) {
+		/* Case for equality. */
+		rhs = tok->next->val;
+		tok = tok->next->next;
+		if (strcmp(get_var(lhs), rhs) == 0) {
+			lhs = "True";
+		} else {
+			lhs = "False";
+		}
+	} else if (strcmp(tok->val, "!=") == 0) {
+		/* Case for inequality. */
+		rhs = tok->next->val;
+		tok = tok->next->next;
+		if (strcmp(get_var(lhs), rhs) == 0) {
+			lhs = "False";
+		} else {
+			lhs = "True";
+		}
+	} else {
+		fputs("Expected == or != after ", stderr);
+		fputs(lhs, stderr);
+		fputs(" in logic\n", stderr);
+		exit(1);
+	}
+
+	if (strcmp(tok->val, ")") == 0) {
+		*val = lhs;
+		return tok;
+	} else if (strcmp(tok->val, "||") == 0) {
+		/* OR */
+		tok = logic(tok->next, &rhs);
+		if (strcmp(lhs, "True") == 0 || strcmp(rhs, "True") == 0) {
+			lhs = "True";
+		} else {
+			lhs = "False";
+		}
+	} else if (strcmp(tok->val, "&&") == 0) {
+		/* AND */
+		tok = logic(tok->next, &rhs);
+		if (strcmp(lhs, "True") == 0 && strcmp(rhs, "True") == 0) {
+			lhs = "True";
+		} else {
+			lhs = "False";
+		}
+	} else {
+		fputs("Expected || or && in logic\n", stderr);
+		exit(1);
+	}
+
+	*val = lhs;
+	return tok;
+}
+
+Token *primary_logic(Token *tok, char **val) {
+	/* Starting ( */
+	if (strcmp(tok->val, "(") != 0) {
+		fputs("Expected logic to begin with (\n", stderr);
+		exit(1);
+	}
+	tok = tok->next;
+
+	tok = logic(tok, val);
+
+	if (strcmp(tok->val, ")") != 0) {
+		fputs("Expected logic to end with )\n", stderr);
+		exit(1);
+	}
+
+	return tok;
+}
+
+int eval_predicate(Token *tok) {
+	char *result;
+	tok = primary_logic(tok, &result);
+	return strcmp(result, "True") == 0;
+}
+
+Token *define(Token *tok, Directive *directive) {
+	/* define = name "=" (logic | constant) */
+	char *name = tok->val;
+	tok = tok->next;
+	if (strcmp(tok->val, "=") != 0) {
+		fputs("define of ", stderr);
+		fputs(name, stderr);
+		fputs(" has a missing equals\n", stderr);
+		exit(1);
+	}
+	tok = tok->next;
+
+	char *val = calloc(MAX_STRING, sizeof(char));
+	if (strcmp(tok->val, "(") == 0) {
+		/* It is a logic. */
+		tok = primary_logic(tok, &val);
+	} else {
+		/* It is a constant. */
+		strcpy(val, tok->val);
+	}
+
+	/* Check for predicate. */
+	tok = tok->next;
+	if (tok != NULL) {
+		if (!eval_predicate(tok)) {
+			/* Nothing more to do. */
+			return tok;
+		}
+	}
+
+	/* Update existing variable, or else, add to the end of variables. */
+	/* Special case: empty variables. */
+	if (variables == NULL) {
+		variables = calloc(1, sizeof(Variable));
+		variables->name = name;
+		variables->val = val;
+	}
+
+	Variable *var;
+	for (var = variables; var->next != NULL; var = var->next) {
+		if (strcmp(var->next->name, name) == 0) {
+			var->next->val = val;
+			break;
+		}
+	}
+	if (var->next == NULL) {
+		/* We did not update an existing variable. */
+		var->next = calloc(1, sizeof(Variable));
+		var->next->name = name;
+		var->next->val = val;
+	}
+
+	return tok;
+}
+
+int interpret(Directive *directive) {
+	/* directive = (build | improve | define | jump | uninstall) predicate? */
+	Token *tok = directive->tok;
+	if (strcmp(tok->val, "build:") == 0) {
+		tok = fill(tok->next, directive, TYPE_BUILD);
+	} else if (strcmp(tok->val, "improve:") == 0) {
+		tok = fill(tok->next, directive, TYPE_IMPROVE);
+	} else if (strcmp(tok->val, "jump:") == 0) {
+		tok = fill(tok->next, directive, TYPE_JUMP);
+	} else if (strcmp(tok->val, "define:") == 0) {
+		tok = define(tok->next, directive);
+		return 1; /* There is no codegen for a define. */
+	} else if (strcmp(tok->val, "uninstall:") == 0) {
+		tok = fill(tok->next, directive, TYPE_UNINSTALL);
+		while (tok != NULL) {
+			if (strcmp(tok->val, "(") == 0) {
+				break;
+			}
+			if (strlen(directive->arg) + strlen(tok->val) + 1 > MAX_STRING) {
+				fputs("somehow you have managed to have too many uninstall arguments.\n", stderr);
+				exit(1);
+			}
+			directive->arg = strcat(directive->arg, " ");
+			directive->arg = strcat(directive->arg, tok->val);
+			tok = tok->next;
+		}
+	}
+
+	if (tok != NULL) {
+		return !eval_predicate(tok);
+	}
+	return 0;
+}
+
+Directive *interpreter(Directive *directives) {
+	Directive *directive;
+	Directive *last = NULL;
+	for (directive = directives; directive != NULL; directive = directive->next) {
+		if (interpret(directive)) {
+			/* This means this directive needs to be removed from the linked list. */
+			if (last == NULL) {
+				/* First directive. */
+				directives = directive->next;
+			} else {
+				last->next = directive->next;
+			}
+		} else {
+			last = directive;
+		}
+	}
+	return directives;
+}
+
+/* Script generator. */
+FILE *start_script(int id, int bash_build) {
+	/* Create the file /steps/$id.sh */
+	char *filename = calloc(MAX_STRING, sizeof(char));
+	strcpy(filename, "/steps/");
+	strcat(filename, int2str(id, 10, 0));
+	strcat(filename, ".sh");
+
+	FILE *out = fopen(filename, "w");
+	if (out == NULL) {
+		fputs("Error opening output file ", stderr);
+		fputs(filename, stderr);
+		fputs("\n", stderr);
+		exit(1);
+	}
+
+	if (bash_build) {
+		fputs("#!/bin/bash\n", out);
+		if (strcmp(get_var("INTERACTIVE"), "True") == 0) {
+			if (bash_build != 1) {
+				fputs("set -E\ntrap 'env PS1=\"[TRAP] \\w # \" bash -i' ERR\n", out);
+			} else {
+				/* FIXME early bash has buggy ERR trap handling */
+				fputs("set -e\ntrap 'bash -c '\"'\"'while true; do printf \""
+				"[TRAP - use Ctrl+D] $(pwd) # \"; eval \"$(cat)\"; done'\"'\"'' EXIT\n",
+				out);
+			}
+		} else {
+			fputs("set -e\n", out);
+		}
+		fputs("cd /steps\n", out);
+		fputs(". ./bootstrap.cfg\n", out);
+		fputs(". ./env\n", out);
+		fputs(". ./helpers.sh\n", out);
+	} else {
+		fputs("set -ex\n", out);
+		fputs("cd /steps\n", out);
+		output_config(out);
+		FILE *env = fopen("/steps/env", "r");
+		char *line = calloc(MAX_STRING, sizeof(char));
+		while (fgets(line, MAX_STRING, env) != 0) {
+			/* Weird M2-Planet behaviour. */
+			if (*line == 0) {
+				break;
+			}
+			fputs(line, out);
+			line = calloc(MAX_STRING, sizeof(char));
+		}
+		fclose(env);
+	}
+
+	return out;
+}
+
+void output_call_script(FILE *out, char *type, char *name, int bash_build, int source) {
+	if (bash_build) {
+		if (source) {
+			fputs(". ", out);
+		} else {
+			fputs("bash ", out);
+		}
+	} else {
+		fputs("kaem --file ", out);
+	}
+	fputs("/steps/", out);
+	if (strlen(type) != 0) {
+		fputs(type, out);
+		fputs("/", out);
+	}
+	fputs(name, out);
+	fputs(".sh\n", out);
+}
+
+void output_build(FILE *out, Directive *directive, int pass_no, int bash_build) {
+	if (bash_build) {
+		fputs("build ", out);
+		fputs(directive->arg, out);
+		fputs(" pass", out);
+		fputs(int2str(pass_no, 10, 0), out);
+		fputs(".sh\n", out);
+	} else {
+		fputs("pkg=", out);
+		fputs(directive->arg, out);
+		fputs("\n", out);
+		fputs("cd ${pkg}\n", out);
+		fputs("kaem --file pass", out);
+		fputs(int2str(pass_no, 10, 0), out);
+		fputs(".kaem\n", out);
+		fputs("cd ..\n", out);
+	}
+}
+
+void generate_preseed_jump(int id) {
+	FILE *out = fopen("/preseed-jump.kaem", "w");
+	fputs("set -ex\n", out);
+	fputs("PATH=/usr/bin\n", out);
+	fputs("bash /steps/", out);
+	fputs(int2str(id, 10, 0), out);
+	fputs(".sh\n", out);
+	fclose(out);
+}
+
+void generate(Directive *directives) {
+	/*
+	 * We are separating the stages given in the mainfest into a bunch of
+	 * smaller scripts. The following conditions call for the creation of
+	 * a new script:
+	 * - a jump
+	 * - build of bash
+	 */
+
+	int counter = 0;
+
+	/* Initially, we use kaem, not bash. */
+	int bash_build = 0;
+
+	FILE *out = start_script(counter, bash_build);
+	counter += 1;
+
+	Directive *directive;
+	Directive *past;
+	char *filename;
+	int pass_no;
+	for (directive = directives; directive != NULL; directive = directive->next) {
+		if (directive->type == TYPE_BUILD) {
+			/* Get what pass number this is. */
+			pass_no = 1;
+			for (past = directives; past != directive; past = past->next) {
+				if (strcmp(past->arg, directive->arg) == 0) {
+					pass_no += 1;
+				}
+			}
+			output_build(out, directive, pass_no, bash_build);
+			if (strncmp(directive->arg, "bash-", 5) == 0) {
+				if (!bash_build) {
+					/*
+					 * We are transitioning from bash to kaem, the point at which "early
+					 * preseed" occurs. So generate the preseed jump script at this point.
+					 */
+					generate_preseed_jump(counter);
+				}
+				bash_build += 1;
+				/* Create call to new script. */
+				output_call_script(out, "", int2str(counter, 10, 0), bash_build, 0);
+				fclose(out);
+				out = start_script(counter, bash_build);
+				counter += 1;
+			}
+		} else if (directive->type == TYPE_IMPROVE) {
+			output_call_script(out, "improve", directive->arg, bash_build, 1);
+		} else if (directive->type == TYPE_JUMP) {
+			/*
+			 * Create /init to call new script.
+			 * We actually do this by creating /init.X for some number X, and then
+			 * moving that to /init at the appropriate time.
+			 */
+			filename = calloc(MAX_STRING, sizeof(char));
+			if (bash_build) {
+				fputs("mv /init /init.bak\n", out);
+				/* Move new init to /init. */
+				strcpy(filename, "/init.");
+				strcat(filename, int2str(counter, 10, 0));
+				fputs("cp ", out);
+				fputs(filename, out);
+				fputs(" /init\n", out);
+				fputs("chmod 755 /init\n", out);
+			} else {
+				strcpy(filename, "/kaem.run.");
+				strcat(filename, int2str(counter, 10, 0));
+				fputs("cp ", out);
+				fputs(filename, out);
+				fputs(" /kaem.run\n", out);
+				fputs("cp /usr/bin/kaem /init\n", out);
+				fputs("chmod 755 /init\n", out);
+			}
+
+			output_call_script(out, "jump", directive->arg, bash_build, 1);
+			fclose(out);
+
+			if (bash_build) {
+				out = fopen(filename, "w");
+				if (out == NULL) {
+					fputs("Error opening /init\n", stderr);
+					exit(1);
+				}
+				fputs("#!/bin/bash\n", out);
+			} else {
+				out = fopen(filename, "w");
+				if (out == NULL) {
+					fputs("Error opening /kaem.run\n", stderr);
+					exit(1);
+				}
+				fputs("set -ex\n", out);
+			}
+			output_call_script(out, "", int2str(counter, 10, 0), bash_build, 0);
+			fclose(out);
+			out = start_script(counter, bash_build);
+			counter += 1;
+		} else if (directive->type == TYPE_UNINSTALL) {
+			fputs("uninstall ", out);
+			fputs(directive->arg, out);
+			fputs("\n", out);
+		}
+	}
+	fclose(out);
+}
+
+void main(int argc, char **argv) {
+	if (argc != 2) {
+		fputs("Usage: script-generator <script>\n", stderr);
+		exit(1);
+	}
+
+	FILE *in = fopen(argv[1], "r");
+	if (in == NULL) {
+		fputs("Error opening input file\n", stderr);
+		exit(1);
+	}
+	Directive *directives = tokenizer(in);
+	fclose(in);
+	load_config();
+	directives = interpreter(directives);
+	generate(directives);
+	FILE *config = fopen("/steps/bootstrap.cfg", "w");
+	output_config(config);
+	fclose(config);
+}

seed/script-generator.riscv64.checksums

@@ -0,0 +1 @@
+1aa31b29ce0999b6a9cace93bc584020ec78059b9ad37d9aa3da9ad6c3d9af59  script-generator

seed/script-generator.x86.checksums

@@ -0,0 +1 @@
+dc6106dbc02839cdc9e3e2348432242eb6d33d840ab74badfd63c3c9997462b9  script-generator

seed/seed.kaem

@@ -0,0 +1,75 @@
+#!/bin/sh
+
+# SPDX-FileCopyrightText: 2021 Andrius Štikonas <andrius@stikonas.eu>
+# SPDX-FileCopyrightText: 2021 Paul Dersey <pdersey@gmail.com>
+# SPDX-FileCopyrightText: 2020-2022 fosslinux <fosslinux@aussies.space>
+# SPDX-FileCopyrightText: 2022 Dor Askayo <dor.askayo@gmail.com>
+#
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+set -ex
+
+mkdir -p ${PREFIX} ${BINDIR} ${LIBDIR} ${INCDIR} ${SRCDIR} ${TMPDIR} /dev
+
+# Temporarily change PATH
+PATH=/${ARCH_DIR}/bin
+
+# Remove remaining dependencies on /bin (stage0-posix directory)
+cp /${ARCH_DIR}/bin/blood-elf ${BINDIR}/blood-elf
+cp /${ARCH_DIR}/bin/catm ${BINDIR}/catm
+cp /${ARCH_DIR}/bin/chmod ${BINDIR}/chmod
+cp /${ARCH_DIR}/bin/get_machine ${BINDIR}/get_machine
+cp /${ARCH_DIR}/bin/hex2 ${BINDIR}/hex2
+cp /${ARCH_DIR}/bin/kaem ${BINDIR}/kaem
+cp /${ARCH_DIR}/bin/match ${BINDIR}/match
+cp /${ARCH_DIR}/bin/M1 ${BINDIR}/M1
+cp /${ARCH_DIR}/bin/M2-Mesoplanet ${BINDIR}/M2-Mesoplanet
+cp /${ARCH_DIR}/bin/M2-Planet ${BINDIR}/M2-Planet
+cp /${ARCH_DIR}/bin/mkdir ${BINDIR}/mkdir
+cp /${ARCH_DIR}/bin/sha256sum ${BINDIR}/sha256sum
+cp /${ARCH_DIR}/bin/unbz2 ${BINDIR}/unbz2
+cp /${ARCH_DIR}/bin/ungz ${BINDIR}/ungz
+cp /${ARCH_DIR}/bin/untar ${BINDIR}/untar
+cp /${ARCH_DIR}/bin/unxz ${BINDIR}/unxz
+cp /${ARCH_DIR}/bin/cp ${BINDIR}/cp
+cp /${ARCH_DIR}/bin/replace ${BINDIR}/replace
+cp /${ARCH_DIR}/bin/rm ${BINDIR}/rm
+
+chmod 755 ${BINDIR}/blood-elf
+chmod 755 ${BINDIR}/catm
+chmod 755 ${BINDIR}/chmod
+chmod 755 ${BINDIR}/cp
+chmod 755 ${BINDIR}/get_machine
+chmod 755 ${BINDIR}/hex2
+chmod 755 ${BINDIR}/kaem
+chmod 755 ${BINDIR}/match
+chmod 755 ${BINDIR}/M1
+chmod 755 ${BINDIR}/M2-Mesoplanet
+chmod 755 ${BINDIR}/M2-Planet
+chmod 755 ${BINDIR}/mkdir
+chmod 755 ${BINDIR}/sha256sum
+chmod 755 ${BINDIR}/unbz2
+chmod 755 ${BINDIR}/ungz
+chmod 755 ${BINDIR}/untar
+chmod 755 ${BINDIR}/unxz
+chmod 755 ${BINDIR}/replace
+chmod 755 ${BINDIR}/rm
+
+PATH=${BINDIR}
+M2LIBC_PATH=/M2libc
+
+# mes envars
+NYACC_PKG=nyacc-1.00.2
+MES_PKG=mes-0.26
+MES_PREFIX=${SRCDIR}/${MES_PKG}/build/${MES_PKG}
+GUILE_LOAD_PATH=${MES_PREFIX}/mes/module:${MES_PREFIX}/module:${SRCDIR}/${MES_PKG}/build/${NYACC_PKG}/module
+
+M2-Mesoplanet --architecture ${ARCH} -f script-generator.c -o script-generator
+# Checksums
+if match x${UPDATE_CHECKSUMS} xTrue; then
+    sha256sum -o script-generator.${ARCH}.checksums script-generator
+else
+    sha256sum -c script-generator.${ARCH}.checksums
+fi
+./script-generator /steps/manifest
+kaem --file /steps/0.sh

seed/stage0-posix

@@ -0,0 +1 @@
+Subproject commit 779e5424d4b55fe9b7faea2285ae8b6486df0433

source_manifest.py

@@ -9,27 +9,11 @@ for the bootstrapping process.
 
 import argparse
 
-from sysa import SysA
-from sysc import SysC
+from lib.generator import Generator
 
 def main():
     """Generate a source manifest for a system"""
-    parser = argparse.ArgumentParser()
-
-    parser.add_argument("-s", "--system",
-                        help="Generate source manifest for the specified systems",
-                        choices=["sysa", "sysc"],
-                        nargs="+",
-                        action="extend",
-                        required=True)
-
-    args = parser.parse_args()
-
-    if "sysa" in args.system:
-        print(SysA.get_source_manifest())
-
-    if "sysc" in args.system:
-        print(SysC.get_source_manifest())
+    print('\n'.join(map(' '.join, Generator.get_source_manifest())))
 
 if __name__ == "__main__":
     main()

steps/SHA256SUMS.pkgs

@@ -0,0 +1,128 @@
+2dac610769286171ca81e736d729f5227938596cf44a0a8a764009715c55d23b  autoconf-2.52_0.tar.bz2
+7345a49227ed69774e01097c514f1ebd3abaeed0b2b9136b7f5ee1b0fe573390  autoconf-2.53_0.tar.bz2
+22418f18b0dbf0476efb1823dcebdf70d7619113ca10076ba9b078ab3597a578  autoconf-2.54_0.tar.bz2
+8e74f93f12599f883ba62fe2bf4f69c70270ce368f71aa782f8f42a36a752d19  autoconf-2.55_0.tar.bz2
+1bd66ca8cc6300e192afe06cd90b1250f02ca64539a4f3e98a872333ec04f44b  autoconf-2.57_0.tar.bz2
+83235c739887f0d801e410e922553a6fd7723cbd0c9b2e96106e20538abb3a3b  autoconf-2.59_0.tar.bz2
+8912ae2a24ea40a7cd3ccab5e97dd109db7be4a86f40458ac836568aa7437b6a  autoconf-2.61_0.tar.bz2
+d5fcce4baea5e11f983a7e4b7148545ab6720bb4e010898c73387f6711be67f4  autoconf-2.64_0.tar.bz2
+298dcf17a439adb306237b308e3c4ed3d376cb863ae0f0f08bfdf1f3199a548b  autoconf-2.69_0.tar.bz2
+8c2401edf1c68910005bbcb8b8e84727a46f7d9643d2ad1267bd0259078dadc4  autoconf-2.71_0.tar.bz2
+6ff691e3ddd4f3d1796d28b07a2a96691dbd9801844c99ccbe660da59dabd74b  autoconf-archive-2021.02.19_0.tar.bz2
+439fc68ab7d0a9a46caca2c7cbefbdb6ffe906f927e07d150e102bb76b393c37  autogen-5.18.16_0.tar.bz2
+02a426e82a52d4a2cd7e73fe3dc5a2c58b3ed100f24a92c57e42511fd86e247c  automake-1.10.3_0.tar.bz2
+34925adeaa2876a5e068adeb17b71fe7c7105244c3cb5439d33f7406310674da  automake-1.11.2_0.tar.bz2
+7e36e9e4b3957c95b6de8ff475c9b37aa007e2f9735b01c0eaf14b711fdd112f  automake-1.15.1_0.tar.bz2
+57100612d902537b72ee2fd3dfecc903f2ea3959f1a480af87c6a1ef9a9d7c03  automake-1.16.3_0.tar.bz2
+1c7e09ac08f156960420fca1f6e8560c1855bf01b001a0648b56153d6ceb9c28  automake-1.6.3_0.tar.bz2
+444131776779d1f27628088a817bfb8c9aa85f4b036b934794a71e484ec9bd70  automake-1.6.3_1.tar.bz2
+e8ff13753943e43972a8cf1672819f001718ed38e5497cf7b554c21a507ba22b  automake-1.7.8_0.tar.bz2
+ef08694bf8e78ffcf56f9c8a63b0f189e301c06f02ab240e15e4c7fe354c2a0d  automake-1.7_0.tar.bz2
+9922651b64ad2e29b944082f26db223008220155048f25d64df1f6d82071ff1c  automake-1.8.5_0.tar.bz2
+c5807850a377e4bb9aa4121d9781d145bdd80327626a9419630a75ce2c8d9c71  automake-1.9.6_0.tar.bz2
+f8a55bf656f950ee3c5cea48af03f552166f6db9926b78c78843f3cc52a6e22d  bash-5.2.15_0.tar.bz2
+ebc631ee8e9382650005984ec5f3910492ccb6fb2656b4c0af0f4afc16998906  bc-1.07.1_0.tar.bz2
+90009c7a2232fce68ad628d6afe1f474d57874462b3ea2c50ab1a3e6c60a5a36  binutils-2.30_0.tar.bz2
+da658346866061324b92a6b734561257acfdb981783ad5794d67a0cd56e14e58  binutils-2.41_0.tar.bz2
+026447a32ab08a8a5150bab82a2f150af4a2ed90c47cfde631e1460eaed019aa  binutils-2.41_1.tar.bz2
+f0bb18831983493b82fde53917e8d16c6a380761ac5e71e14769a82193fee788  bison-2.3_0.tar.bz2
+e4a3c9466d95c58bb072159da3473d71b40eb069b756acefba293588c4495a6f  bison-3.4.1_0.tar.bz2
+fd3a3b855c3cbc42d719a0cbbae773d27e0e12b563641124ba2870b7103da1f8  bison-3.4.1_1.tar.bz2
+5232f6f8f3864b5827a015ce0414d484bb8aeb9f34ba66ceb7093da0a79561a8  bison-3.4.1_2.tar.bz2
+6d5e4debdec03aee096dbcefcad878fb5c6a069f895ee7ade878d9eeb97fb83e  bison-3.4.2_0.tar.bz2
+727b0f28369b78f171dce4e227b05cea474dcb638c357387429c125f0d80fbc8  bzip2-1.0.8_0.tar.bz2
+543214998317d764595d3dd247a1fb2e0803ad77978b8474bd24d64c161b9530  ca-certificates-3.95_0.tar.bz2
+da3a3c7e0e6ce94ccb3d39885bc44207eb94c912b324a4c7669b5a752648b02d  coreutils-5.0_0.tar.bz2
+56b84ab2035847524d595bb417c2b91c290174965550be321cd3d5d1f77aa2e5  coreutils-6.10_0.tar.bz2
+9fa31a4aeaa5132205efb796c8f546c94c1cfef6b5c27e64f6ebe06ca0360e67  coreutils-9.4_0.tar.bz2
+abd629e5675196c39dae00afbe23e2116bbd277be1d82214e2ec557a3b03994f  curl-8.5.0_0.tar.bz2
+db8757b5fe77f56073caa034bfe9b3af6076d7299db966c4348e06f01a34415f  curl-8.5.0_1.tar.bz2
+9534d834b018343b8335d0ef0b5faff3ee18e4ecbecf43a2a80deb972283f5d6  dhcpcd-10.0.1_0.tar.bz2
+e6372051f90c40d1f4a5cca2f8930187b363dfc916bcb973c72bca6c0aba3cd5  diffutils-2.7_0.tar.bz2
+41e3129b974391fe4ace69914fdef9b0fe72c6f5d87d495750fb89bdd999fa01  diffutils-3.10_0.tar.bz2
+bda6afcd3a390a34a57443269a4857ccc470129f7a557981778c145fd841cbd1  dist-3.5-236_0.tar.bz2
+26c0835a8b9be115082a2866e906363947528169c7274bd3b82752df77e99f11  e2fsprogs-1.45.7_0.tar.bz2
+9c6f7e156f299f1d0948537e28f00d40e727a3debae3cce158e9bce827136214  ed-1.4_0.tar.bz2
+0061d240f9bb2bb534db7003ddcc4eae9981954ad72e8f41ff1d76dc449ebd37  file-5.44_0.tar.bz2
+eec07e0d710ed3dd8d0ba6a1ec6ca9f1fa3a611bbb225195435679175317bf90  findutils-4.2.33_0.tar.bz2
+80798628ccb38521f42e1ca1abe4f0244e4b283e6f662cd354ab70e61a9b4675  flex-2.5.11_0.tar.bz2
+b3a8d59229289ca5a1de6a24c9c828d2847144f6662e0d5b7f41e7ff6e57abaa  flex-2.5.33_0.tar.bz2
+e10060b55e2378a065cf8590cae17400c8befc6b3bb0cbbb7093f1b9f46ec4cc  flex-2.6.4_0.tar.bz2
+5619cf9a893902491a9d8a8f8da4d39efb3606d7d0a72cb4aaa9d1ff798b5495  gawk-3.0.4_0.tar.bz2
+62e90abe883f6ccd679adf42d00c2c18a9809bbab599411c81b6c4ac4d446452  gawk-5.3.0_0.tar.bz2
+6de1c1380026ef9948387e665610185b6014f47a80453177a6c81898c95cbbd3  gc-8.0.4_0.tar.bz2
+c99d080cf17ac3f75cbae67095cb76220a92d96d877c92c896a7f05c9c94e92d  gcc-10.4.0_0.tar.bz2
+e0c9e7f4f0cd0d5027f2a38eb262f883133c31ac9ec93ee6afa5abc028096bc0  gcc-13.1.0_0.tar.bz2
+27bb77077559b6c0f8dd7b8790ac48f70410ecc8816b5c493988ad5556da65e2  gcc-4.0.4_0.tar.bz2
+c1bf792e90df13bf7a24d3f8ce3939fd96ff61b8117e4dcef75605974f3fcf25  gcc-4.0.4_1.tar.bz2
+7ac087bf006188154c28401d0c875ede56076c19d41bc54ffa0d7cc18ce40800  gcc-4.7.4_0.tar.bz2
+dc67fc1b132fa3945349efe74c5b2197bd0b6babf4d29d2deddd04f09d9af680  gettext-0.21_0.tar.bz2
+309282ba840f80b5e10897b1221f3ff3329b9e05b982c788d03652f411974524  gmp-6.2.1_0.tar.bz2
+62edfa633f64202b2d29199d11d56a897654a441c7ae39a4d59ca5fe75199f61  gperf-3.1_0.tar.bz2
+c1989fc2855dbb0350c4e5b677028306614a9f4b5cb6c01fd459c5e1abf592b9  grep-2.4_0.tar.bz2
+8e612afb9a439aa8bf5db0fc2589eec890a6017690412d5c8e88a5838730b42c  grep-3.7_0.tar.bz2
+ed16ffe92f50bca0dac71145adc07c08043837457dee46bb3a5f5d0a6dc99221  grub-2.06_0.tar.bz2
+e2a85aad6d51e52c9a30afeed058f95172fde1215f77243549274672990f6237  guile-3.0.9_0.tar.bz2
+6585ae3bc8149ec0e3fba766278fa30e2d7f0e7d1b9a9a4a457e0afa15b109c9  gzip-1.13_0.tar.bz2
+29a1603101cfbe2660e2d695f5b9c08dc9eefd8e3389405a6530c479e271821b  heirloom-devtools-070527_0.tar.bz2
+8d2015b87337abbf287f7a39ee4cf53514120b5d3e90a93fe7d533dcc43f14fa  help2man-1.36.4_0.tar.bz2
+ea5639dc6198c5f243362a1a7adfbea0cb779e085975712a1b410e82d7e0cf84  kbd-1.15_0.tar.bz2
+50a0f881161c68fe7c9ec6836b11a905b0d54e08e99b2476e8d1f5ac3212769e  kexec-linux-1.0.0_0.tar.bz2
+fc784d2c46ffe91e7d54150022c9d2687140e90e3019c1d6799f875f22d99d2e  kexec-tools-2.0.22_0.tar.bz2
+e4dcdabf540a9e51a555616abff8ec2fdee5bb40bb05ce702b5410239dd5d7d8  libarchive-3.5.2_0.tar.bz2
+36550df491767bb24d2ccab304ce70a3b4956e7c0c0e0c343d922fd57cdafbdd  libatomic_ops-7.6.10_0.tar.bz2
+fea96776b929569b98bc1740a9977cf8c0eff1d999a08d766bcc0f40c7b1380c  libbsd-0.11.8_0.tar.bz2
+b39826742e236890f3562cdf19492e7ef4224b271f3e75ddeab1f07982b03ebe  libffi-3.3_0.tar.bz2
+0f6aefeb587b3d14d8ea73d7750f946a23d3e59a76a0ee5cdcdcc8132bccf73d  libmd-1.1.0_0.tar.bz2
+daae709e98d2df2190d1d13b4e86f7f3fe90fa7a975282fe0bb03289b6539f29  libtool-2.2.4_0.tar.bz2
+6cefa575362149620f8008a32c8af54f0198a18bc6ab910bd3cead196c1507d7  libtool-2.4.7_0.tar.bz2
+503007bbcddcf4e49d26514c59b4c9501f8b42f0c994a59dfdc388b1ae6b7900  libunistring-0.9.10_0.tar.bz2
+5787f84a49e1d22560d0398e4f9075d6021017eb2a757697dc2877e7565d0199  linux-4.14.341-openela_0.tar.bz2
+c97644d0db5b3de127b048683afee6d31453441d97ba5dea71df5838b13542a4  linux-headers-4.14.341-openela_0.tar.bz2
+78b0cf6d9312e53c613186cbddd5f747310f375c1f322f33a6ac33682d2f3389  m4-1.4.19_0.tar.bz2
+0e3c21b0a1d8ca0c3f74a98ebe268809def62778ff4a486ff20c1d6e8247dc49  m4-1.4.7_0.tar.bz2
+e69554b0a77b419ddd5d0a0e418ba4005ecd0f6784c92a6928a0270bd929a098  make-3.82_0.tar.bz2
+6d24960d6a987f68a7e0e3abf6edb52d2e0fe4c86f6ba45327e9634dbf7d40b4  make-4.2.1_0.tar.bz2
+17cd976bc0f6e897c6fffe43dd7c55d93ce0adadf1b4dc72925b80e2d266519f  mpc-1.2.1_0.tar.bz2
+98a9f55c01ed375fae21913284e7fa07ff434c0810f34d791f3fbaa1df221302  mpfr-4.1.0_0.tar.bz2
+40fc3497fa8cdd96ac338a10e78504c6dc942e7f4fa459550fb953d52bd53aa7  musl-1.1.24_0.tar.bz2
+20285055d84ac6106965a9d343ba8ebd19209f62782efb8ed863318c0b7d5e7d  musl-1.1.24_1.tar.bz2
+d58d85c4be26d90f111f273929a4998d05294fe5aa37c4842d5aecaa593e6079  musl-1.1.24_2.tar.bz2
+81f79a0faf58e105c5f12d9f538d3ea7578a3b361e633ab8f47324ec23ec9de6  musl-1.2.4_0.tar.bz2
+6dc5e763b747b66f72692c6820d7f46b7f4cf2e2fe05229a1d01c03919c1c936  musl-1.2.4_1.tar.bz2
+820203286127e7c22cee9f1b3cff664431f10d14c3f22f00b3e771fd849fd449  musl-1.2.4_2.tar.bz2
+a18c4b2e5de2bfe5bb3ee9d360484fcfebad3df042f1859d4aa333dd60f55e56  opendoas-6.8.2_0.tar.bz2
+67b536b2f2a6c56e4673e462a3dbe2f42e8a4a5cb0b61b6bc1fdc14adea33f63  openssl-3.0.13_0.tar.bz2
+71864d042cdc564b65eab21360902c714e9b43f80a19689c5600589529b267e7  patch-2.7.6_0.tar.bz2
+287b0a8c0de14a2817e68ab023bfc502a0880d8c34386162351d5ffaf942da70  perl-5.000_0.tar.bz2
+9617799cdc9d4be39d31af54b9bcae47941d68416e9cc9c92cbc2ed08785dbc1  perl-5.003_0.tar.bz2
+74d64a8af080022432fa94dba449090419d25b103d247710dc0b6102a4ad86a6  perl-5.10.1_0.tar.bz2
+d381e910532061ac97e48ff09906bddbcb0956b37fc47d8c666f56a56b50c589  perl-5.32.1_0.tar.bz2
+d63d2fda752759778ae2d0dd08e6c023a81afbfc63ce11db860fc6858ae26c7f  perl-5.6.2_0.tar.bz2
+c39ab70bce0bd09f013109ad7370085e90d77ef2b55abcfdc922d7832524c9cf  perl5.004-05_0.tar.bz2
+2ac211ad3cf966dd89434c175970e10699c363b66228abc1642766aeafe2766a  perl5.005-03_0.tar.bz2
+1b9d4260edf7b2241d10e4c4ad17d0f90047bd4bf42f2487a7133902529e9dfe  pkg-config-0.29.2_0.tar.bz2
+2f7198009e4d021d52ee4ce86241b4936fb88349c20cc8b6c286261368878c3c  python-2.0.1_0.tar.bz2
+b5d86ddc98cfbc684b03f1c84c786caaad810d5e4c7be38089f324eb3c276ad9  python-2.0.1_1.tar.bz2
+396577cdd0cc61d76420a1771c64156e49e8f9d00430c82feb88ad933b341632  python-2.3.7_0.tar.bz2
+2499cb7f10f292c3506fbf1b6a876195179ec98edfe7b8c357140137a1449492  python-2.3.7_1.tar.bz2
+80d8fcaf68c82238ddffb120b62fbc20d58bd9e2416cba080547a9f442e9e229  python-2.5.6_0.tar.bz2
+3508248f299b73c50e3607c4c294d40face05170476a5026b0821aed69025863  python-3.1.5_0.tar.bz2
+12b1ffc7ec98ba8f807160b93ba69a694d5395567c3bcac1e49e8f8d1d50de43  python-3.1.5_1.tar.bz2
+d6a7d3a109f63cd5ed4feaa772c33e272e570e216346ee90f6ca365d3a951615  python-3.11.1_0.tar.bz2
+60b93253a2078f849f81e7e1ed6233e30702f03b1893640eee95671d814f5514  python-3.3.7_0.tar.bz2
+da7c8ec579dd225c0d8bee63d95aeeb27ac2d5a60d4eefe298508cbf86bf506c  python-3.4.10_0.tar.bz2
+af7a8334045fed8bd610b19194c0b46aa670070fc4738c34449ba5e93e49db8a  python-3.8.16_0.tar.bz2
+c313520567176dd9bf61a271ee56e06a62e944e862f9dc4564d94b7fc884c8f9  sed-4.0.9_0.tar.bz2
+e8daec00b2c2de7b18efbec057dc9290eed06668806c6f5a48914d4a5cd95eb4  sed-4.8_0.tar.bz2
+ef2f2c791c1ae74f19c98ebcd7edae6ae28bc1d9367cc3a2cdb1ac302a156cb7  shadow-4.14.3_0.tar.bz2
+912d8f344104f1322255d6210c7c7e1371413ab530b2c6796e6aa565c74bf647  tar-1.34_0.tar.bz2
+60e98c09d9135b0150ed52b71cfbb072335741c0954a9b14bbba850c20564036  tcc-0.9.27_0.tar.bz2
+e2c9ea82a1ebf2d5d4c937fa837ddb6fc6835395cf03f3207df40c96b6cafcd8  tcc-0.9.27_1.tar.bz2
+4860223e6960ce976e6ad59aa90b69c18a1473684bce59ec8a8b36535405792b  tcc-0.9.27_2.tar.bz2
+d4fe9460ca561fc2f546f9730f19f541b17dac0bcc42eb190abba856588c3593  tcc-0.9.27_3.tar.bz2
+50ebaa1d8fcc4a03a43d431eb71e2f435cc8712ae47d400564df3716037d553a  texinfo-6.7_0.tar.bz2
+59bda6faaa95782154b917a6600cdf932e08cd6bf9388ba40d801baf18e5b454  util-linux-2.19.1_0.tar.bz2
+ecdb7ffeb9256f6a9760be70969fe5dea9cde6a538cc88595281fe44340e98a1  which-2.21_0.tar.bz2
+3fade2079cc91f2c5624ff7247220059caee82e7de493332103d7a78155400b2  xz-5.4.1_0.tar.bz2
+ca8ec9876a7334f5f87e1159e0efe343b8b497ffb0dea8b548223035ecd67f9e  zlib-1.2.13_0.tar.bz2

steps/autoconf-2.64/sources

@@ -0,0 +1 @@
+https://mirrors.kernel.org/gnu/autoconf/autoconf-2.64.tar.xz 32d977213320b8ae76c71175305301197f2b0e04e72d70694bc3d3e2ae6c7248

steps/autoconf-2.69/sources

@@ -0,0 +1 @@
+https://mirrors.kernel.org/gnu/autoconf/autoconf-2.69.tar.xz 64ebcec9f8ac5b2487125a86a7760d2591ac9e1d3dbd59489633f9de62a57684

steps/autogen-5.18.16/pass1.sh

@@ -15,13 +15,13 @@ src_prepare() {
 src_compile() {
 (
     set -e
-    export PKG_CONFIG_PATH="${LIBDIR}/pkgconfig"
+    declare -x PKG_CONFIG_PATH="${LIBDIR}/pkgconfig"
     sed -i "s/make install/make install DESTDIR=\${DESTDIR}/" bootstrap_tarball.sh
     sed -i "/make check/d" bootstrap_tarball.sh
-    export FINALPREFIX="${PREFIX}"
-    export GUILE_STATIC="--static"
-    export GNULIBDIR="${PWD}"/../gnulib-8f4538a5
-    export MAN_PAGE_DATE=1970-01-01
+    declare -x FINALPREFIX="${PREFIX}"
+    declare -x GUILE_STATIC="--static"
+    declare -x GNULIBDIR="${PWD}"/../gnulib-8f4538a5
+    declare -x MAN_PAGE_DATE=1970-01-01
 
     SKIP_MAIN=1 . ./bootstrap_tarball.sh
     prepare_tarball
@@ -41,7 +41,7 @@ src_compile() {
 
     SOURCE_DIR="$PWD" ./config/bootstrap
     # Specify timeout to avoid non-reproducibility
-    ./configure \
+    CPPFLAGS=-D_LARGEFILE64_SOURCE=1 ./configure \
 	--prefix="${FINALPREFIX}" \
 	--libdir="${FINALPREFIX}/lib/i386-unknown-linux-musl" \
 	--disable-shared \

steps/autogen-5.18.16/sources

@@ -1,4 +1,6 @@
-https://github.com/schierlm/gnu-autogen-bootstrapping/archive/refs/tags/autogen-5.18.16-v1.0.tar.gz 98ff63400dff5282017c33e4ec3c93da8a92a5260340da253e59bd6eef18d116
+https://github.com/schierlm/gnu-autogen-bootstrapping/archive/refs/tags/autogen-5.18.16-v1.0.1.tar.gz 953ba180b18acff188a0a8700770c7cf2fc97e1683c7b9699a5a748b542ccdd5
 https://mirrors.kernel.org/gnu/autogen/rel5.18.16/autogen-5.18.16.tar.xz f8a13466b48faa3ba99fe17a069e71c9ab006d9b1cfabe699f8c60a47d5bb49a
 https://git.savannah.gnu.org/cgit/autogen.git/snapshot/autogen-5.18.16.tar.gz 0c04ab2f7ce13c4a1c06c4abc7dfe75312aad89b8b0a1068e5e563787eb56632
+https://files.bootstrapping.world/autogen-5.18.16.tar.gz 0c04ab2f7ce13c4a1c06c4abc7dfe75312aad89b8b0a1068e5e563787eb56632
 http://git.savannah.gnu.org/cgit/gnulib.git/snapshot/gnulib-8f4538a5.tar.gz e207c0bb72093c3a72dde302fcfaa1dbda12a62172d47b73565883a92209ebab
+https://files.bootstrapping.world/gnulib-8f4538a5.tar.gz e207c0bb72093c3a72dde302fcfaa1dbda12a62172d47b73565883a92209ebab

steps/automake-1.15.1/pass1.sh

@@ -12,17 +12,17 @@ src_prepare() {
     sed -i '/doc\/Makefile.inc/d' Makefile.am
     sed -i '/t\/Makefile.inc/d' Makefile.am
 
-    AUTOCONF=autoconf-2.69 AUTOM4TE=autom4te-2.69 ./bootstrap
+    AUTOCONF="autoconf-2.69 -f" AUTOM4TE=autom4te-2.69 ./bootstrap
 
     rm doc/automake-history.info doc/automake.info*
 }
 
 src_configure() {
-    AUTORECONF=autoreconf-2.69 AUTOM4TE=autom4te-2.69 AUTOHEADER=autoheader-2.69 AUTOCONF=autoconf-2.69 ./configure --prefix="${PREFIX}"
+    AUTORECONF=autoreconf-2.69 AUTOM4TE=autom4te-2.69 AUTOHEADER=autoheader-2.69 AUTOCONF="autoconf-2.69 -f" ./configure --prefix="${PREFIX}"
 }
 
 src_compile() {
-    AUTORECONF=autoreconf-2.69 AUTOM4TE=autom4te-2.69 AUTOHEADER=autoheader-2.69 AUTOCONF=autoconf-2.69 make -j1 MAKEINFO=true
+    AUTORECONF=autoreconf-2.69 AUTOM4TE=autom4te-2.69 AUTOHEADER=autoheader-2.69 AUTOCONF="autoconf-2.69 -f" make -j1 MAKEINFO=true
 }
 
 src_install() {

steps/automake-1.15.1/sources

@@ -0,0 +1 @@
+https://mirrors.kernel.org/gnu/automake/automake-1.15.1.tar.xz af6ba39142220687c500f79b4aa2f181d9b24e4f8d8ec497cea4ba26c64bedaf

steps/bash-2.05b/bash-2.05b.checksums

@@ -0,0 +1 @@
+467552a7875bae2086ba0fa9a8d2a253a158984e27b2e89acf2e3bcec539ad54  /usr/bin/bash

steps/bash-2.05b/mk/common.mk

@@ -49,6 +49,7 @@ COMMON_CFLAGS  = \
 	-DHAVE_BZERO \
 	-DHAVE_POSIX_SIGNALS \
 	-DHAVE_GETCWD \
+	-DHAVE_RENAME \
 	-DHAVE_SYS_SIGLIST \
 	-Dendpwent\(x\)=0 \
 	-Denable_hostname_completion\(on_or_off\)=0

steps/bash-2.05b/pass1.kaem

@@ -16,8 +16,8 @@ mkdir build src
 cd build
 
 # Extract
-cp ${distfiles}/${pkg}.tar.gz ../src/
-gzip -d -f ../src/${pkg}.tar.gz
+cp ${DISTFILES}/${pkg}.tar.bz2 ../src/
+bzip2 -d -f ../src/${pkg}.tar.bz2
 tar xf ../src/${pkg}.tar
 rm -r ../src/
 cd ${pkg}
@@ -46,11 +46,8 @@ cd ..
 make
 
 # Install
-install bash ${prefix}/bin/
-mkdir /bin/
-ln -s ${prefix}/bin/bash /bin/bash
-ln -s ${prefix}/bin/bash /bin/sh
-ln -s ${prefix}/bin/bash ${prefix}/bin/sh
+install bash ${PREFIX}/bin/
+install bash ${PREFIX}/bin/sh
 
 cd ../..
 
@@ -59,7 +56,7 @@ if match x${UPDATE_CHECKSUMS} xTrue; then
     sha256sum -o ${pkg}.checksums \
         /usr/bin/bash
 
-    install ${pkg}.checksums ${srcdir}
+    install ${pkg}.checksums ${SRCDIR}
 else
     sha256sum -c ${pkg}.checksums
 fi

steps/bash-2.05b/sources

@@ -0,0 +1 @@
+https://src.fedoraproject.org/repo/pkgs/bash/bash-2.05b.tar.bz2/f3e5428ed52a4f536f571a945d5de95d/bash-2.05b.tar.bz2 1ce4e5b47a6354531389f0adefb54dee2823227bf6e1e59a31c0e9317a330822

steps/bash-5.2.15/pass1.sh

@@ -53,9 +53,5 @@ src_compile() {
 
 src_install() {
     install -D bash "${DESTDIR}${PREFIX}/bin/bash"
-    # Work around weird symlink bug
     install bash "${DESTDIR}${PREFIX}/bin/sh"
-
-    # Needs special handling b/c is currently running - tar doesn't like this
-    rm -f "${PREFIX}/bin/bash" "${PREFIX}/bin/sh"
 }

steps/bc-1.07.1/sources

@@ -0,0 +1 @@
+https://mirrors.kernel.org/slackware/slackware64-15.0/source/ap/bc/bc-1.07.1.tar.xz 95396f88fc719a1bf8bd463809119526fef44a42ab9eb708335c2cb79bc801c6