From 5945bdc1e8843182b06d0cea4774932151947f23 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?G=C3=A1bor=20Stefanik?= Date: Mon, 5 Feb 2024 09:55:31 +0100 Subject: [PATCH] Replace Fiwix filelist with parsing builder-hex0's FS structures And while we're at it, use a more sustainable way of getting the address of the next file to be written to. Since builder-hex0 doesn't distinguish directories from zero-length files, this has a limitation that it won't copy any zero-length files or empty directories. Only one such file is important for the bootstrap process (mes/config.h), which is recreated using an improve step. --- builder-hex0 | 2 +- lib/generator.py | 26 --- seed/script-generator.c | 23 --- steps/jump/linux.sh | 3 +- .../early-artifacts-needed-after-fiwix.txt | 177 ------------------ .../files/make_fiwix_initrd.c | 77 +++++--- .../lwext4-1.0.0-lb1.checksums | 2 +- 7 files changed, 51 insertions(+), 259 deletions(-) delete mode 100644 steps/lwext4-1.0.0-lb1/files/early-artifacts-needed-after-fiwix.txt diff --git a/builder-hex0 b/builder-hex0 index d49733d..29e8cff 160000 --- a/builder-hex0 +++ b/builder-hex0 @@ -1 +1 @@ -Subproject commit d49733d3e89538b2845f2be90ca6839deabf0042 +Subproject commit 29e8cffc1b7d6da0dfa3d42204c50ff2cda4a1e4 diff --git a/lib/generator.py b/lib/generator.py index 272f89f..1828fc6 100755 --- a/lib/generator.py +++ b/lib/generator.py @@ -85,8 +85,6 @@ class Generator(): self.distfiles() - self.create_fiwix_file_list() - if self.repo_path: repo_dir = os.path.join(self.external_dir, 'repo-preseeded') shutil.copytree(self.repo_path, repo_dir) @@ -134,30 +132,6 @@ class Generator(): if os.path.isfile(os.path.join(seed_dir, entry)): shutil.copy2(os.path.join(seed_dir, entry), os.path.join(self.target_dir, entry)) - @staticmethod - def add_fiwix_files(file_list_path, dirpath): - """Add files to the list to populate Fiwix file system""" - for root, _, filepaths in os.walk(dirpath): - if 'stage0-posix' in root: - continue - with open(file_list_path, 'a', encoding="utf-8") as file_list: - for filepath in filepaths: - file_list.write(f"/{os.path.join(root, filepath)}\n") - - def create_fiwix_file_list(self): - """Create a list of files to populate Fiwix file system""" - file_list_path = os.path.join(self.target_dir, 'steps', 'lwext4-1.0.0-lb1', - 'files', 'fiwix-file-list.txt') - shutil.copyfile(os.path.join(self.target_dir, 'steps', 'lwext4-1.0.0-lb1', - 'files', 'early-artifacts-needed-after-fiwix.txt'), - file_list_path) - - save_cwd = os.getcwd() - os.chdir(self.target_dir) - self.add_fiwix_files(file_list_path, 'steps') - self.add_fiwix_files(file_list_path, 'distfiles') - os.chdir(save_cwd) - def distfiles(self): """Copy in distfiles""" def copy_no_network_distfiles(out, early): diff --git a/seed/script-generator.c b/seed/script-generator.c index 6bad518..eccc791 100644 --- a/seed/script-generator.c +++ b/seed/script-generator.c @@ -412,22 +412,6 @@ Directive *interpreter(Directive *directives) { return directives; } -void add_to_fiwix_filelist(char *filename) { - /* Add the filename to fiwix-file-list.txt */ - FILE *fiwix_list = fopen("/steps/lwext4-1.0.0-lb1/files/fiwix-file-list.txt", "r"); - fseek(fiwix_list, 0, SEEK_END); - long size = ftell(fiwix_list); - char *contents = calloc(size, sizeof(char)); - fseek(fiwix_list, 0, SEEK_SET); - fread(contents, 1, size, fiwix_list); - fclose(fiwix_list); - fiwix_list = fopen("/steps/lwext4-1.0.0-lb1/files/fiwix-file-list.txt", "w"); - fwrite(contents, 1, size, fiwix_list); - fputs(filename, fiwix_list); - fputc('\n', fiwix_list); - fclose(fiwix_list); -} - /* Script generator. */ FILE *start_script(int id, int bash_build) { /* Create the file /steps/$id.sh */ @@ -435,7 +419,6 @@ FILE *start_script(int id, int bash_build) { strcpy(filename, "/steps/"); strcat(filename, int2str(id, 10, 0)); strcat(filename, ".sh"); - add_to_fiwix_filelist(filename); FILE *out = fopen(filename, "w"); if (out == NULL) { @@ -608,12 +591,6 @@ void generate(Directive *directives) { output_call_script(out, "jump", directive->arg, bash_build, 1); fclose(out); - /* - * This cannot go before here as builder-hex0 does not like having - * multiple files open at once! - */ - add_to_fiwix_filelist(filename); - if (bash_build) { out = fopen(filename, "w"); if (out == NULL) { diff --git a/steps/jump/linux.sh b/steps/jump/linux.sh index e647e8b..7e2043e 100755 --- a/steps/jump/linux.sh +++ b/steps/jump/linux.sh @@ -10,13 +10,14 @@ set -e # Perform the actual kexec if [ "${KERNEL_BOOTSTRAP}" = True ]; then + mv "/High Level Prototypes" "/High_Level_Prototypes" sync # We don't use the gen_initramfs_list.sh script because it is so *SLOW* # This emulates the same thing it does find / -xdev -type d -printf "dir %p %m %U %G\n" >> /initramfs.list find / -xdev -type f -printf "file %p %p %m %U %G\n" >> /initramfs.list find / -xdev -type l -printf "slink %p %l %m %U %G\n" >> /initramfs.list - kexec-linux "/dev/ram1" "/boot/linux-4.9.10" "!$(command -v gen_init_cpio) /initramfs.list" + kexec-linux "/dev/ram1" "/boot/linux-4.9.10" "!gen_init_cpio /initramfs.list | gzip -c" else mkdir /etc # kexec time diff --git a/steps/lwext4-1.0.0-lb1/files/early-artifacts-needed-after-fiwix.txt b/steps/lwext4-1.0.0-lb1/files/early-artifacts-needed-after-fiwix.txt deleted file mode 100644 index 7d0a0ff..0000000 --- a/steps/lwext4-1.0.0-lb1/files/early-artifacts-needed-after-fiwix.txt +++ /dev/null @@ -1,177 +0,0 @@ -# SPDX-FileCopyrightText: 2023 Richard Masters -# SPDX-License-Identifier: MIT -/usr/bin/blood-elf -/usr/bin/catm -/usr/bin/chmod -/usr/bin/get_machine -/usr/bin/hex2 -/usr/bin/kaem -/usr/bin/match -/usr/bin/M1 -/usr/bin/M2-Mesoplanet -/usr/bin/M2-Planet -/usr/bin/mkdir -/usr/bin/sha256sum -/usr/bin/unbz2 -/usr/bin/ungz -/usr/bin/untar -/usr/bin/cp -/usr/bin/replace -/usr/bin/rm -/usr/bin/checksum-transcriber -/usr/bin/tcc -/usr/bin/tcc-0.9.26 -/usr/bin/kexec-linux -/usr/lib/mes/libc.a -/usr/lib/mes/libgetopt.a -/usr/lib/mes/crt1.o -/usr/lib/mes/crti.o -/usr/lib/mes/crtn.o -/usr/lib/mes/tcc/libtcc1.a -/usr/include/mes/alloca.h -/usr/include/mes/argz.h -/usr/include/mes/ar.h -/usr/include/mes/assert.h -/usr/include/mes/ctype.h -/usr/include/mes/dirent.h -/usr/include/mes/dirstream.h -/usr/include/mes/dlfcn.h -/usr/include/mes/endian.h -/usr/include/mes/errno.h -/usr/include/mes/fcntl.h -/usr/include/mes/features.h -/usr/include/mes/float.h -/usr/include/mes/getopt.h -/usr/include/mes/grp.h -/usr/include/mes/inttypes.h -/usr/include/mes/libgen.h -/usr/include/mes/limits.h -/usr/include/mes/locale.h -/usr/include/mes/math.h -/usr/include/mes/memory.h -/usr/include/mes/pwd.h -/usr/include/mes/setjmp.h -/usr/include/mes/signal.h -/usr/include/mes/stdarg.h -/usr/include/mes/stdbool.h -/usr/include/mes/stddef.h -/usr/include/mes/stdint.h -/usr/include/mes/stdio.h -/usr/include/mes/stdlib.h -/usr/include/mes/stdnoreturn.h -/usr/include/mes/string.h -/usr/include/mes/strings.h -/usr/include/mes/termio.h -/usr/include/mes/time.h -/usr/include/mes/unistd.h -/usr/include/mes/arch/kernel-stat.h -/usr/include/mes/arch/syscall.h -/usr/include/mes/linux/syscall.h -/usr/include/mes/linux/x86/syscall.h -/usr/include/mes/mes/builtins.h -/usr/include/mes/mes/cc.h -/usr/include/mes/mes/config.h -/usr/include/mes/mes/constants.h -/usr/include/mes/mes/lib.h -/usr/include/mes/mes/lib-cc.h -/usr/include/mes/mes/lib-mini.h -/usr/include/mes/mes/mes.h -/usr/include/mes/mes/symbols.h -/usr/include/mes/sys/cdefs.h -/usr/include/mes/sys/dir.h -/usr/include/mes/sys/file.h -/usr/include/mes/sys/ioctl.h -/usr/include/mes/sys/mman.h -/usr/include/mes/sys/param.h -/usr/include/mes/sys/resource.h -/usr/include/mes/sys/select.h -/usr/include/mes/sys/stat.h -/usr/include/mes/sys/timeb.h -/usr/include/mes/sys/time.h -/usr/include/mes/sys/times.h -/usr/include/mes/sys/types.h -/usr/include/mes/sys/ucontext.h -/usr/include/mes/sys/user.h -/usr/include/mes/sys/wait.h -/init -/kaem.run -/external/distfiles/make-3.82.tar.bz2 -/external/distfiles/patch-2.5.9.tar.gz -/external/distfiles/gzip-1.2.4.tar.gz -/external/distfiles/tar-1.12.tar.gz -/external/distfiles/sed-4.0.9.tar.gz -/external/distfiles/bzip2-1.0.8.tar.gz -/external/distfiles/coreutils-5.0.tar.bz2 -/external/distfiles/heirloom-devtools-070527.tar.bz2 -/external/distfiles/bash-2.05b.tar.gz -/external/distfiles/flex-2.5.11.tar.gz -/external/distfiles/tcc-0.9.27.tar.bz2 -/external/distfiles/musl-1.1.24.tar.gz -/external/distfiles/tcc-0.9.27.tar.bz2 -/external/distfiles/musl-1.1.24.tar.gz -/external/distfiles/tcc-0.9.27.tar.bz2 -/external/distfiles/sed-4.0.9.tar.gz -/external/distfiles/bzip2-1.0.8.tar.gz -/external/distfiles/m4-1.4.10.tar.gz -/external/distfiles/flex-2.6.4.tar.gz -/external/distfiles/bison-3.4.1.tar.gz -/external/distfiles/bison-3.4.1.tar.gz -/external/distfiles/bison-3.4.1.tar.gz -/external/distfiles/grep-2.4.tar.gz -/external/distfiles/diffutils-2.7.tar.gz -/external/distfiles/coreutils-5.0.tar.bz2 -/external/distfiles/coreutils-6.10.tar.gz -/external/distfiles/gawk-3.0.4.tar.gz -/external/distfiles/perl-5.000.tar.gz -/external/distfiles/perl-5.003.tar.gz -/external/distfiles/perl5.004_05.tar.gz -/external/distfiles/perl5.005_03.tar.gz -/external/distfiles/perl-5.6.2.tar.gz -/external/distfiles/autoconf-2.52.tar.bz2 -/external/distfiles/automake-1.6.3.tar.bz2 -/external/distfiles/automake-1.6.3.tar.bz2 -/external/distfiles/autoconf-2.53.tar.bz2 -/external/distfiles/automake-1.7.tar.bz2 -/external/distfiles/autoconf-2.54.tar.bz2 -/external/distfiles/autoconf-2.55.tar.bz2 -/external/distfiles/automake-1.7.8.tar.bz2 -/external/distfiles/autoconf-2.57.tar.bz2 -/external/distfiles/autoconf-2.59.tar.bz2 -/external/distfiles/automake-1.8.5.tar.bz2 -/external/distfiles/help2man-1.36.4.tar.gz -/external/distfiles/autoconf-2.61.tar.bz2 -/external/distfiles/automake-1.9.6.tar.bz2 -/external/distfiles/automake-1.10.3.tar.bz2 -/external/distfiles/autoconf-2.64.tar.bz2 -/external/distfiles/automake-1.11.2.tar.bz2 -/external/distfiles/autoconf-2.69.tar.gz -/external/distfiles/libtool-2.2.4.tar.bz2 -/external/distfiles/automake-1.15.1.tar.gz -/external/distfiles/binutils-2.30.tar.bz2 -/external/distfiles/musl-1.1.24.tar.gz -/external/distfiles/tcc-0.9.27.tar.bz2 -/external/distfiles/gcc-core-4.0.4.tar.bz2 -/external/distfiles/automake-1.16.3.tar.gz -/external/distfiles/findutils-4.2.33.tar.gz -/external/distfiles/gnulib-8e128e.tar.gz -/external/distfiles/musl-1.2.4.tar.gz -/external/distfiles/gcc-core-4.0.4.tar.bz2 -/external/distfiles/automake-1.16.3.tar.gz -/external/distfiles/util-linux-2.19.1.tar.gz -/external/distfiles/e2fsprogs-1.45.7.tar.gz -/external/distfiles/CaseFolding.txt -/external/distfiles/DerivedAge.txt -/external/distfiles/DerivedCombiningClass.txt -/external/distfiles/DerivedCoreProperties.txt -/external/distfiles/NormalizationCorrections.txt -/external/distfiles/NormalizationTest.txt -/external/distfiles/UnicodeData.txt -/external/distfiles/v10.0.1.tar.gz -/external/distfiles/kbd-1.15.tar.gz -/external/distfiles/make-3.82.tar.bz2 -/external/distfiles/ed-1.4.tar.gz -/external/distfiles/bc-1.07.1.tar.gz -/external/distfiles/v2.0.22.tar.gz -/external/distfiles/linux-4.9.10.tar.gz -/external/distfiles/deblob-4.9 -/external/distfiles/curl-8.5.0.tar.bz2 diff --git a/steps/lwext4-1.0.0-lb1/files/make_fiwix_initrd.c b/steps/lwext4-1.0.0-lb1/files/make_fiwix_initrd.c index e37556e..c141fcb 100644 --- a/steps/lwext4-1.0.0-lb1/files/make_fiwix_initrd.c +++ b/steps/lwext4-1.0.0-lb1/files/make_fiwix_initrd.c @@ -47,7 +47,7 @@ #include "../blockdev/windows/file_windows.h" #define BLOCK_SIZE 1024 -#define FILENAME_LENGTH 256 +#define FILENAME_LENGTH 1024 #define INITRD_MB 1280 const char *input_name = NULL; @@ -131,29 +131,34 @@ bool lwext4_umount(void) bool copy_file(char *src_path, char *dest_path) { + ext4_file dest_file; + int err; + err = ext4_inode_exist(dest_path, EXT4_DE_UNKNOWN); + if (err == EOK) { + /* file already copied or directory already created */ + return EXIT_SUCCESS; + } printf("copy_file: %s\n", src_path); - ext4_file dest_file; + FILE *src_file = fopen(src_path, "rb"); if (!src_file) { - printf("fopen '%s' error.\n", src_path); - return EXIT_FAILURE; + printf("fopen '%s' error.\n", src_path); + return EXIT_FAILURE; } - fseek(src_file, 0, SEEK_END); - int src_len = ftell(src_file); - char * src_mem = malloc(src_len); - int err; + fseek(src_file, 0, SEEK_END); + int src_len = ftell(src_file); - fseek(src_file, 0, SEEK_SET); - if (src_len > 0) { - int read_len = fread(src_mem, src_len, 1, src_file); - fclose(src_file); - if (read_len < 1) { - printf("src fread error file: '%s' read count: %d\n", src_path, read_len); - } + fseek(src_file, 0, SEEK_SET); + + char * src_mem = malloc(src_len); + int read_len = fread(src_mem, src_len, 1, src_file); + fclose(src_file); + if (read_len < 1) { + printf("src fread error file: '%s' read count: %d\n", src_path, read_len); } - err = ext4_fopen(&dest_file, dest_path, "wb"); + err = ext4_fopen(&dest_file, dest_path, "wb"); if (err != EOK) { printf("ext4_open error: %d\n", err); return EXIT_FAILURE; @@ -176,23 +181,23 @@ bool copy_file(char *src_path, char *dest_path) free(src_mem); } -bool copy_file_list(char *file_list_path) +bool copy_file_system() { - char src_filename[FILENAME_LENGTH]; - char dst_filename[FILENAME_LENGTH]; - - FILE *file_list = fopen(file_list_path, "r"); - while(fgets(src_filename, FILENAME_LENGTH, file_list)) { - /* Skip comments */ - if (src_filename[0] == '#') { + int filenum; + unsigned int filename_addr; + /* +4 to account for /mp and null termination */ + char dst_filename[FILENAME_LENGTH + 4]; + for (filenum = 14335, filename_addr = 0xfff000; filenum >= 4; filenum--, filename_addr -= FILENAME_LENGTH) { + /* Avoid including fiwix.ext2 in itself */ + if (((char *) filename_addr)[0] != '/' || + !strcmp((char *) filename_addr, "/") || + !strcmp((char *) filename_addr, "/boot/fiwix.ext2")) { continue; } - src_filename[strlen(src_filename) - 1] = 0; /* strip newline */ strcpy(dst_filename, "/mp"); - strcat(dst_filename, src_filename); - copy_file(src_filename, dst_filename); + strcat(dst_filename, (char *) filename_addr); + copy_file((char *) filename_addr, dst_filename); } - fclose(file_list); } int main(int argc, char **argv) @@ -202,8 +207,20 @@ int main(int argc, char **argv) char zeros[BLOCK_SIZE]; unsigned int next_file_address; + int filenum; + unsigned int filename_addr; + unsigned int file_addr; - next_file_address = *((unsigned int *) 0x7F8D); + next_file_address = 0; + for (filenum = 14335, filename_addr = 0xfff000, file_addr = 0x1037FF4; + filenum >= 4; + filenum--, filename_addr -= FILENAME_LENGTH, file_addr -= 16) { + if (((char *) filename_addr)[0] == '/') { + next_file_address = *((unsigned int *) file_addr); + next_file_address += *((unsigned int *) (file_addr + 4)); + break; + } + } printf("Starting fiwix.ext2 at addr 0x%08x\n", next_file_address); @@ -285,7 +302,7 @@ int main(int argc, char **argv) return EXIT_FAILURE; } - copy_file_list("/steps/lwext4-1.0.0-lb1/files/fiwix-file-list.txt"); + copy_file_system(); if (!lwext4_umount()) return EXIT_FAILURE; diff --git a/steps/lwext4-1.0.0-lb1/lwext4-1.0.0-lb1.checksums b/steps/lwext4-1.0.0-lb1/lwext4-1.0.0-lb1.checksums index cbbef75..788b7e2 100644 --- a/steps/lwext4-1.0.0-lb1/lwext4-1.0.0-lb1.checksums +++ b/steps/lwext4-1.0.0-lb1/lwext4-1.0.0-lb1.checksums @@ -1 +1 @@ -68e17d5d7d372101fb7d9857113b9d9b0d19610b19014a742d054d52998c782d /usr/bin/make_fiwix_initrd +76bc5cca226d1244f1f648656959a955e8ff61fdca3d98d589a46be811628ba7 /usr/bin/make_fiwix_initrd