Implement --bootstrap-mode for preprocessor

- // and # lines always appear in the result of read_all_tokens

- in bootstrap mode, // tokens (but not the subsequent lines) and #
  lines are stripped. no preprocessing happens.

- in non-bootstrap mode, // lines are stripped and # lines stay for the
  preprocessing phase

- updates tests to pass --bootstrap-mode when necessary

cc.c

@@ -24,6 +24,11 @@
 void initialize_types();
 struct token_list* read_all_tokens(FILE* a, struct token_list* current, char* filename);
 struct token_list* reverse_list(struct token_list* head);
+
+struct token_list* remove_line_comments(struct token_list* head);
+struct token_list* remove_line_comment_tokens(struct token_list* head);
+struct token_list* remove_preprocessor_directives(struct token_list* head);
+
 void eat_newline_tokens();
 void preprocess();
 void program();
@@ -154,7 +159,17 @@ int main(int argc, char** argv)
 	}
 	global_token = reverse_list(global_token);
 
-	preprocess();
+	if (BOOTSTRAP_MODE)
+	{
+		global_token = remove_line_comment_tokens(global_token);
+		global_token = remove_preprocessor_directives(global_token);
+	}
+	else
+	{
+		global_token = remove_line_comments(global_token);
+		preprocess();
+	}
+
 	if (PREPROCESSOR_MODE)
 	{
 		file_print("\n/* Preprocessed source */\n", destination_file);

cc_macro.c

@@ -20,6 +20,7 @@
 void require(int bool, char* error);
 int numerate_string(char* a);
 void line_error_token(struct token_list* list);
+struct token_list* eat_token(struct token_list* head);
 
 struct conditional_inclusion
 {
@@ -42,26 +43,14 @@ struct token_list* macro_token;
 
 void eat_current_token()
 {
-	struct token_list* tmp;
+	int update_global_token = FALSE;
+	if (macro_token == global_token)
+		update_global_token = TRUE;
 
-	if(NULL != macro_token->prev)
-	{
-		macro_token->prev->next = macro_token->next;
-	}
-	else
-	{
-		global_token = macro_token->next;
-	}
+	macro_token = eat_token(macro_token);
 
-	/* update backlinks */
-	if(NULL != macro_token->next)
-	{
-		macro_token->next->prev = macro_token->prev;
-	}
-
-	tmp = macro_token;
-	macro_token = macro_token->next;
-	free(tmp);
+	if(update_global_token)
+		global_token = macro_token;
 }
 
 void eat_newline_tokens()

cc_reader.c

@@ -84,6 +84,110 @@ void reset_hold_string()
 	}
 }
 
+/* note if this is the first token in the list, head needs fixing up */
+struct token_list* eat_token(struct token_list* token)
+{
+	if(NULL != token->prev)
+	{
+		token->prev->next = token->next;
+	}
+
+	/* update backlinks */
+	if(NULL != token->next)
+	{
+		token->next->prev = token->prev;
+	}
+
+	return token->next;
+}
+
+struct token_list* eat_until_newline(struct token_list* head)
+{
+	while (NULL != head)
+	{
+		if('\n' == head->s[0])
+		{
+			return head;
+		}
+		else
+		{
+			head = eat_token(head);
+		}
+	}
+
+	return NULL;
+}
+
+struct token_list* remove_line_comments(struct token_list* head)
+{
+	struct token_list* first = NULL;
+
+	while (NULL != head)
+	{
+		if(match("//", head->s))
+		{
+			head = eat_until_newline(head);
+		}
+		else
+		{
+			if(NULL == first)
+			{
+				first = head;
+			}
+			head = head->next;
+		}
+	}
+
+	return first;
+}
+
+struct token_list* remove_line_comment_tokens(struct token_list* head)
+{
+	struct token_list* first = NULL;
+
+	while (NULL != head)
+	{
+		if(match("//", head->s))
+		{
+			head = eat_token(head);
+		}
+		else
+		{
+			if(NULL == first)
+			{
+				first = head;
+			}
+			head = head->next;
+		}
+	}
+
+	return first;
+}
+
+struct token_list* remove_preprocessor_directives(struct token_list* head)
+{
+	struct token_list* first = NULL;
+
+	while (NULL != head)
+	{
+		if('#' == head->s[0])
+		{
+			head = eat_until_newline(head);
+		}
+		else
+		{
+			if(NULL == first)
+			{
+				first = head;
+			}
+			head = head->next;
+		}
+	}
+
+	return first;
+}
+
+
 int get_token(int c)
 {
 	struct token_list* current = calloc(1, sizeof(struct token_list));
@@ -144,8 +248,7 @@ reset:
 		}
 		else if(c == '/')
 		{
-			c = fgetc(input);
-			goto reset;
+			c = consume_byte(c);
 		}
 	}
 	else if (c == '\n')

test/test.answers

@@ -175,8 +175,8 @@ a2a83f42119e646b389b98647cf6cf2aa9597185997c9453db746178c8c4c0bf  test/results/t
 698853b79efb30865a663c4863c050639eb21c7400008f7840830503928973d4  test/results/test0106-knight-native-binary
 45c2ba61dc209d7ffa39de9ff0f0a7f8f3ea4d7e38598c72f982fcaf9a05c84a  test/results/test0106-knight-posix-binary
 944580ff4aae38aafac139faf6eed5bfe4ff68b01a7a3adfa346de8803101182  test/results/test0106-x86-binary
-770ff703d257edfe01563649b14f73aa8cc844460d7e1dbc27420f407c7e8b2f  test/results/test1000-aarch64-binary
-057b66ec17170ec1e13a457bc2064d13b6b57e6b6f0abb7aa260f1fa1099edc9  test/results/test1000-amd64-binary
-63d09ed177486cb7fe23f541d5e4f753359b05ad3bc61703efde555ec5793a03  test/results/test1000-armv7l-binary
-b14e247056af6c0f0a8b983444a8a04b4c9221f0a66b8d89b218c6cfeed55cf1  test/results/test1000-knight-posix-binary
-e2944312b3a46b9dc5111b078655436b731f8f841b3d0aeb8fe43a6d85ca3ae1  test/results/test1000-x86-binary
+d4ea011b32eabe55d30d9872fb62472a7941845cf2c9198bd84378fce9098505  test/results/test1000-aarch64-binary
+0c21df6c75998ef0d2e4188e591759b147d1d3011675c2243b9d5d90f93745c9  test/results/test1000-amd64-binary
+b97927188f7683005fb97d82f88a232284a7b472446f4afb2c5ccdef59f335f8  test/results/test1000-armv7l-binary
+d67c1c38a4d3e8b4461df71754a7d0ae556e549a7718fd190964e0a3c1835ecf  test/results/test1000-knight-posix-binary
+42dd021d8a0c8979a8796b1f459f29adca28b293b73530d17f34753b5a17d016  test/results/test1000-x86-binary

test/test0008/hello-aarch64.sh

@@ -23,6 +23,7 @@ bin/M2-Planet --architecture aarch64 \
 	-f test/common_aarch64/functions/exit.c \
 	-f test/common_aarch64/functions/malloc.c \
 	-f test/test0008/struct.c \
+	--bootstrap-mode \
 	-o test/test0008/struct.M1 || exit 1
 
 # Macro assemble with libc written in M1-Macro

test/test0008/hello-amd64.sh

@@ -21,6 +21,7 @@ bin/M2-Planet --architecture amd64 -f test/common_amd64/functions/putchar.c \
 	-f test/common_amd64/functions/exit.c \
 	-f test/common_amd64/functions/malloc.c \
 	-f test/test0008/struct.c \
+	--bootstrap-mode \
 	-o test/test0008/struct.M1 || exit 1
 
 # Macro assemble with libc written in M1-Macro

test/test0008/hello-armv7l.sh

@@ -21,6 +21,7 @@ bin/M2-Planet --architecture armv7l -f test/common_armv7l/functions/putchar.c \
 	-f test/common_armv7l/functions/exit.c \
 	-f test/common_armv7l/functions/malloc.c \
 	-f test/test0008/struct.c \
+	--bootstrap-mode \
 	-o test/test0008/struct.M1 || exit 1
 
 # Macro assemble with libc written in M1-Macro

test/test0008/hello-knight-native.sh

@@ -21,6 +21,7 @@ bin/M2-Planet --architecture knight-native -f test/common_knight/functions/putch
 	-f test/common_knight/functions/exit-native.c \
 	-f test/common_knight/functions/malloc.c \
 	-f test/test0008/struct.c \
+	--bootstrap-mode \
 	-o test/test0008/struct.M1 || exit 1
 
 # Macro assemble with libc written in M1-Macro

test/test0008/hello-knight-posix.sh

@@ -21,6 +21,7 @@ bin/M2-Planet --architecture knight-posix -f test/common_knight/functions/putcha
 	-f test/common_knight/functions/exit.c \
 	-f test/common_knight/functions/malloc.c \
 	-f test/test0008/struct.c \
+	--bootstrap-mode \
 	-o test/test0008/struct.M1 || exit 1
 
 # Macro assemble with libc written in M1-Macro

test/test0008/hello-x86.sh

@@ -21,6 +21,7 @@ bin/M2-Planet --architecture x86 -f test/common_x86/functions/putchar.c \
 	-f test/common_x86/functions/exit.c \
 	-f test/common_x86/functions/malloc.c \
 	-f test/test0008/struct.c \
+	--bootstrap-mode \
 	-o test/test0008/struct.M1 || exit 1
 
 # Macro assemble with libc written in M1-Macro

test/test0010/hello-aarch64.sh

@@ -23,6 +23,7 @@ bin/M2-Planet --architecture aarch64 \
 	-f test/common_aarch64/functions/exit.c \
 	-f test/common_aarch64/functions/malloc.c \
 	-f test/test0010/nested_struct.c \
+	--bootstrap-mode \
 	-o test/test0010/nested_struct.M1 || exit 1
 
 # Macro assemble with libc written in M1-Macro

test/test0010/hello-amd64.sh

@@ -21,6 +21,7 @@ bin/M2-Planet --architecture amd64 -f test/common_amd64/functions/putchar.c \
 	-f test/common_amd64/functions/exit.c \
 	-f test/common_amd64/functions/malloc.c \
 	-f test/test0010/nested_struct.c \
+	--bootstrap-mode \
 	-o test/test0010/nested_struct.M1 || exit 1
 
 # Macro assemble with libc written in M1-Macro

test/test0010/hello-armv7l.sh

@@ -21,6 +21,7 @@ bin/M2-Planet --architecture armv7l -f test/common_armv7l/functions/putchar.c \
 	-f test/common_armv7l/functions/exit.c \
 	-f test/common_armv7l/functions/malloc.c \
 	-f test/test0010/nested_struct.c \
+	--bootstrap-mode \
 	-o test/test0010/nested_struct.M1 || exit 1
 
 # Macro assemble with libc written in M1-Macro

test/test0010/hello-knight-native.sh

@@ -21,6 +21,7 @@ bin/M2-Planet --architecture knight-native -f test/common_knight/functions/putch
 	-f test/common_knight/functions/exit-native.c \
 	-f test/common_knight/functions/malloc.c \
 	-f test/test0010/nested_struct.c \
+	--bootstrap-mode \
 	-o test/test0010/nested_struct.M1 || exit 1
 
 # Macro assemble with libc written in M1-Macro

test/test0010/hello-knight-posix.sh

@@ -21,6 +21,7 @@ bin/M2-Planet --architecture knight-posix -f test/common_knight/functions/putcha
 	-f test/common_knight/functions/exit.c \
 	-f test/common_knight/functions/malloc.c \
 	-f test/test0010/nested_struct.c \
+	--bootstrap-mode \
 	-o test/test0010/nested_struct.M1 || exit 1
 
 # Macro assemble with libc written in M1-Macro

test/test0010/hello-x86.sh

@@ -21,6 +21,7 @@ bin/M2-Planet --architecture x86 -f test/common_x86/functions/putchar.c \
 	-f test/common_x86/functions/exit.c \
 	-f test/common_x86/functions/malloc.c \
 	-f test/test0010/nested_struct.c \
+	--bootstrap-mode \
 	-o test/test0010/nested_struct.M1 || exit 1
 
 # Macro assemble with libc written in M1-Macro

test/test0017/hello-aarch64.sh

@@ -23,6 +23,7 @@ bin/M2-Planet --architecture aarch64 \
 	-f functions/calloc.c \
 	-f test/common_aarch64/functions/putchar.c \
 	-f test/test0017/memset.c \
+	--bootstrap-mode \
 	-o test/test0017/memset.M1 || exit 1
 
 # Macro assemble with libc written in M1-Macro

test/test0017/hello-amd64.sh

@@ -21,6 +21,7 @@ bin/M2-Planet --architecture amd64 -f test/common_amd64/functions/malloc.c \
 	-f functions/calloc.c \
 	-f test/common_amd64/functions/putchar.c \
 	-f test/test0017/memset.c \
+	--bootstrap-mode \
 	-o test/test0017/memset.M1 || exit 1
 
 # Macro assemble with libc written in M1-Macro

test/test0017/hello-armv7l.sh

@@ -21,6 +21,7 @@ bin/M2-Planet --architecture armv7l -f test/common_armv7l/functions/malloc.c \
 	-f functions/calloc.c \
 	-f test/common_armv7l/functions/putchar.c \
 	-f test/test0017/memset.c \
+	--bootstrap-mode \
 	-o test/test0017/memset.M1 || exit 1
 
 # Macro assemble with libc written in M1-Macro

test/test0017/hello-knight-native.sh

@@ -21,6 +21,7 @@ bin/M2-Planet --architecture knight-native -f test/common_knight/functions/mallo
 	-f functions/calloc.c \
 	-f test/common_knight/functions/putchar-native.c \
 	-f test/test0017/memset.c \
+	--bootstrap-mode \
 	-o test/test0017/memset.M1 || exit 1
 
 # Macro assemble with libc written in M1-Macro

test/test0017/hello-knight-posix.sh

@@ -21,6 +21,7 @@ bin/M2-Planet --architecture knight-posix -f test/common_knight/functions/malloc
 	-f functions/calloc.c \
 	-f test/common_knight/functions/putchar.c \
 	-f test/test0017/memset.c \
+	--bootstrap-mode \
 	-o test/test0017/memset.M1 || exit 1
 
 # Macro assemble with libc written in M1-Macro

test/test0017/hello-x86.sh

@@ -21,6 +21,7 @@ bin/M2-Planet --architecture x86 -f test/common_x86/functions/malloc.c \
 	-f functions/calloc.c \
 	-f test/common_x86/functions/putchar.c \
 	-f test/test0017/memset.c \
+	--bootstrap-mode \
 	-o test/test0017/memset.M1 || exit 1
 
 # Macro assemble with libc written in M1-Macro

test/test0020/hello-aarch64.sh

@@ -23,6 +23,7 @@ bin/M2-Planet --architecture aarch64 \
 	-f test/common_aarch64/functions/exit.c \
 	-f test/common_aarch64/functions/malloc.c \
 	-f test/test0020/struct.c \
+	--bootstrap-mode \
 	-o test/test0020/struct.M1 || exit 1
 
 # Macro assemble with libc written in M1-Macro

test/test0020/hello-amd64.sh

@@ -21,6 +21,7 @@ bin/M2-Planet --architecture amd64 -f test/common_amd64/functions/putchar.c \
 	-f test/common_amd64/functions/exit.c \
 	-f test/common_amd64/functions/malloc.c \
 	-f test/test0020/struct.c \
+	--bootstrap-mode \
 	-o test/test0020/struct.M1 || exit 1
 
 # Macro assemble with libc written in M1-Macro

test/test0020/hello-armv7l.sh

@@ -21,6 +21,7 @@ bin/M2-Planet --architecture armv7l -f test/common_armv7l/functions/putchar.c \
 	-f test/common_armv7l/functions/exit.c \
 	-f test/common_armv7l/functions/malloc.c \
 	-f test/test0020/struct.c \
+	--bootstrap-mode \
 	-o test/test0020/struct.M1 || exit 1
 
 # Macro assemble with libc written in M1-Macro

test/test0020/hello-knight-native.sh

@@ -21,6 +21,7 @@ bin/M2-Planet --architecture knight-native -f test/common_knight/functions/putch
 	-f test/common_knight/functions/exit-native.c \
 	-f test/common_knight/functions/malloc.c \
 	-f test/test0020/struct.c \
+	--bootstrap-mode \
 	-o test/test0020/struct.M1 || exit 1
 
 # Macro assemble with libc written in M1-Macro

test/test0020/hello-knight-posix.sh

@@ -21,6 +21,7 @@ bin/M2-Planet --architecture knight-posix -f test/common_knight/functions/putcha
 	-f test/common_knight/functions/exit.c \
 	-f test/common_knight/functions/malloc.c \
 	-f test/test0020/struct.c \
+	--bootstrap-mode \
 	-o test/test0020/struct.M1 || exit 1
 
 # Macro assemble with libc written in M1-Macro

test/test0020/hello-x86.sh

@@ -21,6 +21,7 @@ bin/M2-Planet --architecture x86 -f test/common_x86/functions/putchar.c \
 	-f test/common_x86/functions/exit.c \
 	-f test/common_x86/functions/malloc.c \
 	-f test/test0020/struct.c \
+	--bootstrap-mode \
 	-o test/test0020/struct.M1 || exit 1
 
 # Macro assemble with libc written in M1-Macro

test/test0106/hello-aarch64.sh

@@ -24,6 +24,7 @@ bin/M2-Planet --architecture aarch64 \
 	-f test/common_aarch64/functions/exit.c \
 	-f test/common_aarch64/functions/malloc.c \
 	-f test/test0106/cc500.c \
+	--bootstrap-mode \
 	-o test/test0106/cc0.M1 || exit 1
 
 # Macro assemble with libc written in M1-Macro

test/test0106/hello-amd64.sh

@@ -23,6 +23,7 @@ bin/M2-Planet --architecture amd64 \
 	-f test/common_amd64/functions/exit.c \
 	-f test/common_amd64/functions/malloc.c \
 	-f test/test0106/cc500.c \
+	--bootstrap-mode \
 	-o test/test0106/cc0.M1 || exit 1
 
 # Macro assemble with libc written in M1-Macro

test/test0106/hello-armv7l.sh

@@ -23,6 +23,7 @@ bin/M2-Planet --architecture armv7l \
 	-f test/common_armv7l/functions/exit.c \
 	-f test/common_armv7l/functions/malloc.c \
 	-f test/test0106/cc500.c \
+	--bootstrap-mode \
 	-o test/test0106/cc0.M1 || exit 1
 
 # Macro assemble with libc written in M1-Macro

test/test0106/hello-knight-native.sh

@@ -23,6 +23,7 @@ bin/M2-Planet --architecture knight-native \
 	-f test/common_knight/functions/exit-native.c \
 	-f test/common_knight/functions/malloc.c \
 	-f test/test0106/cc500.c \
+	--bootstrap-mode \
 	-o test/test0106/cc0.M1 || exit 1
 
 # Macro assemble with libc written in M1-Macro

test/test0106/hello-knight-posix.sh

@@ -23,6 +23,7 @@ bin/M2-Planet --architecture knight-posix \
 	-f test/common_knight/functions/exit.c \
 	-f test/common_knight/functions/malloc.c \
 	-f test/test0106/cc500.c \
+	--bootstrap-mode \
 	-o test/test0106/cc0.M1 || exit 1
 
 # Macro assemble with libc written in M1-Macro

test/test0106/hello-x86.sh

@@ -23,6 +23,7 @@ bin/M2-Planet --architecture x86 \
 	-f test/common_x86/functions/exit.c \
 	-f test/common_x86/functions/malloc.c \
 	-f test/test0106/cc500.c \
+	--bootstrap-mode \
 	-o test/test0106/cc0.M1 || exit 1
 
 # Macro assemble with libc written in M1-Macro

test/test1000/proof.answer

@@ -1 +1 @@
-58da283d4215ba779d3f5de527f6d918cfae5dc2a3c999fc92fb76dfc7cb97ad  test/test1000/proof
+5717733f915e9c4251f773783e210ca4b25e6a3c2f78c182fd905209911c974e  test/test1000/proof