Add newline tokens to parse stream

- needed to determine start and end of macro directives
- the main parser doesn't need them, so strip out the newline tokens
  before parsing to avoid changing it

cc.c

@@ -24,6 +24,7 @@
 void initialize_types();
 struct token_list* read_all_tokens(FILE* a, struct token_list* current, char* filename);
 struct token_list* reverse_list(struct token_list* head);
+void eat_newline_tokens();
 void program();
 void recursive_output(struct token_list* i, FILE* out);
 int numerate_string(char *a);
@@ -145,6 +146,9 @@ int main(int argc, char** argv)
 	}
 	global_token = reverse_list(global_token);
 
+	/* the main parser doesn't know how to handle newline tokens */
+	eat_newline_tokens();
+
 	initialize_types();
 	reset_hold_string();
 	output_list = NULL;

cc_core.c

@@ -1845,6 +1845,7 @@ new_type:
 	}
 	else
 	{
+		require(!match("\n", global_token->s), "unexpected newline token\n");
 		type_size = type_name();
 		if(NULL == type_size)
 		{

cc_macro.c

@@ -0,0 +1,56 @@
+/* Copyright (C) 2021 Sanne Wouda
+ * This file is part of M2-Planet.
+ *
+ * M2-Planet is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * M2-Planet is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with M2-Planet.  If not, see <http://www.gnu.org/licenses/>.
+ */
+#include "cc.h"
+#include "gcc_req.h"
+
+/* point where we are currently modifying the global_token list */
+struct token_list* macro_token;
+
+void eat_current_token()
+{
+    struct token_list* tmp;
+
+    if (NULL != macro_token->prev)
+        macro_token->prev->next = macro_token->next;
+    else
+        global_token = macro_token->next;
+
+    /* update backlinks */
+    if (NULL != macro_token->next)
+        macro_token->next->prev = macro_token->prev;
+
+    tmp = macro_token;
+    macro_token = macro_token->next;
+    free(tmp);
+}
+
+void eat_newline_tokens()
+{
+    macro_token = global_token;
+
+    while (TRUE)
+    {
+        if (NULL == macro_token)
+            return;
+
+        if(match("\n", macro_token->s))
+            eat_current_token();
+        else
+            macro_token = macro_token->next;
+    }
+}
+

cc_reader.c

@@ -26,11 +26,6 @@ char* file;
 int clearWhiteSpace(int c)
 {
 	if((32 == c) || (9 == c)) return clearWhiteSpace(fgetc(input));
-	else if (10 == c)
-	{
-		line = line + 1;
-		return clearWhiteSpace(fgetc(input));
-	}
 	return c;
 }
 
@@ -163,6 +158,11 @@ reset:
 			goto reset;
 		}
 	}
+	else if (c == '\n')
+	{
+		line = line + 1;
+		c = consume_byte(c);
+	}
 	else
 	{
 		c = consume_byte(c);

makefile

@@ -39,6 +39,7 @@ M2-Planet: bin results cc.h cc_reader.c cc_strings.c cc_types.c cc_core.c cc.c c
 	cc_strings.c \
 	cc_types.c \
 	cc_core.c \
+	cc_macro.c \
 	cc.c \
 	cc.h \
 	cc_globals.c \

test/test.answers

@@ -169,8 +169,8 @@ a2a83f42119e646b389b98647cf6cf2aa9597185997c9453db746178c8c4c0bf  test/results/t
 698853b79efb30865a663c4863c050639eb21c7400008f7840830503928973d4  test/results/test0106-knight-native-binary
 45c2ba61dc209d7ffa39de9ff0f0a7f8f3ea4d7e38598c72f982fcaf9a05c84a  test/results/test0106-knight-posix-binary
 944580ff4aae38aafac139faf6eed5bfe4ff68b01a7a3adfa346de8803101182  test/results/test0106-x86-binary
-b20b981b871539b73ebb5034a12ddbc6da478ccc1e66f3d607f1f6133a6d8bd0  test/results/test1000-aarch64-binary
-5307ff61ac4adb237d8ff403647283bb127b7226f61e07e68161bd3a8f8fec13  test/results/test1000-amd64-binary
-b35fae63b64e85cf7e70f867f6eb0c38a7b70b616fc66b9dc900b2a04c2807fa  test/results/test1000-armv7l-binary
-3ce668b5d32a36198be73deccc9a4c3fefdb431c13e3f7adc2603241b92f717d  test/results/test1000-knight-posix-binary
-744b64abdeb09f2e8963e3189bfde1174a21943818853d2759b3d8a9ba34f7d1  test/results/test1000-x86-binary
+0ab012de93d082dc205137aa7e767f4c39d2609d94c9bc1bdcf08dcb9417d5cd  test/results/test1000-aarch64-binary
+b8b05d03088b7cff4128b618051c45f26ef386a33513d7d48c64bbdf8f8aad63  test/results/test1000-amd64-binary
+bb3cb53f2717e230ea773e6366a553f834421c1427985ea101451a89d426e8c3  test/results/test1000-armv7l-binary
+2a165debb8f2dbe877a6d246267e181ad49035eee07a4190a81c195098fb1abb  test/results/test1000-knight-posix-binary
+f9435eaaadf614936dbe2cab8fa5eb4f088b4e6bb2d25d003083671e9fb2658c  test/results/test1000-x86-binary

test/test1000/hello-aarch64.sh

@@ -37,6 +37,7 @@ set -ex
 	-f cc_strings.c \
 	-f cc_types.c \
 	-f cc_core.c \
+	-f cc_macro.c \
 	-f cc.c \
 	--debug \
 	--bootstrap-mode \
@@ -86,6 +87,7 @@ then
 		-f cc_strings.c \
 		-f cc_types.c \
 		-f cc_core.c \
+		-f cc_macro.c \
 		-f cc.c \
 		--bootstrap-mode \
 		-o test/test1000/proof || exit 5

test/test1000/hello-amd64.sh

@@ -36,6 +36,7 @@ set -ex
 	-f cc_strings.c \
 	-f cc_types.c \
 	-f cc_core.c \
+	-f cc_macro.c \
 	-f cc.c \
 	--debug \
 	--bootstrap-mode \
@@ -85,6 +86,7 @@ then
 		-f cc_strings.c \
 		-f cc_types.c \
 		-f cc_core.c \
+		-f cc_macro.c \
 		-f cc.c \
 		--bootstrap-mode \
 		-o test/test1000/proof || exit 5

test/test1000/hello-armv7l.sh

@@ -36,6 +36,7 @@ set -ex
 	-f cc_strings.c \
 	-f cc_types.c \
 	-f cc_core.c \
+	-f cc_macro.c \
 	-f cc.c \
 	--debug \
 	--bootstrap-mode \
@@ -85,6 +86,7 @@ then
 		-f cc_strings.c \
 		-f cc_types.c \
 		-f cc_core.c \
+		-f cc_macro.c \
 		-f cc.c \
 		--bootstrap-mode \
 		-o test/test1000/proof || exit 5

test/test1000/hello-knight-posix.sh

@@ -36,6 +36,7 @@ set -ex
 	-f cc_strings.c \
 	-f cc_types.c \
 	-f cc_core.c \
+	-f cc_macro.c \
 	-f cc.c \
 	--debug \
 	--bootstrap-mode \
@@ -79,6 +80,7 @@ then
 		-f cc_strings.c \
 		-f cc_types.c \
 		-f cc_core.c \
+		-f cc_macro.c \
 		-f cc.c \
 		--bootstrap-mode \
 		-o test/test1000/proof || exit 4

test/test1000/hello-x86.sh

@@ -36,6 +36,7 @@ set -ex
 	-f cc_strings.c \
 	-f cc_types.c \
 	-f cc_core.c \
+	-f cc_macro.c \
 	-f cc.c \
 	--debug \
 	--bootstrap-mode \
@@ -85,6 +86,7 @@ then
 		-f cc_strings.c \
 		-f cc_types.c \
 		-f cc_core.c \
+		-f cc_macro.c \
 		-f cc.c \
 		--bootstrap-mode \
 		-o test/test1000/proof || exit 5

test/test1000/proof.answer

@@ -1 +1 @@
-7d38cf1d5cd25d976cc39dcef8b46f95bb7ea4eefd15e97e3d8c3e66fa06281c  test/test1000/proof
+12ed7170da8642ed218a9eef2c11f65dae37cc65a331431aa7ff58063ed8734a  test/test1000/proof