Pass through literal space values from replacement lists.

This makes test 15 pass and also dramatically simplifies the lexer.

We were previously using a CONTROL state in the lexer to only emit
SPACE tokens when on text lines. But that's not actually what we
want. We need SPACE tokens in the replacement lists as well. Instead
of a lexer state for this, we now simply set a "space_tokens" flag
whenever we start constructing a pp_tokens list and clear the flag
whenever we see a '#' introducing a directive.

Much cleaner this way.

1 files changed, 19 insertions, 66 deletions

diff --git a/glcpp-lex.l b/glcpp-lex.l
index b1980742d3..f6d0c8b7d6 100644
--- a/glcpp-lex.l
+++ b/glcpp-lex.l
@@ -32,21 +32,6 @@
 %option reentrant noyywrap
 %option extra-type="glcpp_parser_t *"
 
-	/* This lexer has two states:
-	 *
-	 * The CONTROL state is for control lines (directives)
-	 * It lexes exactly as specified in the C99 specification.
-	 *
-	 * The INITIAL state is for input lines. In this state, we
-	 * make the OTHER token much more broad in that it now
-	 * includes tokens consisting entirely of whitespace. This
-	 * allows us to pass text through verbatim. It avoids the
-	 * "inadvertent token pasting" problem that would occur if we
-	 * just printed tokens, while also avoiding excess whitespace
-	 * insertion in the output.*/
-
-%x CONTROL
-
 SPACE		[[:space:]]
 NONSPACE	[^[:space:]]
 NEWLINE		[\n]
@@ -63,116 +48,84 @@ HEXADECIMAL_INTEGER	0[xX][0-9a-fA-F]+[uU]?
 %%
 
 {HASH}define{HSPACE}+/{IDENTIFIER}"(" {
-	BEGIN CONTROL;
+	yyextra->space_tokens = 0;
 	return HASH_DEFINE_FUNC;
 }
 
 {HASH}define {
-	BEGIN CONTROL;
+	yyextra->space_tokens = 0;
 	return HASH_DEFINE_OBJ;
 }
 
 {HASH}undef {
-	BEGIN CONTROL;
+	yyextra->space_tokens = 0;
 	return HASH_UNDEF;
 }
 
 {HASH} {
-	BEGIN CONTROL;
+	yyextra->space_tokens = 0;
 	return HASH;
 }
 
-<CONTROL>{IDENTIFIER} {
+{IDENTIFIER} {
 	yylval.str = xtalloc_strdup (yyextra, yytext);
 	return IDENTIFIER;
 }
 
-<CONTROL>"<<"  {
+"<<"  {
 	return LEFT_SHIFT;
 }
 
-<CONTROL>">>" {
+">>" {
 	return RIGHT_SHIFT;
 }
 
-<CONTROL>"<=" {
+"<=" {
 	return LESS_OR_EQUAL;
 }
 
-<CONTROL>">=" {
+">=" {
 	return GREATER_OR_EQUAL;
 }
 
-<CONTROL>"==" {
+"==" {
 	return EQUAL;
 }
 
-<CONTROL>"!=" {
+"!=" {
 	return NOT_EQUAL;
 }
 
-<CONTROL>"&&" {
+"&&" {
 	return AND;
 }
 
-<CONTROL>"||" {
+"||" {
 	return OR;
 }
 
-<CONTROL>"##" {
+"##" {
 	return PASTE;
 }
 
-<CONTROL>{PUNCTUATION} {
+{PUNCTUATION} {
 	return yytext[0];
 }
 
-<CONTROL>{OTHER} {
-	yylval.str = xtalloc_strdup (yyextra, yytext);
-	return OTHER;
-}
-
-<CONTROL>{HSPACE}+
-
-<CONTROL>\n {
-	BEGIN INITIAL;
-	return NEWLINE;
-}
-
-{IDENTIFIER} {
-	yylval.str = xtalloc_strdup (yyextra, yytext);
-	return IDENTIFIER;
-}
-
-"(" {
-	return '(';
-}
-
-")" {
-	return ')';
-}
-
-"," {
-	return ',';
-}
-
 {OTHER}+ {
 	yylval.str = xtalloc_strdup (yyextra, yytext);
 	return OTHER;
 }
 
 {HSPACE}+ {
-	yylval.str = xtalloc_strdup (yyextra, yytext);
-	return SPACE;
+	if (yyextra->space_tokens) {
+		yylval.str = xtalloc_strdup (yyextra, yytext);
+		return SPACE;
+	}
 }
 
 \n {
 	return NEWLINE;
 }
 
-. {
-	yylval.str = xtalloc_strdup (yyextra, yytext);
-	return OTHER;
-}
-
 %%