From 3ff81670848abb29b92e78f45080ad36cc85001c Mon Sep 17 00:00:00 2001 From: Carl Worth Date: Tue, 25 May 2010 13:09:03 -0700 Subject: Starting over with the C99 grammar for the preprocessor. This is a fresh start with a much simpler approach for the flex/bison portions of the preprocessor. This isn't functional yet, (produces no output), but can at least read all of our test cases without any parse errors. The grammar here is based on the grammar provided for the preprocessor in the C99 specification. --- tests/glcpp-test | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) (limited to 'tests') diff --git a/tests/glcpp-test b/tests/glcpp-test index 022a236712..868b03cce8 100755 --- a/tests/glcpp-test +++ b/tests/glcpp-test @@ -1,9 +1,12 @@ #!/bin/sh +set -e + +echo "Caution: These results are just verifying parse-ability, not correctness!" for test in *.c; do echo "Testing $test" ../glcpp < $test > $test.out gcc -E $test -o $test.gcc grep -v '^#' < $test.gcc > $test.expected - diff -B -u $test.expected $test.out +# diff -B -u $test.expected $test.out done -- cgit v1.2.3 From 808401fd79eea9fa2c965f9f235a753c0cb0d920 Mon Sep 17 00:00:00 2001 From: Carl Worth Date: Tue, 25 May 2010 14:52:43 -0700 Subject: Store parsed tokens as token list and print all text lines. Still not doing any macro expansion just yet. But it should be fairly easy from here. --- glcpp-parse.y | 227 +++++++++++++++++++++++++++++++++++++++++-------------- glcpp.h | 27 +++++-- tests/glcpp-test | 5 +- 3 files changed, 195 insertions(+), 64 deletions(-) (limited to 'tests') diff --git a/glcpp-parse.y b/glcpp-parse.y index c53370a89a..991b8a0b85 100644 --- a/glcpp-parse.y +++ b/glcpp-parse.y @@ -77,15 +77,29 @@ _argument_list_length (argument_list_t *list); token_list_t * _argument_list_member_at (argument_list_t *list, int index); +/* Note: This function talloc_steal()s the str pointer. */ +token_t * +_token_create_str (void *ctx, int type, char *str); + +token_t * +_token_create_ival (void *ctx, int type, int ival); + token_list_t * _token_list_create (void *ctx); +/* Note: This function add a talloc_reference() to token. + * + * You may want to talloc_unlink any current reference if you no + * longer need it. */ void -_token_list_append (token_list_t *list, int type, const char *value); +_token_list_append (token_list_t *list, token_t *token); void _token_list_append_list (token_list_t *list, token_list_t *tail); +void +_token_list_print (token_list_t *list); + static void glcpp_parser_pop_expansion (glcpp_parser_t *parser); @@ -107,12 +121,9 @@ glcpp_parser_lex (glcpp_parser_t *parser); %} %union { - intmax_t imaxval; int ival; char *str; - argument_list_t *argument_list; - string_list_t *string_list; - token_t token; + token_t *token; token_list_t *token_list; } @@ -121,6 +132,10 @@ glcpp_parser_lex (glcpp_parser_t *parser); %token HASH_DEFINE_FUNC HASH_DEFINE_OBJ HASH IDENTIFIER NEWLINE OTHER HASH_UNDEF %token LEFT_SHIFT RIGHT_SHIFT LESS_OR_EQUAL GREATER_OR_EQUAL EQUAL NOT_EQUAL AND OR PASTE +%type punctuator +%type IDENTIFIER OTHER +%type preprocessing_token +%type pp_tokens replacement_list text_line /* Stale stuff just to allow code to compile. */ %token IDENTIFIER_FINALIZED FUNC_MACRO OBJ_MACRO @@ -134,7 +149,11 @@ input: line: control_line -| text_line +| text_line { + _token_list_print ($1); + printf ("\n"); + talloc_free ($1); + } | HASH non_directive ; @@ -152,7 +171,7 @@ identifier_list: ; text_line: - NEWLINE + NEWLINE { $$ = NULL; } | pp_tokens NEWLINE ; @@ -161,55 +180,68 @@ non_directive: ; replacement_list: - /* empty */ + /* empty */ { $$ = NULL; } | pp_tokens ; pp_tokens: - preprocessing_token -| pp_tokens preprocessing_token + preprocessing_token { + $$ = _token_list_create (parser); + _token_list_append ($$, $1); + talloc_unlink (parser, $1); + } +| pp_tokens preprocessing_token { + $$ = $1; + _token_list_append ($$, $2); + talloc_unlink (parser, $2); + } ; preprocessing_token: - IDENTIFIER -| punctuator -| OTHER + IDENTIFIER { + $$ = _token_create_str (parser, IDENTIFIER, $1); + } +| punctuator { + $$ = _token_create_ival (parser, $1, $1); + } +| OTHER { + $$ = _token_create_str (parser, OTHER, $1); + } ; punctuator: - '[' -| ']' -| '(' -| ')' -| '{' -| '}' -| '.' -| '&' -| '*' -| '+' -| '-' -| '~' -| '!' -| '/' -| '%' -| LEFT_SHIFT -| RIGHT_SHIFT -| '<' -| '>' -| LESS_OR_EQUAL -| GREATER_OR_EQUAL -| EQUAL -| NOT_EQUAL -| '^' -| '|' -| AND -| OR -| ';' -| ',' -| PASTE + '[' { $$ = '['; } +| ']' { $$ = ']'; } +| '(' { $$ = '('; } +| ')' { $$ = ')'; } +| '{' { $$ = '{'; } +| '}' { $$ = '}'; } +| '.' { $$ = '.'; } +| '&' { $$ = '&'; } +| '*' { $$ = '*'; } +| '+' { $$ = '+'; } +| '-' { $$ = '-'; } +| '~' { $$ = '~'; } +| '!' { $$ = '!'; } +| '/' { $$ = '/'; } +| '%' { $$ = '%'; } +| LEFT_SHIFT { $$ = LEFT_SHIFT; } +| RIGHT_SHIFT { $$ = RIGHT_SHIFT; } +| '<' { $$ = '<'; } +| '>' { $$ = '>'; } +| LESS_OR_EQUAL { $$ = LESS_OR_EQUAL; } +| GREATER_OR_EQUAL { $$ = GREATER_OR_EQUAL; } +| EQUAL { $$ = EQUAL; } +| NOT_EQUAL { $$ = NOT_EQUAL; } +| '^' { $$ = '^'; } +| '|' { $$ = '|'; } +| AND { $$ = AND; } +| OR { $$ = OR; } +| ';' { $$ = ';'; } +| ',' { $$ = ','; } +| PASTE { $$ = PASTE; } ; - %% string_list_t * @@ -361,6 +393,77 @@ _argument_list_member_at (argument_list_t *list, int index) return NULL; } +/* Note: This function talloc_steal()s the str pointer. */ +token_t * +_token_create_str (void *ctx, int type, char *str) +{ + token_t *token; + + token = xtalloc (ctx, token_t); + token->type = type; + token->value.str = talloc_steal (token, str); + + return token; +} + +token_t * +_token_create_ival (void *ctx, int type, int ival) +{ + token_t *token; + + token = xtalloc (ctx, token_t); + token->type = type; + token->value.ival = ival; + + return token; +} + +void +_token_print (token_t *token) +{ + if (token->type < 256) { + printf ("%c", token->type); + return; + } + + switch (token->type) { + case IDENTIFIER: + case OTHER: + printf ("%s", token->value.str); + break; + case LEFT_SHIFT: + printf ("<<"); + break; + case RIGHT_SHIFT: + printf (">>"); + break; + case LESS_OR_EQUAL: + printf ("<="); + break; + case GREATER_OR_EQUAL: + printf (">="); + break; + case EQUAL: + printf ("=="); + break; + case NOT_EQUAL: + printf ("!="); + break; + case AND: + printf ("&&"); + break; + case OR: + printf ("||"); + break; + case PASTE: + printf ("##"); + break; + default: + fprintf (stderr, "Error: Don't know how to print token type %d\n", token->type); + break; + } +} + token_list_t * _token_list_create (void *ctx) { @@ -374,13 +477,12 @@ _token_list_create (void *ctx) } void -_token_list_append (token_list_t *list, int type, const char *value) +_token_list_append (token_list_t *list, token_t *token) { token_node_t *node; node = xtalloc (list, token_node_t); - node->type = type; - node->value = xtalloc_strdup (list, value); + node->token = xtalloc_reference (list, token); node->next = NULL; @@ -405,6 +507,21 @@ _token_list_append_list (token_list_t *list, token_list_t *tail) list->tail = tail->tail; } +void +_token_list_print (token_list_t *list) +{ + token_node_t *node; + + if (list == NULL) + return; + + for (node = list->head; node; node = node->next) { + _token_print (node->token); + if (node->next) + printf (" "); + } +} + void yyerror (void *scanner, const char *error) { @@ -598,7 +715,8 @@ _expand_function_macro (glcpp_parser_t *parser, expanded = _token_list_create (macro); for (i = macro->replacements->head; i; i = i->next) { - if (_string_list_contains (macro->parameters, i->value, + if (_string_list_contains (macro->parameters, + i->token->value.str, ¶meter_index)) { token_list_t *argument; @@ -606,11 +724,10 @@ _expand_function_macro (glcpp_parser_t *parser, parameter_index); for (j = argument->head; j; j = j->next) { - _token_list_append (expanded, j->type, - j->value); + _token_list_append (expanded, j->token); } } else { - _token_list_append (expanded, i->type, i->value); + _token_list_append (expanded, i->token); } } @@ -644,10 +761,10 @@ glcpp_parser_lex (glcpp_parser_t *parser) expansion->replacements = replacements->next; - token = replacements->value; + token = replacements->token->value.str; /* Implement token pasting. */ - if (replacements->next && strcmp (replacements->next->value, "##") == 0) { + if (replacements->next && strcmp (replacements->next->token->value.str, "##") == 0) { token_node_t *next_node; next_node = replacements->next->next; @@ -658,7 +775,7 @@ glcpp_parser_lex (glcpp_parser_t *parser) } token = xtalloc_asprintf (parser, "%s%s", - token, next_node->value); + token, next_node->token->value.str); expansion->replacements = next_node->next; } @@ -671,7 +788,7 @@ glcpp_parser_lex (glcpp_parser_t *parser) yylval.str = xtalloc_strdup (parser, token); /* Carefully refuse to expand any finalized identifier. */ - if (replacements->type == IDENTIFIER_FINALIZED) + if (replacements->token->type == IDENTIFIER_FINALIZED) return IDENTIFIER_FINALIZED; switch (glcpp_parser_classify_token (parser, yylval.str, diff --git a/glcpp.h b/glcpp.h index 6171ce8b4a..261254a17c 100644 --- a/glcpp.h +++ b/glcpp.h @@ -44,21 +44,34 @@ typedef struct string_list { string_node_t *tail; } string_list_t; -typedef struct token { +typedef struct token token_t; +typedef struct token_list token_list_t; + +typedef union YYSTYPE +{ + int ival; + char *str; + token_t *token; + token_list_t *token_list; +} YYSTYPE; + +# define YYSTYPE_IS_TRIVIAL 1 +# define YYSTYPE_IS_DECLARED 1 + +struct token { int type; - char *value; -} token_t; + YYSTYPE value; +}; typedef struct token_node { - int type; - const char *value; + token_t *token; struct token_node *next; } token_node_t; -typedef struct token_list { +struct token_list { token_node_t *head; token_node_t *tail; -} token_list_t; +}; typedef struct argument_node { token_list_t *argument; diff --git a/tests/glcpp-test b/tests/glcpp-test index 868b03cce8..34cca88330 100755 --- a/tests/glcpp-test +++ b/tests/glcpp-test @@ -7,6 +7,7 @@ for test in *.c; do echo "Testing $test" ../glcpp < $test > $test.out gcc -E $test -o $test.gcc - grep -v '^#' < $test.gcc > $test.expected -# diff -B -u $test.expected $test.out +# grep -v '^#' < $test.gcc > $test.expected + grep -v '^[ ]*#' < $test > $test.expected + diff -w -u $test.expected $test.out done -- cgit v1.2.3 From 9fb8b7a495c9dc6f9a62cf82300fae5925af92fc Mon Sep 17 00:00:00 2001 From: Carl Worth Date: Tue, 25 May 2010 15:04:32 -0700 Subject: Make the lexer pass whitespace through (as OTHER tokens) for text lines. With this change, we can recreate the original text-line input exactly. Previously we were inserting a space between every pair of tokens so our output had a lot more whitespace than our input. With this change, we can drop the "-b" option to diff and match the input exactly. --- glcpp-lex.l | 72 +++++++++++++++++++++++++++++++++++++++++++++----------- glcpp-parse.y | 2 -- tests/glcpp-test | 2 +- 3 files changed, 59 insertions(+), 17 deletions(-) (limited to 'tests') diff --git a/glcpp-lex.l b/glcpp-lex.l index f1dd11ea9b..7b5cdd57a0 100644 --- a/glcpp-lex.l +++ b/glcpp-lex.l @@ -32,6 +32,21 @@ %option reentrant noyywrap %option extra-type="glcpp_parser_t *" + /* This lexer has two states: + * + * The CONTROL state is for control lines (directives) + * It lexes exactly as specified in the C99 specification. + * + * The INITIAL state is for input lines. In this state, we + * make the OTHER token much more broad in that it now + * includes tokens consisting entirely of whitespace. This + * allows us to pass text through verbatim. It avoids the + * "inadvertent token pasting" problem that would occur if we + * just printed tokens, while also avoiding excess whitespace + * insertion in the output.*/ + +%x CONTROL + SPACE [[:space:]] NONSPACE [^[:space:]] NEWLINE [\n] @@ -48,75 +63,104 @@ HEXADECIMAL_INTEGER 0[xX][0-9a-fA-F]+[uU]? %% {HASH}define{HSPACE}+/{IDENTIFIER}"(" { + BEGIN CONTROL; return HASH_DEFINE_FUNC; } {HASH}define { + BEGIN CONTROL; return HASH_DEFINE_OBJ; } {HASH}undef { + BEGIN CONTROL; return HASH_UNDEF; } {HASH} { + BEGIN CONTROL; return HASH; } -{IDENTIFIER} { +{IDENTIFIER} { yylval.str = xtalloc_strdup (yyextra, yytext); return IDENTIFIER; } -"<<" { +"<<" { return LEFT_SHIFT; } -">>" { +">>" { return RIGHT_SHIFT; } -"<=" { +"<=" { return LESS_OR_EQUAL; } -">=" { +">=" { return GREATER_OR_EQUAL; } -"==" { +"==" { return EQUAL; } -"!=" { +"!=" { return NOT_EQUAL; } -"&&" { +"&&" { return AND; } -"||" { +"||" { return OR; } -"##" { +"##" { return PASTE; } -{PUNCTUATION} { +{PUNCTUATION} { return yytext[0]; } -\n { +{OTHER} { + yylval.str = xtalloc_strdup (yyextra, yytext); + return OTHER; +} + +{HSPACE}+ + +\n { + BEGIN INITIAL; return NEWLINE; } -{OTHER} { +{IDENTIFIER} { + yylval.str = xtalloc_strdup (yyextra, yytext); + return IDENTIFIER; +} + +{OTHER}+ { + yylval.str = xtalloc_strdup (yyextra, yytext); + return OTHER; +} + +{HSPACE}+ { yylval.str = xtalloc_strdup (yyextra, yytext); return OTHER; } -{HSPACE}+ +\n { + return NEWLINE; +} + +. { + yylval.str = xtalloc_strdup (yyextra, yytext); + return OTHER; +} %% diff --git a/glcpp-parse.y b/glcpp-parse.y index 991b8a0b85..957421b864 100644 --- a/glcpp-parse.y +++ b/glcpp-parse.y @@ -517,8 +517,6 @@ _token_list_print (token_list_t *list) for (node = list->head; node; node = node->next) { _token_print (node->token); - if (node->next) - printf (" "); } } diff --git a/tests/glcpp-test b/tests/glcpp-test index 34cca88330..8074e47119 100755 --- a/tests/glcpp-test +++ b/tests/glcpp-test @@ -9,5 +9,5 @@ for test in *.c; do gcc -E $test -o $test.gcc # grep -v '^#' < $test.gcc > $test.expected grep -v '^[ ]*#' < $test > $test.expected - diff -w -u $test.expected $test.out + diff -u $test.expected $test.out done -- cgit v1.2.3 From ae6517f4a83981ae363bbbfe439ec23e8deb04b1 Mon Sep 17 00:00:00 2001 From: Carl Worth Date: Tue, 25 May 2010 15:24:59 -0700 Subject: Implement expansion of object-like macros. For this we add an "active" string_list_t to the parser. This makes the current expansion_list_t in the parser obsolete, but we don't remove that yet. With this change we can now start passing some actual tests, so we turn on real testing in the test suite again. I expect to implement things more or less in the same order as before, so the test suite now halts on first error. With this change the first 8 tests in the suite pass, (object-like macros with chaining and recursion). --- glcpp-parse.y | 128 +++++++++++++++++++++++++++++++++++++++++++++++-------- glcpp.h | 1 + tests/glcpp-test | 5 +-- 3 files changed, 112 insertions(+), 22 deletions(-) (limited to 'tests') diff --git a/glcpp-parse.y b/glcpp-parse.y index 957421b864..b3ef177a6d 100644 --- a/glcpp-parse.y +++ b/glcpp-parse.y @@ -59,6 +59,12 @@ _string_list_append_item (string_list_t *list, const char *str); void _string_list_append_list (string_list_t *list, string_list_t *tail); +void +_string_list_push (string_list_t *list, const char *str); + +void +_string_list_pop (string_list_t *list); + int _string_list_contains (string_list_t *list, const char *member, int *index); @@ -98,7 +104,8 @@ void _token_list_append_list (token_list_t *list, token_list_t *tail); void -_token_list_print (token_list_t *list); +_glcpp_parser_print_expanded_token_list (glcpp_parser_t *parser, + token_list_t *list); static void glcpp_parser_pop_expansion (glcpp_parser_t *parser); @@ -144,21 +151,24 @@ glcpp_parser_lex (glcpp_parser_t *parser); input: /* empty */ -| input line +| input line { + printf ("\n"); + } ; line: control_line | text_line { - _token_list_print ($1); - printf ("\n"); + _glcpp_parser_print_expanded_token_list (parser, $1); talloc_free ($1); } | HASH non_directive ; control_line: - HASH_DEFINE_OBJ IDENTIFIER replacement_list NEWLINE + HASH_DEFINE_OBJ IDENTIFIER replacement_list NEWLINE { + _define_object_macro (parser, $2, $3); + } | HASH_DEFINE_FUNC IDENTIFIER '(' ')' replacement_list NEWLINE | HASH_DEFINE_FUNC IDENTIFIER '(' identifier_list ')' replacement_list NEWLINE | HASH_UNDEF IDENTIFIER NEWLINE @@ -287,6 +297,42 @@ _string_list_append_item (string_list_t *list, const char *str) list->tail = node; } +void +_string_list_push (string_list_t *list, const char *str) +{ + string_node_t *node; + + node = xtalloc (list, string_node_t); + node->str = xtalloc_strdup (node, str); + node->next = list->head; + + if (list->tail == NULL) { + list->tail = node; + } + list->head = node; +} + +void +_string_list_pop (string_list_t *list) +{ + string_node_t *node; + + node = list->head; + + if (node == NULL) { + fprintf (stderr, "Internal error: _string_list_pop called on an empty list.\n"); + exit (1); + } + + list->head = node->next; + if (list->tail == node) { + assert (node->next == NULL); + list->tail = NULL; + } + + talloc_free (node); +} + int _string_list_contains (string_list_t *list, const char *member, int *index) { @@ -507,19 +553,6 @@ _token_list_append_list (token_list_t *list, token_list_t *tail) list->tail = tail->tail; } -void -_token_list_print (token_list_t *list) -{ - token_node_t *node; - - if (list == NULL) - return; - - for (node = list->head; node; node = node->next) { - _token_print (node->token); - } -} - void yyerror (void *scanner, const char *error) { @@ -536,6 +569,7 @@ glcpp_parser_create (void) glcpp_lex_init_extra (parser, &parser->scanner); parser->defines = hash_table_ctor (32, hash_table_string_hash, hash_table_string_compare); + parser->active = _string_list_create (parser); parser->expansions = NULL; parser->just_printed_separator = 1; @@ -605,6 +639,64 @@ glcpp_parser_classify_token (glcpp_parser_t *parser, return TOKEN_CLASS_OBJ_MACRO; } +void +_glcpp_parser_print_expanded_token (glcpp_parser_t *parser, + token_t *token) +{ + const char *identifier; + macro_t *macro; + + /* We only expand identifiers */ + if (token->type != IDENTIFIER) { + _token_print (token); + return; + } + + /* Look up this identifier in the hash table. */ + identifier = token->value.str; + macro = hash_table_find (parser->defines, identifier); + + /* Not a macro, so just print directly. */ + if (macro == NULL) { + printf ("%s", identifier); + return; + } + + /* We're not (yet) supporting function-like macros. */ + if (macro->is_function) { + printf ("%s", identifier); + return; + } + + /* Finally, don't expand this macro if we're already actively + * expanding it, (to avoid infinite recursion). */ + if (_string_list_contains (parser->active, identifier, NULL)) { + printf ("%s", identifier); + return; + } + + _string_list_push (parser->active, identifier); + _glcpp_parser_print_expanded_token_list (parser, + macro->replacements); + _string_list_pop (parser->active); +} + +void +_glcpp_parser_print_expanded_token_list (glcpp_parser_t *parser, + token_list_t *list) +{ + token_node_t *node; + + if (list == NULL) + return; + + for (node = list->head; node; node = node->next) { + _glcpp_parser_print_expanded_token (parser, node->token); + if (node->next) + printf (" "); + } +} + void _define_object_macro (glcpp_parser_t *parser, const char *identifier, diff --git a/glcpp.h b/glcpp.h index 261254a17c..bd599d7301 100644 --- a/glcpp.h +++ b/glcpp.h @@ -124,6 +124,7 @@ typedef struct skip_node { struct glcpp_parser { yyscan_t scanner; struct hash_table *defines; + string_list_t *active; expansion_node_t *expansions; int just_printed_separator; int need_newline; diff --git a/tests/glcpp-test b/tests/glcpp-test index 8074e47119..6304155210 100755 --- a/tests/glcpp-test +++ b/tests/glcpp-test @@ -1,13 +1,10 @@ #!/bin/sh set -e -echo "Caution: These results are just verifying parse-ability, not correctness!" - for test in *.c; do echo "Testing $test" ../glcpp < $test > $test.out gcc -E $test -o $test.gcc -# grep -v '^#' < $test.gcc > $test.expected - grep -v '^[ ]*#' < $test > $test.expected + grep -v '^#' < $test.gcc > $test.expected diff -u $test.expected $test.out done -- cgit v1.2.3 From f8ec4e0be86eee05f5a661a01864247fcd1a6b30 Mon Sep 17 00:00:00 2001 From: Carl Worth Date: Tue, 25 May 2010 17:06:17 -0700 Subject: Add a test #0 to ensure that we don't do any inadvertent token pasting. This simply ensures that spaces in input line are preserved. --- tests/000-content-with-spaces.c | 1 + 1 file changed, 1 insertion(+) create mode 100644 tests/000-content-with-spaces.c (limited to 'tests') diff --git a/tests/000-content-with-spaces.c b/tests/000-content-with-spaces.c new file mode 100644 index 0000000000..a7fc918c90 --- /dev/null +++ b/tests/000-content-with-spaces.c @@ -0,0 +1 @@ +this is four tokens -- cgit v1.2.3 From e9397867ddce20a4263949f4b3a488fa99af3041 Mon Sep 17 00:00:00 2001 From: Carl Worth Date: Tue, 25 May 2010 17:08:07 -0700 Subject: Collapse multiple spaces in input down to a single space. This is what gcc does, and it's actually less work to do this. Previously we were having to save the contents of space tokens as a string, but we don't need to do that now. We extend test #0 to exercise this feature here. --- glcpp-lex.l | 1 - glcpp-parse.y | 10 ++++++---- tests/000-content-with-spaces.c | 2 +- 3 files changed, 7 insertions(+), 6 deletions(-) (limited to 'tests') diff --git a/glcpp-lex.l b/glcpp-lex.l index f6d0c8b7d6..516f42dee3 100644 --- a/glcpp-lex.l +++ b/glcpp-lex.l @@ -119,7 +119,6 @@ HEXADECIMAL_INTEGER 0[xX][0-9a-fA-F]+[uU]? {HSPACE}+ { if (yyextra->space_tokens) { - yylval.str = xtalloc_strdup (yyextra, yytext); return SPACE; } } diff --git a/glcpp-parse.y b/glcpp-parse.y index a1981995fd..0460f71f74 100644 --- a/glcpp-parse.y +++ b/glcpp-parse.y @@ -132,8 +132,8 @@ glcpp_parser_lex (glcpp_parser_t *parser); %token HASH HASH_DEFINE_FUNC HASH_DEFINE_OBJ HASH_UNDEF IDENTIFIER NEWLINE OTHER SPACE %token LEFT_SHIFT RIGHT_SHIFT LESS_OR_EQUAL GREATER_OR_EQUAL EQUAL NOT_EQUAL AND OR PASTE -%type punctuator -%type IDENTIFIER OTHER SPACE +%type punctuator SPACE +%type IDENTIFIER OTHER %type identifier_list %type preprocessing_token %type pp_tokens replacement_list text_line @@ -235,7 +235,7 @@ preprocessing_token: $$ = _token_create_str (parser, OTHER, $1); } | SPACE { - $$ = _token_create_str (parser, SPACE, $1); + $$ = _token_create_ival (parser, SPACE, SPACE); } ; @@ -495,9 +495,11 @@ _token_print (token_t *token) switch (token->type) { case IDENTIFIER: case OTHER: - case SPACE: printf ("%s", token->value.str); break; + case SPACE: + printf (" "); + break; case LEFT_SHIFT: printf ("<<"); break; diff --git a/tests/000-content-with-spaces.c b/tests/000-content-with-spaces.c index a7fc918c90..696cb3a74f 100644 --- a/tests/000-content-with-spaces.c +++ b/tests/000-content-with-spaces.c @@ -1 +1 @@ -this is four tokens +this is four tokens -- cgit v1.2.3 From 039739b2da0ce8496f6e8d38127c0b3793607afa Mon Sep 17 00:00:00 2001 From: Carl Worth Date: Wed, 26 May 2010 07:58:59 -0700 Subject: Defer test 26 until much later (to test 55). Supporting embedded newlines in a macro invocation is going to be tricky with our current approach to lexing and parsing. Since this isn't really an important feature for us, we can defer this until more important things are resolved. With this test out of the way, tests 27 through 31 are passing. --- tests/026-define-func-extra-newlines.c | 6 ------ tests/055-define-func-extra-newlines.c | 6 ++++++ 2 files changed, 6 insertions(+), 6 deletions(-) delete mode 100644 tests/026-define-func-extra-newlines.c create mode 100644 tests/055-define-func-extra-newlines.c (limited to 'tests') diff --git a/tests/026-define-func-extra-newlines.c b/tests/026-define-func-extra-newlines.c deleted file mode 100644 index 0d83740530..0000000000 --- a/tests/026-define-func-extra-newlines.c +++ /dev/null @@ -1,6 +0,0 @@ -#define foo(a) bar - -foo -( -1 -) diff --git a/tests/055-define-func-extra-newlines.c b/tests/055-define-func-extra-newlines.c new file mode 100644 index 0000000000..0d83740530 --- /dev/null +++ b/tests/055-define-func-extra-newlines.c @@ -0,0 +1,6 @@ +#define foo(a) bar + +foo +( +1 +) -- cgit v1.2.3 From f6914fd37b2b66d7be1ba0c31450d89d1785ccce Mon Sep 17 00:00:00 2001 From: Carl Worth Date: Wed, 26 May 2010 09:32:57 -0700 Subject: Implement #if and friends. With this change, tests 41 through 49 all pass. (The defined operator appears to be somehow broken so that test 50 doesn't pass yet.) --- glcpp.h | 2 +- tests/049-if-expression-precedence.c | 1 - tests/050-if-defined.c | 2 -- tests/glcpp-test | 2 +- 4 files changed, 2 insertions(+), 5 deletions(-) (limited to 'tests') diff --git a/glcpp.h b/glcpp.h index 21db918cdc..36ab0e7ca5 100644 --- a/glcpp.h +++ b/glcpp.h @@ -49,7 +49,7 @@ typedef struct token_list token_list_t; typedef union YYSTYPE { - int ival; + intmax_t ival; char *str; string_list_t *string_list; token_t *token; diff --git a/tests/049-if-expression-precedence.c b/tests/049-if-expression-precedence.c index cea935220f..833ea03882 100644 --- a/tests/049-if-expression-precedence.c +++ b/tests/049-if-expression-precedence.c @@ -3,4 +3,3 @@ failure with operator precedence #else success #endif - diff --git a/tests/050-if-defined.c b/tests/050-if-defined.c index 9838cc747d..34f0f95140 100644 --- a/tests/050-if-defined.c +++ b/tests/050-if-defined.c @@ -15,5 +15,3 @@ failure_3 #else success_3 #endif - - diff --git a/tests/glcpp-test b/tests/glcpp-test index 6304155210..bf88d4462e 100755 --- a/tests/glcpp-test +++ b/tests/glcpp-test @@ -6,5 +6,5 @@ for test in *.c; do ../glcpp < $test > $test.out gcc -E $test -o $test.gcc grep -v '^#' < $test.gcc > $test.expected - diff -u $test.expected $test.out + diff -B -u $test.expected $test.out done -- cgit v1.2.3 From 16c1e980e2e3c8852ce9bea85afe094c24e420fa Mon Sep 17 00:00:00 2001 From: Carl Worth Date: Wed, 26 May 2010 09:35:34 -0700 Subject: Fix lexing of "defined" as an operator, not an identifier. Simply need to move the rule for IDENTIFIER to be after "defined" and everything is happy. With this change, tests 50 through 53 all pass now. --- glcpp-lex.l | 11 +++++------ tests/053-if-divide-and-shift.c | 1 - 2 files changed, 5 insertions(+), 7 deletions(-) (limited to 'tests') diff --git a/glcpp-lex.l b/glcpp-lex.l index 97f01d0636..d6b7726d36 100644 --- a/glcpp-lex.l +++ b/glcpp-lex.l @@ -102,12 +102,6 @@ HEXADECIMAL_INTEGER 0[xX][0-9a-fA-F]+[uU]? return INTEGER; } - -{IDENTIFIER} { - yylval.str = xtalloc_strdup (yyextra, yytext); - return IDENTIFIER; -} - "<<" { return LEFT_SHIFT; } @@ -148,6 +142,11 @@ HEXADECIMAL_INTEGER 0[xX][0-9a-fA-F]+[uU]? return DEFINED; } +{IDENTIFIER} { + yylval.str = xtalloc_strdup (yyextra, yytext); + return IDENTIFIER; +} + {PUNCTUATION} { return yytext[0]; } diff --git a/tests/053-if-divide-and-shift.c b/tests/053-if-divide-and-shift.c index ddc1573ab2..d24c54a88d 100644 --- a/tests/053-if-divide-and-shift.c +++ b/tests/053-if-divide-and-shift.c @@ -13,4 +13,3 @@ failure_3 #else success_3 #endif - -- cgit v1.2.3 From 8e82fcb070d5fae0ec2c763cee4cea225b459664 Mon Sep 17 00:00:00 2001 From: Carl Worth Date: Wed, 26 May 2010 11:15:21 -0700 Subject: Implement (and test) support for macro expansion within conditional expressions. To do this we have split the existing "HASH_IF expression" into two productions: First is HASH_IF pp_tokens which simply constructs a list of tokens. Then, with that resulting token list, we first evaluate all DEFINED operator tokens, then expand all macros, and finally start lexing from the resulting token list. This brings us to the second production, IF_EXPANDED expression This final production works just like our previous "HASH_IF expression", evaluating a constant integer expression. The new test (54) added for this case now passes. --- glcpp-parse.y | 155 ++++++++++++++++++++++++++++++++++++++------- glcpp.h | 2 + tests/054-if-with-macros.c | 34 ++++++++++ 3 files changed, 169 insertions(+), 22 deletions(-) create mode 100644 tests/054-if-with-macros.c (limited to 'tests') diff --git a/glcpp-parse.y b/glcpp-parse.y index 58e1e655fd..cce8a70156 100644 --- a/glcpp-parse.y +++ b/glcpp-parse.y @@ -96,6 +96,10 @@ _token_list_append (token_list_t *list, token_t *token); static void _token_list_append_list (token_list_t *list, token_list_t *tail); +static void +_glcpp_parser_evaluate_defined (glcpp_parser_t *parser, + token_list_t *list); + static void _glcpp_parser_print_expanded_token_list (glcpp_parser_t *parser, token_list_t *list); @@ -120,14 +124,17 @@ _glcpp_parser_skip_stack_pop (glcpp_parser_t *parser); static int glcpp_parser_lex (glcpp_parser_t *parser); +static void +glcpp_parser_lex_from (glcpp_parser_t *parser, token_list_t *list); + %} %parse-param {glcpp_parser_t *parser} %lex-param {glcpp_parser_t *parser} -%token DEFINED HASH HASH_DEFINE_FUNC HASH_DEFINE_OBJ HASH_ELIF HASH_ELSE HASH_ENDIF HASH_IF HASH_IFDEF HASH_IFNDEF HASH_UNDEF IDENTIFIER INTEGER NEWLINE OTHER SPACE +%token DEFINED ELIF_EXPANDED HASH HASH_DEFINE_FUNC HASH_DEFINE_OBJ HASH_ELIF HASH_ELSE HASH_ENDIF HASH_IF HASH_IFDEF HASH_IFNDEF HASH_UNDEF IDENTIFIER IF_EXPANDED INTEGER NEWLINE OTHER SPACE %token PASTE -%type expression INTEGER punctuator SPACE +%type expression INTEGER operator SPACE %type IDENTIFIER OTHER %type identifier_list %type preprocessing_token @@ -148,28 +155,39 @@ glcpp_parser_lex (glcpp_parser_t *parser); input: /* empty */ -| input line { +| input line +; + +line: + control_line { if (parser->skip_stack == NULL || parser->skip_stack->type == SKIP_NO_SKIP) { printf ("\n"); } } -; - -line: - control_line | text_line { if (parser->skip_stack == NULL || parser->skip_stack->type == SKIP_NO_SKIP) { _glcpp_parser_print_expanded_token_list (parser, $1); + printf ("\n"); } talloc_free ($1); } +| expanded_line | HASH non_directive ; +expanded_line: + IF_EXPANDED expression NEWLINE { + _glcpp_parser_skip_stack_push_if (parser, $2); + } +| ELIF_EXPANDED expression NEWLINE { + _glcpp_parser_skip_stack_change_if (parser, "elif", $2); + } +; + control_line: HASH_DEFINE_OBJ IDENTIFIER replacement_list NEWLINE { _define_object_macro (parser, $2, $3); @@ -191,8 +209,17 @@ control_line: } talloc_free ($2); } -| HASH_IF expression NEWLINE { - _glcpp_parser_skip_stack_push_if (parser, $2); +| HASH_IF pp_tokens NEWLINE { + token_list_t *expanded; + token_t *token; + + expanded = _token_list_create (parser); + token = _token_create_ival (parser, IF_EXPANDED, IF_EXPANDED); + _token_list_append (expanded, token); + talloc_unlink (parser, token); + _glcpp_parser_evaluate_defined (parser, $2); + _glcpp_parser_expand_token_list_onto (parser, $2, expanded); + glcpp_parser_lex_from (parser, expanded); } | HASH_IFDEF IDENTIFIER NEWLINE { string_list_t *macro = hash_table_find (parser->defines, $2); @@ -204,8 +231,17 @@ control_line: talloc_free ($2); _glcpp_parser_skip_stack_push_if (parser, macro == NULL); } -| HASH_ELIF expression NEWLINE { - _glcpp_parser_skip_stack_change_if (parser, "#elif", $2); +| HASH_ELIF pp_tokens NEWLINE { + token_list_t *expanded; + token_t *token; + + expanded = _token_list_create (parser); + token = _token_create_ival (parser, ELIF_EXPANDED, ELIF_EXPANDED); + _token_list_append (expanded, token); + talloc_unlink (parser, token); + _glcpp_parser_evaluate_defined (parser, $2); + _glcpp_parser_expand_token_list_onto (parser, $2, expanded); + glcpp_parser_lex_from (parser, expanded); } | HASH_ELSE NEWLINE { _glcpp_parser_skip_stack_change_if (parser, "else", 1); @@ -286,14 +322,6 @@ expression: | '+' expression %prec UNARY { $$ = + $2; } -| DEFINED IDENTIFIER %prec UNARY { - string_list_t *macro = hash_table_find (parser->defines, $2); - talloc_free ($2); - if (macro) - $$ = 1; - else - $$ = 0; - } | '(' expression ')' { $$ = $2; } @@ -347,7 +375,7 @@ preprocessing_token: | INTEGER { $$ = _token_create_ival (parser, INTEGER, $1); } -| punctuator { +| operator { $$ = _token_create_ival (parser, $1, $1); } | OTHER { @@ -358,7 +386,7 @@ preprocessing_token: } ; -punctuator: +operator: '[' { $$ = '['; } | ']' { $$ = ']'; } | '(' { $$ = '('; } @@ -389,6 +417,7 @@ punctuator: | ';' { $$ = ';'; } | ',' { $$ = ','; } | PASTE { $$ = PASTE; } +| DEFINED { $$ = DEFINED; } ; %% @@ -830,6 +859,9 @@ glcpp_parser_create (void) parser->skip_stack = NULL; + parser->lex_from_list = NULL; + parser->lex_from_node = NULL; + return parser; } @@ -849,6 +881,39 @@ glcpp_parser_destroy (glcpp_parser_t *parser) talloc_free (parser); } +/* Replace any occurences of DEFINED tokens in 'list' with either a + * '0' or '1' INTEGER token depending on whether the next token in the + * list is defined or not. */ +static void +_glcpp_parser_evaluate_defined (glcpp_parser_t *parser, + token_list_t *list) +{ + token_node_t *node, *next; + string_list_t *macro; + + if (list == NULL) + return; + + for (node = list->head; node; node = node->next) { + if (node->token->type != DEFINED) + continue; + next = node->next; + while (next && next->token->type == SPACE) + next = next->next; + if (next == NULL || next->token->type != IDENTIFIER) { + fprintf (stderr, "Error: operator \"defined\" requires an identifier\n"); + exit (1); + } + macro = hash_table_find (parser->defines, + next->token->value.str); + + node->token->type = INTEGER; + node->token->value.ival = (macro != NULL); + node->next = next->next; + } +} + + /* Appends onto 'expansion' a non-macro token or the expansion of an * object-like macro. * @@ -1206,7 +1271,53 @@ _define_function_macro (glcpp_parser_t *parser, static int glcpp_parser_lex (glcpp_parser_t *parser) { - return glcpp_lex (parser->scanner); + token_node_t *node; + int ret; + + if (parser->lex_from_list == NULL) + return glcpp_lex (parser->scanner); + + node = parser->lex_from_node; + + if (node == NULL) { + talloc_free (parser->lex_from_list); + parser->lex_from_list = NULL; + return NEWLINE; + } + + yylval = node->token->value; + ret = node->token->type; + + parser->lex_from_node = node->next; + + return ret; +} + +static void +glcpp_parser_lex_from (glcpp_parser_t *parser, token_list_t *list) +{ + token_node_t *node; + + assert (parser->lex_from_list == NULL); + + /* Copy list, eliminating any space tokens. */ + parser->lex_from_list = _token_list_create (parser); + + for (node = list->head; node; node = node->next) { + if (node->token->type == SPACE) + continue; + _token_list_append (parser->lex_from_list, node->token); + } + + talloc_free (list); + + parser->lex_from_node = parser->lex_from_list->head; + + /* It's possible the list consisted of nothing but whitespace. */ + if (parser->lex_from_node == NULL) { + talloc_free (parser->lex_from_list); + parser->lex_from_list = NULL; + } } static void diff --git a/glcpp.h b/glcpp.h index 36ab0e7ca5..e5be1a6cd6 100644 --- a/glcpp.h +++ b/glcpp.h @@ -129,6 +129,8 @@ struct glcpp_parser { string_list_t *active; int space_tokens; skip_node_t *skip_stack; + token_list_t *lex_from_list; + token_node_t *lex_from_node; }; glcpp_parser_t * diff --git a/tests/054-if-with-macros.c b/tests/054-if-with-macros.c new file mode 100644 index 0000000000..3da79a0d96 --- /dev/null +++ b/tests/054-if-with-macros.c @@ -0,0 +1,34 @@ +#define one 1 +#define two 2 +#define three 3 +#define five 5 +#if five < two +failure_1 +#else +success_1 +#endif +#if three >= two +success_2 +#else +failure_2 +#endif +#if two + three <= five +success_3 +#else +failure_3 +#endif +#if five - two == three +success_4 +#else +failure_4 +#endif +#if one > three +failure_5 +#else +success_5 +#endif +#if one != five +success_6 +#else +failure_6 +#endif -- cgit v1.2.3 From 95951ea7bb8728cf54ae4136cb59d0af9e8a06bd Mon Sep 17 00:00:00 2001 From: Carl Worth Date: Wed, 26 May 2010 15:57:10 -0700 Subject: Treat newlines as space when invoking a function-like macro invocation. This adds three new pieces of state to the parser, (is_control_line, newline_as_space, and paren_count), and a large amount of messy code. I'd definitely like to see a cleaner solution for this. With this fix, the "define-func-extra-newlines" now passes so we put it back to test #26 where it was originally (lately it has been known as test #55). Also, we tweak test 25 slightly. Previously this test was ending a file function-like macro name that was not actually a macro (not followed by a left parenthesis). As is, this fix was making that test fail because the text_line production expects to see a terminating NEWLINE, but that NEWLINE is now getting turned into a SPACE here. This seems unlikely to be a problem in the wild, (function macros being used in a non-macro sense seems rare enough---but more than likely they won't happen at the end of a file). Still, we document this shortcoming in the README. --- README | 4 +++ glcpp-parse.y | 61 ++++++++++++++++++++++++++++++++-- glcpp.h | 3 ++ tests/025-func-macro-as-non-macro.c | 2 +- tests/026-define-func-extra-newlines.c | 6 ++++ tests/055-define-func-extra-newlines.c | 6 ---- 6 files changed, 73 insertions(+), 9 deletions(-) create mode 100644 tests/026-define-func-extra-newlines.c delete mode 100644 tests/055-define-func-extra-newlines.c (limited to 'tests') diff --git a/README b/README index f0f64c2644..ab42a3ffe1 100644 --- a/README +++ b/README @@ -24,3 +24,7 @@ parentheses. The #error, #pragma, #extension, #version, and #line macros are not yet supported. + +A file that ends with a function-like macro name as the last +non-whitespace token will result in a parse error, (where it should be +passed through as is). \ No newline at end of file diff --git a/glcpp-parse.y b/glcpp-parse.y index a809ebf3af..1346b65aff 100644 --- a/glcpp-parse.y +++ b/glcpp-parse.y @@ -856,6 +856,9 @@ glcpp_parser_create (void) hash_table_string_compare); parser->active = _string_list_create (parser); parser->space_tokens = 1; + parser->newline_as_space = 0; + parser->in_control_line = 0; + parser->paren_count = 0; parser->skip_stack = NULL; @@ -1274,8 +1277,62 @@ glcpp_parser_lex (glcpp_parser_t *parser) token_node_t *node; int ret; - if (parser->lex_from_list == NULL) - return glcpp_lex (parser->scanner); + if (parser->lex_from_list == NULL) { + ret = glcpp_lex (parser->scanner); + + /* XXX: This ugly block of code exists for the sole + * purpose of converting a NEWLINE token into a SPACE + * token, but only in the case where we have seen a + * function-like macro name, but have not yet seen its + * closing parenthesis. + * + * There's perhaps a more compact way to do this with + * mid-rule actions in the grammar. + * + * I'm definitely not pleased with the complexity of + * this code here. + */ + if (parser->newline_as_space) + { + if (ret == '(') { + parser->paren_count++; + } else if (ret == ')') { + parser->paren_count--; + if (parser->paren_count == 0) + parser->newline_as_space = 0; + } else if (ret == NEWLINE) { + ret = SPACE; + } else if (ret != SPACE) { + if (parser->paren_count == 0) + parser->newline_as_space = 0; + } + } + else if (parser->in_control_line) + { + if (ret == NEWLINE) + parser->in_control_line = 0; + } + else if (ret == HASH_DEFINE_OBJ || ret == HASH_DEFINE_FUNC || + ret == HASH_UNDEF || ret == HASH_IF || + ret == HASH_IFDEF || ret == HASH_IFNDEF || + ret == HASH_ELIF || ret == HASH_ELSE || + ret == HASH_ENDIF || ret == HASH) + { + parser->in_control_line = 1; + } + else if (ret == IDENTIFIER) + { + macro_t *macro; + macro = hash_table_find (parser->defines, + yylval.str); + if (macro && macro->is_function) { + parser->newline_as_space = 1; + parser->paren_count = 0; + } + } + + return ret; + } node = parser->lex_from_node; diff --git a/glcpp.h b/glcpp.h index e5be1a6cd6..5c8c304a9c 100644 --- a/glcpp.h +++ b/glcpp.h @@ -128,6 +128,9 @@ struct glcpp_parser { struct hash_table *defines; string_list_t *active; int space_tokens; + int newline_as_space; + int in_control_line; + int paren_count; skip_node_t *skip_stack; token_list_t *lex_from_list; token_node_t *lex_from_node; diff --git a/tests/025-func-macro-as-non-macro.c b/tests/025-func-macro-as-non-macro.c index 3dbe026d9d..b433671d1b 100644 --- a/tests/025-func-macro-as-non-macro.c +++ b/tests/025-func-macro-as-non-macro.c @@ -1,2 +1,2 @@ #define foo(bar) bar -foo +foo bar diff --git a/tests/026-define-func-extra-newlines.c b/tests/026-define-func-extra-newlines.c new file mode 100644 index 0000000000..0d83740530 --- /dev/null +++ b/tests/026-define-func-extra-newlines.c @@ -0,0 +1,6 @@ +#define foo(a) bar + +foo +( +1 +) diff --git a/tests/055-define-func-extra-newlines.c b/tests/055-define-func-extra-newlines.c deleted file mode 100644 index 0d83740530..0000000000 --- a/tests/055-define-func-extra-newlines.c +++ /dev/null @@ -1,6 +0,0 @@ -#define foo(a) bar - -foo -( -1 -) -- cgit v1.2.3 From 7db2402a8009772a3f10d19cfc7f30be9ee79295 Mon Sep 17 00:00:00 2001 From: Carl Worth Date: Wed, 26 May 2010 17:01:57 -0700 Subject: Add support (and test) for an object-to-function chain with the parens in the content. That is, the following case: #define foo(x) (x) #define bar bar(baz) which now works with this (ugly) commit. I definitely want to come up with something cleaner than this. --- glcpp-parse.y | 65 ++++++++++++++++------ .../055-define-chain-obj-to-func-parens-in-text.c | 3 + 2 files changed, 52 insertions(+), 16 deletions(-) create mode 100644 tests/055-define-chain-obj-to-func-parens-in-text.c (limited to 'tests') diff --git a/glcpp-parse.y b/glcpp-parse.y index 1346b65aff..abdcd1ed5d 100644 --- a/glcpp-parse.y +++ b/glcpp-parse.y @@ -926,9 +926,9 @@ _glcpp_parser_evaluate_defined (glcpp_parser_t *parser, * needs further expansion. */ static int -_glcpp_parser_expand_token_onto (glcpp_parser_t *parser, - token_t *token, - token_list_t *result) +_expand_token_onto (glcpp_parser_t *parser, + token_t *token, + token_list_t *result) { const char *identifier; macro_t *macro; @@ -1075,10 +1075,10 @@ _arguments_parse (argument_list_t *arguments, token_node_t **node_ret) /* Prints the expansion of *node (consuming further tokens from the * list as necessary). Upon return *node will be the last consumed * node, such that further processing can continue with node->next. */ -static void -_glcpp_parser_expand_function_onto (glcpp_parser_t *parser, - token_node_t **node_ret, - token_list_t *result) +static function_status_t +_expand_function_onto (glcpp_parser_t *parser, + token_node_t **node_ret, + token_list_t *result) { macro_t *macro; token_node_t *node; @@ -1103,7 +1103,7 @@ _glcpp_parser_expand_function_onto (glcpp_parser_t *parser, break; case FUNCTION_NOT_A_FUNCTION: _token_list_append (result, node->token); - return; + return FUNCTION_NOT_A_FUNCTION; case FUNCTION_UNBALANCED_PARENTHESES: fprintf (stderr, "Error: Macro %s call has unbalanced parentheses\n", identifier); @@ -1112,7 +1112,7 @@ _glcpp_parser_expand_function_onto (glcpp_parser_t *parser, if (macro->replacements == NULL) { talloc_free (arguments); - return; + return FUNCTION_STATUS_SUCCESS; } if (_argument_list_length (arguments) != @@ -1123,7 +1123,7 @@ _glcpp_parser_expand_function_onto (glcpp_parser_t *parser, identifier, _argument_list_length (arguments), _string_list_length (macro->parameters)); - return; + exit (1); } /* Perform argument substitution on the replacement list. */ @@ -1191,6 +1191,8 @@ _glcpp_parser_expand_function_onto (glcpp_parser_t *parser, _string_list_pop (parser->active); talloc_free (arguments); + + return FUNCTION_STATUS_SUCCESS; } static void @@ -1199,19 +1201,50 @@ _glcpp_parser_expand_token_list_onto (glcpp_parser_t *parser, token_list_t *result) { token_node_t *node; + token_list_t *intermediate, *list_orig = list; + int i, need_rescan = 0; if (list == NULL) return; - for (node = list->head; node; node = node->next) - { - if (_glcpp_parser_expand_token_onto (parser, node->token, - result)) + intermediate = _token_list_create (parser); + + /* XXX: The two-pass expansion here is really ugly. The + * problem this is solving is that we can expand a macro into + * a function-like macro name, and then we need to recognize + * that as a function-like macro, but perhaps the parentheses + * and arguments aren't on the token list yet, (since they are + * in the actual content so they are part of what we are + * expanding. + * + * This ugly hack works, but is messy, fragile, and hard to + * maintain. I think a cleaner solution would separate the + * notions of expanding and appending and avoid this problem + * altogether. + */ + + for (i = 0; i < 2; i++) { + if (i == 1) { + list = intermediate; + intermediate = _token_list_create (parser); + } + for (node = list->head; node; node = node->next) { - _glcpp_parser_expand_function_onto (parser, &node, - result); + if (_expand_token_onto (parser, node->token, + intermediate)) + { + if (_expand_function_onto (parser, &node, + intermediate)) + { + need_rescan = 1; + } + } } + if (list != list_orig) + talloc_free (list); } + + _token_list_append_list (result, intermediate); } void diff --git a/tests/055-define-chain-obj-to-func-parens-in-text.c b/tests/055-define-chain-obj-to-func-parens-in-text.c new file mode 100644 index 0000000000..00f2c2346d --- /dev/null +++ b/tests/055-define-chain-obj-to-func-parens-in-text.c @@ -0,0 +1,3 @@ +#define failure() success +#define foo failure +foo() -- cgit v1.2.3 From dd7490093d84ce74a99922c3544b51c3f5d43345 Mon Sep 17 00:00:00 2001 From: Carl Worth Date: Thu, 27 May 2010 10:12:33 -0700 Subject: Avoid treating an expanded comma as an argument separator. That is, a function-like invocation foo(x) is valid as a single-argument invocation even if 'x' is a macro that expands into a value with a comma. Add a new COMMA_FINAL token type to handle this, and add a test for this case, (which passes). --- glcpp-parse.y | 18 ++++++++++++++++-- tests/056-macro-argument-with-comma.c | 4 ++++ 2 files changed, 20 insertions(+), 2 deletions(-) create mode 100644 tests/056-macro-argument-with-comma.c (limited to 'tests') diff --git a/glcpp-parse.y b/glcpp-parse.y index abdcd1ed5d..b2684d06d9 100644 --- a/glcpp-parse.y +++ b/glcpp-parse.y @@ -132,7 +132,7 @@ glcpp_parser_lex_from (glcpp_parser_t *parser, token_list_t *list); %parse-param {glcpp_parser_t *parser} %lex-param {glcpp_parser_t *parser} -%token DEFINED ELIF_EXPANDED HASH HASH_DEFINE_FUNC HASH_DEFINE_OBJ HASH_ELIF HASH_ELSE HASH_ENDIF HASH_IF HASH_IFDEF HASH_IFNDEF HASH_UNDEF IDENTIFIER IF_EXPANDED INTEGER NEWLINE OTHER SPACE +%token COMMA_FINAL DEFINED ELIF_EXPANDED HASH HASH_DEFINE_FUNC HASH_DEFINE_OBJ HASH_ELIF HASH_ELSE HASH_ENDIF HASH_IF HASH_IFDEF HASH_IFNDEF HASH_UNDEF IDENTIFIER IF_EXPANDED INTEGER NEWLINE OTHER SPACE %token PASTE %type expression INTEGER operator SPACE %type IDENTIFIER OTHER @@ -740,6 +740,9 @@ _token_print (token_t *token) case PASTE: printf ("##"); break; + case COMMA_FINAL: + printf (","); + break; default: fprintf (stderr, "Error: Don't know how to print token type %d\n", token->type); break; @@ -936,7 +939,18 @@ _expand_token_onto (glcpp_parser_t *parser, /* We only expand identifiers */ if (token->type != IDENTIFIER) { - _token_list_append (result, token); + /* We change any COMMA into a COMMA_FINAL to prevent + * it being mistaken for an argument separator + * later. */ + if (token->type == ',') { + token_t *new_token; + + new_token = _token_create_ival (result, COMMA_FINAL, + COMMA_FINAL); + _token_list_append (result, new_token); + } else { + _token_list_append (result, token); + } return 0; } diff --git a/tests/056-macro-argument-with-comma.c b/tests/056-macro-argument-with-comma.c new file mode 100644 index 0000000000..58701d1f25 --- /dev/null +++ b/tests/056-macro-argument-with-comma.c @@ -0,0 +1,4 @@ +#define bar with,embedded,commas +#define function(x) success +#define foo function +foo(bar) -- cgit v1.2.3 From a19297b26e971e5a9dbe00b4254931505da4b5a9 Mon Sep 17 00:00:00 2001 From: Carl Worth Date: Thu, 27 May 2010 13:29:19 -0700 Subject: Provide support for empty arguments in macro invocations. For this we always add a new argument to the argument list as soon as possible, without waiting until we see some argument token. This does mean we need to take some extra care when comparing the number of arguments with the number of expected arguments. In addition to matching numbers, we also support one (empty) argument when zero arguments are expected. Add a test case here for this, which does pass. --- glcpp-parse.y | 20 +++++++++++--------- tests/057-empty-arguments.c | 6 ++++++ 2 files changed, 17 insertions(+), 9 deletions(-) create mode 100644 tests/057-empty-arguments.c (limited to 'tests') diff --git a/glcpp-parse.y b/glcpp-parse.y index ba79a611f6..3e0a96528b 100644 --- a/glcpp-parse.y +++ b/glcpp-parse.y @@ -1044,7 +1044,8 @@ _arguments_parse (argument_list_t *arguments, token_node_t **node_ret) last = node; node = node->next; - argument = NULL; + argument = _token_list_create (arguments); + _argument_list_append (arguments, argument); for (paren_count = 1; node; last = node, node = node->next) { if (node->token->type == '(') @@ -1064,18 +1065,16 @@ _arguments_parse (argument_list_t *arguments, token_node_t **node_ret) if (node->token->type == ',' && paren_count == 1) { - if (argument) - _token_list_trim_trailing_space (argument); - argument = NULL; + _token_list_trim_trailing_space (argument); + argument = _token_list_create (arguments); + _argument_list_append (arguments, argument); } else { - if (argument == NULL) { + if (argument->head == NULL) { /* Don't treat initial whitespace as * part of the arguement. */ if (node->token->type == SPACE) continue; - argument = _token_list_create (arguments); - _argument_list_append (arguments, argument); } _token_list_append (argument, node->token); } @@ -1132,8 +1131,11 @@ _expand_function_onto (glcpp_parser_t *parser, return FUNCTION_STATUS_SUCCESS; } - if (_argument_list_length (arguments) != - _string_list_length (macro->parameters)) + if (! ((_argument_list_length (arguments) == + _string_list_length (macro->parameters)) || + (_string_list_length (macro->parameters) == 0 && + _argument_list_length (arguments) == 1 && + arguments->head->argument->head == NULL))) { fprintf (stderr, "Error: macro %s invoked with %d arguments (expected %d)\n", diff --git a/tests/057-empty-arguments.c b/tests/057-empty-arguments.c new file mode 100644 index 0000000000..6140232865 --- /dev/null +++ b/tests/057-empty-arguments.c @@ -0,0 +1,6 @@ +#define zero() success +zero() +#define one(x) success +one() +#define two(x,y) success +two(,) -- cgit v1.2.3 From 85b50e840d969c4d9ebcfcc3df1df7a95e07e34e Mon Sep 17 00:00:00 2001 From: Carl Worth Date: Thu, 27 May 2010 14:01:18 -0700 Subject: Add placeholder tokens to support pasting with empty arguments. Along with a passing test to verify that this works. --- glcpp-parse.y | 36 +++++++++++++++++++++++++------ tests/058-token-pasting-empty-arguments.c | 5 +++++ 2 files changed, 35 insertions(+), 6 deletions(-) create mode 100644 tests/058-token-pasting-empty-arguments.c (limited to 'tests') diff --git a/glcpp-parse.y b/glcpp-parse.y index 3e0a96528b..d587a4bf33 100644 --- a/glcpp-parse.y +++ b/glcpp-parse.y @@ -132,7 +132,7 @@ glcpp_parser_lex_from (glcpp_parser_t *parser, token_list_t *list); %parse-param {glcpp_parser_t *parser} %lex-param {glcpp_parser_t *parser} -%token COMMA_FINAL DEFINED ELIF_EXPANDED HASH HASH_DEFINE_FUNC HASH_DEFINE_OBJ HASH_ELIF HASH_ELSE HASH_ENDIF HASH_IF HASH_IFDEF HASH_IFNDEF HASH_UNDEF IDENTIFIER IF_EXPANDED INTEGER NEWLINE OTHER SPACE +%token COMMA_FINAL DEFINED ELIF_EXPANDED HASH HASH_DEFINE_FUNC HASH_DEFINE_OBJ HASH_ELIF HASH_ELSE HASH_ENDIF HASH_IF HASH_IFDEF HASH_IFNDEF HASH_UNDEF IDENTIFIER IF_EXPANDED INTEGER NEWLINE OTHER PLACEHOLDER SPACE %token PASTE %type expression INTEGER operator SPACE %type IDENTIFIER OTHER @@ -746,6 +746,9 @@ _token_print (token_t *token) case COMMA_FINAL: printf (","); break; + case PLACEHOLDER: + /* Nothing to print. */ + break; default: fprintf (stderr, "Error: Don't know how to print token type %d\n", token->type); break; @@ -756,6 +759,17 @@ _token_print (token_t *token) static void _token_paste (token_t *token, token_t *other) { + /* Pasting a placeholder onto anything makes no change. */ + if (other->type == PLACEHOLDER) + return; + + /* When 'token' is a placeholder, just return contents of 'other'. */ + if (token->type == PLACEHOLDER) { + token->type = other->type; + token->value = other->value; + return; + } + /* A very few single-character punctuators can be combined * with another to form a multi-character punctuator. */ switch (token->type) { @@ -1159,10 +1173,20 @@ _expand_function_onto (glcpp_parser_t *parser, argument = _argument_list_member_at (arguments, parameter_index); /* Before substituting, we expand the argument - * tokens. */ - _glcpp_parser_expand_token_list_onto (parser, - argument, - substituted); + * tokens, or append a placeholder token for + * an empty argument. */ + if (argument->head) { + _glcpp_parser_expand_token_list_onto (parser, + argument, + substituted); + } else { + token_t *new_token; + + new_token = _token_create_ival (substituted, + PLACEHOLDER, + PLACEHOLDER); + _token_list_append (substituted, new_token); + } } else { _token_list_append (substituted, node->token); } @@ -1196,7 +1220,7 @@ _expand_function_onto (glcpp_parser_t *parser, if (next_non_space == NULL) { fprintf (stderr, "Error: '##' cannot appear at either end of a macro expansion\n"); - exit (1); + return FUNCTION_STATUS_SUCCESS; } _token_paste (node->token, next_non_space->token); diff --git a/tests/058-token-pasting-empty-arguments.c b/tests/058-token-pasting-empty-arguments.c new file mode 100644 index 0000000000..8ac260c76b --- /dev/null +++ b/tests/058-token-pasting-empty-arguments.c @@ -0,0 +1,5 @@ +#define paste(x,y) x ## y +paste(a,b) +paste(a,) +paste(,b) +paste(,) -- cgit v1.2.3 From 050e3ded1ea05cfe336dd0cd20212d17d7960c9e Mon Sep 17 00:00:00 2001 From: Carl Worth Date: Thu, 27 May 2010 14:36:29 -0700 Subject: Implement token pasting of integers. To do this correctly, we change the lexer to lex integers as string values, (new token type of INTEGER_STRING), and only convert to integer values when evaluating an expression value. Add a new test case for this, (which does pass now). --- Makefile | 2 +- glcpp-lex.l | 12 ++++++------ glcpp-parse.y | 32 ++++++++++++++++++++++---------- tests/059-token-pasting-integer.c | 4 ++++ 4 files changed, 33 insertions(+), 17 deletions(-) create mode 100644 tests/059-token-pasting-integer.c (limited to 'tests') diff --git a/Makefile b/Makefile index 88116128f8..0c06aa880f 100644 --- a/Makefile +++ b/Makefile @@ -7,7 +7,7 @@ CFLAGS = -g override CFLAGS += -Wall -Wextra -Wwrite-strings -Wswitch-enum -Wno-unused glcpp: glcpp.o glcpp-lex.o glcpp-parse.o hash_table.o xtalloc.o - gcc -o $@ -ltalloc $^ + gcc -o $@ -ltalloc -lm $^ %.c %.h: %.y bison --debug --defines=$*.h --output=$*.c $^ diff --git a/glcpp-lex.l b/glcpp-lex.l index d6b7726d36..70d47d2497 100644 --- a/glcpp-lex.l +++ b/glcpp-lex.l @@ -88,18 +88,18 @@ HEXADECIMAL_INTEGER 0[xX][0-9a-fA-F]+[uU]? } {DECIMAL_INTEGER} { - yylval.ival = strtoll (yytext, NULL, 10); - return INTEGER; + yylval.str = xtalloc_strdup (yyextra, yytext); + return INTEGER_STRING; } {OCTAL_INTEGER} { - yylval.ival = strtoll (yytext + 1, NULL, 8); - return INTEGER; + yylval.str = xtalloc_strdup (yyextra, yytext); + return INTEGER_STRING; } {HEXADECIMAL_INTEGER} { - yylval.ival = strtoll (yytext + 2, NULL, 16); - return INTEGER; + yylval.str = xtalloc_strdup (yyextra, yytext); + return INTEGER_STRING; } "<<" { diff --git a/glcpp-parse.y b/glcpp-parse.y index d587a4bf33..5b2d0d3927 100644 --- a/glcpp-parse.y +++ b/glcpp-parse.y @@ -132,10 +132,10 @@ glcpp_parser_lex_from (glcpp_parser_t *parser, token_list_t *list); %parse-param {glcpp_parser_t *parser} %lex-param {glcpp_parser_t *parser} -%token COMMA_FINAL DEFINED ELIF_EXPANDED HASH HASH_DEFINE_FUNC HASH_DEFINE_OBJ HASH_ELIF HASH_ELSE HASH_ENDIF HASH_IF HASH_IFDEF HASH_IFNDEF HASH_UNDEF IDENTIFIER IF_EXPANDED INTEGER NEWLINE OTHER PLACEHOLDER SPACE +%token COMMA_FINAL DEFINED ELIF_EXPANDED HASH HASH_DEFINE_FUNC HASH_DEFINE_OBJ HASH_ELIF HASH_ELSE HASH_ENDIF HASH_IF HASH_IFDEF HASH_IFNDEF HASH_UNDEF IDENTIFIER IF_EXPANDED INTEGER INTEGER_STRING NEWLINE OTHER PLACEHOLDER SPACE %token PASTE %type expression INTEGER operator SPACE -%type IDENTIFIER OTHER +%type IDENTIFIER INTEGER_STRING OTHER %type identifier_list %type preprocessing_token %type pp_tokens replacement_list text_line @@ -253,7 +253,16 @@ control_line: ; expression: - INTEGER { + INTEGER_STRING { + if (strlen ($1) >= 3 && strncmp ($1, "0x", 2) == 0) { + $$ = strtoll ($1 + 2, NULL, 16); + } else if ($1[0] == '0') { + $$ = strtoll ($1, NULL, 8); + } else { + $$ = strtoll ($1, NULL, 10); + } + } +| INTEGER { $$ = $1; } | expression OR expression { @@ -372,8 +381,8 @@ preprocessing_token: IDENTIFIER { $$ = _token_create_str (parser, IDENTIFIER, $1); } -| INTEGER { - $$ = _token_create_ival (parser, INTEGER, $1); +| INTEGER_STRING { + $$ = _token_create_str (parser, INTEGER_STRING, $1); } | operator { $$ = _token_create_ival (parser, $1, $1); @@ -710,6 +719,7 @@ _token_print (token_t *token) printf ("%" PRIxMAX, token->value.ival); break; case IDENTIFIER: + case INTEGER_STRING: case OTHER: printf ("%s", token->value.str); break; @@ -828,11 +838,13 @@ _token_paste (token_t *token, token_t *other) /* Two string-valued tokens can usually just be mashed * together. * - * XXX: Since our 'OTHER' case is currently so loose, this may - * allow some things thruogh that should be treated as - * errors. */ - if ((token->type == IDENTIFIER || token->type == OTHER) && - (other->type == IDENTIFIER || other->type == OTHER)) + * XXX: This isn't actually legitimate. Several things here + * should result in a diagnostic since the result cannot be a + * valid, single pre-processing token. For example, pasting + * "123" and "abc" is not legal, but we don't catch that + * here. */ + if ((token->type == IDENTIFIER || token->type == OTHER || token->type == INTEGER_STRING) && + (other->type == IDENTIFIER || other->type == OTHER || other->type == INTEGER_STRING)) { token->value.str = talloc_strdup_append (token->value.str, other->value.str); diff --git a/tests/059-token-pasting-integer.c b/tests/059-token-pasting-integer.c new file mode 100644 index 0000000000..37b895a423 --- /dev/null +++ b/tests/059-token-pasting-integer.c @@ -0,0 +1,4 @@ +#define paste(x,y) x ## y +paste(1,2) +paste(1,000) +paste(identifier,2) -- cgit v1.2.3 From baa17c87485b5e776ec142844f5df38a3df9dccc Mon Sep 17 00:00:00 2001 From: Carl Worth Date: Thu, 27 May 2010 14:53:51 -0700 Subject: Remove blank lines from output files before comparing. Recently I'm seeing cases where "gcc -E" mysteriously omits blank lines, (even though it prints the blank lines in other very similar cases). Rather than trying to decipher and imitate this, just get rid of the blank lines. This approach with sed to kill the lines before the diff is better than "diff -B" since when there is an actual difference, the presence of blank lines won't make the diff harder to read. --- .gitignore | 1 + tests/glcpp-test | 7 ++++--- 2 files changed, 5 insertions(+), 3 deletions(-) (limited to 'tests') diff --git a/.gitignore b/.gitignore index d67bd38c93..b88f0cc75c 100644 --- a/.gitignore +++ b/.gitignore @@ -6,4 +6,5 @@ glcpp-parse.h *~ tests/*.expected tests/*.gcc +tests/*.glcpp tests/*.out diff --git a/tests/glcpp-test b/tests/glcpp-test index bf88d4462e..92c994979a 100755 --- a/tests/glcpp-test +++ b/tests/glcpp-test @@ -3,8 +3,9 @@ set -e for test in *.c; do echo "Testing $test" - ../glcpp < $test > $test.out + ../glcpp < $test > $test.glcpp + grep -v '^$' < $test.glcpp > $test.out || true gcc -E $test -o $test.gcc - grep -v '^#' < $test.gcc > $test.expected - diff -B -u $test.expected $test.out + grep -v '^#' < $test.gcc | grep -v '^$' > $test.expected || true + diff -u $test.expected $test.out done -- cgit v1.2.3 From 9b519f9c7997e0ec02c66d39edc12912aebb9eca Mon Sep 17 00:00:00 2001 From: Carl Worth Date: Fri, 28 May 2010 08:04:13 -0700 Subject: Stop interrupting the test suite at the first failure. This behavior was useful when starting the implementation over ("take-2") where the whole test suite was failing. This made it easy to focus on one test at a time and get each working. More recently, we got the whole suite working, so we don't need this feature anymore. And in the previous commit, we regressed a couple of tests, so it's nice to be able to see all the failures with a single run of the suite. --- tests/glcpp-test | 1 - 1 file changed, 1 deletion(-) (limited to 'tests') diff --git a/tests/glcpp-test b/tests/glcpp-test index 92c994979a..ba398af0d5 100755 --- a/tests/glcpp-test +++ b/tests/glcpp-test @@ -1,5 +1,4 @@ #!/bin/sh -set -e for test in *.c; do echo "Testing $test" -- cgit v1.2.3 From 681afbc855c86df8c3521ccdfadb7f16b9729baa Mon Sep 17 00:00:00 2001 From: Carl Worth Date: Fri, 28 May 2010 15:06:02 -0700 Subject: Perform macro by replacing tokens in original list. We take the results of macro expansion and splice them into the original token list over which we are iterating. This makes it easy for function-like macro invocations to find their arguments since they are simply subsequent tokens on the list. This fixes the recently-introduced regressions (tests 55 and 56) and also passes new tests 60 and 61 introduced to strees this feature, (with macro-argument parentheses split between a macro value and the textual input). --- glcpp-parse.y | 278 +++++++++++++-------- .../060-left-paren-in-macro-right-paren-in-text.c | 3 + tests/061-define-chain-obj-to-func-multi.c | 5 + 3 files changed, 187 insertions(+), 99 deletions(-) create mode 100644 tests/060-left-paren-in-macro-right-paren-in-text.c create mode 100644 tests/061-define-chain-obj-to-func-multi.c (limited to 'tests') diff --git a/glcpp-parse.y b/glcpp-parse.y index 9f97b2a282..c89d7bf159 100644 --- a/glcpp-parse.y +++ b/glcpp-parse.y @@ -101,13 +101,12 @@ _glcpp_parser_evaluate_defined (glcpp_parser_t *parser, token_list_t *list); static void -_glcpp_parser_print_expanded_token_list (glcpp_parser_t *parser, - token_list_t *list); +_glcpp_parser_expand_token_list (glcpp_parser_t *parser, + token_list_t *list); static void -_glcpp_parser_expand_token_list_onto (glcpp_parser_t *parser, - token_list_t *list, - token_list_t *result); +_glcpp_parser_print_expanded_token_list (glcpp_parser_t *parser, + token_list_t *list); static void _glcpp_parser_skip_stack_push_if (glcpp_parser_t *parser, int condition); @@ -218,7 +217,8 @@ control_line: _token_list_append (expanded, token); talloc_unlink (parser, token); _glcpp_parser_evaluate_defined (parser, $2); - _glcpp_parser_expand_token_list_onto (parser, $2, expanded); + _glcpp_parser_expand_token_list (parser, $2); + _token_list_append_list (expanded, $2); glcpp_parser_lex_from (parser, expanded); } | HASH_IFDEF IDENTIFIER NEWLINE { @@ -240,7 +240,8 @@ control_line: _token_list_append (expanded, token); talloc_unlink (parser, token); _glcpp_parser_evaluate_defined (parser, $2); - _glcpp_parser_expand_token_list_onto (parser, $2, expanded); + _glcpp_parser_expand_token_list (parser, $2); + _token_list_append_list (expanded, $2); glcpp_parser_lex_from (parser, expanded); } | HASH_ELSE NEWLINE { @@ -688,6 +689,22 @@ _token_list_append_list (token_list_t *list, token_list_t *tail) list->non_space_tail = tail->non_space_tail; } +token_list_t * +_token_list_copy (void *ctx, token_list_t *other) +{ + token_list_t *copy; + token_node_t *node; + + if (other == NULL) + return NULL; + + copy = _token_list_create (ctx); + for (node = other->head; node; node = node->next) + _token_list_append (copy, node->token); + + return copy; +} + void _token_list_trim_trailing_space (token_list_t *list) { @@ -956,9 +973,12 @@ typedef enum function_status } function_status_t; /* Find a set of function-like macro arguments by looking for a - * balanced set of parentheses. Upon return *node will be the last - * consumed node, such that further processing can continue with - * node->next. + * balanced set of parentheses. + * + * When called, 'node' should be the opening-parenthesis token, (or + * perhaps preceeding SPACE tokens). Upon successful return *last will + * be the last consumed node, (corresponding to the closing right + * parenthesis). * * Return values: * @@ -976,13 +996,13 @@ typedef enum function_status * Macro name is not followed by a balanced set of parentheses. */ static function_status_t -_arguments_parse (argument_list_t *arguments, token_node_t **node_ret) +_arguments_parse (argument_list_t *arguments, + token_node_t *node, + token_node_t **last) { token_list_t *argument; - token_node_t *node = *node_ret, *last; int paren_count; - last = node; node = node->next; /* Ignore whitespace before first parenthesis. */ @@ -992,13 +1012,12 @@ _arguments_parse (argument_list_t *arguments, token_node_t **node_ret) if (node == NULL || node->token->type != '(') return FUNCTION_NOT_A_FUNCTION; - last = node; node = node->next; argument = _token_list_create (arguments); _argument_list_append (arguments, argument); - for (paren_count = 1; node; last = node, node = node->next) { + for (paren_count = 1; node; node = node->next) { if (node->token->type == '(') { paren_count++; @@ -1006,11 +1025,8 @@ _arguments_parse (argument_list_t *arguments, token_node_t **node_ret) else if (node->token->type == ')') { paren_count--; - if (paren_count == 0) { - last = node; - node = node->next; + if (paren_count == 0) break; - } } if (node->token->type == ',' && @@ -1031,32 +1047,44 @@ _arguments_parse (argument_list_t *arguments, token_node_t **node_ret) } } - if (node && paren_count) + if (paren_count) return FUNCTION_UNBALANCED_PARENTHESES; - *node_ret = last; + *last = node; return FUNCTION_STATUS_SUCCESS; } -/* Appends expansion of *node (consuming further tokens from the list - * as necessary) onto result. Upon return *node will be the last - * consumed node, such that further processing can continue with - * node->next. */ -static void -_glcpp_parser_expand_function_onto (glcpp_parser_t *parser, - token_node_t **node_ret, - token_list_t *result) +/* This is a helper function that's essentially part of the + * implementation of _glcpp_parser_expand_node. It shouldn't be called + * except for by that function. + * + * Returns NULL if node is a simple token with no expansion, (that is, + * although 'node' corresponds to an identifier defined as a + * function-like macro, it is not followed with a parenthesized + * argument list). + * + * Compute the complete expansion of node (which is a function-like + * macro) and subsequent nodes which are arguments. + * + * Returns the token list that results from the expansion and sets + * *last to the last node in the list that was consumed by the + * expansion. Specificallty, *last will be set as follows: as the + * token of the closing right parenthesis. + */ +static token_list_t * +_glcpp_parser_expand_function (glcpp_parser_t *parser, + token_node_t *node, + token_node_t **last) + { macro_t *macro; - token_node_t *node; const char *identifier; argument_list_t *arguments; function_status_t status; token_list_t *substituted; int parameter_index; - node = *node_ret; identifier = node->token->value.str; macro = hash_table_find (parser->defines, identifier); @@ -1064,23 +1092,20 @@ _glcpp_parser_expand_function_onto (glcpp_parser_t *parser, assert (macro->is_function); arguments = _argument_list_create (parser); - status = _arguments_parse (arguments, node_ret); + status = _arguments_parse (arguments, node, last); switch (status) { case FUNCTION_STATUS_SUCCESS: break; case FUNCTION_NOT_A_FUNCTION: - _token_list_append (result, node->token); - return; + return NULL; case FUNCTION_UNBALANCED_PARENTHESES: - fprintf (stderr, "Error: Macro %s call has unbalanced parentheses\n", - identifier); - exit (1); + return NULL; } if (macro->replacements == NULL) { talloc_free (arguments); - return; + return _token_list_create (parser); } if (! ((_argument_list_length (arguments) == @@ -1094,7 +1119,7 @@ _glcpp_parser_expand_function_onto (glcpp_parser_t *parser, identifier, _argument_list_length (arguments), _string_list_length (macro->parameters)); - return; + return NULL; } /* Perform argument substitution on the replacement list. */ @@ -1114,9 +1139,9 @@ _glcpp_parser_expand_function_onto (glcpp_parser_t *parser, * tokens, or append a placeholder token for * an empty argument. */ if (argument->head) { - _glcpp_parser_expand_token_list_onto (parser, - argument, - substituted); + _glcpp_parser_expand_token_list (parser, + argument); + _token_list_append_list (substituted, argument); } else { token_t *new_token; @@ -1158,7 +1183,7 @@ _glcpp_parser_expand_function_onto (glcpp_parser_t *parser, if (next_non_space == NULL) { fprintf (stderr, "Error: '##' cannot appear at either end of a macro expansion\n"); - return; + return NULL; } _token_paste (node->token, next_non_space->token); @@ -1168,22 +1193,33 @@ _glcpp_parser_expand_function_onto (glcpp_parser_t *parser, } _string_list_push (parser->active, identifier); - _glcpp_parser_expand_token_list_onto (parser, substituted, result); + _glcpp_parser_expand_token_list (parser, substituted); _string_list_pop (parser->active); - talloc_free (arguments); + return substituted; } - -/* Appends the expansion of the token in *node onto result. - * Upon return *node will be the last consumed node, such that further - * processing can continue with node->next. */ -static void -_glcpp_parser_expand_token_onto (glcpp_parser_t *parser, - token_node_t **node, - token_list_t *result) +/* Compute the complete expansion of node, (and subsequent nodes after + * 'node' in the case that 'node' is a function-like macro and + * subsequent nodes are arguments). + * + * Returns NULL if node is a simple token with no expansion. + * + * Otherwise, returns the token list that results from the expansion + * and sets *last to the last node in the list that was consumed by + * the expansion. Specificallty, *last will be set as follows: + * + * As 'node' in the case of object-like macro expansion. + * + * As the token of the closing right parenthesis in the case of + * function-like macro expansion. + */ +static token_list_t * +_glcpp_parser_expand_node (glcpp_parser_t *parser, + token_node_t *node, + token_node_t **last) { - token_t *token = (*node)->token; + token_t *token = node->token; const char *identifier; macro_t *macro; token_list_t *expansion; @@ -1194,90 +1230,134 @@ _glcpp_parser_expand_token_onto (glcpp_parser_t *parser, * it being mistaken for an argument separator * later. */ if (token->type == ',') { - token_t *new_token; - - new_token = _token_create_ival (result, COMMA_FINAL, - COMMA_FINAL); - _token_list_append (result, new_token); - } else { - _token_list_append (result, token); + token->type = COMMA_FINAL; + token->value.ival = COMMA_FINAL; } - return; + + return NULL; } /* Look up this identifier in the hash table. */ identifier = token->value.str; macro = hash_table_find (parser->defines, identifier); - /* Not a macro, so just append. */ - if (macro == NULL) { - _token_list_append (result, token); - return; - } + /* Not a macro, so no expansion needed. */ + if (macro == NULL) + return NULL; /* Finally, don't expand this macro if we're already actively * expanding it, (to avoid infinite recursion). */ - if (_string_list_contains (parser->active, identifier, NULL)) - { + if (_string_list_contains (parser->active, identifier, NULL)) { /* We change the token type here from IDENTIFIER to * OTHER to prevent any future expansion of this * unexpanded token. */ char *str; - token_t *new_token; + token_list_t *expansion; + token_t *final; - str = xtalloc_strdup (result, token->value.str); - new_token = _token_create_str (result, OTHER, str); - _token_list_append (result, new_token); - return; + str = xtalloc_strdup (parser, token->value.str); + final = _token_create_str (parser, OTHER, str); + expansion = _token_list_create (parser); + _token_list_append (expansion, final); + *last = node; + return expansion; } - if (macro->is_function) { - _glcpp_parser_expand_function_onto (parser, node, result); - } else { + if (! macro->is_function) + { + *last = node; + + if (macro->replacements == NULL) + return _token_list_create (parser); + + expansion = _token_list_copy (parser, macro->replacements); + _string_list_push (parser->active, identifier); - _glcpp_parser_expand_token_list_onto (parser, - macro->replacements, - result); + _glcpp_parser_expand_token_list (parser, expansion); _string_list_pop (parser->active); + + return expansion; } + + return _glcpp_parser_expand_function (parser, node, last); } +/* Walk over the token list replacing nodes with their expansion. + * Whenever nodes are expanded the walking will walk over the new + * nodes, continuing to expand as necessary. The results are placed in + * 'list' itself; + */ static void -_glcpp_parser_expand_token_list_onto (glcpp_parser_t *parser, - token_list_t *list, - token_list_t *result) +_glcpp_parser_expand_token_list (glcpp_parser_t *parser, + token_list_t *list) { - token_node_t *node; + token_node_t *node_prev; + token_node_t *node, *last; + token_list_t *expansion; - if (list == NULL || list->head == NULL) + if (list == NULL) return; - for (node = list->head; node; node = node->next) - { - _glcpp_parser_expand_token_onto (parser, &node, result); + _token_list_trim_trailing_space (list); + + node_prev = NULL; + node = list->head; + + while (node) { + /* Find the expansion for node, which will replace all + * nodes from node to last, inclusive. */ + expansion = _glcpp_parser_expand_node (parser, node, &last); + if (expansion) { + /* Splice expansion into list, supporting a + * simple deletion if the expansion is + * empty. */ + if (expansion->head) { + if (node_prev) + node_prev->next = expansion->head; + else + list->head = expansion->head; + expansion->tail->next = last->next; + if (last == list->tail) + list->tail = expansion->tail; + } else { + if (node_prev) + node_prev->next = last->next; + else + list->head = last->next; + if (last == list->tail) + list->tail == NULL; + } + } else { + node_prev = node; + } + node = node_prev ? node_prev->next : list->head; } + + list->non_space_tail = list->tail; +} + +static void +_glcpp_parser_expand_token_list_onto (glcpp_parser_t *parser, + token_list_t *list, + token_list_t *result) +{ + _glcpp_parser_expand_token_list (parser, list); + + _token_list_append_list (result, list); } void _glcpp_parser_print_expanded_token_list (glcpp_parser_t *parser, token_list_t *list) { - token_list_t *expanded; - token_node_t *node; - function_status_t function_status; - if (list == NULL) return; - expanded = _token_list_create (parser); - - _glcpp_parser_expand_token_list_onto (parser, list, expanded); - - _token_list_trim_trailing_space (expanded); + _glcpp_parser_expand_token_list (parser, list); - _token_list_print (expanded); + _token_list_trim_trailing_space (list); - talloc_free (expanded); + _token_list_print (list); } void diff --git a/tests/060-left-paren-in-macro-right-paren-in-text.c b/tests/060-left-paren-in-macro-right-paren-in-text.c new file mode 100644 index 0000000000..ed80ea879c --- /dev/null +++ b/tests/060-left-paren-in-macro-right-paren-in-text.c @@ -0,0 +1,3 @@ +#define double(a) a*2 +#define foo double( +foo 5) diff --git a/tests/061-define-chain-obj-to-func-multi.c b/tests/061-define-chain-obj-to-func-multi.c new file mode 100644 index 0000000000..6dbfd1f62d --- /dev/null +++ b/tests/061-define-chain-obj-to-func-multi.c @@ -0,0 +1,5 @@ +#define foo(x) success +#define bar foo +#define baz bar +#define joe baz +joe (failure) -- cgit v1.2.3 From 631016946ca8134244c4e58bef6863d204b1119b Mon Sep 17 00:00:00 2001 From: Carl Worth Date: Sat, 29 May 2010 05:07:24 -0700 Subject: Fix pass-through of '=' and add a test for it. Previously '=' was not included in our PUNCTUATION regeular expression, but it *was* excldued from our OTHER regular expression, so we were getting the default (and hamful) lex action of just printing it. The test we add here is named "punctuator" with the idea that we can extend it as needed for other punctuator testing. --- glcpp-lex.l | 2 +- glcpp-parse.y | 1 + tests/071-punctuator.c | 1 + 3 files changed, 3 insertions(+), 1 deletion(-) create mode 100644 tests/071-punctuator.c (limited to 'tests') diff --git a/glcpp-lex.l b/glcpp-lex.l index 70d47d2497..52269c6b30 100644 --- a/glcpp-lex.l +++ b/glcpp-lex.l @@ -38,7 +38,7 @@ NEWLINE [\n] HSPACE [ \t] HASH ^{HSPACE}*#{HSPACE}* IDENTIFIER [_a-zA-Z][_a-zA-Z0-9]* -PUNCTUATION [][(){}.&*~!/%<>^|;,+-] +PUNCTUATION [][(){}.&*~!/%<>^|;,=+-] OTHER [^][(){}.&*~!/%<>^|;,=#[:space:]+-]+ DECIMAL_INTEGER [1-9][0-9]*[uU]? diff --git a/glcpp-parse.y b/glcpp-parse.y index c89d7bf159..01ca08ec74 100644 --- a/glcpp-parse.y +++ b/glcpp-parse.y @@ -426,6 +426,7 @@ operator: | OR { $$ = OR; } | ';' { $$ = ';'; } | ',' { $$ = ','; } +| '=' { $$ = '='; } | PASTE { $$ = PASTE; } | DEFINED { $$ = DEFINED; } ; diff --git a/tests/071-punctuator.c b/tests/071-punctuator.c new file mode 100644 index 0000000000..959d682598 --- /dev/null +++ b/tests/071-punctuator.c @@ -0,0 +1 @@ +a = b -- cgit v1.2.3 From b06096e86eda1257769156523b5738044c6a2b10 Mon Sep 17 00:00:00 2001 From: Carl Worth Date: Sat, 29 May 2010 05:54:19 -0700 Subject: Add test and fix bugs with multiple token-pasting on the same line. The list replacement when token pasting was broken, (failing to properly update the list's tail pointer). Also, memory management when pasting was broken, (modifying the original token's string which would cause problems with multiple calls to a macro which pasted a literal string). We didn't catch this with previous tests because they only pasted argument values. --- glcpp-parse.y | 92 +++++++++++++++++-------------------- tests/072-token-pasting-same-line.c | 2 + 2 files changed, 43 insertions(+), 51 deletions(-) create mode 100644 tests/072-token-pasting-same-line.c (limited to 'tests') diff --git a/glcpp-parse.y b/glcpp-parse.y index 01ca08ec74..f4c834e038 100644 --- a/glcpp-parse.y +++ b/glcpp-parse.y @@ -783,73 +783,53 @@ _token_print (token_t *token) } } -/* Change 'token' into a new token formed by pasting 'other'. */ -static void +/* Return a new token (talloc()ed off of 'token') formed by pasting + * 'token' and 'other'. Note that this function may return 'token' or + * 'other' directly rather than allocating anything new. + * + * Caution: Only very cursory error-checking is performed to see if + * the final result is a valid single token. */ +static token_t * _token_paste (token_t *token, token_t *other) { /* Pasting a placeholder onto anything makes no change. */ if (other->type == PLACEHOLDER) - return; + return token; - /* When 'token' is a placeholder, just return contents of 'other'. */ - if (token->type == PLACEHOLDER) { - token->type = other->type; - token->value = other->value; - return; - } + /* When 'token' is a placeholder, just return 'other'. */ + if (token->type == PLACEHOLDER) + return other; /* A very few single-character punctuators can be combined * with another to form a multi-character punctuator. */ switch (token->type) { case '<': - if (other->type == '<') { - token->type = LEFT_SHIFT; - token->value.ival = LEFT_SHIFT; - return; - } else if (other->type == '=') { - token->type = LESS_OR_EQUAL; - token->value.ival = LESS_OR_EQUAL; - return; - } + if (other->type == '<') + return _token_create_ival (token, LEFT_SHIFT, LEFT_SHIFT); + else if (other->type == '=') + return _token_create_ival (token, LESS_OR_EQUAL, LESS_OR_EQUAL); break; case '>': - if (other->type == '>') { - token->type = RIGHT_SHIFT; - token->value.ival = RIGHT_SHIFT; - return; - } else if (other->type == '=') { - token->type = GREATER_OR_EQUAL; - token->value.ival = GREATER_OR_EQUAL; - return; - } + if (other->type == '>') + return _token_create_ival (token, RIGHT_SHIFT, RIGHT_SHIFT); + else if (other->type == '=') + return _token_create_ival (token, GREATER_OR_EQUAL, GREATER_OR_EQUAL); break; case '=': - if (other->type == '=') { - token->type = EQUAL; - token->value.ival = EQUAL; - return; - } + if (other->type == '=') + return _token_create_ival (token, EQUAL, EQUAL); break; case '!': - if (other->type == '=') { - token->type = NOT_EQUAL; - token->value.ival = NOT_EQUAL; - return; - } + if (other->type == '=') + return _token_create_ival (token, NOT_EQUAL, NOT_EQUAL); break; case '&': - if (other->type == '&') { - token->type = AND; - token->value.ival = AND; - return; - } + if (other->type == '&') + return _token_create_ival (token, AND, AND); break; case '|': - if (other->type == '|') { - token->type = OR; - token->value.ival = OR; - return; - } + if (other->type == '|') + return _token_create_ival (token, OR, OR); break; } @@ -864,9 +844,11 @@ _token_paste (token_t *token, token_t *other) if ((token->type == IDENTIFIER || token->type == OTHER || token->type == INTEGER_STRING) && (other->type == IDENTIFIER || other->type == OTHER || other->type == INTEGER_STRING)) { - token->value.str = talloc_strdup_append (token->value.str, - other->value.str); - return; + char *str; + + str = xtalloc_asprintf (token, "%s%s", + token->value.str, other->value.str); + return _token_create_str (token, token->type, str); } printf ("Error: Pasting \""); @@ -874,6 +856,8 @@ _token_paste (token_t *token, token_t *other) printf ("\" and \""); _token_print (other); printf ("\" does not give a valid preprocessing token.\n"); + + return token; } static void @@ -1159,6 +1143,8 @@ _glcpp_parser_expand_function (glcpp_parser_t *parser, /* After argument substitution, and before further expansion * below, implement token pasting. */ + _token_list_trim_trailing_space (substituted); + node = substituted->head; while (node) { @@ -1187,12 +1173,16 @@ _glcpp_parser_expand_function (glcpp_parser_t *parser, return NULL; } - _token_paste (node->token, next_non_space->token); + node->token = _token_paste (node->token, next_non_space->token); node->next = next_non_space->next; + if (next_non_space == substituted->tail) + substituted->tail = node; node = node->next; } + substituted->non_space_tail = substituted->tail; + _string_list_push (parser->active, identifier); _glcpp_parser_expand_token_list (parser, substituted); _string_list_pop (parser->active); diff --git a/tests/072-token-pasting-same-line.c b/tests/072-token-pasting-same-line.c new file mode 100644 index 0000000000..e421e9d5e2 --- /dev/null +++ b/tests/072-token-pasting-same-line.c @@ -0,0 +1,2 @@ +#define paste(x) success_ ## x +paste(1) paste(2) paste(3) -- cgit v1.2.3 From 75ef1c75dd47a0b4054a767fd94f7c3cf68d2331 Mon Sep 17 00:00:00 2001 From: Carl Worth Date: Sat, 29 May 2010 05:57:22 -0700 Subject: Add killer test case from the C99 specification. Happily, this passes now, (since many of the previously added test cases were extracted from this one). --- tests/099-c99-example.c | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) create mode 100644 tests/099-c99-example.c (limited to 'tests') diff --git a/tests/099-c99-example.c b/tests/099-c99-example.c new file mode 100644 index 0000000000..d1976b1f26 --- /dev/null +++ b/tests/099-c99-example.c @@ -0,0 +1,17 @@ +#define x 3 +#define f(a) f(x * (a)) +#undef x +#define x 2 +#define g f +#define z z[0] +#define h g(~ +#define m(a) a(w) +#define w 0,1 +#define t(a) a +#define p() int +#define q(x) x +#define r(x,y) x ## y +f(y+1) + f(f(z)) % t(t(g)(0) + t)(1); +g(x +(3,4)-w) | h 5) & m + (f)^m(m); +p() i[q()] = { q(1), r(2,3), r(4,), r(,5), r(,)}; -- cgit v1.2.3