glsl: Add a preprocessor tokeniser.

3 files changed, 509 insertions, 0 deletions

diff --git a/src/glsl/pp/SConscript b/src/glsl/pp/SConscript
index ac58a3e5fd..a08f5cf632 100644
--- a/src/glsl/pp/SConscript
+++ b/src/glsl/pp/SConscript
@@ -9,6 +9,7 @@ glsl = env.StaticLibrary(
     target = 'glsl',
     source = [
         'sl_pp_purify.c',
+        'sl_pp_token.c',
     ],
 )
 Export('glsl')
diff --git a/src/glsl/pp/sl_pp_token.c b/src/glsl/pp/sl_pp_token.c
new file mode 100644
index 0000000000..6402c6a95b
--- /dev/null
+++ b/src/glsl/pp/sl_pp_token.c
@@ -0,0 +1,401 @@
+/**************************************************************************
+ * 
+ * Copyright 2009 VMware, Inc.
+ * All Rights Reserved.
+ * 
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the
+ * "Software"), to deal in the Software without restriction, including
+ * without limitation the rights to use, copy, modify, merge, publish,
+ * distribute, sub license, and/or sell copies of the Software, and to
+ * permit persons to whom the Software is furnished to do so, subject to
+ * the following conditions:
+ * 
+ * The above copyright notice and this permission notice (including the
+ * next paragraph) shall be included in all copies or substantial portions
+ * of the Software.
+ * 
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
+ * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT.
+ * IN NO EVENT SHALL TUNGSTEN GRAPHICS AND/OR ITS SUPPLIERS BE LIABLE FOR
+ * ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
+ * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+ * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ * 
+ **************************************************************************/
+
+#include <stdlib.h>
+#include "sl_pp_directive.h"
+#include "sl_pp_token.h"
+
+
+static int
+_tokenise_identifier(const char **pinput,
+                     struct sl_pp_token_info *info)
+{
+   const char *input = *pinput;
+   char identifier[256];   /* XXX: Remove this artifical limit. */
+   unsigned int i = 0;
+
+   info->token = SL_PP_IDENTIFIER;
+   info->data.identifier = NULL;
+
+   identifier[i++] = *input++;
+   while ((*input >= 'a' && *input <= 'z') ||
+          (*input >= 'A' && *input <= 'Z') ||
+          (*input >= '0' && *input <= '9') ||
+          (*input == '_')) {
+      if (i >= sizeof(identifier) - 1) {
+         return -1;
+      }
+      identifier[i++] = *input++;
+   }
+   identifier[i++] = '\0';
+
+   info->data.identifier = malloc(i);
+   if (!info->data.identifier) {
+      return -1;
+   }
+   memcpy(info->data.identifier, identifier, i);
+
+   *pinput = input;
+   return 0;
+}
+
+
+static int
+_tokenise_number(const char **pinput,
+                 struct sl_pp_token_info *info)
+{
+   const char *input = *pinput;
+   char number[256];   /* XXX: Remove this artifical limit. */
+   unsigned int i = 0;
+
+   info->token = SL_PP_NUMBER;
+   info->data.number = NULL;
+
+   number[i++] = *input++;
+   while ((*input >= '0' && *input <= '9') ||
+          (*input >= 'a' && *input <= 'f') ||
+          (*input >= 'A' && *input <= 'F') ||
+          (*input == 'x') ||
+          (*input == 'X') ||
+          (*input == '+') ||
+          (*input == '-') ||
+          (*input == '.')) {
+      if (i >= sizeof(number) - 1) {
+         return -1;
+      }
+      number[i++] = *input++;
+   }
+   number[i++] = '\0';
+
+   info->data.number = malloc(i);
+   if (!info->data.number) {
+      return -1;
+   }
+   memcpy(info->data.number, number, i);
+
+   *pinput = input;
+   return 0;
+}
+
+
+int
+sl_pp_tokenise(const char *input,
+               struct sl_pp_token_info **output)
+{
+   struct sl_pp_token_info *out = NULL;
+   unsigned int out_len = 0;
+   unsigned int out_max = 0;
+
+   for (;;) {
+      struct sl_pp_token_info info;
+
+      switch (*input) {
+      case ' ':
+      case '\t':
+         input++;
+         info.token = SL_PP_WHITESPACE;
+         break;
+
+      case '\n':
+         input++;
+         info.token = SL_PP_NEWLINE;
+         break;
+
+      case '#':
+         input++;
+         info.token = SL_PP_HASH;
+         break;
+
+      case ',':
+         input++;
+         info.token = SL_PP_COMMA;
+         break;
+
+      case ';':
+         input++;
+         info.token = SL_PP_SEMICOLON;
+         break;
+
+      case '{':
+         input++;
+         info.token = SL_PP_LBRACE;
+         break;
+
+      case '}':
+         input++;
+         info.token = SL_PP_RBRACE;
+         break;
+
+      case '(':
+         input++;
+         info.token = SL_PP_LPAREN;
+         break;
+
+      case ')':
+         input++;
+         info.token = SL_PP_RPAREN;
+         break;
+
+      case '[':
+         input++;
+         info.token = SL_PP_LBRACKET;
+         break;
+
+      case ']':
+         input++;
+         info.token = SL_PP_RBRACKET;
+         break;
+
+      case '.':
+         if (input[1] >= '0' && input[1] <= '9') {
+            if (_tokenise_number(&input, &info)) {
+               free(out);
+               return -1;
+            }
+         } else {
+            input++;
+            info.token = SL_PP_DOT;
+         }
+         break;
+
+      case '+':
+         input++;
+         if (*input == '+') {
+            input++;
+            info.token = SL_PP_INCREMENT;
+         } else if (*input == '=') {
+            input++;
+            info.token = SL_PP_ADDASSIGN;
+         } else {
+            info.token = SL_PP_PLUS;
+         }
+         break;
+
+      case '-':
+         input++;
+         if (*input == '-') {
+            input++;
+            info.token = SL_PP_DECREMENT;
+         } else if (*input == '=') {
+            input++;
+            info.token = SL_PP_SUBASSIGN;
+         } else {
+            info.token = SL_PP_MINUS;
+         }
+         break;
+
+      case '~':
+         input++;
+         info.token = SL_PP_BITNOT;
+         break;
+
+      case '!':
+         input++;
+         if (*input == '=') {
+            input++;
+            info.token = SL_PP_NOTEQUAL;
+         } else {
+            info.token = SL_PP_NOT;
+         }
+         break;
+
+      case '*':
+         input++;
+         if (*input == '=') {
+            input++;
+            info.token = SL_PP_MULASSIGN;
+         } else {
+            info.token = SL_PP_STAR;
+         }
+         break;
+
+      case '/':
+         input++;
+         if (*input == '=') {
+            input++;
+            info.token = SL_PP_DIVASSIGN;
+         } else {
+            info.token = SL_PP_SLASH;
+         }
+         break;
+
+      case '%':
+         input++;
+         if (*input == '=') {
+            input++;
+            info.token = SL_PP_MODASSIGN;
+         } else {
+            info.token = SL_PP_MODULO;
+         }
+         break;
+
+      case '<':
+         input++;
+         if (*input == '<') {
+            input++;
+            if (*input == '=') {
+               input++;
+               info.token = SL_PP_LSHIFTASSIGN;
+            } else {
+               info.token = SL_PP_LSHIFT;
+            }
+         } else if (*input == '=') {
+            input++;
+            info.token = SL_PP_LESSEQUAL;
+         } else {
+            info.token = SL_PP_LESS;
+         }
+         break;
+
+      case '>':
+         input++;
+         if (*input == '>') {
+            input++;
+            if (*input == '=') {
+               input++;
+               info.token = SL_PP_RSHIFTASSIGN;
+            } else {
+               info.token = SL_PP_RSHIFT;
+            }
+         } else if (*input == '=') {
+            input++;
+            info.token = SL_PP_GREATEREQUAL;
+         } else {
+            info.token = SL_PP_GREATER;
+         }
+         break;
+
+      case '=':
+         input++;
+         if (*input == '=') {
+            input++;
+            info.token = SL_PP_EQUAL;
+         } else {
+            info.token = SL_PP_ASSIGN;
+         }
+         break;
+
+      case '&':
+         input++;
+         if (*input == '&') {
+            input++;
+            info.token = SL_PP_AND;
+         } else if (*input == '=') {
+            input++;
+            info.token = SL_PP_BITANDASSIGN;
+         } else {
+            info.token = SL_PP_BITAND;
+         }
+         break;
+
+      case '^':
+         input++;
+         if (*input == '^') {
+            input++;
+            info.token = SL_PP_XOR;
+         } else if (*input == '=') {
+            input++;
+            info.token = SL_PP_BITXORASSIGN;
+         } else {
+            info.token = SL_PP_BITXOR;
+         }
+         break;
+
+      case '|':
+         input++;
+         if (*input == '|') {
+            input++;
+            info.token = SL_PP_OR;
+         } else if (*input == '=') {
+            input++;
+            info.token = SL_PP_BITORASSIGN;
+         } else {
+            info.token = SL_PP_BITOR;
+         }
+         break;
+
+      case '?':
+         input++;
+         info.token = SL_PP_QUESTION;
+         break;
+
+      case ':':
+         input++;
+         info.token = SL_PP_COLON;
+         break;
+
+      case '\0':
+         info.token = SL_PP_EOF;
+         break;
+
+      default:
+         if ((*input >= 'a' && *input <= 'z') ||
+             (*input >= 'A' && *input <= 'Z') ||
+             (*input == '_')) {
+            if (_tokenise_identifier(&input, &info)) {
+               free(out);
+               return -1;
+            }
+         } else if (*input >= '0' && *input <= '9') {
+            if (_tokenise_number(&input, &info)) {
+               free(out);
+               return -1;
+            }
+         } else {
+            info.data.other = *input++;
+            info.token = SL_PP_OTHER;
+         }
+      }
+
+      if (out_len >= out_max) {
+         unsigned int new_max = out_max;
+
+         if (new_max < 0x100) {
+            new_max = 0x100;
+         } else if (new_max < 0x10000) {
+            new_max *= 2;
+         } else {
+            new_max += 0x10000;
+         }
+
+         out = realloc(out, new_max * sizeof(struct sl_pp_token_info));
+         if (!out) {
+            return -1;
+         }
+         out_max = new_max;
+      }
+
+      out[out_len++] = info;
+
+      if (info.token == SL_PP_EOF) {
+         break;
+      }
+   }
+
+   *output = out;
+   return 0;
+}
diff --git a/src/glsl/pp/sl_pp_token.h b/src/glsl/pp/sl_pp_token.h
new file mode 100644
index 0000000000..5e7fae7d29
--- /dev/null
+++ b/src/glsl/pp/sl_pp_token.h
@@ -0,0 +1,107 @@
+/**************************************************************************
+ * 
+ * Copyright 2009 VMware, Inc.
+ * All Rights Reserved.
+ * 
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the
+ * "Software"), to deal in the Software without restriction, including
+ * without limitation the rights to use, copy, modify, merge, publish,
+ * distribute, sub license, and/or sell copies of the Software, and to
+ * permit persons to whom the Software is furnished to do so, subject to
+ * the following conditions:
+ * 
+ * The above copyright notice and this permission notice (including the
+ * next paragraph) shall be included in all copies or substantial portions
+ * of the Software.
+ * 
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
+ * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT.
+ * IN NO EVENT SHALL TUNGSTEN GRAPHICS AND/OR ITS SUPPLIERS BE LIABLE FOR
+ * ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
+ * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+ * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ * 
+ **************************************************************************/
+
+#ifndef SL_PP_TOKEN_H
+#define SL_PP_TOKEN_H
+
+
+enum sl_pp_token {
+   SL_PP_WHITESPACE,
+   SL_PP_NEWLINE,
+   SL_PP_HASH,             /* #   */
+
+   SL_PP_COMMA,            /* ,   */
+   SL_PP_SEMICOLON,        /* ;   */
+   SL_PP_LBRACE,           /* {   */
+   SL_PP_RBRACE,           /* }   */
+   SL_PP_LPAREN,           /* (   */
+   SL_PP_RPAREN,           /* )   */
+   SL_PP_LBRACKET,         /* [   */
+   SL_PP_RBRACKET,         /* ]   */
+   SL_PP_DOT,              /* .   */
+   SL_PP_INCREMENT,        /* ++  */
+   SL_PP_ADDASSIGN,        /* +=  */
+   SL_PP_PLUS,             /* +   */
+   SL_PP_DECREMENT,        /* --  */
+   SL_PP_SUBASSIGN,        /* -=  */
+   SL_PP_MINUS,            /* -   */
+   SL_PP_BITNOT,           /* ~   */
+   SL_PP_NOTEQUAL,         /* !=  */
+   SL_PP_NOT,              /* !   */
+   SL_PP_MULASSIGN,        /* *=  */
+   SL_PP_STAR,             /* *   */
+   SL_PP_DIVASSIGN,        /* /=  */
+   SL_PP_SLASH,            /* /   */
+   SL_PP_MODASSIGN,        /* %=  */
+   SL_PP_MODULO,           /* %   */
+   SL_PP_LSHIFTASSIGN,     /* <<= */
+   SL_PP_LSHIFT,           /* <<  */
+   SL_PP_LESSEQUAL,        /* <=  */
+   SL_PP_LESS,             /* <   */
+   SL_PP_RSHIFTASSIGN,     /* >>= */
+   SL_PP_RSHIFT,           /* >>  */
+   SL_PP_GREATEREQUAL,     /* >=  */
+   SL_PP_GREATER,          /* >   */
+   SL_PP_EQUAL,            /* ==  */
+   SL_PP_ASSIGN,           /* =   */
+   SL_PP_AND,              /* &&  */
+   SL_PP_BITANDASSIGN,     /* &=  */
+   SL_PP_BITAND,           /* &   */
+   SL_PP_XOR,              /* ^^  */
+   SL_PP_BITXORASSIGN,     /* ^=  */
+   SL_PP_BITXOR,           /* ^   */
+   SL_PP_OR,               /* ||  */
+   SL_PP_BITORASSIGN,      /* |=  */
+   SL_PP_BITOR,            /* |   */
+   SL_PP_QUESTION,         /* ?   */
+   SL_PP_COLON,            /* :   */
+
+   SL_PP_IDENTIFIER,
+
+   SL_PP_NUMBER,
+
+   SL_PP_OTHER,
+
+   SL_PP_EOF
+};
+
+union sl_pp_token_data {
+   char *identifier;
+   char *number;
+   char other;
+};
+
+struct sl_pp_token_info {
+   enum sl_pp_token token;
+   union sl_pp_token_data data;
+};
+
+int
+sl_pp_tokenise(const char *input,
+               struct sl_pp_token_info **output);
+
+#endif /* SL_PP_TOKEN_H */