working scanner

src/Makefile.am

-CFLAGS += -Wall 	\
+CFLAGS += -g -Wall 	\
 	-Wshadow             -Wpointer-arith  	\
 	-Wmissing-prototypes -Wmissing-declarations 
 
-bin_PROGRAMS = gnumeric test-format
+bin_PROGRAMS = gnumeric test-format test-token
 
 INCLUDES = 			\
 	-DGNOMELOCALEDIR=\""$(datadir)/locale"\" \
@@ -40,4 +40,14 @@ test_format_SOURCES = format.c
 
 test_format_LDADD = \
 	$(GNOME_LIBDIR) \
-	$(GNOMEUI_LIBS)
\ No newline at end of file
+	$(GNOMEUI_LIBS)
+
+test_token_SOURCES = 		\
+	test-token.c		\
+	token.c			\
+	token.h
+
+test_token_LDADD = \
+	$(GNOME_LIBDIR)	\
+	$(GNOMEUI_LIBS) -lgmp
+

src/token.c

+#include <gmp.h>
 #include <glib.h>
 #include <ctype.h>
+#include "token.h"
+#include "string.h"
 
 #define CANSKIP(x) ((x) == ' ' || (x) == '\t')
 
 int
-token_get_next (char **str, result_t *res)
+token_get_next (char **str, token_result_t *res)
 {
-	char *p, *q;
-	char *res;
+	char *p, *q, *start;
+	int mp_res, len;
 	
 	g_return_val_if_fail (str != NULL, TOKEN_ERROR);
 	g_return_val_if_fail (*str != NULL, TOKEN_ERROR);
@@ -16,65 +19,93 @@ token_get_next (char **str, result_t *res)
 	p = *str;
 	
 	while (*p && CANSKIP (*p))
-		*p++;
+		p++;
 
-	if (*p == '\''){
+	if (*p == '.' && *(p+1) == '.'){
+		*str = p+2;
+		return TOKEN_ELLIPSIS;
+	}
+	if (*p == '\''|| isalpha (*p)){
 		q = p+1;
-		while (*q && *q != '\'')
-			q++;
-		if (*q){
+		if (*p == '\''){
+			start = p + 1;
+			while (*q && *q != '\'')
+				q++;
 			q++;
-
-			/* Store result */
-			res->v.str_val = g_new (char, p - q + 1);
-			strncpy (res->v.str_val, p, p - q);
-			res->v.str_val [p-q] = 0;
-			res->type = R_STRING;
-			
-			*str = q;
-			return TOKEN_IDENTIFIER;
 		} else {
-			return TOKEN_ERROR;
+			start = p;
+			while (*q && isalnum (*q))
+				q++;
 		}
-	}
-
-	if (isalpha (*p)){
-		q = p+1;
+		
+		/* Store result */
+		len = q - p;
+		if (len < SMALL_STR_TOKEN)
+			res->v.str_value = res->str_inline;
+		else
+			res->v.str_value = g_new (char, len + 1);
 
-		while (*q && isalnum (*q))
-			q++;
-		res->v.str_val = g_new (char, p - q + 1);
-		strncpy (res->v.str_val, p, p - q);
-		res->v.str_val [p-q] = 0;
 		res->type = R_STRING;
+		
+		strncpy (res->v.str_value, p, len);
+		res->v.str_value [len] = 0;
 		*str = q;
 		return TOKEN_IDENTIFIER;
 	}
 
-	if (isnum (*p)){
+	if (isdigit (*p)){
 		int is_float = 0;
+		char *copy;
 
+		/* Scan for the whole number */
 		q = p;
-		while (isnum (*q) || *q == '.'){
-			if (q == '.')
+		while (isdigit (*q) || *q == '.'){
+			if (*q == '.')
 				is_float = 1;
+			q++;
 		}
+
+		copy = alloca (q - p + 1);
+		strncpy (copy, p, q - p);
+		copy [q-p] = 0;
+
+		/* Convert to MP number */
+		if (is_float){
+			res->type = R_FLOAT;
+			mp_res = mpf_init_set_str (res->v.float_value, copy, 10);
+		} else {
+			res->type = R_INTEGER;
+			mp_res = mpz_init_set_str (res->v.int_value, copy, 10);
+		}
+		*str = q;
 		
+		return TOKEN_NUMBER;
+	}
+	if (*p)
+		*str = p + 1;
+	else
+		*str = p;
+
+	res->type = R_CHAR;
+	return *p;
 }
 
 /*
  * Should be invoked when a token has been used
  */
 void
-token_done (result_t *r)
+token_done (token_result_t *r)
 {
 	g_return_if_fail (r != NULL);
-	g_return_if_fail (r->type != R_NONE;
 			  
-	if (r->type == R_STRING)
-		g_free (r->v.str_value);
+	if (r->type == R_STRING){
+		if (r->v.str_value != &r->str_inline)
+			g_free (r->v.str_value);
+	} else if (r->type == R_FLOAT)
+		mpf_clear (r->v.float_value);
+	else if (r->type == R_INTEGER)
+		mpz_clear (r->v.int_value);
+
 	r->type = R_NONE;
 }
 	    
-
-

src/token.h

 #ifndef TOKEN_H
 #define TOKEN_H
 
+typedef enum {
+	TOKEN_IDENTIFIER = -1,
+	TOKEN_NUMBER     = -2,
+	TOKEN_ELLIPSIS   = -4,
+	TOKEN_ERROR      = -5
+} TokenType;
+
+#define SMALL_STR_TOKEN 20
+
 typedef enum {
 	R_NONE,
+	R_CHAR,
 	R_INTEGER,
 	R_STRING,
-	R_STRING_STATIC,	/* Just an optimization */
 	R_FLOAT
 } token_tag_t;
 
 typedef struct {
 	token_tag_t  type;
 	union {
-		int  int_value;
-		char *str_value;
-		char str_value_small [20];
-		double float_value;
+		mpz_t  int_value;
+		char   *str_value;
+		mpf_t  float_value;
 	} v;
+	char str_inline [SMALL_STR_TOKEN+1];
 } token_result_t;
 
-int  token_get_next (char **str, result_t *res);
-void token_done (result_t *);
+int    token_get_next (char **str, token_result_t *res);
+void   token_done (token_result_t *res);
 
 #endif