// joycub - an interpeter for the programming language Joy
// (c) 2004-2006 Sergey Zubkov 
// $Id: token_list.cc,v 1.8 2006/02/27 18:44:01 cubbi Exp $
// token_list.cc
//  Implementation of the Token_list methods
//
#include <sstream>
#include <iostream>
#include <vector>
#include <cctype>
#include <string>
#include <algorithm>
#include <functional>
#include "token_list.h"
#include "globals.h"
#include "exceptions.h"

using namespace std;

static char RCSID[] ="$Id: token_list.cc,v 1.8 2006/02/27 18:44:01 cubbi Exp $";

// Default constructor - creates empty list.
Token_list::Token_list(void) : state(STATE_SPACE) {};

// Constructor from an input stream, such as std::cin.
// reads one line, tokenizes it, and stores in the Token_list
Token_list::Token_list(istream& ss, Parse_state init_state) {
//cout << " creating token list with starting state " << init_state << endl;
	state=init_state;
//	string s;
	char s[2000];
	ss.getline(s, 2000);
	if(ss.eof()) throw "EOF";
	if(s[0]=='#') return; // skip comments
//	cout << " parser: string " << s << endl;
	LINENUM++;
	switch(ECHO_SETTING) {
		case ECHO_ECHO:
			cout << s << endl;
			break;
		case ECHO_TAB:
			cout << "\t" << s << endl;
			break;
		case ECHO_LINENUM:
			cout << "   " << LINENUM << "\t" << s << endl;
			break;
		case ECHO_NONE:
		default: break;
	}
	contents=tokenize(s);
//cout << " in the end of constructor, state=" << state << endl ;
};

// Constructor from string (calls tokenize)
Token_list::Token_list(string s) {
	state=STATE_SPACE;
	contents=tokenize(s);
}

// DEBUG print out the contents of the token list
void Token_list::debug_print(void) {
	cout << to_string() << endl;
}

// returns a single string representation of the token list
string Token_list::to_string(void) {
	ostringstream s;
	for(vector<pair<obj_type,string> >::iterator i = contents.begin();
			i<contents.end(); i++) {
		s << (*i).second << ":" << (*i).first << " " ;
	}
	return s.str();
}

void Token_list::concat(Token_list& t) {
	for(vector<pair<obj_type,string> >::iterator i = t.contents.begin();
			i<t.contents.end(); i++) {
				contents.push_back(*i);
	}
	state = t.state;
}
void Token_list::erase(void) {
	contents.erase(contents.begin(), contents.end());
}

bool Token_list::has_dot(void) const {
	for(vector<pair<obj_type,string> >::const_iterator i = contents.begin();
			i<contents.end(); i++) {
			if( (*i).second=="." ) return true;
	}
	return false;
}

struct Isspace {
	bool operator() (char c) const {
		return isspace(c);
	}
};

/*
// This function breaks a string into words separated by one
// or more isspace() characters. It returns a vector of strings
// where each string contains one word.
vector<string> Token_list::old_tokenize(const string s) {
	vector<string> v;

	string::const_iterator it = s.begin();
	string::const_iterator itTokenEnd = s.begin();
	while(it != s.end()) {
		while(isspace(*it)) it++; //Eat seperators
		//Find next token
		itTokenEnd = find_if(it, s.end(), Isspace());
		//Append token to result
		if(it < itTokenEnd)
			v.push_back(string(it, itTokenEnd));
		it = itTokenEnd;
	}
	return v;
}


// Read string and return  pair containing the first word and the
// rest of the input string.
pair<string,string> Token_list::old_tokenizeonce(const string s) {
	vector<string> v;

	string::const_iterator it = s.begin();
	string::const_iterator itTokenEnd = s.begin();
	while(isspace(*it)) it++; //Eat leading separators
	//Find next token
	itTokenEnd = find_if(it, s.end(), Isspace());
	//token is the result
	if(it < itTokenEnd)
		return pair<string,string>(string(it, itTokenEnd),
									 string(itTokenEnd,s.end()));
	else
		return pair<string,string>("",s);
}
*/


// NEW tokenizer
vector<pair<obj_type,string> > Token_list::tokenize(const string s) {
	string space("  ");
//	state = STATE_SPACE;			// DON'T reset state
	parse_step_buffer.erase();		// clean buffer
	parse_buffer.clear();			// clean output buffer
	list_counter=0;					// start outside lists

	string::const_iterator it = s.begin();
	while( it != s.end() )
		it = tokenize_step(it);			// this fills out parse_buffer
	tokenize_step(space.begin());	// parse a string  consisting of a space to reset the parser to whitespace mode in the end of the input
	return parse_buffer;
}

// NEW tokenizer: parse a single character
// This will produce a list of pairs <type, string>, where
// list open/close will be brackets of list type, etc
 // I need a table
 //   table[STATE][char] -> save_char? , new_state , new_object_type(if>0)
class Parse_Table {

 struct Parse_table_entry {
 	bool save_char;
 	Parse_state next_state;
	obj_type new_object; // If this is a valid type, token is complete
						// if this is NO_TYPE, token is not complete
 };

 static Parse_table_entry PARSE_TABLE[127][VALID_STATES];

 public:
  inline bool save_char(Parse_state state, char c) {
	  return PARSE_TABLE[c-' '][state].save_char;
  }
  inline Parse_state next_state(Parse_state state, char c) {
	  return PARSE_TABLE[c-' '][state].next_state;
  }
  inline obj_type new_token_type(Parse_state state, char c) {
	  return PARSE_TABLE[c-' '][state].new_object;
  }
};

static Parse_Table PT;

string::const_iterator Token_list::tokenize_step(string::const_iterator it) {
//cout << "parsing char " << (*it) << " in state " << state << endl;
//	if( *it < 32 ) { it++; return it; } // ignore codes below 32
	char c = *it;
	if( c < 32 ) c = 32 ; // ignore codes below 32
	obj_type new_obj_type = PT.new_token_type(state,c);

	if(new_obj_type != NO_TYPE ) {
		parse_buffer.push_back(
			pair<obj_type,string>(new_obj_type,parse_step_buffer) );
		parse_step_buffer.erase();
	}
	if(PT.save_char(state,c)) 
			parse_step_buffer.push_back(c);

	state = PT.next_state(state,c);
//cout << " new state = " << state << endl;
 	it++; return it;
}




Parse_Table::Parse_table_entry Parse_Table::PARSE_TABLE[][VALID_STATES] = {
// CHARS = 96
//    ! " # $ % & ' ( ) * + , - . /
//  0 1 2 3 4 5 6 7 8 9 : ; < = > ?
//  @ A B C D E F G H I J K L M N O
//  P Q R S T U V W X Y Z [ \ ] ^ _
//  ` a b c d e f g h i j k l m n o
//  p q r s t u v w x y z { | } ~

// VALID STATES = 15
//
// STATE_SPACE   empty space
// STATE_M_P_NUM - or + (start of number, float, or operator)
// STATE_D_FLOAT . (start of float or operator)
// STATE_NUM     digit (start of number, float, or operator)
// STATE_OP      letter (start of operator)
// STATE_SET     { or }
// STATE_LIST    [ or ]
// STATE_CHAR    character after '
// STATE_CHAR_E0  fist digit after 'backslash
// STATE_CHAR_E1  second digit after 'backslash
// STATE_CHAR_E2  third digit after 'backslash
// STATE_FLOAT   digits after . (float or operator)
// STATE_FLOAT_E0 digits or +/- after nn.nnnE (float or operator)
// STATE_FLOAT_E1 digits after nn.nnnE+/- (float or operator)
// STATE_STRING  data in "
// STATE_STRING_ESC  data in "
// STATE_COMMENT_B,    // after (, before *
// STATE_COMMENT,      // comments, delimited by (* *)
// STATE_COMMENT_E,    // after * in a comment
//

#define ALPHA \
		{{true,STATE_OP,NO_TYPE},	/* STATE_SPACE */ \
		 {true,STATE_OP,NO_TYPE},	/* STATE_M_P_NUM */ \
		 {true,STATE_OP,NO_TYPE},	/* STATE_D_FLOAT */ \
		 {true,STATE_OP,NO_TYPE},	/* STATE_NUM turns into an op such as 2dup*/ \
		 {true,STATE_OP,NO_TYPE},	/* STATE_OP */ \
		 {true,STATE_OP,SET},		/* STATE_SET */ \
		 {true,STATE_OP,LIST},		/* STATE_LIST */ \
		 {true,STATE_CHAR,NO_TYPE},	/* STATE_CHAR */ \
		 {true,STATE_CHAR,NO_TYPE},	/* STATE_CHAR_E0 */\
		 {true,STATE_OP,NO_TYPE},	/* STATE_CHAR_E1 */\
		 {true,STATE_OP,NO_TYPE},	/* STATE_CHAR_E2 */\
		 {true,STATE_OP,NO_TYPE},	/* STATE_FLOAT */ \
		 {true,STATE_OP,NO_TYPE},	/* STATE_FLOAT_E0 */\
		 {true,STATE_OP,NO_TYPE},	/* STATE_FLOAT_E1 */\
		 {true,STATE_STRING,NO_TYPE},	/* STATE_STRING */ \
		 {true,STATE_STRING,NO_TYPE},	/* STATE_STRING_ESC */ \
		 {false,STATE_ERROR,NO_TYPE},	/* STATE_COMMENT_B */ \
		 {false,STATE_COMMENT,NO_TYPE},	/* STATE_COMMENT */ \
		 {false,STATE_COMMENT,NO_TYPE}	/* STATE_COMMENT_E */ }

#define ALPHA_E \
		{{true,STATE_OP,NO_TYPE},	/* STATE_SPACE */ \
		 {true,STATE_OP,NO_TYPE},	/* STATE_M_P_NUM */ \
		 {true,STATE_FLOAT_E0,NO_TYPE},	/* STATE_D_FLOAT */ \
		 {true,STATE_FLOAT_E0,NO_TYPE},	/* STATE_NUM */ \
		 {true,STATE_OP,NO_TYPE},	/* STATE_OP */ \
		 {true,STATE_OP,SET},		/* STATE_SET */ \
		 {true,STATE_OP,LIST},		/* STATE_LIST */ \
		 {true,STATE_CHAR,NO_TYPE},	/* STATE_CHAR */ \
		 {true,STATE_CHAR,NO_TYPE},	/* STATE_CHAR_E0 */\
		 {true,STATE_OP,NO_TYPE},	/* STATE_CHAR_E1 */\
		 {true,STATE_OP,NO_TYPE},	/* STATE_CHAR_E2 */\
		 {true,STATE_FLOAT_E0,NO_TYPE},	/* STATE_FLOAT */ \
		 {true,STATE_OP,NO_TYPE},	/* STATE_FLOAT_E0 */\
		 {true,STATE_OP,NO_TYPE},	/* STATE_FLOAT_E1 */\
		 {true,STATE_STRING,NO_TYPE},	/* STATE_STRING */ \
		 {true,STATE_STRING,NO_TYPE},	/* STATE_STRING_ESC */ \
		 {false,STATE_ERROR,NO_TYPE},	/* STATE_COMMENT_B */ \
		 {false,STATE_COMMENT,NO_TYPE},	/* STATE_COMMENT */ \
		 {false,STATE_COMMENT,NO_TYPE}	/* STATE_COMMENT_E */ }

#define DIGIT \
		{{true,STATE_NUM,NO_TYPE}, /* STATE_SPACE */ \
		 {true,STATE_NUM,NO_TYPE}, /* STATE_M_P_NUM */ \
		 {true,STATE_FLOAT,NO_TYPE}, /* STATE_D_FLOAT */ \
		 {true,STATE_NUM,NO_TYPE}, /* STATE_NUM */ \
		 {true,STATE_OP,NO_TYPE}, /* STATE_OP */ \
		 {true,STATE_NUM,SET}, /* STATE_SET */ \
		 {true,STATE_NUM,LIST}, /* STATE_LIST */ \
		 {true,STATE_CHAR,NO_TYPE},	/* STATE_CHAR */ \
		 {true,STATE_CHAR_E1,NO_TYPE}, /* STATE_CHAR_E0 */\
		 {true,STATE_CHAR_E2,NO_TYPE}, /* STATE_CHAR_E1 */\
		 {true,STATE_CHAR,NO_TYPE},      /* STATE_CHAR_E2 */\
		 {true,STATE_FLOAT,NO_TYPE}, /* STATE_FLOAT */ \
		 {true,STATE_FLOAT_E1,NO_TYPE}, /* STATE_FLOAT_E0 */\
		 {true,STATE_FLOAT_E1,NO_TYPE}, /* STATE_FLOAT_E1 */\
		 {true,STATE_STRING,NO_TYPE}, /* STATE_STRING */ \
		 {true,STATE_STRING,NO_TYPE}, /* STATE_STRING_ESC */ \
		 {false,STATE_ERROR,NO_TYPE},	/* STATE_COMMENT_B */ \
		 {false,STATE_COMMENT,NO_TYPE},	/* STATE_COMMENT */ \
		 {false,STATE_COMMENT,NO_TYPE}	/* STATE_COMMENT_E */ }

#define M_OR_P \
		{{true,STATE_M_P_NUM,NO_TYPE}, /* STATE_SPACE */ \
		 {true,STATE_OP,NO_TYPE}, /* STATE_M_P_NUM */ \
		 {true,STATE_OP,NO_TYPE}, /* STATE_D_FLOAT */ \
		 {true,STATE_OP,NO_TYPE}, /* STATE_NUM */ \
		 {true,STATE_OP,NO_TYPE}, /* STATE_OP */ \
		 {true,STATE_M_P_NUM,SET}, /* STATE_SET */ \
		 {true,STATE_M_P_NUM,LIST}, /* STATE_LIST */ \
		 {true,STATE_SPACE,CHAR}, /* STATE_CHAR */ \
		 {true,STATE_OP,NO_TYPE}, /* STATE_CHAR_E0 */\
		 {true,STATE_OP,NO_TYPE}, /* STATE_CHAR_E1 */\
		 {true,STATE_OP,NO_TYPE}, /* STATE_CHAR_E2 */\
		 {true,STATE_OP,NO_TYPE}, /* STATE_FLOAT */ \
		 {true,STATE_FLOAT_E1,NO_TYPE}, /* STATE_FLOAT_E0 */\
		 {true,STATE_OP,NO_TYPE}, /* STATE_FLOAT_E1 */\
		 {true,STATE_STRING,NO_TYPE}, /* STATE_STRING */ \
		 {true,STATE_STRING,NO_TYPE}, /* STATE_STRING_ESC */ \
		 {false,STATE_ERROR,NO_TYPE},	/* STATE_COMMENT_B */ \
		 {false,STATE_COMMENT,NO_TYPE},	/* STATE_COMMENT */ \
		 {false,STATE_COMMENT,NO_TYPE}	/* STATE_COMMENT_E */ }

#define BACKSLASH \
		{{true,STATE_OP,NO_TYPE}, /* STATE_SPACE */ \
		 {true,STATE_OP,NO_TYPE}, /* STATE_M_P_NUM */ \
		 {true,STATE_OP,NO_TYPE}, /* STATE_D_FLOAT */ \
		 {true,STATE_OP,NO_TYPE}, /* STATE_NUM */ \
		 {true,STATE_OP,NO_TYPE}, /* STATE_OP */ \
		 {true,STATE_OP,SET}, /* STATE_SET */ \
		 {true,STATE_OP,LIST}, /* STATE_LIST */ \
		 {true,STATE_CHAR_E0,NO_TYPE}, /* STATE_CHAR */ \
		 {true,STATE_CHAR,NO_TYPE}, /* STATE_CHAR_E0 */\
		 {true,STATE_OP,NO_TYPE}, /* STATE_CHAR_E1 */\
		 {true,STATE_OP,NO_TYPE}, /* STATE_CHAR_E2 */\
		 {true,STATE_OP,NO_TYPE}, /* STATE_FLOAT */ \
		 {true,STATE_OP,NO_TYPE}, /* STATE_FLOAT_E0 */\
		 {true,STATE_OP,NO_TYPE}, /* STATE_FLOAT_E1 */\
		 {true,STATE_STRING_ESC,NO_TYPE}, /* STATE_STRING */ \
		 {true,STATE_STRING,NO_TYPE}, /* STATE_STRING_ESC */ \
		 {false,STATE_ERROR,NO_TYPE},	/* STATE_COMMENT_B */ \
		 {false,STATE_COMMENT,NO_TYPE},	/* STATE_COMMENT */ \
		 {false,STATE_COMMENT,NO_TYPE}	/* STATE_COMMENT_E */ }

#define TICK \
		{{false,STATE_CHAR,NO_TYPE}, /* STATE_SPACE */ \
		 {false,STATE_CHAR,OPERATOR}, /* STATE_M_P_NUM */ \
		 {false,STATE_CHAR,OPERATOR}, /* STATE_D_FLOAT */ \
		 {false,STATE_CHAR,INT}, /* STATE_NUM */ \
		 {false,STATE_CHAR,OPERATOR}, /* STATE_OP */ \
		 {false,STATE_CHAR,SET}, /* STATE_SET */ \
		 {false,STATE_CHAR,LIST}, /* STATE_LIST */ \
		 {true,STATE_SPACE,CHAR}, /* STATE_CHAR */ \
		 {false,STATE_ERROR,NO_TYPE}, /* STATE_CHAR_E0 */\
		 {false,STATE_ERROR,NO_TYPE}, /* STATE_CHAR_E1 */\
		 {false,STATE_CHAR,CHAR}, /* STATE_CHAR_E2 */\
		 {false,STATE_CHAR,FLOAT}, /* STATE_FLOAT */ \
		 {true,STATE_ERROR,NO_TYPE}, /* STATE_FLOAT_E0 */\
		 {false,STATE_CHAR,FLOAT}, /* STATE_FLOAT_E1 */\
		 {true,STATE_STRING,NO_TYPE}, /* STATE_STRING */ \
		 {true,STATE_STRING,NO_TYPE}, /* STATE_STRING_ESC */ \
		 {false,STATE_ERROR,NO_TYPE},	/* STATE_COMMENT_B */ \
		 {false,STATE_COMMENT,NO_TYPE},	/* STATE_COMMENT */ \
		 {false,STATE_COMMENT,NO_TYPE}	/* STATE_COMMENT_E */ }

#define DOT \
		{{true,STATE_D_FLOAT,NO_TYPE}, /* STATE_SPACE */ \
		 {true,STATE_FLOAT,NO_TYPE}, /* STATE_M_P_NUM */ \
		 {true,STATE_OP,NO_TYPE}, /* STATE_D_FLOAT */ \
		 {true,STATE_FLOAT,NO_TYPE}, /* STATE_NUM */ \
		 {true,STATE_OP,OPERATOR}, /* STATE_OP */ \
		 {true,STATE_OP,SET}, /* STATE_SET */ \
		 {true,STATE_OP,LIST}, /* STATE_LIST */ \
		 {true,STATE_SPACE,CHAR}, /* STATE_CHAR */ \
		 {true,STATE_OP,NO_TYPE}, /* STATE_CHAR_E0 */\
		 {true,STATE_OP,NO_TYPE}, /* STATE_CHAR_E1 */\
		 {true,STATE_OP,NO_TYPE}, /* STATE_CHAR_E2 */\
		 {true,STATE_OP,NO_TYPE}, /* STATE_FLOAT */ \
		 {true,STATE_OP,NO_TYPE}, /* STATE_FLOAT_E0 */\
		 {true,STATE_OP,NO_TYPE}, /* STATE_FLOAT_E1 */\
		 {true,STATE_STRING,NO_TYPE}, /* STATE_STRING */ \
		 {true,STATE_STRING,NO_TYPE}, /* STATE_STRING_ESC */ \
		 {false,STATE_ERROR,NO_TYPE},	/* STATE_COMMENT_B */ \
		 {false,STATE_COMMENT,NO_TYPE},	/* STATE_COMMENT */ \
		 {false,STATE_COMMENT,NO_TYPE}	/* STATE_COMMENT_E */ }

#define SEMICOLON \
		{{true,STATE_OP,NO_TYPE},	/* STATE_SPACE */ \
		 {true,STATE_OP,OPERATOR},	/* STATE_M_P_NUM */ \
		 {true,STATE_OP,OPERATOR},	/* STATE_D_FLOAT */ \
		 {true,STATE_OP,INT},	/* STATE_NUM turns into an op such as 2dup*/ \
		 {true,STATE_OP,OPERATOR},	/* STATE_OP */ \
		 {true,STATE_OP,SET},		/* STATE_SET */ \
		 {true,STATE_OP,LIST},		/* STATE_LIST */ \
		 {true,STATE_CHAR,NO_TYPE},	/* STATE_CHAR */ \
		 {true,STATE_OP,NO_TYPE},	/* STATE_CHAR_E0 */\
		 {true,STATE_OP,NO_TYPE},	/* STATE_CHAR_E1 */\
		 {true,STATE_OP,CHAR},	/* STATE_CHAR_E2 */\
		 {true,STATE_OP,FLOAT},	/* STATE_FLOAT */ \
		 {true,STATE_OP,NO_TYPE},	/* STATE_FLOAT_E0 */\
		 {true,STATE_OP,FLOAT},	/* STATE_FLOAT_E1 */\
		 {true,STATE_STRING,NO_TYPE},	/* STATE_STRING */ \
		 {true,STATE_STRING,NO_TYPE},	/* STATE_STRING_ESC */ \
		 {false,STATE_ERROR,NO_TYPE},	/* STATE_COMMENT_B */ \
		 {false,STATE_COMMENT,NO_TYPE},	/* STATE_COMMENT */ \
		 {false,STATE_COMMENT,NO_TYPE}	/* STATE_COMMENT_E */ }

#define LISTCHAR \
		{{true,STATE_LIST,NO_TYPE}, /* STATE_SPACE */ \
		 {true,STATE_LIST,OPERATOR}, /* STATE_M_P_NUM */ \
		 {true,STATE_LIST,OPERATOR}, /* STATE_D_FLOAT */ \
		 {true,STATE_LIST,INT}, /* STATE_NUM */ \
		 {true,STATE_LIST,OPERATOR}, /* STATE_OP */ \
		 {true,STATE_LIST,SET}, /* STATE_SET */ \
		 {true,STATE_LIST,LIST}, /* STATE_LIST */ \
		 {true,STATE_SPACE,CHAR}, /* STATE_CHAR */ \
		 {true,STATE_ERROR,NO_TYPE}, /* STATE_CHAR_E0 */\
		 {true,STATE_ERROR,NO_TYPE}, /* STATE_CHAR_E1 */\
		 {true,STATE_LIST,CHAR}, /* STATE_CHAR_E2 */\
		 {true,STATE_LIST,FLOAT}, /* STATE_FLOAT */ \
		 {true,STATE_ERROR,NO_TYPE}, /* STATE_FLOAT_E0 */\
		 {true,STATE_LIST,FLOAT}, /* STATE_FLOAT_E1 */\
		 {true,STATE_STRING,NO_TYPE}, /* STATE_STRING */ \
		 {true,STATE_STRING,NO_TYPE}, /* STATE_STRING_ESC */ \
		 {false,STATE_ERROR,NO_TYPE},	/* STATE_COMMENT_B */ \
		 {false,STATE_COMMENT,NO_TYPE},	/* STATE_COMMENT */ \
		 {false,STATE_COMMENT,NO_TYPE}	/* STATE_COMMENT_E */ }

#define SETCHAR \
		{{true,STATE_SET,NO_TYPE}, /* STATE_SPACE */ \
		 {true,STATE_SET,OPERATOR}, /* STATE_M_P_NUM */ \
		 {true,STATE_SET,OPERATOR}, /* STATE_D_FLOAT */ \
		 {true,STATE_SET,INT}, /* STATE_NUM */ \
		 {true,STATE_SET,OPERATOR}, /* STATE_OP */ \
		 {true,STATE_SET,SET}, /* STATE_SET */ \
		 {true,STATE_SET,LIST}, /* STATE_LIST */ \
		 {true,STATE_SPACE,CHAR}, /* STATE_CHAR */ \
		 {true,STATE_ERROR,NO_TYPE}, /* STATE_CHAR_E0 */\
		 {true,STATE_ERROR,NO_TYPE}, /* STATE_CHAR_E1 */\
		 {true,STATE_SET,CHAR}, /* STATE_CHAR_E2 */\
		 {true,STATE_SET,FLOAT}, /* STATE_FLOAT */ \
		 {true,STATE_ERROR,NO_TYPE}, /* STATE_FLOAT_E0 */\
		 {true,STATE_SET,FLOAT}, /* STATE_FLOAT_E1 */\
		 {true,STATE_STRING,NO_TYPE}, /* STATE_STRING */ \
		 {true,STATE_STRING,NO_TYPE}, /* STATE_STRING_ESC */ \
		 {false,STATE_ERROR,NO_TYPE},	/* STATE_COMMENT_B */ \
		 {false,STATE_COMMENT,NO_TYPE},	/* STATE_COMMENT */ \
		 {false,STATE_COMMENT,NO_TYPE}	/* STATE_COMMENT_E */ }

#define UNSUPPORTED {{false,STATE_ERROR,NO_TYPE}, /* STATE_SPACE */ \
		 {false,STATE_ERROR,NO_TYPE}, /* STATE_M_P_NUM */ \
		 {false,STATE_ERROR,NO_TYPE}, /* STATE_D_FLOAT */ \
		 {false,STATE_ERROR,NO_TYPE}, /* STATE_NUM */ \
		 {false,STATE_ERROR,NO_TYPE}, /* STATE_OP */ \
		 {false,STATE_ERROR,NO_TYPE}, /* STATE_SET */ \
		 {false,STATE_ERROR,NO_TYPE}, /* STATE_LIST */ \
		 {false,STATE_ERROR,NO_TYPE}, /* STATE_CHAR */ \
		 {false,STATE_ERROR,NO_TYPE}, /* STATE_CHAR_E0 */ \
		 {false,STATE_ERROR,NO_TYPE}, /* STATE_CHAR_E1 */ \
		 {false,STATE_ERROR,NO_TYPE}, /* STATE_CHAR_E2 */ \
		 {false,STATE_ERROR,NO_TYPE}, /* STATE_FLOAT */ \
		 {false,STATE_ERROR,NO_TYPE}, /* STATE_FLOAT_E0 */ \
		 {false,STATE_ERROR,NO_TYPE}, /* STATE_FLOAT_E1 */ \
		 {false,STATE_STRING,NO_TYPE}, /* STATE_STRING */ \
		 {false,STATE_STRING,NO_TYPE}, /* STATE_STRING_ESC */ \
		 {false,STATE_ERROR,NO_TYPE},	/* STATE_COMMENT_B */ \
		 {false,STATE_COMMENT,NO_TYPE},	/* STATE_COMMENT */ \
		 {false,STATE_COMMENT,NO_TYPE}	/* STATE_COMMENT_E */ }

	 { // CHAR=" "
		 {false,STATE_SPACE,NO_TYPE}, // STATE_SPACE
		 {false,STATE_SPACE,OPERATOR}, // STATE_M_P_NUM
		 {false,STATE_SPACE,OPERATOR}, // STATE_D_FLOAT
		 {false,STATE_SPACE,INT}, // STATE_NUM
		 {false,STATE_SPACE,OPERATOR}, // STATE_OP
		 {false,STATE_SPACE,SET}, // STATE_SET
		 {false,STATE_SPACE,LIST}, // STATE_LIST
		 {false,STATE_SPACE,CHAR}, // STATE_CHAR
		 {false,STATE_ERROR,NO_TYPE}, // STATE_CHAR_E0
		 {false,STATE_ERROR,NO_TYPE}, // STATE_CHAR_E1
		 {false,STATE_SPACE,CHAR}, // STATE_CHAR_E2
		 {false,STATE_SPACE,FLOAT}, // STATE_FLOAT
		 {false,STATE_ERROR,NO_TYPE}, // STATE_FLOAT_E0
		 {false,STATE_SPACE,FLOAT}, // STATE_FLOAT_E1
		 {true,STATE_STRING,NO_TYPE}, // STATE_STRING
		 {true,STATE_STRING,NO_TYPE}, // STATE_STRING_ESC
		 {false,STATE_ERROR,NO_TYPE},	/* STATE_COMMENT_B */ 
		 {false,STATE_COMMENT,NO_TYPE},	/* STATE_COMMENT */ 
		 {false,STATE_COMMENT,NO_TYPE}	/* STATE_COMMENT_E */
	 },
	   /*"!"*/ ALPHA,
	 { // CHAR="""
		 {false,STATE_STRING,NO_TYPE}, // STATE_SPACE
		 {false,STATE_STRING,OPERATOR}, // STATE_M_P_NUM
		 {false,STATE_STRING,OPERATOR}, // STATE_D_FLOAT
		 {false,STATE_STRING,INT}, // STATE_NUM
		 {false,STATE_STRING,OPERATOR}, // STATE_OP
		 {false,STATE_STRING,SET}, // STATE_SET
		 {false,STATE_STRING,LIST}, // STATE_LIST
		 {false,STATE_SPACE,CHAR}, // STATE_CHAR
		 {false,STATE_ERROR,NO_TYPE}, // STATE_CHAR_E0
		 {false,STATE_ERROR,NO_TYPE}, // STATE_CHAR_E1
		 {false,STATE_STRING,CHAR}, // STATE_CHAR_E2
		 {false,STATE_STRING,FLOAT}, // STATE_FLOAT
		 {false,STATE_ERROR,NO_TYPE}, // STATE_FLOAT_E0
		 {false,STATE_STRING,FLOAT}, // STATE_FLOAT_E1
		 {false,STATE_SPACE,STRING}, // STATE_STRING
		 {true,STATE_STRING,NO_TYPE}, // STATE_STRING_ESC
		 {false,STATE_ERROR,NO_TYPE},	/* STATE_COMMENT_B */ 
		 {false,STATE_COMMENT,NO_TYPE},	/* STATE_COMMENT */ 
		 {false,STATE_COMMENT,NO_TYPE}	/* STATE_COMMENT_E */
	 },
	 /* CHAR = " # " */ ALPHA,
	 /* CHAR = " $ " */ ALPHA,
	 /* CHAR = " % " */ ALPHA,
	 /* CHAR = " & " */ ALPHA,
	 /* CHAR = " ' " */ TICK,
     /* CHAR = " ( " */ 
	 { // CHAR="("
         {false,STATE_COMMENT_B,NO_TYPE}, /* STATE_SPACE */ 
		 {false,STATE_COMMENT_B,NO_TYPE}, /* STATE_M_P_NUM */ 
		 {false,STATE_COMMENT_B,NO_TYPE}, /* STATE_D_FLOAT */ 
		 {false,STATE_COMMENT_B,NO_TYPE}, /* STATE_NUM */ 
		 {false,STATE_COMMENT_B,OPERATOR}, /* STATE_OP */ 
		 {false,STATE_COMMENT_B,NO_TYPE}, /* STATE_SET */ 
		 {false,STATE_COMMENT_B,NO_TYPE}, /* STATE_LIST */ 
		 {true,STATE_CHAR,NO_TYPE},	/* STATE_CHAR */
		 {false,STATE_COMMENT_B,NO_TYPE}, /* STATE_CHAR_E0 */ 
		 {false,STATE_COMMENT_B,NO_TYPE}, /* STATE_CHAR_E1 */ 
		 {false,STATE_COMMENT_B,NO_TYPE}, /* STATE_CHAR_E2 */ 
		 {false,STATE_COMMENT_B,NO_TYPE}, /* STATE_FLOAT */ 
		 {false,STATE_COMMENT_B,NO_TYPE}, /* STATE_FLOAT_E0 */
		 {false,STATE_COMMENT_B,NO_TYPE}, /* STATE_FLOAT_E1 */
		 {true,STATE_STRING,NO_TYPE}, /* STATE_STRING */ 
		 {true,STATE_STRING,NO_TYPE}, /* STATE_STRING */ 
		 {false,STATE_ERROR,NO_TYPE},	/* STATE_COMMENT_B */ 
		 {false,STATE_COMMENT,NO_TYPE},	/* STATE_COMMENT */ 
		 {false,STATE_COMMENT,NO_TYPE}	/* STATE_COMMENT_E */
	 },
     /* CHAR = " ) " */
	 { // CHAR=")"
         {false,STATE_ERROR,NO_TYPE}, /* STATE_SPACE */ 
		 {false,STATE_ERROR,NO_TYPE}, /* STATE_M_P_NUM */ 
		 {false,STATE_ERROR,NO_TYPE}, /* STATE_D_FLOAT */ 
		 {false,STATE_ERROR,NO_TYPE}, /* STATE_NUM */ 
		 {false,STATE_COMMENT_B,OPERATOR}, /* STATE_OP */ 
		 {false,STATE_ERROR,NO_TYPE}, /* STATE_SET */ 
		 {false,STATE_ERROR,NO_TYPE}, /* STATE_LIST */ 
		 {true,STATE_CHAR,NO_TYPE},	/* STATE_CHAR */ 
		 {false,STATE_ERROR,NO_TYPE}, /* STATE_CHAR_E0 */ 
		 {false,STATE_ERROR,NO_TYPE}, /* STATE_CHAR_E1 */ 
		 {false,STATE_ERROR,NO_TYPE}, /* STATE_CHAR_E2 */ 
		 {false,STATE_ERROR,NO_TYPE}, /* STATE_FLOAT */ 
		 {false,STATE_ERROR,NO_TYPE}, /* STATE_FLOAT_E0 */
		 {false,STATE_ERROR,NO_TYPE}, /* STATE_FLOAT_E1 */
		 {true,STATE_STRING,NO_TYPE}, /* STATE_STRING */ 
		 {true,STATE_STRING,NO_TYPE}, /* STATE_STRING_ESC */ 
		 {false,STATE_ERROR,NO_TYPE},	/* STATE_COMMENT_B */ 
		 {false,STATE_COMMENT,NO_TYPE},	/* STATE_COMMENT */ 
		 {false,STATE_SPACE,NO_TYPE}	/* STATE_COMMENT_E */
	 },
	 { /* CHAR = " * " */
		 {true,STATE_OP,NO_TYPE},	/* STATE_SPACE */ 
		 {true,STATE_OP,NO_TYPE},	/* STATE_M_P_NUM */ 
		 {true,STATE_OP,NO_TYPE},	/* STATE_D_FLOAT */ 
		 {true,STATE_OP,NO_TYPE},	/* STATE_NUM turns into an op such as 2dup*/ 
		 {true,STATE_OP,NO_TYPE},	/* STATE_OP */ 
		 {true,STATE_OP,SET},		/* STATE_SET */ 
		 {true,STATE_OP,LIST},		/* STATE_LIST */ 
		 {true,STATE_CHAR,NO_TYPE},	/* STATE_CHAR */ 
		 {true,STATE_OP,NO_TYPE},	/* STATE_CHAR_E0 */
		 {true,STATE_OP,NO_TYPE},	/* STATE_CHAR_E1 */
		 {true,STATE_OP,NO_TYPE},	/* STATE_CHAR_E2 */
		 {true,STATE_OP,NO_TYPE},	/* STATE_FLOAT */ 
		 {true,STATE_OP,NO_TYPE},	/* STATE_FLOAT_E0 */
		 {true,STATE_OP,NO_TYPE},	/* STATE_FLOAT_E1 */
		 {true,STATE_STRING,NO_TYPE},	/* STATE_STRING */ 
		 {true,STATE_STRING,NO_TYPE},	/* STATE_STRING_ESC */ 
		 {false,STATE_COMMENT,NO_TYPE},	/* STATE_COMMENT_B */ 
		 {false,STATE_COMMENT_E,NO_TYPE},/* STATE_COMMENT */ 
		 {false,STATE_COMMENT_E,NO_TYPE}	/* STATE_COMMENT_E */
	 },
	 /* CHAR = " + " */ M_OR_P,
	 /* CHAR = " , " */ ALPHA,
	 /* CHAR = " - " */ M_OR_P,
	 /* CHAR = " . " */ DOT,
	 /* CHAR = " / " */ ALPHA,
///////////////////////////////////
//  0 1 2 3 4 5 6 7 8 9 : ; < = > ?
	 /* CHAR = " 0 " */ DIGIT,
	 /* CHAR = " 1 " */ DIGIT,
	 /* CHAR = " 2 " */ DIGIT,
	 /* CHAR = " 3 " */ DIGIT,
	 /* CHAR = " 4 " */ DIGIT,
	 /* CHAR = " 5 " */ DIGIT,
	 /* CHAR = " 6 " */ DIGIT,
	 /* CHAR = " 7 " */ DIGIT,
	 /* CHAR = " 8 " */ DIGIT,
	 /* CHAR = " 9 " */ DIGIT,
	 /* CHAR = " : " */ ALPHA,
	 /* CHAR = " ; " */ SEMICOLON,
	 /* CHAR = " < " */ ALPHA,
	 /* CHAR = " = " */ ALPHA,
	 /* CHAR = " > " */ ALPHA,
	 /* CHAR = " ? " */ ALPHA,
///////////////////////////////////
//  @ A B C D E F G H I J K L M N O
	 /* CHAR = " @ " */ ALPHA,
	 /* CHAR = " A " */ ALPHA,
	 /* CHAR = " B " */ ALPHA,
	 /* CHAR = " C " */ ALPHA,
	 /* CHAR = " D " */ ALPHA,
	 /* CHAR = " E " */ ALPHA_E,
	 /* CHAR = " F " */ ALPHA,
	 /* CHAR = " G " */ ALPHA,
	 /* CHAR = " H " */ ALPHA,
	 /* CHAR = " I " */ ALPHA,
	 /* CHAR = " J " */ ALPHA,
	 /* CHAR = " K " */ ALPHA,
	 /* CHAR = " L " */ ALPHA,
	 /* CHAR = " M " */ ALPHA,
	 /* CHAR = " N " */ ALPHA,
	 /* CHAR = " O " */ ALPHA,
///////////////////////////////////
//  P Q R S T U V W X Y Z [ \ ] ^ _
	 /* CHAR = " P " */ ALPHA,
	 /* CHAR = " Q " */ ALPHA,
	 /* CHAR = " R " */ ALPHA,
	 /* CHAR = " S " */ ALPHA,
	 /* CHAR = " T " */ ALPHA,
	 /* CHAR = " U " */ ALPHA,
	 /* CHAR = " V " */ ALPHA,
	 /* CHAR = " W " */ ALPHA,
	 /* CHAR = " X " */ ALPHA,
	 /* CHAR = " Y " */ ALPHA,
	 /* CHAR = " Z " */ ALPHA,
	 /* CHAR = " [ " */ LISTCHAR,
	 /* CHAR = " \ " */ BACKSLASH, // special meaning in CHAR0
	 /* CHAR = " ] " */ LISTCHAR,
	 /* CHAR = " ^ " */ ALPHA,
	 /* CHAR = " _ " */ ALPHA,
///////////////////////////////////
//  ` a b c d e f g h i j k l m n o
	 /* CHAR = " ` " */ ALPHA,
	 /* CHAR = " a " */ ALPHA,
	 /* CHAR = " b " */ ALPHA,
	 /* CHAR = " c " */ ALPHA,
	 /* CHAR = " d " */ ALPHA,
	 /* CHAR = " e " */ ALPHA_E,
	 /* CHAR = " f " */ ALPHA,
	 /* CHAR = " g " */ ALPHA,
	 /* CHAR = " h " */ ALPHA,
	 /* CHAR = " i " */ ALPHA,
	 /* CHAR = " j " */ ALPHA,
	 /* CHAR = " k " */ ALPHA,
	 /* CHAR = " l " */ ALPHA,
	 /* CHAR = " m " */ ALPHA,
	 /* CHAR = " n " */ ALPHA,
	 /* CHAR = " o " */ ALPHA,
///////////////////////////////////
//  p q r s t u v w x y z { | } ~
	 /* CHAR = " p " */ ALPHA,
	 /* CHAR = " 1 " */ ALPHA,
	 /* CHAR = " r " */ ALPHA,
	 /* CHAR = " s " */ ALPHA,
	 /* CHAR = " t " */ ALPHA,
	 /* CHAR = " u " */ ALPHA,
	 /* CHAR = " v " */ ALPHA,
	 /* CHAR = " w " */ ALPHA,
	 /* CHAR = " x " */ ALPHA,
	 /* CHAR = " y " */ ALPHA,
	 /* CHAR = " z " */ ALPHA,
	 /* CHAR = " { " */ SETCHAR,
	 /* CHAR = " | " */ ALPHA,
	 /* CHAR = " } " */ SETCHAR,
	 /* CHAR = " ~ " */ ALPHA,
	 /* CHAR=del */ UNSUPPORTED
 };
