Added comments, began work on ID

tests/test_constants_literals.txt

@@ -1,23 +1,50 @@
-This is a test
-9combined 7okens
-12345
-893247892
-combined'DueToUnknownChar _validtoken __validtoken1 _valid_token2 validToken3_
-true false
-null while !wrong if when
-else	type	function
-return  external        as
-string _NOte_that_was_not_reserved
-([)]{}:;,->"\
-+-*/%
-<=
-:=
-"This is not a valid
-String"
-"This is a valid String"
-!|
-..
-(* this is a comment *)
-(*Not a comment
-$^&
->
+
+'''
+'\'
+false
+'''
+nullfalse
+"nulltrue
+null
+'7'
+true
+'189
+'\t'
+'"'
+'/'
+'\n'
+'\''
+'\t'
+'\\'
+'n'
+'\'
+'fdsf'
+'
+'
+'	'
+'''
+"STRINGwithnotSPaces"
+' '
+'\ '
+
+"J"
+""
+" "
+\"\"
+"this is a string" 721398 'g' '/n' 
+12893 "this is not a string
+"{SCHAR}"
+"SCHAR"
+"[SCHAR]"
+"I'd think this is a legal \"string\" that contains \n \t several escaped characters, isn't it?"
+"I'd think this is a legal \"string\" that contains several \\n \\t escaped characters, isn't it?"
+
+(* hello *)
+(* hello *)
+(* I'd think this is a legal "string" that contains several \n \t 
+escaped characters, isn't it? *)
+(* \ *)
+(* *)
+(*{COMMENT}+ *)
+(* * *)
+(* (hello) *)

tests/test_variables.txt

@@ -0,0 +1,10 @@
+valid1
+Valid2
+_valid3
+_valid_name_4
+VALID
+0Invalid
+1invalid
+"invalid
+invalid=
+String