6.9.6.1. Test Tokeniser

6.9.6.1. Test Tokeniser

Start python section to interscript/parsers/lalr1.py[23]
   776: #line 1032 "lalr1_parser.ipk"
   777: 
   778: import string
   779: def tokenise(s):
   780:   i = 0
   781:   s = s + '\0'
   782:   toks = []
   783:   while 1:
   784:     while s[i] == ' ': i = i + 1
   785:     if s[i] in '#\0': return toks
   786:     lexeme = ''
   787:     if s[i] in string.letters+'_':
   788:       while s[i] in string.letters+'_'+string.digits:
   789:         lexeme = lexeme+s[i]
   790:         i = i + 1
   791:       toks.append(('NAME',lexeme))
   792:       continue
   793:     if s[i] == '"':
   794:       i = i + 1
   795:       while s[i] not in '"\0':
   796:         lexeme = lexeme + s[i]
   797:         i = i + 1
   798:       toks.append(('STRING',lexeme))
   799:       if s[i]=='\0': return toks
   800:       i = i + 1
   801:       continue
   802:     if s[i] == "'":
   803:       i = i + 1
   804:       while s[i] not in "'\0":
   805:         lexeme = lexeme + s[i]
   806:         i = i + 1
   807:       toks.append(('STRING',lexeme))
   808:       if s[i]=='\0': return toks
   809:       i = i + 1
   810:       continue
   811:     if s[i] in '{}=':
   812:       toks.append((s[i],s[i]))
   813:       i = i + 1
   814:       continue
   815:     if s[i] in '#\n':
   816:       toks.append(('NEWLINE','\n'))
   817:       i = i + 1
   818:       continue
   819:     if s[i]=='\\':
   820:       i = i + 1
   821:       if s[i] == '\0': return toks
   822:       toks.append(('STRING',s[i]))
   823:       i = i + 1
   824:       continue
   825:     toks.append(('STRING',s[i]))
   826:     i = i + 1
   827:     continue
   828: 
End python section to interscript/parsers/lalr1.py[23]