BlitzMax Source Tokenizer

-=+=- -=+=- -=+=- -=+=- -=+=- -=+=- -=+=- -=+=- -=+=- -=+=- -=+=- -=+=- -=+=- -=+=- -=+=- -=+=- -=+=- -=+=- -=+=- -=+=- -=+=- -=+=- -=+=- -=+=- -=+=- -=+=- -=+=- -=+=- -=+=- -=+=- (c) WidthPadding Industries 1987 0\|519\|0 -=+=- -=+=- -=+=- -=+=- -=+=- -=+=- -=+=- -=+=- -=+=- -=+=- -=+=- -=+=- -=+=- -=+=- -=+=- -=+=- -=+=- -=+=- -=+=- -=+=- -=+=- -=+=- -=+=- -=+=- -=+=- -=+=- -=+=- -=+=- -=+=- -=+=-
SoCoder -> Snippet Home -> Misc

Cower	Created : 25 December 2009 Language : Delphi/Pascal BlitzMax Source Tokenizer Title just about does it
Bit of code I wrote today for tokenizing BlitzMax code. No parser yet, although you can sort of see where I thought I would put it (the only TODO I bothered to write). Probably going to have to write a TokenReader... Anyhow, if for some horrifying reason you need to tokenize BlitzMax code, this should do the trick. You might want to comment out the extension bits (in the tuples, they're marked by "(x)", in the single tokens they're under the extensions comment). This script requires Ruby 1.9 as a minimum due to the way the [] operator is implemented for Strings in Ruby 1.9. Ruby 1.8 will not work, I can guarantee this. #!/usr/bin/env ruby19 -w # Copyright (c) 2009 Noel R. Cower # # Permission is hereby granted, free of charge, to any person obtaining a copy # of this software and associated documentation files (the "Software"), to deal # in the Software without restriction, including without limitation the rights # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell # copies of the Software, and to permit persons to whom the Software is # furnished to do so, subject to the following conditions: # # The above copyright notice and this permission notice shall be included in # all copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN # THE SOFTWARE. BLITZ_TOKEN_TUPLES = { [:END_KW, :REM_KW] => {:kind => :ENDREM_KW, :range => 1}, # "End Rem" [:END_KW, :METHOD_KW] => {:kind => :ENDMETHOD_KW, :range => 1}, # "End Method" [:END_KW, :FUNCTION_KW] => {:kind => :ENDFUNCTION_KW, :range => 1}, # "End Function" [:END_KW, :TYPE_KW] => {:kind => :ENDTYPE_KW, :range => 1}, # "End Type" [:END_KW, :EXTERN_KW] => {:kind => :ENDEXTERN_KW, :range => 1}, # "End Extern" [:END_KW, :IF_KW] => {:kind => :ENDIF_KW, :range => 1}, # "End If" [:END_KW, :SELECT_KW] => {:kind => :ENDSELECT_KW, :range => 1}, # "End Select" [:END_KW, :WHILE_KW] => {:kind => :ENDWHILE_KW, :range => 1}, # "End While" [:END_KW, :END_PROTOCOL] => {:kind => :ENDPROTOCOL_KW, :range => 1}, # "End Protocol" (x) [:COLON, :PLUS] => {:kind => :ASSIGN_ADD, :range => 0}, # :+ [:COLON, :MINUS] => {:kind => :ASSIGN_SUBTRACT, :range => 0}, # :- [:COLON, :SLASH] => {:kind => :ASSIGN_DIVIDE, :range => 0}, # :/ [:COLON, :ASTERISK] => {:kind => :ASSIGN_MULTIPLY, :range => 0}, # :* [:COLON, :CARET] => {:kind => :ASSIGN_POWER, :range => 0}, # :^ [:COLON, :SHL_KW] => {:kind => :ASSIGN_SHL, :range => 0}, # :Shl [:COLON, :SHR_KW] => {:kind => :ASSIGN_SHR, :range => 0}, # :Shr [:COLON, :SAR_KW] => {:kind => :ASSIGN_SAR, :range => 0}, # :Sar [:COLON, :MOD_KW] => {:kind => :ASSIGN_MOD, :range => 0}, # :Mod [:COLON, :TILDE] => {:kind => :ASSIGN_XOR, :range => 0}, # :~ [:COLON, :AMPERSAND] => {:kind => :ASSIGN_AND, :range => 0}, # :& [:COLON, :PIPE] => {:kind => :ASSIGN_OR, :range => 0}, # :\| [:COLON, :EQUALS] => {:kind => :ASSIGN_AUTO, :range => 0}, # := (x) [:MINUS, :MINUS] => {:kind => :DOUBLEMINUS, :range => 0}, # -- (x) [:PLUS, :PLUS] => {:kind => :DOUBLEPLUS, :range => 0}, # ++ (x) [:MINUS, :NUMBER_LIT] => {:kind => :NUMBER_LIT, :range => 0}, # -NUMBER } BLITZ_TOKEN_SINGLES = { 'end' => :END_KW, 'function' => :FUNCTION_KW, 'endfunction' => :ENDFUNCTION_KW, 'method' => :METHOD_KW, 'endmethod' => :ENDMETHOD_KW, 'type' => :TYPE_KW, 'extends' => :EXTENDS_KW, 'abstract' => :ABSTRACT_KW, 'final' => :FINAL_KW, 'nodebug' => :NODEBUG_KW, 'endtype' => :ENDTYPE_KW, 'extern' => :EXTERN_KW, 'endextern' => :ENDEXTERN_KW, 'rem' => :REM_KW, 'endrem' => :ENDREM_KW, #required data types 'float' => :FLOAT_KW, 'double' => :DOUBLE_KW, 'byte' => :BYTE_KW, 'short' => :SHORT_KW, 'int' => :INT_KW, 'string' => :STRING_KW, 'object' => :OBJECT_KW, 'local' => :LOCAL_KW, 'global' => :GLOBAL_KW, 'const' => :CONST_KW, 'varptr' => :VARPTR_KW, 'ptr' => :PTR_KW, 'var' => :VAR_KW, 'null' => :NULL_KW, 'strict' => :STRICT_KW, 'superstrict' => :SUPERSTRICT_KW, 'framework' => :FRAMEWORK_KW, 'module' => :MODULE_KW, 'moduleinfo' => :MODULEINFO_KW, 'import' => :IMPORT_KW, 'include' => :INCLUDE_KW, 'private' => :PRIVATE_KW, 'public' => :PUBLIC_KW, 'or' => :OR_KW, 'and' => :AND_KW, 'shr' => :SHR_KW, 'shl' => :SHL_KW, 'sar' => :SAR_KW, 'mod' => :MOD_KW, 'not' => :NOT_KW, 'while' => :WHILE_KW, 'wend' => :WEND_KW, 'endwhile' => :ENDWHILE_KW, #why the hell does this exist? 'for' => :FOR_KW, 'next' => :NEXT_KW, 'until' => :UNTIL_KW, 'to' => :TO_KW, 'eachin' => :EACHIN_KW, 'repeat' => :REPEAT_KW, 'forever' => :FOREVER_KW, 'if' => :IF_KW, 'endif' => :ENDIF_KW, 'else' => :ELSE_KW, 'elseif' => :ELSEIF_KW, 'then' => :THEN_KW, 'select' => :SELECT_KW, 'case' => :CASE_KW, 'default' => :DEFAULT_KW, 'endselect' => :ENDSELECT_KW, 'self' => :SELF_KW, 'super' => :SUPER_KW, 'pi' => :PI_KW, 'new' => :NEW_KW, #extensions 'protocol' => :PROTOCOL_KW, 'endprotocol' => :ENDPROTOCOL_KW, 'auto' => :AUTO_KW, ?: => :COLON, ?? => :QUESTION, ?! => :BANG, ?# => :HASH, # ?@ => :AT, ?$ => :DOLLAR, ?% => :PERCENT, ?' => :SINGLEQUOTE, ?( => :OPENPAREN, ?) => :CLOSEPAREN, ?[ => :OPENBRACKET, ?] => :CLOSEBRACKET, ?{ => :OPENCURL, ?} => :CLOSECURL, ?> => :GREATERTHAN, ?< => :LESSTHAN, ?= => :EQUALS, ?- => :MINUS, ?+ => :PLUS, ?* => :ASTERISK, ?^ => :CARET, ?~ => :TILDE, ?` => :GRAVE, "\\" => :BACKSLASH, ?/ => :SLASH, # ?. => :DOT, ?, => :COMMA, ?; => :SEMICOLON, ?\| => :PIPE, ?& => :AMPERSAND, "\n" => :NEWLINE } class BlitzToken attr_accessor :kind attr_accessor :from, :to attr_accessor :line, :column def initialize(source) @source = source @line = 0 @column = 0 end def value return @source[@from .. @to] end def follows(other, range=0) return (other.from < (self.to+1)+range) end def precedes(other, range=0) other.follows?(self, range) end #debugging def linecol "[#{@line}:#{@column}]" end end class BlitzTokenizer attr_reader :tokens def initialize(source) @source = source.contents @sourceIndex = 0 @mark = 0 @tokens = [] end protected def next unless @source[@sourceIndex].nil? @column += 1 @sourceIndex += 1 return self.current end return nil end def peek(at=nil, caseSens=false) return @source[@sourceIndex + 1] unless at if at then str = @source[@sourceIndex, at.length] return (at == (caseSense ? str : str.downcase)) end end def next? return (@sourceIndex+1 < @source.length) end def current @source[@sourceIndex] end def skipWhitespace while self.current and self.current.match(/[ \t\r]/) { self.next } do end end def position @sourceIndex end def position=(p) @sourceIndex = p end def mark @mark = self.position end def skip(n=1) if n.kind_of? String then until self.peek(n) do self.next end else while 0 < n do self.next n -= 1 end end end def reset(to=nil) self.position = (to or @mark) end def linecol "[#{@line}:#{@column}]" end public def tokenize @column = 0 @line = 1 @sourceIndex = 0 @mark = 0 @tokens = [] comment = nil while self.current do self.skipWhitespace token = nil index = self.mark char = self.current #debug # puts "#{self.linecol} #{index} '#{char}'" if comment.nil? then # read @ and @@ tokens if char == ?@ then token = BlitzToken.new(@source) token.kind = :AT token.from = index token.line = @line token.column = @column self.next if self.current == ?@ then self.next token.kind = :DOUBLEAT end token.to = self.position-1 end # read ., .., and number tokens if char == ?. then if self.peek.match(/[0-9]/) then token = self.readNumber else token = BlitzToken.new(@source) token.kind = :DOT token.from = index token.line = @line token.column = @column self.next if self.current == ?. then token.kind = :DOUBLEDOT end token.to = self.position-1 end end # binary number tokens if char == ?% and self.peek.match(reg = /[01]/) then token = self.readBaseNumber(:BINARY_LIT, reg) end # hexadecimal number tokens if char == ?$ and self.peek.match(reg = /[0-9a-fA-F]/) then token = self.readBaseNumber(:HEXNUM_LIT, reg) end # string literals if char == ?" then token = self.readString end # numbers char.match(/[0-9]/) do token = self.readNumber end # single character tokens if token.nil? and sym = BLITZ_TOKEN_SINGLES[char] then token = BlitzToken.new(@source) token.kind = sym token.from = index token.to = index token.line = @line token.column = @column self.next end end # identifiers, keywords, etc. char.match(/[a-zA-Z_]/) do token = self.readWord end if char == "\n" then @line += 1 @column = 0 end # the ugly business of block comments in the language of the whitespace devil if comment then if token then if token.kind == :ENDREM_KW then @tokens << blockToken = BlitzToken.new(@source) blockToken.from = comment.to+1 blockToken.line = comment.line blockToken.column = comment.column blockToken.kind = :COMMENT_BLOCK blockToken.to = token.from-1 comment = nil elsif token.kind == :END_KW then self.skipWhitespace char = self.current index = self.mark if char.match(/[a-zA-Z_]/) nexttoken = self.readWord if nexttoken.kind == :REM_KW and nexttoken.follows(token, 1) then @tokens << blockToken = BlitzToken.new(@source) blockToken.from = comment.to+1 blockToken.line = comment.line blockToken.column = comment.column blockToken.kind = :COMMENT_BLOCK blockToken.to = token.from-1 comment = nil token.kind = :ENDREM_KW token.to = nexttoken.to end end end else self.skip self.skipWhitespace end end # Let me reiterate: fuck "End Rem." EndRem? Fine. "End Rem?" Fuck you. @tokens << token if token and comment.nil? if comment.nil? and token.kind == :REM_KW then comment = token end end #if there are any tokens that, when combined, produce other tokens, merge them newtokens = [] index = 0 while index < @tokens.length do merged = false BLITZ_TOKEN_TUPLES.each { \|tuple, merger\| range = merger[:range] kind = merger[:kind] tuptok = @tokens[index, tuple.length] next unless tuptok.inject { \|m,t\| t.nil? ? nil : (t.follows(m, range) ? t : nil) } symbols = Array.new(tuptok.length) {\|i\| tuptok[i].kind} if tuple == symbols then merged = true newtoken = tuptok.first newtoken.to = tuptok.last.to @tokens[index, tuple.length] = newtoken break end } index += 1 unless merged end return self.tokens end def readBaseNumber(kind, reg) token = BlitzToken.new(@source) token.kind = kind token.line = @line token.column = @column token.from = self.mark while self.next? and self.next.match(reg) do end token.to = self.position-1 return token end def readNumber char = self.current index = self.mark token = BlitzToken.new(@source) token.line = @line token.column = @column token.from = index isDec = (char == ?.) isExp = false while self.next? and char = self.next do break if (isDec and char == ?.) if char == ?. then isDec = true next end if char.downcase == ?e then raise "#{self.linecol} Malformed number literal - exponent already provided (\"#{@source[index .. self.position]}\")" if isExp isExp = true char = self.peek if char == ?- or char == ?+ then self.next char = self.peek end raise "#{self.linecol} Malformed number literal - no exponent provided (\"#{@source[index .. self.position]}\")" if char.match(/[^0-9]/) end break if char.match(/[^0-9]/) end token.kind = :NUMBER_LIT token.to = self.position-1 return token end def readWord char = self.current index = self.mark token = BlitzToken.new(@source) token.line = @line token.column = @column token.kind = :IDENTIFIER token.from = index while self.next? and self.peek.match(/[a-zA-Z0-9_]/) do self.next end self.next token.to = self.position-1 if sym = BLITZ_TOKEN_SINGLES[@source[token.from .. token.to].downcase] then token.kind = sym end return token end def readString char = self.current index = self.mark token = BlitzToken.new(@source) token.kind = :STRING_LIT token.line = @line token.column = @column token.from = index until self.next? and (char = self.next) == ?" do raise "#{self.linecol} String literal does not terminate before newline or EOF (\"#{@source[index .. self.position]}\")" if char == "\n" or char == nil end token.to = self.position self.next return token end def readBlockComment end def readLineComment char = self.current index = self.mark token = BlitzToken.new(@source) token.kind = :LINE_COMMENT token.line = @line token.column = @column token.from = index loop do char = self.next if char == nil or char == "\n" then break end end token.to = self.position-1 return token end end class BlitzSource attr_reader :path attr_reader :contents attr_reader :tokens #includes are translated to imports that inherit strictness/imports (excluding self) from the importing source code attr_reader :imports attr_reader :isStrict attr_reader :isModule attr_reader :functions attr_reader :types attr_reader :globals attr_reader :constants def initialize(path, isSource=false) if isSource then @path = Pathname.getwd @contents = path.clone.freeze else @path = Pathname.new(path).realpath @contents = File.new(@path).read.freeze end @tokenizer = BlitzTokenizer.new(self) @tokens = @tokenizer.tokenize.clone end def parse tokens.each do # TODO end end end --v

Comments

Comments