1from fontTools.voltLib.error import VoltLibError 2 3 4class Lexer(object): 5 NUMBER = "NUMBER" 6 STRING = "STRING" 7 NAME = "NAME" 8 NEWLINE = "NEWLINE" 9 10 CHAR_WHITESPACE_ = " \t" 11 CHAR_NEWLINE_ = "\r\n" 12 CHAR_DIGIT_ = "0123456789" 13 CHAR_UC_LETTER_ = "ABCDEFGHIJKLMNOPQRSTUVWXYZ" 14 CHAR_LC_LETTER_ = "abcdefghijklmnopqrstuvwxyz" 15 CHAR_UNDERSCORE_ = "_" 16 CHAR_PERIOD_ = "." 17 CHAR_NAME_START_ = ( 18 CHAR_UC_LETTER_ + CHAR_LC_LETTER_ + CHAR_PERIOD_ + CHAR_UNDERSCORE_ 19 ) 20 CHAR_NAME_CONTINUATION_ = CHAR_NAME_START_ + CHAR_DIGIT_ 21 22 def __init__(self, text, filename): 23 self.filename_ = filename 24 self.line_ = 1 25 self.pos_ = 0 26 self.line_start_ = 0 27 self.text_ = text 28 self.text_length_ = len(text) 29 30 def __iter__(self): 31 return self 32 33 def next(self): # Python 2 34 return self.__next__() 35 36 def __next__(self): # Python 3 37 while True: 38 token_type, token, location = self.next_() 39 if token_type not in {Lexer.NEWLINE}: 40 return (token_type, token, location) 41 42 def location_(self): 43 column = self.pos_ - self.line_start_ + 1 44 return (self.filename_ or "<volt>", self.line_, column) 45 46 def next_(self): 47 self.scan_over_(Lexer.CHAR_WHITESPACE_) 48 location = self.location_() 49 start = self.pos_ 50 text = self.text_ 51 limit = len(text) 52 if start >= limit: 53 raise StopIteration() 54 cur_char = text[start] 55 next_char = text[start + 1] if start + 1 < limit else None 56 57 if cur_char == "\n": 58 self.pos_ += 1 59 self.line_ += 1 60 self.line_start_ = self.pos_ 61 return (Lexer.NEWLINE, None, location) 62 if cur_char == "\r": 63 self.pos_ += 2 if next_char == "\n" else 1 64 self.line_ += 1 65 self.line_start_ = self.pos_ 66 return (Lexer.NEWLINE, None, location) 67 if cur_char == '"': 68 self.pos_ += 1 69 self.scan_until_('"\r\n') 70 if self.pos_ < self.text_length_ and self.text_[self.pos_] == '"': 71 self.pos_ += 1 72 return (Lexer.STRING, text[start + 1 : self.pos_ - 1], location) 73 else: 74 raise VoltLibError("Expected '\"' to terminate string", location) 75 if cur_char in Lexer.CHAR_NAME_START_: 76 self.pos_ += 1 77 self.scan_over_(Lexer.CHAR_NAME_CONTINUATION_) 78 token = text[start : self.pos_] 79 return (Lexer.NAME, token, location) 80 if cur_char in Lexer.CHAR_DIGIT_: 81 self.scan_over_(Lexer.CHAR_DIGIT_) 82 return (Lexer.NUMBER, int(text[start : self.pos_], 10), location) 83 if cur_char == "-" and next_char in Lexer.CHAR_DIGIT_: 84 self.pos_ += 1 85 self.scan_over_(Lexer.CHAR_DIGIT_) 86 return (Lexer.NUMBER, int(text[start : self.pos_], 10), location) 87 raise VoltLibError("Unexpected character: '%s'" % cur_char, location) 88 89 def scan_over_(self, valid): 90 p = self.pos_ 91 while p < self.text_length_ and self.text_[p] in valid: 92 p += 1 93 self.pos_ = p 94 95 def scan_until_(self, stop_at): 96 p = self.pos_ 97 while p < self.text_length_ and self.text_[p] not in stop_at: 98 p += 1 99 self.pos_ = p 100