Lines Matching refs:source
18 """Tokenize C++ source code."""
67 start contains the index of the first char of the token in the source
68 end contains the index of the last char of the token in the source
86 def _GetString(source, start, i):
87 i = source.find('"', i+1)
88 while source[i-1] == '\\':
92 while source[j] == '\\':
98 i = source.find('"', i+1)
102 def _GetChar(source, start, i):
104 i = source.find("'", i+1)
105 while source[i-1] == '\\':
107 if (i - 2) > start and source[i-2] == '\\':
109 i = source.find("'", i+1)
116 def GetTokens(source):
120 source: string of C++ source code.
123 Token that represents the next token in the source.
136 end = len(source)
139 while i < end and source[i].isspace():
146 c = source[i]
149 while source[i] in valid_identifier_chars:
153 if (source[i] == "'" and (i - start) == 1 and
154 source[start:i] in 'uUL'):
157 i = _GetChar(source, start, i)
158 elif source[i] == "'" and source[start:i] in _STR_PREFIXES:
160 i = _GetString(source, start, i)
161 elif c == '/' and source[i+1] == '/': # Find // comments.
162 i = source.find('\n', i)
166 elif c == '/' and source[i+1] == '*': # Find /* comments. */
167 i = source.find('*/', i) + 2
172 new_ch = source[i]
182 if c == '.' and source[i].isdigit():
185 while source[i] in int_or_float_digits:
189 if suffix == source[i:i+1].lower():
194 if c == '0' and source[i+1] in 'xX':
197 while source[i] in hex_digits:
200 while source[i] in int_or_float_digits2:
205 if suffix == source[i:i+size].lower():
210 i = _GetString(source, start, i)
213 i = _GetChar(source, start, i)
216 got_if = source[i:i+3] == '#if' and source[i+3:i+4].isspace()
219 elif source[i:i+6] == '#endif':
226 i1 = source.find('\n', i)
227 i2 = source.find('//', i)
228 i3 = source.find('/*', i)
229 i4 = source.find('"', i)
235 if source[i] == '"':
236 i = source.find('"', i+1) + 1
240 if not (i == i1 and source[i-1] == '\\'):
242 condition = source[start+4:i].lstrip()
261 ('?', i, c, source[i-10:i+10]))
267 yield Token(token_type, source[start:i], start, i)
274 source = utils.ReadFile(filename)
275 if source is None:
278 for token in GetTokens(source):