Lines Matching refs:line
8 repeatedly to get the next line of input (or "" for EOF). It generates
15 the original line (string)
83 # Single-line ' or " string.
102 # First (or only) line of ' or " string.
142 def printtoken(type, token, xxx_todo_changeme, xxx_todo_changeme1, line): # for testing
155 Each call to the function should return one line of input as a string.
190 tok_type, token, start, end, line = t
268 def find_cookie(line):
270 line_string = line.decode('ascii')
340 should return one line of input as a string. Alternately, readline
348 and the line on which the token was found. The line passed is the
349 physical line.
364 line = readline()
366 line = ''
368 pos, max = 0, len(line)
371 if not line:
372 raise TokenError("EOF in multi-line string", strstart)
373 endmatch = endprog.match(line)
376 yield (STRING, contstr + line[:end],
377 strstart, (lnum, end), contline + line)
380 elif needcont and line[-2:] != '\\\n' and line[-3:] != '\\\r\n':
381 yield (ERRORTOKEN, contstr + line,
382 strstart, (lnum, len(line)), contline)
387 contstr = contstr + line
388 contline = contline + line
392 if not line: break
395 if line[pos] == ' ': column = column + 1
396 elif line[pos] == '\t': column = (column//tabsize + 1)*tabsize
397 elif line[pos] == '\f': column = 0
406 if line[pos] in '#\r\n': # skip comments or blank lines
407 if line[pos] == '#':
408 comment_token = line[pos:].rstrip('\r\n')
411 (lnum, pos), (lnum, pos + len(comment_token)), line)
412 yield (NL, line[nl_pos:],
413 (lnum, nl_pos), (lnum, len(line)), line)
415 yield ((NL, COMMENT)[line[pos] == '#'], line[pos:],
416 (lnum, pos), (lnum, len(line)), line)
421 yield (INDENT, line[:pos], (lnum, 0), (lnum, pos), line)
426 ("<tokenize>", lnum, pos, line))
434 yield (DEDENT, '', (lnum, pos), (lnum, pos), line)
442 if not line:
443 raise TokenError("EOF in multi-line statement", (lnum, 0))
447 pseudomatch = pseudoprog.match(line, pos)
451 token, initial = line[start:end], line[start]
455 yield (NUMBER, token, spos, epos, line)
465 yield (newline, token, spos, epos, line)
472 yield (COMMENT, token, spos, epos, line)
475 endmatch = endprog.match(line, pos)
476 if endmatch: # all on one line
478 token = line[start:pos]
482 yield (STRING, token, spos, (lnum, pos), line)
485 contstr = line[start:]
486 contline = line
495 contstr, needcont = line[start:], 1
496 contline = line
502 yield (STRING, token, spos, epos, line)
507 token, spos, epos, line)
510 tok = (NAME, token, spos, epos, line)
539 yield (NL, token, spos, (lnum, pos), line)
547 yield (OP, token, spos, epos, line)
549 yield (ERRORTOKEN, line[pos],
550 (lnum, pos), (lnum, pos+1), line)