Lines Matching refs:line
84 We allow that a user-agent: line is not preceded by
89 # 1: saw user-agent line
90 # 2: saw an allow or disallow line
95 for line in lines:
96 if not line:
104 # remove optional comment and strip line
105 i = line.find('#')
107 line = line[:i]
108 line = line.strip()
109 if not line:
111 line = line.split(':', 1)
112 if len(line) == 2:
113 line[0] = line[0].strip().lower()
114 line[1] = urllib.parse.unquote(line[1].strip())
115 if line[0] == "user-agent":
119 entry.useragents.append(line[1])
121 elif line[0] == "disallow":
123 entry.rulelines.append(RuleLine(line[1], False))
125 elif line[0] == "allow":
127 entry.rulelines.append(RuleLine(line[1], True))
129 elif line[0] == "crawl-delay":
134 if line[1].strip().isdigit():
135 entry.delay = int(line[1])
137 elif line[0] == "request-rate":
139 numbers = line[1].split('/')
145 elif line[0] == "sitemap":
147 # "This directive is independent of the user-agent line,
150 self.sitemaps.append(line[1])
216 """A rule line is a single "Allow:" (allowance==True) or "Disallow:"
270 for line in self.rulelines:
271 if line.applies_to(filename):
272 return line.allowance