Lines Matching refs:useragent
154 def can_fetch(self, useragent, url):
155 """using the parsed robots.txt decide if useragent can fetch url"""
175 if entry.applies_to(useragent):
183 def crawl_delay(self, useragent):
187 if entry.applies_to(useragent):
193 def request_rate(self, useragent):
197 if entry.applies_to(useragent):
253 def applies_to(self, useragent):
256 useragent = useragent.split("/")[0].lower()
262 if agent in useragent: