Lines Matching refs:url
24 def get_agent_and_url(self, url):
25 if isinstance(url, tuple):
26 agent, url = url
27 return agent, url
28 return self.agent, url
31 for url in self.good:
32 agent, url = self.get_agent_and_url(url)
33 with self.subTest(url=url, agent=agent):
34 self.assertTrue(self.parser.can_fetch(agent, url))
37 for url in self.bad:
38 agent, url = self.get_agent_and_url(url)
39 with self.subTest(url=url, agent=agent):
40 self.assertFalse(self.parser.can_fetch(agent, url))
107 for url in self.good + self.bad:
108 agent, url = self.get_agent_and_url(url)
109 with self.subTest(url=url, agent=agent):
341 url = 'http://' + socket_helper.HOST + ':' + str(addr[1])
342 robots_url = url + "/robots.txt"
344 parser.set_url(url)
362 def url(self, path):
375 self.assertTrue(self.parser.can_fetch('*', self.url('elsewhere')))
377 self.assertFalse(self.parser.can_fetch('Nutch', self.url('brian')))
378 self.assertFalse(self.parser.can_fetch('Nutch', self.url('webstats')))
379 self.assertFalse(self.parser.can_fetch('*', self.url('webstats')))
383 parser = urllib.robotparser.RobotFileParser(self.url('i-robot.txt'))