Lines Matching refs:parser
21 self.parser = urllib.robotparser.RobotFileParser()
22 self.parser.parse(lines)
34 self.assertTrue(self.parser.can_fetch(agent, url))
40 self.assertFalse(self.parser.can_fetch(agent, url))
43 self.assertEqual(self.parser.site_maps(), self.site_maps)
106 parser = self.parser
110 self.assertEqual(parser.crawl_delay(agent), self.crawl_delay)
112 parsed_request_rate = parser.request_rate(agent)
299 self.assertEqual(str(self.parser), self.expected_output)
343 parser = urllib.robotparser.RobotFileParser()
344 parser.set_url(url)
345 parser.read()
346 self.assertFalse(parser.can_fetch("*", robots_url))
359 cls.parser = urllib.robotparser.RobotFileParser(cls.robots_txt)
360 cls.parser.read()
368 self.assertFalse(self.parser.disallow_all)
369 self.assertFalse(self.parser.allow_all)
370 self.assertGreater(self.parser.mtime(), 0)
371 self.assertFalse(self.parser.crawl_delay('*'))
372 self.assertFalse(self.parser.request_rate('*'))
375 self.assertTrue(self.parser.can_fetch('*', self.url('elsewhere')))
376 self.assertFalse(self.parser.can_fetch('Nutch', self.base_url))
377 self.assertFalse(self.parser.can_fetch('Nutch', self.url('brian')))
378 self.assertFalse(self.parser.can_fetch('Nutch', self.url('webstats')))
379 self.assertFalse(self.parser.can_fetch('*', self.url('webstats')))
380 self.assertTrue(self.parser.can_fetch('*', self.base_url))
383 parser = urllib.robotparser.RobotFileParser(self.url('i-robot.txt'))
384 parser.read()
385 self.assertTrue(parser.allow_all)
386 self.assertFalse(parser.disallow_all)
387 self.assertEqual(parser.mtime(), 0)
388 self.assertIsNone(parser.crawl_delay('*'))
389 self.assertIsNone(parser.request_rate('*'))