From 0281e2fc10479831667976f2174ca2cce4ecd2fc Mon Sep 17 00:00:00 2001 From: Alexander Sibiryakov Date: Tue, 25 Apr 2017 16:48:08 +0200 Subject: [PATCH] fixing test cases --- tests/test_frontera_scheduler.py | 28 ++++++++++++---------------- tests/test_seed_loader.py | 6 +++--- 2 files changed, 15 insertions(+), 19 deletions(-) diff --git a/tests/test_frontera_scheduler.py b/tests/test_frontera_scheduler.py index 3b6e1ed04..ac68340bc 100644 --- a/tests/test_frontera_scheduler.py +++ b/tests/test_frontera_scheduler.py @@ -11,15 +11,15 @@ # test requests -r1 = Request('http://www.example.com') -r2 = Request('https://www.example.com/some/page') -r3 = Request('http://example1.com') +r1 = Request('http://www.example.com', meta={'seed': True}) +r2 = Request('https://www.example.com/some/page', meta={'seed': True}) +r3 = Request('http://example1.com', meta={'seed': True}) # test requests with redirects -rr1 = Request('http://www.example.com', meta={b'redirect_times': 1}) -rr2 = Request('https://www.example.com/some/page', meta={b'redirect_times': 4}) -rr3 = Request('http://example1.com', meta={b'redirect_times': 0}) +rr1 = Request('http://www.example.com', meta={'redirect_times': 1}) +rr2 = Request('https://www.example.com/some/page', meta={'redirect_times': 4}) +rr3 = Request('http://example1.com', meta={'redirect_times': 0}) # test frontier requests @@ -49,11 +49,10 @@ def test_redirect_disabled_enqueue_requests(self): fs.open(Spider) assert fs.enqueue_request(rr1) is False assert fs.enqueue_request(rr2) is False - assert fs.enqueue_request(rr3) is True - assert isinstance(fs.frontier.manager.seeds[0], FRequest) - assert len(fs.frontier.manager.seeds) == 1 - assert fs.frontier.manager.seeds[0].url == rr3.url - assert fs.stats_manager.stats.get_value('frontera/seeds_count') == 1 + assert fs.enqueue_request(rr3) is False + assert len(fs.frontier.manager.seeds) == 0 + assert fs.stats_manager.stats.get_value('frontera/seeds_count') == None + def test_redirect_enabled_enqueue_requests(self): settings = Settings() @@ -63,13 +62,10 @@ def test_redirect_enabled_enqueue_requests(self): fs.open(Spider) assert fs.enqueue_request(rr1) is True assert fs.enqueue_request(rr2) is True - assert fs.enqueue_request(rr3) is True - assert len(fs.frontier.manager.seeds) == 1 - assert isinstance(fs.frontier.manager.seeds[0], FRequest) - assert fs.frontier.manager.seeds[0].url == rr3.url + assert fs.enqueue_request(rr3) is False assert set([request.url for request in fs._pending_requests]) == set([rr1.url, rr2.url]) assert all([isinstance(request, Request) for request in fs._pending_requests]) - assert fs.stats_manager.stats.get_value('frontera/seeds_count') == 1 + assert fs.stats_manager.stats.get_value('frontera/seeds_count') == None assert fs.stats_manager.stats.get_value('frontera/redirected_requests_count') == 2 def test_next_request(self): diff --git a/tests/test_seed_loader.py b/tests/test_seed_loader.py index bc512e2a9..7aea5a082 100644 --- a/tests/test_seed_loader.py +++ b/tests/test_seed_loader.py @@ -49,7 +49,7 @@ def test_load_seeds(self): def test_process_start_requests(self): seed_loader = self.seed_loader_setup() - requests = seed_loader.process_start_requests(None, Spider(name='spider')) + requests = seed_loader.process_start_requests((), Spider(name='spider')) self.assertEqual([r.url for r in requests], ['https://www.example.com', 'https://www.scrapy.org']) def test_process_start_requests_ignore_comments(self): @@ -60,7 +60,7 @@ def test_process_start_requests_ignore_comments(self): # https://www.test.com """ seed_loader = self.seed_loader_setup(seeds_content) - requests = seed_loader.process_start_requests(None, Spider(name='spider')) + requests = seed_loader.process_start_requests((), Spider(name='spider')) self.assertEqual([r.url for r in requests], ['https://www.example.com', 'https://www.scrapy.org']) @@ -123,5 +123,5 @@ def mocked_connect_s3(*args, **kwargs): with mock.patch('frontera.contrib.scrapy.middlewares.seeds.s3.connect_s3', side_effect=mocked_connect_s3): - requests = self.seed_loader.process_start_requests(None, Spider(name='spider')) + requests = self.seed_loader.process_start_requests((), Spider(name='spider')) self.assertEqual(set([r.url for r in requests]), set(urls))