gerrit-402355-22

dev-requirements.txt

@@ -24,6 +24,8 @@ codecov
 coverage
 mock ; python_version < '3'
 
+vcrpy-unittest
+
 # pywin32 & pywinauto>=0.4.0 are Win32 UI test dependencies that have been
 # excluded from this file as they are quite expensive to install, and they
 # are not useful on the Appveyor Win32 builds since the relevant UI tests

tests/README.rst

@@ -226,6 +226,10 @@ To enable 'write' tests, set PYWIKIBOT2_TEST_WRITE=1
 Enabling only 'edit failure' tests or 'write' tests won't enable the other tests
 automatically.
 
+Some tests are also using VCR.py, which means that these tests are dry and all
+requests are recorded and used later, being stored in 'tests/cassettes'.
+To avoid using recorded requests and to run live tests set PYWIKIBOT_LIVE_TESTS=1
+
 Decorators
 =====================
 
@@ -356,3 +360,4 @@ Other class attributes
 - ``user = True`` : test class needs to login to site
 - ``sysop = True`` : test class needs to login to site as a sysop
 - ``write = True`` : test class needs to write to a site
+- ``vcr = True``: test class uses VCR.py to record requests for dry testing

tests/api_tests.py

@@ -476,6 +476,8 @@ class TestParaminfoModules(DefaultSiteTestCase):
 
     """Test loading all paraminfo modules."""
 
+    vcr = True
+
     def test_action_modules(self):
         """Test loading all action modules."""
         self.site._paraminfo.fetch(self.site._paraminfo.action_modules)
@@ -821,9 +823,11 @@ class TestCachedRequest(DefaultSiteTestCase):
     """Test API Request caching.
 
     This test class does not use the forced test caching.
+    This class contains test cases with requests which are mocked with VCR.
     """
 
     cached = False
+    vcr = True
 
     def test_normal_use(self):
         """Test the caching of CachedRequest with an ordinary request."""
@@ -851,6 +855,18 @@ def test_normal_use(self):
         self.assertIsNotNone(req3._cachetime)
         self.assertEqual(req2._cachetime, req3._cachetime)
 
+
+class TestCachedRequestNonVCR(DefaultSiteTestCase):
+
+    """Test API Request caching (live without VCR)
+
+    This test class does not use the forced test caching.
+    This class contains test cases with requests which can't be mocked
+    with VCR.
+    """
+
+    cached = False
+
     def test_internals(self):
         """Test the caching of CachedRequest by faking a unique request."""
         mysite = self.get_site()

tests/aspects.py

@@ -57,6 +57,8 @@
     WarningSourceSkipContextManager, AssertAPIErrorContextManager,
 )
 
+import vcr_unittest
+
 try:
     import pytest_httpbin
     optional_pytest_httpbin_cls_decorator = pytest_httpbin.use_class_based_httpbin
@@ -707,6 +709,76 @@ def get_userpage(self, site=None):
         return userpage
 
 
+class VCRMixin(vcr_unittest.VCRMixin):
+
+    """Run dry tests using vcrpy to record and mock requests."""
+
+    @classmethod
+    def setUpClass(cls):
+        """Set up cassette recording for setUp."""
+        vcr = VCRMixin()._get_vcr()
+
+        func = getattr(cls, 'setUp')
+
+        def wrapped_setup(arg):
+            arg.vcr_enabled = False
+            with vcr.use_cassette(cls.__name__ + '.setUp.yaml'):
+                func(arg)
+            arg.vcr_enabled = True
+            vcr_unittest.VCRMixin.setUp(arg)
+        setattr(cls, 'setUp', wrapped_setup)
+
+        super(VCRMixin, cls).setUpClass()
+
+    def _get_vcr(self, **kwargs):
+        """Configure VCR."""
+        vcr = super(VCRMixin, self)._get_vcr(**kwargs)
+        vcr.record_mode = 'once'
+        vcr.decode_compressed_response = True
+        vcr.before_record_request = self.before_record_request
+        vcr.before_record_response = self.before_record_response
+        return vcr
+
+    def _get_cassette_name(self):
+        """Get name of VCR cassette."""
+        if hasattr(self, 'sites'):
+            # Multiple sites are used in this class
+            # Let's guess which of them is used for this test method by name
+            for name in self.sites:
+                if self._testMethodName.endswith('_' + name.replace('-', '_')):
+                    break
+            family = self.sites[name]['family']
+            code = self.sites[name]['code']
+            return '{0}.{1}/{2}.{3}.yaml'.format(family, code,
+                                                 self.__class__.__name__,
+                                                 self._testMethodName)
+        else:
+            return '{0}/{1}.{2}.yaml'.format(str(self.site).replace(':', '.'),
+                                             self.__class__.__name__,
+                                             self._testMethodName)
+
+    def before_record_request(self, request):
+        """Remove headers containing sensitive data from request."""
+        # TODO: custom cookie processing if user=True
+        if 'Cookie' in request.headers:
+            del request.headers['Cookie']
+        if 'user-agent' in request.headers:
+            del request.headers['user-agent']
+
+        return request
+
+    def before_record_response(self, response):
+        """Remove headers containing sensitive data from response."""
+        # TODO: custom cookie processing if user=True
+        for header in ['Set-Cookie', 'X-Client-IP']:
+            if header in response['headers']:
+                del response['headers'][header]
+            if header.lower() in response['headers']:
+                del response['headers'][header.lower()]
+
+        return response
+
+
 class MetaTestCaseClass(type):
 
     """Test meta class."""
@@ -850,7 +922,11 @@ def wrapped_method(self):
         if 'cached' in dct and dct['cached']:
             bases = cls.add_base(bases, ForceCacheMixin)
 
-        if 'net' in dct and dct['net']:
+        if ('vcr' in dct and dct['vcr'] and
+           'PYWIKIBOT_LIVE_TESTS' not in os.environ):
+            bases = cls.add_base(bases, VCRMixin)
+            del dct['net']
+        elif 'net' in dct and dct['net']:
             bases = cls.add_base(bases, CheckHostnameMixin)
         else:
             assert not hostnames, 'net must be True with hostnames defined'

tests/wikibase_tests.py

@@ -1846,6 +1846,8 @@ class TestOwnClient(TestCase):
         },
     }
 
+    vcr = True
+
     def test_own_client(self, key):
         """Test that a data repository family can be its own client."""
         site = self.get_site(key)

tox.ini

@@ -98,6 +98,7 @@ deps =
     nose-detecthttp
     unicodecsv
     mock
+    vcrpy-unittest
 
 [testenv:nose34]
 basepython = python3
@@ -112,6 +113,7 @@ deps =
     nose-detecthttp>=0.1.3
     six
     mock
+    vcrpy-unittest
 
 [testenv:doctest]
 commands =