fix whole object detection

changelog.d/20250611_120003_jb_PL_133453_rbd_whole_object.rst

@@ -0,0 +1,3 @@
+.. A new scriv changelog fragment.
+
+- Fix whole object detection (PL-133453)

pyproject.toml

@@ -13,6 +13,17 @@ version = "literal: pyproject.toml: tool.poetry.version"
 entry_title_template = "{% if version %}{{ version }} {% endif %}({{ date.strftime('%Y-%m-%d') }})"
 categories = ""
 
+
+[tool.pytest.ini_options]
+addopts = "--timeout=30 --tb=native --cov=src --cov-report=html --cov-config=pyproject.toml src -r w"
+markers = "slow: This is a non-unit test and thus is not run by default. Use ``-m slow`` to run these, or ``-m 1`` to run all tests."
+log_level = "NOTSET"
+asyncio_mode = "auto"
+
+[tool.coverage.run]
+branch = true
+omit = [ "*/tests/*" ]
+
 [tool.poetry]
 name = "backy"
 version = "2.6.0.dev0"
@@ -64,7 +75,7 @@ pytest = "^7.4.0"
 pytest-aiohttp = "^1.0.4"
 pytest-asyncio = "^0.23.3"
 pytest-cache = "^1.0"
-pytest-cov = "^4.1.0"
+pytest-cov = "^6.1.0"
 pytest-flake8 = "^1.1.1"
 pytest-timeout = "^2.1.0"
 scriv = "^1.3.1"

src/backy/rbd/rbd.py

@@ -1,4 +1,5 @@
 import contextlib
+import functools
 import json
 import struct
 import subprocess
@@ -8,23 +9,7 @@
 from structlog.stdlib import BoundLogger
 
 from backy.ext_deps import RBD
-from backy.utils import CHUNK_SIZE, punch_hole
-
-
-def detect_whole_object_support():
-    result = subprocess.run(
-        ["rbd", "help", "export-diff"],
-        stdout=subprocess.PIPE,
-        stderr=subprocess.PIPE,
-        check=True,
-    )
-    return "--whole-object" in result.stdout.decode("ascii")
-
-
-try:
-    CEPH_RBD_SUPPORTS_WHOLE_OBJECT_DIFF = detect_whole_object_support()
-except Exception:
-    CEPH_RBD_SUPPORTS_WHOLE_OBJECT_DIFF = False
+from backy.utils import CHUNK_SIZE
 
 
 class RBDClient(object):
@@ -80,6 +65,10 @@ def _rbd_stream(self, cmd: list[str]) -> Iterator[IO[bytes]]:
             if rc:
                 raise subprocess.CalledProcessError(rc, proc.args)
 
+    @functools.cached_property
+    def _supports_whole_object(self):
+        return "--whole-object" in self._rbd(["help", "export-diff"])
+
     def exists(self, snapspec: str):
         try:
             return self._rbd(["info", snapspec], format="json")
@@ -148,7 +137,7 @@ def snap_rm(self, image):
 
     @contextlib.contextmanager
     def export_diff(self, new: str, old: str) -> Iterator["RBDDiffV1"]:
-        if CEPH_RBD_SUPPORTS_WHOLE_OBJECT_DIFF:
+        if self._supports_whole_object:
             EXPORT_WHOLE_OBJECT = ["--whole-object"]
         else:
             EXPORT_WHOLE_OBJECT = []
@@ -316,7 +305,7 @@ def integrate(self, target, snapshot_from, snapshot_to):
         for record in self.read_data():
             target.seek(record.start)
             if isinstance(record, Zero):
-                punch_hole(target, target.tell(), record.length)
+                target.write(b"\x00" * record.length)
             elif isinstance(record, Data):
                 for chunk in record.stream():
                     target.write(chunk)

src/backy/rbd/tests/conftest.py

@@ -4,7 +4,6 @@
 
 import pytest
 
-import backy.rbd.rbd
 from backy.rbd import RBDClient
 
 
@@ -242,11 +241,8 @@ def unmap(self, device):
 
 @pytest.fixture(params=[CephJewelCLI, CephLuminousCLI, CephNautilusCLI])
 def rbdclient(request, tmp_path, monkeypatch, log):
-    monkeypatch.setattr(
-        backy.rbd.rbd, "CEPH_RBD_SUPPORTS_WHOLE_OBJECT_DIFF", True
-    )
-
     client = RBDClient(log)
+    client._supports_whole_object = True
     client._ceph_cli = request.param(tmp_path)
 
     return client

src/backy/rbd/tests/test_ceph.py

@@ -15,7 +15,7 @@
 from backy.rbd.rbd import RBDDiffV1
 from backy.revision import Revision
 
-BLOCK = backy.utils.PUNCH_SIZE
+BLOCK = backy.utils.CHUNK_SIZE
 
 with open(Path(__file__).parent / "nodata.rbddiff", "rb") as f:
     SAMPLE_RBDDIFF = f.read()

src/backy/tests/test_utils.py

@@ -1,7 +1,6 @@
 import asyncio
 import datetime
 import os
-import sys
 from zoneinfo import ZoneInfo
 
 import pytest
@@ -13,11 +12,8 @@
     SafeFile,
     TimeOut,
     TimeOutError,
-    _fake_fallocate,
-    copy_overwrite,
     files_are_equal,
     files_are_roughly_equal,
-    punch_hole,
 )
 
 
@@ -324,80 +320,13 @@ def test_roughly_compare_files_timeout(tmp_path):
     assert not files_are_roughly_equal(open("a", "rb"), open("b", "rb"))
 
 
-def test_copy_overwrite_correctly_makes_sparse_file(tmp_path):
-    # Create a test file that contains random data, then we insert
-    # blocks of zeroes. copy_overwrite will not break them and will make the
-    # file sparse.
-    source_name = str(tmp_path / "input")
-    with open(source_name, "wb") as f:
-        f.write(b"12345" * 1024 * 100)
-        f.seek(1024 * 16)
-        f.write(b"\x00" * 1024 * 16)
-    with open(source_name, "rb") as source:
-        target_name = str(tmp_path / "output")
-        with open(target_name, "wb") as target:
-            # To actually ensure that we punch holes and truncate, lets
-            # fill the file with a predictable pattern that is non-zero and
-            # longer than the source.
-            target.write(b"1" * 1024 * 150)
-        with open(target_name, "r+b") as target:
-            copy_overwrite(source, target)
-    with open(source_name, "rb") as source_current:
-        with open(target_name, "rb") as target_current:
-            assert source_current.read() == target_current.read()
-    if sys.platform == "linux2":
-        assert os.stat(source_name).st_blocks > os.stat(target_name).st_blocks
-    else:
-        assert os.stat(source_name).st_blocks >= os.stat(target_name).st_blocks
-
-
 def test_unmocked_now_returns_time_time_float():
     before = datetime.datetime.now(ZoneInfo("UTC"))
     now = backy.utils.now()
     after = datetime.datetime.now(ZoneInfo("UTC"))
     assert before <= now <= after
 
 
-@pytest.fixture
-def testfile(tmp_path):
-    fn = str(tmp_path / "myfile")
-    with open(fn, "wb") as f:
-        f.write(b"\xde\xad\xbe\xef" * 32)
-    return fn
-
-
-def test_punch_hole(testfile):
-    with open(testfile, "r+b") as f:
-        f.seek(0)
-        punch_hole(f, 2, 4)
-        f.seek(0)
-        assert f.read(8) == b"\xde\xad\x00\x00\x00\x00\xbe\xef"
-
-
-def test_punch_hole_needs_length(testfile):
-    with pytest.raises(IOError):
-        with open(testfile, "r+b") as f:
-            punch_hole(f, 10, 0)
-
-
-def test_punch_hole_needs_writable_file(testfile):
-    with pytest.raises(OSError):
-        with open(testfile, "rb") as f:
-            punch_hole(f, 0, 1)
-
-
-def test_punch_hole_needs_nonnegative_offset(testfile):
-    with pytest.raises(OSError):
-        with open(testfile, "r+b") as f:
-            punch_hole(f, -1, 1)
-
-
-def test_fake_fallocate_only_punches_holes(testfile):
-    with pytest.raises(NotImplementedError):
-        with open(testfile, "r+b") as f:
-            _fake_fallocate(f, 0, 0, 10)
-
-
 def test_timeout(capsys):
     timeout = TimeOut(0.05, 0.01)
     while timeout.tick():