Add fsfreeze related integration test

Longhorn 2187

Signed-off-by: Eric Weber <eric.weber@suse.com>

Dockerfile.dapper

@@ -12,7 +12,10 @@ ENV PATH /go/bin:$PATH
 ENV DAPPER_DOCKER_SOCKET true
 ENV DAPPER_ENV TAG REPO DRONE_REPO DRONE_PULL_REQUEST DRONE_COMMIT_REF SKIP_TASKS
 ENV DAPPER_OUTPUT bin coverage.out
-ENV DAPPER_RUN_ARGS --privileged --tmpfs /go/src/github.com/longhorn/longhorn-engine/integration/.venv:exec --tmpfs /go/src/github.com/longhorn/longhorn-engine/integration/.tox:exec -v /dev:/host/dev -v /proc:/host/proc
+# For file system freeze tests, our container must be able to see the file system mounted in the host mount namespace.
+# Usually, instance-manager runs with the equivalent of "-v /:/host". For integration testing, use 
+# "-v /tmp/mnt:host/tmp/mnt" and mount file systems to /tmp/mnt to simulate this without bind mounting everything.
+ENV DAPPER_RUN_ARGS --privileged --tmpfs /go/src/github.com/longhorn/longhorn-engine/integration/.venv:exec --tmpfs /go/src/github.com/longhorn/longhorn-engine/integration/.tox:exec -v /dev:/host/dev -v /proc:/host/proc --mount type=bind,source=/tmp/mnt,destination=/host/tmp/mnt,bind-propagation=rslave
 ENV DAPPER_SOURCE /go/src/github.com/longhorn/longhorn-engine
 WORKDIR ${DAPPER_SOURCE}
 

integration/common/cmd.py

@@ -27,8 +27,9 @@ def info_get(url):
     return json.loads(subprocess.check_output(cmd, encoding='utf-8'))
 
 
-def snapshot_create(url):
-    cmd = [_bin(), '--url', url, '--debug', 'snapshot', 'create']
+def snapshot_create(url, freeze_fs = False):
+    cmd = [_bin(), '--url', url, '--debug', 'snapshot', 'create',
+           f'--freeze-fs={freeze_fs}']
     return subprocess.check_output(cmd, encoding='utf-8').strip()
 
 

integration/common/constants.py

@@ -79,3 +79,5 @@
 EXPANSION_DISK_NAME = "volume-snap-expand-%d.img" % EXPANDED_SIZE
 
 MESSAGE_TYPE_ERROR = "error"
+
+LOGS_DIR = "/var/log/instances/"

integration/common/util.py

@@ -1,6 +1,7 @@
 import os
 import hashlib
 
+from common.constants import LOGS_DIR
 
 def file(f):
     return os.path.join(_base(), '../../{}'.format(f))
@@ -37,3 +38,10 @@ def read_file(file_path, offset, length):
 
 def checksum_data(data):
     return hashlib.sha512(data).hexdigest()
+
+def get_process_log_lines(process_name):
+    file_path = LOGS_DIR + process_name + '.log'
+    assert os.path.exists(file_path)
+    with open(file_path, 'r') as file:
+        lines = file.readlines()
+        return lines

integration/data/test_snapshot.py

@@ -28,6 +28,8 @@
     RETRY_COUNTS_SHORT, RETRY_INTERVAL_SHORT
 )
 
+from common.util import get_process_log_lines
+
 from data.snapshot_tree import snapshot_tree_build, snapshot_tree_verify_node
 
 
@@ -402,7 +404,7 @@ def test_snapshot_mounted_filesystem(grpc_controller,  # NOQA
     6. Take snapshot 2 (test file included)
     7. Overwrite test file on the filesystem
     8. Take snapshot 3 (test file changed)
-    9. Observe that logs of engine `Filesystem frozen / unfrozen`
+    9. Observe that logs of engine include "Freezing" and "Unfreezing"
     10. Validate snapshots are created
     11. Restore snapshot 1, validate test file missing
     12. Restore snapshot 2, validate test file original content
@@ -415,7 +417,10 @@ def test_snapshot_mounted_filesystem(grpc_controller,  # NOQA
 
 def snapshot_mounted_filesystem_test(volume_name, dev, address, engine_name):  # NOQA
     dev_path = dev.dev
-    mnt_path = "/tmp/mnt-" + volume_name
+    # /tmp/mnt is bind mounted into the container at /host/tmp/mnt. This
+    # simulates the typical running environment of instance-manager with
+    # "-v /:/host".
+    mnt_path = "/tmp/mnt/" + volume_name
     test_file = mnt_path + "/test"
     length = 128
 
@@ -435,15 +440,29 @@ def snapshot_mounted_filesystem_test(volume_name, dev, address, engine_name):  #
     # create snapshot1 with empty fs
     # NOTE: we cannot use checksum_dev since it assumes
     #  asci data for device data instead of raw bytes
-    snap1 = cmd.snapshot_create(address)
+    snap1 = cmd.snapshot_create(address, True)
 
     # create snapshot2 with a new test file
     test2_checksum = write_filesystem_file(length, test_file)
-    snap2 = cmd.snapshot_create(address)
+    snap2 = cmd.snapshot_create(address, True)
 
     # create snapshot3 overwriting the test file
     test3_checksum = write_filesystem_file(length, test_file)
-    snap3 = cmd.snapshot_create(address)
+    snap3 = cmd.snapshot_create(address, True)
+
+    # Observe that logs of engine include "Freezing" and "Unfreezing"
+    freeze_lines_count = unfreeze_lines_count = 0
+    for line in get_process_log_lines(engine_name):
+        if 'Freezing file system' in line:
+            freeze_lines_count += 1
+        if 'Unfreezing file system' in line:
+            unfreeze_lines_count += 1
+    # We only log these if we have detected a mounted Longhorn file system,
+    # done a bind mount, and are immediately preparing to freeze. If we did
+    # this for all three snapshots (and the snapshots themselves didn't fail),
+    # it's a good guarantee that whole of the freezing logic is working.
+    assert freeze_lines_count == 3
+    assert unfreeze_lines_count == 3
 
     # verify existence of the snapshots
     snapshots = cmd.snapshot_ls(address)