Automation to set size for log rotate and parse dbg logs for specific strings

Suma R · Suma R · commit 63a67a9e370f · 2025-01-08T12:47:06.000+05:30
Signed-off-by: Suma R &lt;sumar@Sumas-MacBook-Pro.local&gt;
diff --git a/suites/squid/cephfs/tier-3_cephfs_system_test.yaml b/suites/squid/cephfs/tier-3_cephfs_system_test.yaml
@@ -215,7 +215,15 @@ tests:
       config:
        ENABLE_LOGS : 1
        daemon_list : ['mds','client','osd','mgr','mon']
-       daemon_dbg_level : {'mds':20,'client':20,'osd':10,'mgr':10,'mon':10}
+       daemon_dbg_level : {'mds':10,'client':10,'osd':5,'mgr':5,'mon':5}
+  - test:
+      abort-on-fail: false
+      desc: "Set size limit for log rotation"
+      name: size limit for log rotation
+      module: cephfs_logs_util.py
+      config:
+        LOG_ROTATE_SIZE : 1
+        log_size : '200M'
   -
     test:
       abort-on-fail: false
@@ -294,3 +302,12 @@ tests:
       config:
        DISABLE_LOGS : 1
        daemon_list : ['mds','client','osd','mgr','mon']
+  - 
+    test:
+      name: Parse dbg logs for specific strings
+      module: cephfs_logs_util.py
+      config:
+        LOG_PARSER : 1
+        daemon : 'mds'
+        expect_list : ['issue_new_caps','get_allowed_caps','"sending MClientCaps"','client_caps\(revoke']
+        unexpect_list: ['Exception','assert']
diff --git a/tests/cephfs/cephfs_logs_util.py b/tests/cephfs/cephfs_logs_util.py
@@ -1,6 +1,7 @@
 import os
 import traceback
 
+from tests.cephfs.cephfs_system.cephfs_system_utils import CephFSSystemUtils
 from tests.cephfs.cephfs_utilsV1 import FsUtils
 from utility.log import Log
 
@@ -9,8 +10,9 @@
 
 def run(ceph_cluster, **kw):
     """
-    This script is a wrapper to Logs enablement and Logs collection module available in cephfs_uitlsV1
-    It can be included prior to test case execution to enable debug logs and post testcase execution to collect logs,
+    This script is a wrapper to Logs enablement, and Logs collection module available in cephfs_uitlsV1, Log Rotation
+    for size limit available in cephfs_system_utils
+    It can be included prior to test case execution to enable debug logs and log rotation and post testcase execution to collect logs,
     PRETEST: To enable logs
     -----------------------
     - test:
@@ -19,6 +21,14 @@ def run(ceph_cluster, **kw):
       config:
        ENABLE_LOGS : 1
        daemon_dbg_level : {'mds':5}
+    PRETEST: To rotate logs after size limit
+    ---------------------------------------
+    - test:
+      name: Set size limit for log rotation
+      module: cephfs_logs_util.py
+      config:
+       LOGS_ROTATE_SIZE : 1
+       log_size : '200M'
     POSTTEST: To collect logs
     -------------------------
     - test:
@@ -35,13 +45,24 @@ def run(ceph_cluster, **kw):
       config:
        DISABLE_LOGS : 1
        daemon_list : ['mds']
+    POSTTEST: To parse debug logs for specific strings
+    -------------------------
+    - test:
+      name: parse debug logs for specific strings
+      module: cephfs_logs_util.py
+      config:
+       LOG_PARSER : 1
+       daemon : 'mds'
+       expect_list : ['issue_new_caps','get_allowed_caps','sending MClientCaps','client_caps(revoke']
+       unexpect_list: ['Exception','assert']
 
     This script will read input params ENABLE_LOGS,UPLOAD_LOGS and DISABLE_LOGS and invoke corresponding
     cephfs_utilsV1 module to perform the task. If UPLOAD_LOGS, script will print the path were logs are uploadded.
 
     """
     try:
         fs_util = FsUtils(ceph_cluster)
+        fs_sys_util = CephFSSystemUtils(ceph_cluster)
         config = kw.get("config")
         clients = ceph_cluster.get_ceph_objects("client")
         log.info("checking Pre-requisites")
@@ -56,8 +77,14 @@ def run(ceph_cluster, **kw):
         daemon_dbg_level = config.get("daemon_dbg_level", {"mds": 5})
         daemon_list = config.get("daemon_list", ["mds"])
         enable_logs = config.get("ENABLE_LOGS", 0)
+        log_rotate_size = config.get("LOG_ROTATE_SIZE", 0)
+        log_size = config.get("log_size", "200M")
         disable_logs = config.get("DISABLE_LOGS", 0)
         upload_logs = config.get("UPLOAD_LOGS", 0)
+        log_parser = config.get("LOG_PARSER", 0)
+        log_daemon = config.get("daemon")
+        expect_list = config.get("expect_list", [])
+        unexpect_list = config.get("unexpect_list", [])
         log_str = (
             f"Test Params : ENABLE_LOGS : {enable_logs}, UPLOAD_LOGS:{upload_logs}"
         )
@@ -69,7 +96,10 @@ def run(ceph_cluster, **kw):
             log.info(f"Enabling debug logs on daemons : {daemon_dbg_level}")
             if fs_util.enable_logs(client1, daemon_dbg_level):
                 assert False, "Enable logs failed"
-
+        if log_rotate_size == 1:
+            log.info(f"Enable log rotation after size limit : {log_size}")
+            if fs_sys_util.log_rotate_size(client1, size_str=log_size):
+                assert False, f"log rotate enablement for size {log_size} failed"
         if upload_logs == 1:
             log_dir = os.path.dirname(log.logger.handlers[0].baseFilename)
             log.info(f"log path:{log_dir}")
@@ -80,6 +110,14 @@ def run(ceph_cluster, **kw):
             log.info(f"Disabling debug logs on daemons : {daemon_list}")
             if fs_util.disable_logs(client1, daemon_list):
                 assert False, "Disable logs failed"
+        if log_parser == 1:
+            log.info(
+                f"Parse {log_daemon} dbg logs for expected {expect_list} and unexpected {unexpect_list}"
+            )
+            if fs_sys_util.log_parser(
+                client1, expect_list, unexpect_list, daemon=log_daemon
+            ):
+                assert False, f"{log_daemon} log parser failed"
 
         log.info("Testcase Results:")
         for res in results:
diff --git a/tests/cephfs/cephfs_system/cephfs_system_utils.py b/tests/cephfs/cephfs_system/cephfs_system_utils.py
@@ -25,6 +25,7 @@ def __init__(self, ceph_cluster):
         Args:
             ceph_cluster (ceph.ceph.Ceph): ceph cluster
         """
+        self.ceph_cluster = ceph_cluster
         self.mons = ceph_cluster.get_ceph_objects("mon")
         self.mgrs = ceph_cluster.get_ceph_objects("mgr")
         self.mdss = ceph_cluster.get_ceph_objects("mds")
@@ -273,3 +274,99 @@ def wait_for_two_active_mds(
                 time.sleep(retry_interval)  # Retry after the specified interval
 
         return False
+
+    def log_rotate_size(self, client, size_str="200M"):
+        """
+        This mutility will enable log rotation when debug log file size reached the limit mentioned in 'size_str'
+        Required_param : size_str, it should be size with units recognised by Ceph Cluster such as,
+        200M , 500M, 1G ...
+        """
+        out, rc = client.exec_command(sudo=True, cmd="ceph fsid -f json")
+        fsid_out = json.loads(out)
+        fsid = fsid_out["fsid"]
+        mds_nodes = self.ceph_cluster.get_ceph_objects("mds")
+        mgr_nodes = self.ceph_cluster.get_ceph_objects("mgr")
+        osd_nodes = self.ceph_cluster.get_ceph_objects("osd")
+        mon_nodes = self.ceph_cluster.get_ceph_objects("mon")
+        log_rotate_file = f"/etc/logrotate.d/ceph-{fsid}"
+        log_rotate_file_bkp = f"/etc/logrotate.d/ceph-{fsid}.backup"
+        log_rotate_tmp = "/home/cephuser/log_rotate_tmp"
+        crontab_str = f"2 * * * * /usr/sbin/logrotate {log_rotate_file} >/dev/null 2>&1"
+        log_complete = []
+        for log_node_list in [mds_nodes, mgr_nodes, osd_nodes, mon_nodes]:
+            for log_node in log_node_list:
+                if log_node.node.hostname not in log_complete:
+                    # on each node
+                    cmd = f"cp {log_rotate_file} {log_rotate_file_bkp}"
+                    out, _ = log_node.exec_command(sudo=True, cmd=cmd)
+                    cmd = f"sed '/compress/i \    \size {size_str}' {log_rotate_file} > {log_rotate_tmp}"
+                    out, _ = log_node.exec_command(sudo=True, cmd=cmd)
+                    cmd = f"yes | cp {log_rotate_tmp} {log_rotate_file}"
+                    out, _ = log_node.exec_command(sudo=True, cmd=cmd)
+                    cmd = f"echo {crontab_str} > /home/cephuser/log_cron_file"
+                    out, _ = log_node.exec_command(sudo=True, cmd=cmd)
+                    cmd = f"crontab /home/cephuser/log_cron_file"
+                    out, _ = log_node.exec_command(sudo=True, cmd=cmd)
+                    log_complete.append(log_node.node.hostname)
+        return 0
+
+    def log_parser(self, client, expect_list, unexpect_list, daemon="mds"):
+        """
+        This utility parsers through daemon debug logs mentioned in daemon_list and checks for
+        expected and unexpected strings in logs.
+        If expected strings found and unexpected strings not found, return pass as 0
+        If unexpected strrings found and expected strings not found, return fail as 1
+        Example usage:
+        expect_list = ['issue_new_caps','get_allowed_caps','sending MClientCaps','client_caps(revoke']
+        unexpect_list = ['Exception','assert']
+        log_parser(expect_list,unexpect_list)
+        """
+        out, rc = client.exec_command(sudo=True, cmd="ceph fsid -f json")
+        fsid_out = json.loads(out)
+        fsid = fsid_out["fsid"]
+        daemon_nodes = self.ceph_cluster.get_ceph_objects(daemon)
+        log_path = f"/var/log/ceph/{fsid}"
+        results = {"expect": {}, "unexpect": {}}
+        for node in daemon_nodes:
+            for search_str in expect_list:
+                cmd = f"grep {search_str} {log_path}/*{daemon}*"
+                try:
+                    out = node.exec_command(sudo=True, cmd=cmd)
+                    if len(out) > 0:
+                        log.info(
+                            f"Found {search_str} in {daemon} log in {log_path} on {node.node.hostname}:\n {out}"
+                        )
+                        results["expect"].update({search_str: node})
+                except BaseException as ex:
+                    log.info(ex)
+            for search_str in unexpect_list:
+                cmd = f"grep {search_str} {log_path}/*{daemon}*"
+                try:
+                    out, _ = node.exec_command(sudo=True, cmd=cmd)
+                    if len(out) > 0:
+                        log.error(
+                            f"Found {search_str} in {daemon} log in {log_path} on {node.node.hostname}:\n {out}"
+                        )
+                        results["unexpect"].update({search_str: node})
+                except BaseException as ex:
+                    log.info(ex)
+        expect_not_found = []
+        unexpect_found = []
+        for exp_str in expect_list:
+            if exp_str not in results["expect"]:
+                expect_not_found.append(exp_str)
+        for unexp_str in unexpect_list:
+            if unexp_str in results["expect"]:
+                unexpect_found.append(unexp_str)
+        test_status = 0
+        if len(expect_not_found):
+            log.error(
+                f"Some of expected strings not found in debug logs for daemon {daemon}:{expect_not_found}"
+            )
+            test_status = 1
+        if len(unexpect_found):
+            log.error(
+                f"Some of unexpected strings found in debug logs for daemon {daemon}:{unexpect_found}"
+            )
+            test_status = 1
+        return test_status
