Extended profile data expiration from 7 to 30 days (#349)

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "redisbench-admin"
-version = "0.8.5"
+version = "0.8.6"
 description = "Redis benchmark run helper. A wrapper around Redis and Redis Modules benchmark tools ( ftsb_redisearch, memtier_benchmark, redis-benchmark, aibench, etc... )."
 authors = ["filipecosta90 <filipecosta.90@gmail.com>","Redis Performance Group <performance@redis.com>"]
 readme = "README.md"

redisbench_admin/run/grafana.py

@@ -9,8 +9,8 @@
 import pytablewriter
 from pytablewriter import MarkdownTableWriter
 
-# 7 days expire
-STALL_INFO_DAYS = 7
+# 30 days expire
+STALL_INFO_DAYS = 30
 EXPIRE_TIME_SECS_PROFILE_KEYS = 60 * 60 * 24 * STALL_INFO_DAYS
 EXPIRE_TIME_MSECS_PROFILE_KEYS = EXPIRE_TIME_SECS_PROFILE_KEYS * 1000
 
@@ -65,33 +65,21 @@ def generate_artifacts_table_grafana_redis(
     profile_markdown_str = profile_markdown_str.replace("\n", "")
     profile_id = "{}_{}_hash_{}".format(start_time_str, setup_name, tf_github_sha)
     profile_string_testcase_markdown_key = "profile:{}:{}".format(profile_id, test_name)
-    zset_profiles = "profiles:{}_{}_{}".format(
-        tf_github_org, tf_github_repo, setup_name
-    )
-    zset_profiles_setup = "profiles:setups:{}_{}".format(
-        tf_github_org,
-        tf_github_repo,
-    )
-    profile_set_redis_key = "profile:{}:testcases".format(profile_id)
-    zset_profiles_setups_testcases = "profiles:testcases:{}_{}_{}".format(
-        tf_github_org,
-        tf_github_repo,
-        setup_name,
-    )
-    zset_profiles_setups_testcases_profileid = "profiles:ids:{}_{}_{}_{}_{}".format(
-        tf_github_org,
-        tf_github_repo,
+    (
+        profile_set_redis_key,
+        zset_profiles,
+        zset_profiles_setup,
+        zset_profiles_setups_testcases,
+        zset_profiles_setups_testcases_branches,
+        zset_profiles_setups_testcases_branches_latest_link,
+        zset_profiles_setups_testcases_profileid,
+    ) = get_profile_zset_names(
+        profile_id,
         setup_name,
         test_name,
         tf_github_branch,
-    )
-    zset_profiles_setups_testcases_branches = "profiles:branches:{}_{}_{}_{}".format(
-        tf_github_org, tf_github_repo, setup_name, test_name
-    )
-    zset_profiles_setups_testcases_branches_latest_link = (
-        "latest_profiles:by.branch:{}_{}_{}_{}".format(
-            tf_github_org, tf_github_repo, setup_name, test_name
-        )
+        tf_github_org,
+        tf_github_repo,
     )
     https_link = "{}?var-org={}&var-repo={}&var-setup={}&var-branch={}".format(
         grafana_profile_dashboard,
@@ -103,13 +91,31 @@ def generate_artifacts_table_grafana_redis(
         test_name,
         profile_id,
     )
-    if push_results_redistimeseries:
+    if push_results_redistimeseries is True:
+        sorted_set_keys = [
+            zset_profiles,
+            zset_profiles_setups_testcases_profileid,
+            zset_profiles_setups_testcases,
+            zset_profiles_setup,
+            zset_profiles_setups_testcases_branches_latest_link,
+        ]
+        logging.info(
+            "Propulating the profile helper ZSETs: {}".format(" ".join(sorted_set_keys))
+        )
         current_time = time.time() * 1000
         timeframe_by_branch = current_time - EXPIRE_TIME_MSECS_PROFILE_KEYS
-        redis_conn.zadd(
+        res = redis_conn.zadd(
             zset_profiles_setups_testcases_branches,
             {tf_github_branch: start_time_ms},
         )
+        logging.info(
+            "Result of ZADD {} {} {} = {}".format(
+                zset_profiles_setups_testcases_branches,
+                start_time_ms,
+                tf_github_branch,
+                res,
+            )
+        )
         redis_conn.zadd(
             zset_profiles_setups_testcases_branches_latest_link,
             {https_link: start_time_ms},
@@ -130,14 +136,13 @@ def generate_artifacts_table_grafana_redis(
             zset_profiles,
             {profile_id: start_time_ms},
         )
-        sorted_set_keys = [
-            zset_profiles,
-            zset_profiles_setups_testcases_profileid,
-            zset_profiles_setups_testcases,
-            zset_profiles_setup,
-            zset_profiles_setups_testcases_branches_latest_link,
-        ]
+
         for keyname in sorted_set_keys:
+            logging.info(
+                "Expiring all elements with score between 0 and {}".format(
+                    int(timeframe_by_branch)
+                )
+            )
             redis_conn.zremrangebyscore(keyname, 0, int(timeframe_by_branch))
 
         redis_conn.sadd(profile_set_redis_key, test_name)
@@ -153,3 +158,45 @@ def generate_artifacts_table_grafana_redis(
             )
         )
     return https_link
+
+
+def get_profile_zset_names(
+    profile_id, setup_name, test_name, tf_github_branch, tf_github_org, tf_github_repo
+):
+    profile_set_redis_key = "profile:{}:testcases".format(profile_id)
+    zset_profiles = "profiles:{}_{}_{}".format(
+        tf_github_org, tf_github_repo, setup_name
+    )
+    zset_profiles_setup = "profiles:setups:{}_{}".format(
+        tf_github_org,
+        tf_github_repo,
+    )
+    zset_profiles_setups_testcases = "profiles:testcases:{}_{}_{}".format(
+        tf_github_org,
+        tf_github_repo,
+        setup_name,
+    )
+    zset_profiles_setups_testcases_profileid = "profiles:ids:{}_{}_{}_{}_{}".format(
+        tf_github_org,
+        tf_github_repo,
+        setup_name,
+        test_name,
+        tf_github_branch,
+    )
+    zset_profiles_setups_testcases_branches = "profiles:branches:{}_{}_{}_{}".format(
+        tf_github_org, tf_github_repo, setup_name, test_name
+    )
+    zset_profiles_setups_testcases_branches_latest_link = (
+        "latest_profiles:by.branch:{}_{}_{}_{}".format(
+            tf_github_org, tf_github_repo, setup_name, test_name
+        )
+    )
+    return (
+        profile_set_redis_key,
+        zset_profiles,
+        zset_profiles_setup,
+        zset_profiles_setups_testcases,
+        zset_profiles_setups_testcases_branches,
+        zset_profiles_setups_testcases_branches_latest_link,
+        zset_profiles_setups_testcases_profileid,
+    )

tests/test_grafana.py

@@ -0,0 +1,97 @@
+#  BSD 3-Clause License
+#
+#  Copyright (c) 2022., Redis Labs Modules
+#  All rights reserved.
+#
+import os
+
+import redis
+
+from redisbench_admin.run.grafana import (
+    generate_artifacts_table_grafana_redis,
+    get_profile_zset_names,
+)
+
+
+def test_generate_artifacts_table_grafana_redis():
+    rts_host = os.getenv("RTS_DATASINK_HOST", None)
+    rts_port = 16379
+    if rts_host is None:
+        assert False
+    redis_conn = redis.Redis(port=rts_port, host=rts_host, decode_responses=True)
+    redis_conn.ping()
+    redis_conn.flushall()
+    push_results_redistimeseries = True
+    grafana_profile_dashboard = (
+        "https://benchmarksrediscom.grafana.net/d/uRPZar57k/ci-profiler-viewer"
+    )
+    setup_name = "oss-standalone"
+    test_name = (
+        "memtier_benchmark-1Mkeys-load-stream-1-fields-with-100B-values-pipeline-10"
+    )
+    start_time_ms = 1650018944037
+    start_time_str = "2022-04-22-10-34-25"
+    tf_github_org = "redis"
+    tf_github_repo = "redis"
+    tf_github_sha = "96c8751069a89ecfa62e4291d8f879882bc0f0aa"
+    tf_github_branch = "unstable"
+    profile_artifacts = [
+        {
+            "artifact_name": "artifact 1",
+            "s3_link": "s3:212312",
+        }
+    ]
+    generate_artifacts_table_grafana_redis(
+        push_results_redistimeseries,
+        grafana_profile_dashboard,
+        profile_artifacts,
+        redis_conn,
+        setup_name,
+        start_time_ms,
+        start_time_str,
+        test_name,
+        tf_github_org,
+        tf_github_repo,
+        tf_github_sha,
+        tf_github_branch,
+    )
+    profile_id = "{}_{}_hash_{}".format(start_time_str, setup_name, tf_github_sha)
+    (
+        profile_set_redis_key,
+        zset_profiles,
+        zset_profiles_setup,
+        zset_profiles_setups_testcases,
+        zset_profiles_setups_testcases_branches,
+        zset_profiles_setups_testcases_branches_latest_link,
+        zset_profiles_setups_testcases_profileid,
+    ) = get_profile_zset_names(
+        profile_id,
+        setup_name,
+        test_name,
+        tf_github_branch,
+        tf_github_org,
+        tf_github_repo,
+    )
+    assert redis_conn.exists(zset_profiles)
+    assert redis_conn.exists(profile_set_redis_key)
+    assert redis_conn.exists(zset_profiles_setup)
+    assert redis_conn.exists(zset_profiles_setups_testcases)
+    assert redis_conn.exists(zset_profiles_setups_testcases_branches)
+    assert redis_conn.exists(zset_profiles_setups_testcases_branches_latest_link)
+    assert redis_conn.exists(zset_profiles_setups_testcases_profileid)
+    assert redis_conn.type(zset_profiles) == "zset"
+    assert redis_conn.zcard(zset_profiles) == 1
+    assert redis_conn.zcard(zset_profiles_setup) == 1
+    assert redis_conn.zcard(zset_profiles_setups_testcases) == 1
+    assert redis_conn.zcard(zset_profiles_setups_testcases_branches) == 1
+    assert redis_conn.zcard(zset_profiles_setups_testcases_branches_latest_link) == 1
+    assert redis_conn.zcard(zset_profiles_setups_testcases_profileid) == 1
+    assert redis_conn.zrangebyscore(
+        zset_profiles_setups_testcases, start_time_ms - 1, start_time_ms + 1
+    ) == [test_name]
+    assert redis_conn.zrangebyscore(
+        zset_profiles_setup, start_time_ms - 1, start_time_ms + 1
+    ) == [setup_name]
+    assert redis_conn.zrangebyscore(
+        zset_profiles_setups_testcases_branches, start_time_ms - 1, start_time_ms + 1
+    ) == [tf_github_branch]