Skip to content

Commit a718ba8

Browse files
filipecosta90filipecosta90
authored
Extended profile data expiration from 7 to 30 days (#349)
1 parent f332159 commit a718ba8

File tree

3 files changed

+180
-36
lines changed

3 files changed

+180
-36
lines changed

‎pyproject.toml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
[tool.poetry]
22
name = "redisbench-admin"
3-
version = "0.8.5"
3+
version = "0.8.6"
44
description = "Redis benchmark run helper. A wrapper around Redis and Redis Modules benchmark tools ( ftsb_redisearch, memtier_benchmark, redis-benchmark, aibench, etc... )."
55
authors = ["filipecosta90 <filipecosta.90@gmail.com>","Redis Performance Group <performance@redis.com>"]
66
readme = "README.md"

‎redisbench_admin/run/grafana.py

Lines changed: 82 additions & 35 deletions
Original file line numberDiff line numberDiff line change
@@ -9,8 +9,8 @@
99
import pytablewriter
1010
from pytablewriter import MarkdownTableWriter
1111

12-
# 7 days expire
13-
STALL_INFO_DAYS = 7
12+
# 30 days expire
13+
STALL_INFO_DAYS = 30
1414
EXPIRE_TIME_SECS_PROFILE_KEYS = 60 * 60 * 24 * STALL_INFO_DAYS
1515
EXPIRE_TIME_MSECS_PROFILE_KEYS = EXPIRE_TIME_SECS_PROFILE_KEYS * 1000
1616

@@ -65,33 +65,21 @@ def generate_artifacts_table_grafana_redis(
6565
profile_markdown_str = profile_markdown_str.replace("\n", "")
6666
profile_id = "{}_{}_hash_{}".format(start_time_str, setup_name, tf_github_sha)
6767
profile_string_testcase_markdown_key = "profile:{}:{}".format(profile_id, test_name)
68-
zset_profiles = "profiles:{}_{}_{}".format(
69-
tf_github_org, tf_github_repo, setup_name
70-
)
71-
zset_profiles_setup = "profiles:setups:{}_{}".format(
72-
tf_github_org,
73-
tf_github_repo,
74-
)
75-
profile_set_redis_key = "profile:{}:testcases".format(profile_id)
76-
zset_profiles_setups_testcases = "profiles:testcases:{}_{}_{}".format(
77-
tf_github_org,
78-
tf_github_repo,
79-
setup_name,
80-
)
81-
zset_profiles_setups_testcases_profileid = "profiles:ids:{}_{}_{}_{}_{}".format(
82-
tf_github_org,
83-
tf_github_repo,
68+
(
69+
profile_set_redis_key,
70+
zset_profiles,
71+
zset_profiles_setup,
72+
zset_profiles_setups_testcases,
73+
zset_profiles_setups_testcases_branches,
74+
zset_profiles_setups_testcases_branches_latest_link,
75+
zset_profiles_setups_testcases_profileid,
76+
) = get_profile_zset_names(
77+
profile_id,
8478
setup_name,
8579
test_name,
8680
tf_github_branch,
87-
)
88-
zset_profiles_setups_testcases_branches = "profiles:branches:{}_{}_{}_{}".format(
89-
tf_github_org, tf_github_repo, setup_name, test_name
90-
)
91-
zset_profiles_setups_testcases_branches_latest_link = (
92-
"latest_profiles:by.branch:{}_{}_{}_{}".format(
93-
tf_github_org, tf_github_repo, setup_name, test_name
94-
)
81+
tf_github_org,
82+
tf_github_repo,
9583
)
9684
https_link = "{}?var-org={}&var-repo={}&var-setup={}&var-branch={}".format(
9785
grafana_profile_dashboard,
@@ -103,13 +91,31 @@ def generate_artifacts_table_grafana_redis(
10391
test_name,
10492
profile_id,
10593
)
106-
if push_results_redistimeseries:
94+
if push_results_redistimeseries is True:
95+
sorted_set_keys = [
96+
zset_profiles,
97+
zset_profiles_setups_testcases_profileid,
98+
zset_profiles_setups_testcases,
99+
zset_profiles_setup,
100+
zset_profiles_setups_testcases_branches_latest_link,
101+
]
102+
logging.info(
103+
"Propulating the profile helper ZSETs: {}".format(" ".join(sorted_set_keys))
104+
)
107105
current_time = time.time() * 1000
108106
timeframe_by_branch = current_time - EXPIRE_TIME_MSECS_PROFILE_KEYS
109-
redis_conn.zadd(
107+
res = redis_conn.zadd(
110108
zset_profiles_setups_testcases_branches,
111109
{tf_github_branch: start_time_ms},
112110
)
111+
logging.info(
112+
"Result of ZADD {} {} {} = {}".format(
113+
zset_profiles_setups_testcases_branches,
114+
start_time_ms,
115+
tf_github_branch,
116+
res,
117+
)
118+
)
113119
redis_conn.zadd(
114120
zset_profiles_setups_testcases_branches_latest_link,
115121
{https_link: start_time_ms},
@@ -130,14 +136,13 @@ def generate_artifacts_table_grafana_redis(
130136
zset_profiles,
131137
{profile_id: start_time_ms},
132138
)
133-
sorted_set_keys = [
134-
zset_profiles,
135-
zset_profiles_setups_testcases_profileid,
136-
zset_profiles_setups_testcases,
137-
zset_profiles_setup,
138-
zset_profiles_setups_testcases_branches_latest_link,
139-
]
139+
140140
for keyname in sorted_set_keys:
141+
logging.info(
142+
"Expiring all elements with score between 0 and {}".format(
143+
int(timeframe_by_branch)
144+
)
145+
)
141146
redis_conn.zremrangebyscore(keyname, 0, int(timeframe_by_branch))
142147

143148
redis_conn.sadd(profile_set_redis_key, test_name)
@@ -153,3 +158,45 @@ def generate_artifacts_table_grafana_redis(
153158
)
154159
)
155160
return https_link
161+
162+
163+
def get_profile_zset_names(
164+
profile_id, setup_name, test_name, tf_github_branch, tf_github_org, tf_github_repo
165+
):
166+
profile_set_redis_key = "profile:{}:testcases".format(profile_id)
167+
zset_profiles = "profiles:{}_{}_{}".format(
168+
tf_github_org, tf_github_repo, setup_name
169+
)
170+
zset_profiles_setup = "profiles:setups:{}_{}".format(
171+
tf_github_org,
172+
tf_github_repo,
173+
)
174+
zset_profiles_setups_testcases = "profiles:testcases:{}_{}_{}".format(
175+
tf_github_org,
176+
tf_github_repo,
177+
setup_name,
178+
)
179+
zset_profiles_setups_testcases_profileid = "profiles:ids:{}_{}_{}_{}_{}".format(
180+
tf_github_org,
181+
tf_github_repo,
182+
setup_name,
183+
test_name,
184+
tf_github_branch,
185+
)
186+
zset_profiles_setups_testcases_branches = "profiles:branches:{}_{}_{}_{}".format(
187+
tf_github_org, tf_github_repo, setup_name, test_name
188+
)
189+
zset_profiles_setups_testcases_branches_latest_link = (
190+
"latest_profiles:by.branch:{}_{}_{}_{}".format(
191+
tf_github_org, tf_github_repo, setup_name, test_name
192+
)
193+
)
194+
return (
195+
profile_set_redis_key,
196+
zset_profiles,
197+
zset_profiles_setup,
198+
zset_profiles_setups_testcases,
199+
zset_profiles_setups_testcases_branches,
200+
zset_profiles_setups_testcases_branches_latest_link,
201+
zset_profiles_setups_testcases_profileid,
202+
)

‎tests/test_grafana.py

Lines changed: 97 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,97 @@
1+
# BSD 3-Clause License
2+
#
3+
# Copyright (c) 2022., Redis Labs Modules
4+
# All rights reserved.
5+
#
6+
import os
7+
8+
import redis
9+
10+
from redisbench_admin.run.grafana import (
11+
generate_artifacts_table_grafana_redis,
12+
get_profile_zset_names,
13+
)
14+
15+
16+
def test_generate_artifacts_table_grafana_redis():
17+
rts_host = os.getenv("RTS_DATASINK_HOST", None)
18+
rts_port = 16379
19+
if rts_host is None:
20+
assert False
21+
redis_conn = redis.Redis(port=rts_port, host=rts_host, decode_responses=True)
22+
redis_conn.ping()
23+
redis_conn.flushall()
24+
push_results_redistimeseries = True
25+
grafana_profile_dashboard = (
26+
"https://benchmarksrediscom.grafana.net/d/uRPZar57k/ci-profiler-viewer"
27+
)
28+
setup_name = "oss-standalone"
29+
test_name = (
30+
"memtier_benchmark-1Mkeys-load-stream-1-fields-with-100B-values-pipeline-10"
31+
)
32+
start_time_ms = 1650018944037
33+
start_time_str = "2022-04-22-10-34-25"
34+
tf_github_org = "redis"
35+
tf_github_repo = "redis"
36+
tf_github_sha = "96c8751069a89ecfa62e4291d8f879882bc0f0aa"
37+
tf_github_branch = "unstable"
38+
profile_artifacts = [
39+
{
40+
"artifact_name": "artifact 1",
41+
"s3_link": "s3:212312",
42+
}
43+
]
44+
generate_artifacts_table_grafana_redis(
45+
push_results_redistimeseries,
46+
grafana_profile_dashboard,
47+
profile_artifacts,
48+
redis_conn,
49+
setup_name,
50+
start_time_ms,
51+
start_time_str,
52+
test_name,
53+
tf_github_org,
54+
tf_github_repo,
55+
tf_github_sha,
56+
tf_github_branch,
57+
)
58+
profile_id = "{}_{}_hash_{}".format(start_time_str, setup_name, tf_github_sha)
59+
(
60+
profile_set_redis_key,
61+
zset_profiles,
62+
zset_profiles_setup,
63+
zset_profiles_setups_testcases,
64+
zset_profiles_setups_testcases_branches,
65+
zset_profiles_setups_testcases_branches_latest_link,
66+
zset_profiles_setups_testcases_profileid,
67+
) = get_profile_zset_names(
68+
profile_id,
69+
setup_name,
70+
test_name,
71+
tf_github_branch,
72+
tf_github_org,
73+
tf_github_repo,
74+
)
75+
assert redis_conn.exists(zset_profiles)
76+
assert redis_conn.exists(profile_set_redis_key)
77+
assert redis_conn.exists(zset_profiles_setup)
78+
assert redis_conn.exists(zset_profiles_setups_testcases)
79+
assert redis_conn.exists(zset_profiles_setups_testcases_branches)
80+
assert redis_conn.exists(zset_profiles_setups_testcases_branches_latest_link)
81+
assert redis_conn.exists(zset_profiles_setups_testcases_profileid)
82+
assert redis_conn.type(zset_profiles) == "zset"
83+
assert redis_conn.zcard(zset_profiles) == 1
84+
assert redis_conn.zcard(zset_profiles_setup) == 1
85+
assert redis_conn.zcard(zset_profiles_setups_testcases) == 1
86+
assert redis_conn.zcard(zset_profiles_setups_testcases_branches) == 1
87+
assert redis_conn.zcard(zset_profiles_setups_testcases_branches_latest_link) == 1
88+
assert redis_conn.zcard(zset_profiles_setups_testcases_profileid) == 1
89+
assert redis_conn.zrangebyscore(
90+
zset_profiles_setups_testcases, start_time_ms - 1, start_time_ms + 1
91+
) == [test_name]
92+
assert redis_conn.zrangebyscore(
93+
zset_profiles_setup, start_time_ms - 1, start_time_ms + 1
94+
) == [setup_name]
95+
assert redis_conn.zrangebyscore(
96+
zset_profiles_setups_testcases_branches, start_time_ms - 1, start_time_ms + 1
97+
) == [tf_github_branch]

0 commit comments

Comments
 (0)