-
Notifications
You must be signed in to change notification settings - Fork 5
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
* feature/callback: add callback in to run_remote logic * feature/callback: test fix for gnupg * feature/callback: remove cat gpg.conf command from action * feature/callback: try another way of fix gnupg problem * feature/callback: remove actions-setup-docker * feature/callback: different poetry install * feature/callback: shell way to install poetry * feature/callback: try another actions lib to install poetry * feature/callback: fix file * feature/callback: poetry version * feature/callback: test ci without python3.6 * feature/callback: fix formatting * feature/callback: fix what flake8 found * feature/callback: add callback url * feature/callback: change github env variable name to avoid problem with github actions env * feature/callback: remove branch, workflow id from request, make github token not required * feature/callback: fix callback url formatting * feature/callback: callback_headers statement in function * feature/async-run: add async run * feature/async-run: private key fix * feature/async-run: return actual poetry lock * feature/async-run: delete poetry.lock file * feature/async-run: poetry.lock from master * feature/async-run: remove callback action * feature/async-run: fix path * feature/async-run: fix path * feature/async-run: fix path for python * feature/async-run: command line, terraform install * feature/async-run: add sudo * feature/async-run: add /dev/null redirect for useless stdout from gpg * feature/async-run: add copy of private key in right directory * feature/async-run: rename pem file naming, file writing change * feature/async-run: fixing pem formatting * feature/async-run: module path * feature/async-run: module copy fix and module path fix * feature/async-run: docker redis, add info in output of copy files * feature/async-run: docker redis install fix * feature/async-run: fix for branches with slash * feature/async-run: fix for branches with slash, add sudo to docker run * feature/async-run: remove fix for branch with slashes * feature/async-run: add run for service, add github token pass to run variables * feature/async-run: token env variable pass, code format * feature/async-run: remove duplicate code * feature/async-run: testing infrastructure repo url * feature/async-run: cleanup code * feature/async-run: cleanup and fix diff between master * feature/async-run: remove unused files * feature/async-run: fix formatting * feature/async-run: flake8 formatting * feature/async-run: fix commands * feature/async-run: fix commands * feature/async-run: return commands * feature/async-run: add job name env pass * feature/async-run: add job name parameter * Updated run-async not to run stack * Bumping version from 0.9.55 to 0.10.0 --------- Co-authored-by: filipe oliveira <filipecosta.90@gmail.com>
- Loading branch information
1 parent
ed20d5c
commit afab8fe
Showing
9 changed files
with
1,021 additions
and
1 deletion.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,5 @@ | ||
# Apache License Version 2.0 | ||
# | ||
# Copyright (c) 2021., Redis Labs Modules | ||
# All rights reserved. | ||
# |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,91 @@ | ||
# BSD 3-Clause License | ||
# | ||
# Copyright (c) 2021., Redis Labs Modules | ||
# All rights reserved. | ||
# | ||
import logging | ||
import tarfile | ||
|
||
from redisbench_admin.run_async.terraform import ( | ||
retrieve_inventory_info, | ||
terraform_spin_or_reuse_env, | ||
) | ||
|
||
|
||
def tar_files(filename="archive.tar", path="./../../"): | ||
logging.info("Creating tar archive: {} for path {}".format(filename, path)) | ||
with tarfile.open(filename, "w") as tar: | ||
tar.add(path, arcname=".") | ||
return filename | ||
|
||
|
||
def async_env_setup( | ||
args, | ||
benchmark_config, | ||
remote_envs, | ||
repetition, | ||
test_name, | ||
tf_bin_path, | ||
tf_github_actor, | ||
tf_github_org, | ||
tf_github_repo, | ||
tf_github_sha, | ||
tf_setup_name_sufix, | ||
tf_triggering_env, | ||
tf_timeout_secs=7200, | ||
tf_override_name=None, | ||
tf_folder_path=None, | ||
): | ||
server_plaintext_port = args.db_port | ||
db_ssh_port = args.db_ssh_port | ||
client_ssh_port = args.client_ssh_port | ||
username = args.user | ||
if args.inventory is not None: | ||
( | ||
status, | ||
client_public_ip, | ||
server_private_ip, | ||
server_public_ip, | ||
) = retrieve_inventory_info(args.inventory) | ||
if status is False: | ||
logging.error( | ||
"Missing one of the required keys for inventory usage. Exiting..." | ||
) | ||
exit(1) | ||
logging.info("Using the following connection addresses.") | ||
logging.info("client_public_ip={}".format(client_public_ip)) | ||
logging.info("server_public_ip={}".format(server_public_ip)) | ||
logging.info("server_private_ip={}".format(server_private_ip)) | ||
else: | ||
( | ||
client_public_ip, | ||
_, | ||
_, | ||
server_private_ip, | ||
server_public_ip, | ||
username, | ||
) = terraform_spin_or_reuse_env( | ||
benchmark_config, | ||
remote_envs, | ||
repetition, | ||
test_name, | ||
tf_bin_path, | ||
tf_github_actor, | ||
tf_github_org, | ||
tf_github_repo, | ||
tf_github_sha, | ||
tf_setup_name_sufix, | ||
tf_triggering_env, | ||
tf_timeout_secs, | ||
tf_override_name, | ||
tf_folder_path, | ||
) | ||
return ( | ||
client_public_ip, | ||
server_plaintext_port, | ||
server_private_ip, | ||
server_public_ip, | ||
db_ssh_port, | ||
client_ssh_port, | ||
username, | ||
) |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,49 @@ | ||
from redisbench_admin.utils.benchmark_config import ( | ||
prepare_benchmark_definitions, | ||
process_benchmark_definitions_remote_timeouts, | ||
) | ||
|
||
|
||
class BenchmarkClass: | ||
def __init__( | ||
self, | ||
benchmark_defs_result=None, | ||
benchmark_definitions=None, | ||
default_metrics=None, | ||
exporter_timemetric_path=None, | ||
default_specs=None, | ||
clusterconfig=None, | ||
remote_envs_timeout=None, | ||
benchmark_runs_plan=None, | ||
): | ||
self.benchmark_defs_result = benchmark_defs_result | ||
self.benchmark_definitions = benchmark_definitions | ||
self.default_metrics = default_metrics | ||
self.exporter_timemetric_path = exporter_timemetric_path | ||
self.default_specs = default_specs | ||
self.clusterconfig = clusterconfig | ||
self.benchmark_artifacts_table_headers = [ | ||
"Setup", | ||
"Test-case", | ||
"Artifact", | ||
"link", | ||
] | ||
self.benchmark_artifacts_table_name = "Benchmark client artifacts" | ||
self.benchmark_runs_plan = benchmark_runs_plan | ||
|
||
self.remote_envs_timeout = remote_envs_timeout | ||
|
||
def prepare_benchmark_definitions(self, args): | ||
( | ||
self.benchmark_defs_result, | ||
self.benchmark_definitions, | ||
self.default_metrics, | ||
self.exporter_timemetric_path, | ||
self.default_specs, | ||
self.clusterconfig, | ||
) = prepare_benchmark_definitions(args) | ||
|
||
def populate_remote_envs_timeout(self): | ||
self.remote_envs_timeout = process_benchmark_definitions_remote_timeouts( | ||
self.benchmark_definitions | ||
) |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,5 @@ | ||
# BSD 3-Clause License | ||
# | ||
# Copyright (c) 2021., Redis Labs Modules | ||
# All rights reserved. | ||
# |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,79 @@ | ||
import jinja2 | ||
|
||
|
||
def renderServiceFile(access_key, region, secret_key, gh_token, job_name, args, argv): | ||
environment = jinja2.Environment() | ||
template = environment.from_string( | ||
"""[Unit] | ||
Description=Redisbench-admin run service | ||
[Service] | ||
Type=oneshot | ||
Environment="PATH=/home/ubuntu/.local/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" | ||
Environment="AWS_ACCESS_KEY_ID={{access_key}}" | ||
Environment="AWS_DEFAULT_REGION={{region}}" | ||
Environment="AWS_SECRET_ACCESS_KEY={{secret_key}}" | ||
{% if gh_token != None %}Environment="GH_TOKEN={{gh_token}}"{% endif %} | ||
{% if job_name != None %}Environment="CIRCLE_JOB={{job_name}}"{% endif %} | ||
WorkingDirectory=/home/ubuntu/work_dir/tests/benchmarks | ||
Type=simple | ||
User=ubuntu | ||
ExecStart=/home/ubuntu/work_dir/redisbench-admin/.venv/bin/python /home/ubuntu/work_dir/redisbench-admin/run.py run-remote {{args}} | ||
[Install] | ||
WantedBy=multi-user.target | ||
""" | ||
) | ||
if "--private_key" not in argv: | ||
argv.append("--private_key") | ||
argv.append("/home/ubuntu/work_dir/tests/benchmarks/benchmarks.redislabs.pem") | ||
else: | ||
argv[ | ||
argv.index(args.private_key) | ||
] = "/home/ubuntu/work_dir/tests/benchmarks/benchmarks.redislabs.pem" | ||
if len(args.module_path) != 0: | ||
argv[argv.index(args.module_path[0])] = ( | ||
"/home/ubuntu/work_dir/tests/benchmarks/" | ||
+ args.module_path[0].split("/")[-1] | ||
) | ||
argv_str = " ".join(argv) | ||
with open("redisbench-admin.service", mode="w", encoding="utf-8") as results: | ||
results.write( | ||
template.render( | ||
access_key=access_key, | ||
region=region, | ||
secret_key=secret_key, | ||
gh_token=gh_token, | ||
job_name=job_name, | ||
args=argv_str, | ||
) | ||
) | ||
|
||
|
||
def renderRunFile(): | ||
with open("run.py", mode="w", encoding="utf-8") as run_file: | ||
run_file.write( | ||
"""#!/usr/local/bin/python | ||
# -*- coding: utf-8 -*- | ||
import re | ||
import sys | ||
from redisbench_admin.cli import main | ||
if __name__ == "__main__": | ||
sys.argv[0] = re.sub(r"(-script\\.pyw|\\.exe)?$", "", sys.argv[0]) | ||
sys.exit(main()) | ||
""" | ||
) | ||
|
||
|
||
def savePemFile(pem_data): | ||
with open("benchmarks.redislabs.pem", mode="w", encoding="utf-8") as pem_file: | ||
pem_data = pem_data.replace("-----BEGIN RSA PRIVATE KEY-----", "") | ||
pem_data = pem_data.replace("-----END RSA PRIVATE KEY-----", "") | ||
pem_data = pem_data.replace(" ", "\n") | ||
pem_data = ( | ||
"-----BEGIN RSA PRIVATE KEY-----" | ||
+ pem_data | ||
+ "-----END RSA PRIVATE KEY-----" | ||
) | ||
pem_file.write(pem_data) |
Oops, something went wrong.