From 3173757a4170c50548090060d98bae78dfbadd76 Mon Sep 17 00:00:00 2001 From: MaxGelbakhiani Date: Tue, 12 Mar 2024 13:45:49 +0100 Subject: [PATCH 1/3] Added replace_objects and replace_container_contents options Fixes #2 and #15 Signed-off-by: MaxGelbakhiani --- action.yml | 11 +++ push-to-neofs.py | 227 ++++++++++++++++++++++++++++++++++++++--------- 2 files changed, 196 insertions(+), 42 deletions(-) diff --git a/action.yml b/action.yml index 467a52e..848bc78 100644 --- a/action.yml +++ b/action.yml @@ -35,6 +35,14 @@ inputs: description: Number of epochs for object to stay valid required: false default: 0 + REPLACE_OBJECTS: + description: Replace existing objects with the same attributes in the container + required: false + default: 'True' + REPLACE_CONTAINER_CONTENTS: + description: Remove all the old existing objects in the container after the new objects are uploaded + required: false + default: 'False' outputs: OUTPUT_CONTAINER_URL: @@ -93,11 +101,14 @@ runs: NEOFS_ATTRIBUTES: ${{ inputs.NEOFS_ATTRIBUTES }} URL_PREFIX: ${{ inputs.URL_PREFIX }} LIFETIME: ${{ inputs.LIFETIME }} + REPLACE_OBJECTS: ${{ inputs.REPLACE_OBJECTS }} + REPLACE_CONTAINER_CONTENTS: ${{ inputs.REPLACE_CONTAINER_CONTENTS }} GITHUB_ACTION_PATH: ${{ github.action_path }} run: | source "$GITHUB_ACTION_PATH/venv/bin/activate" && NEOFS_CLI_PASSWORD=$NEOFS_WALLET_PASSWORD python "$GITHUB_ACTION_PATH/push-to-neofs.py" \ --lifetime "$LIFETIME" --neofs_domain "$NEOFS_NETWORK_DOMAIN" --attributes "$NEOFS_ATTRIBUTES" \ --cid "$STORE_OBJECTS_CID" --files-dir "$PATH_TO_FILES_DIR" --url_path_prefix "$URL_PREFIX" \ + --replace-objects "$REPLACE_OBJECTS" --replace-container-contents "$REPLACE_CONTAINER_CONTENTS" \ --wallet "$GITHUB_ACTION_PATH/wallet.json" BASE_URL="https://$NEOFS_HTTP_GATE/$STORE_OBJECTS_CID" if [ -z "$URL_PREFIX" ]; then diff --git a/push-to-neofs.py b/push-to-neofs.py index c32cb60..0c75de8 100644 --- a/push-to-neofs.py +++ b/push-to-neofs.py @@ -1,7 +1,10 @@ import os +import re import subprocess import argparse import magic +import json +import distutils.util FILE_PATH = "FilePath" # the key for the attribute, is the path for the static page and allure report zip files CONTENT_TYPE = "ContentType" @@ -9,6 +12,14 @@ PORT_8080 = 8080 +def str_to_bool(value): + """Convert a string representation of a boolean value to a boolean.""" + try: + return bool(distutils.util.strtobool(value)) + except ValueError: + raise argparse.ArgumentTypeError(f"Invalid boolean value: {value}") + + def parse_args(): parser = argparse.ArgumentParser(description="Process allure reports") parser.add_argument( @@ -69,6 +80,20 @@ def parse_args(): help="Timeout for the put each file to neofs, in seconds. Default is 600 seconds", default=600, ) + parser.add_argument( + "--replace-objects", + required=False, + type=str_to_bool, + help="Replace existing objects with the same attributes in the container", + default=True, + ) + parser.add_argument( + "--replace-container-contents", + required=False, + type=str_to_bool, + help="Remove all the old existing objects in the container after the new objects are uploaded", + default=False, + ) return parser.parse_args() @@ -88,45 +113,34 @@ def get_rpc_endpoint(neofs_domain: str) -> str: return f"{neofs_domain}:{PORT_8080}" -def push_file( - directory: str, - subdir: str, - url_path_prefix: str, - filename: str, - attributes: str, - base_cmd: str, - put_timeout: int, -) -> None: - filepath = os.path.join(subdir, filename) - mime_type = magic.from_file(filepath, mime=True) - relative_path = os.path.relpath(filepath, os.path.dirname(directory)) - - if url_path_prefix is not None and url_path_prefix != "": - neofs_path_attr = os.path.join(url_path_prefix, relative_path) - else: - neofs_path_attr = relative_path - - base_cmd_with_file = f"{base_cmd} --file {filepath} --attributes {FILE_PATH}={neofs_path_attr},{CONTENT_TYPE}={mime_type}" - - if attributes is not None and attributes != "": - base_cmd_with_file += f",{attributes}" +def neofs_cli_execute(cmd: str, json_output: bool = False, timeout: int = None): + """ + Executes a given command and returns its output. - print(f"Neofs cli cmd is: {base_cmd_with_file}") + :param cmd: Command to execute. + :param json_output: Specifies if the command output is JSON. + :param timeout: Optional timeout for command execution. + :return: Command output as a string or a JSON object. + """ try: compl_proc = subprocess.run( - base_cmd_with_file, + cmd, check=True, text=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, - timeout=put_timeout, + timeout=timeout, shell=True, ) print(f"RC: {compl_proc.returncode}") print(f"Output: {compl_proc.stdout}") - print(f"Error: {compl_proc.stderr}") + + if json_output: + return json.loads(compl_proc.stdout) + else: + return compl_proc.stdout.splitlines() except subprocess.CalledProcessError as e: raise Exception( @@ -138,6 +152,117 @@ def push_file( ) +def search_objects_in_container(endpoint: str, + wallet: str, + password: str, + cid: str, + filters: str) -> list[str]: + cmd = ( + f"NEOFS_CLI_PASSWORD={password} neofs-cli --rpc-endpoint {endpoint} " + f"--wallet {wallet} object search --cid {cid} --filters '{filters}'" + ) + output_filter_re = re.compile(r"^Found \d+ objects\.$") + stdout_list = neofs_cli_execute(cmd) + filtered_lines = [line for line in stdout_list if not output_filter_re.search(line)] + return filtered_lines + + +def list_objects_in_container(endpoint: str, + wallet: str, + password: str, + cid: str) -> list[str]: + cmd = ( + f"NEOFS_CLI_PASSWORD={password} neofs-cli --rpc-endpoint {endpoint} " + f"--wallet {wallet} container list-objects --cid {cid}" + ) + return neofs_cli_execute(cmd) + + +def delete_objects( + endpoint: str, + wallet: str, + password: str, + cid: str, + oids: list[str], +) -> None: + for oid in oids: + cmd = ( + f"NEOFS_CLI_PASSWORD={password} neofs-cli --rpc-endpoint {endpoint} " + f"--wallet {wallet} object delete --cid {cid} --oid '{oid}'" + ) + neofs_cli_execute(cmd) + + +def compile_attributes(file_path: str, content_type: str = None, + attributes: str = None, output_format: str = "str") -> str: + attrs = { + FILE_PATH: file_path, + } + if content_type: + attrs[CONTENT_TYPE] = content_type + if attributes: + attrs.update(dict([attr.split('=') for attr in attributes.split(',')])) + if output_format == "str": + return ','.join([f"{k}={v}" for k, v in attrs.items()]) + elif output_format == "filter_str": + return ','.join([f"{k} EQ {v}" for k, v in attrs.items()]) + elif output_format == "dict": + return attrs + + +def get_file_info(directory: str, url_path_prefix: str): + base_path = os.path.abspath(directory) + file_infos = [] + + for subdir, dirs, files in os.walk(base_path): + for filename in files: + filepath = os.path.join(subdir, filename) + mime_type = magic.from_file(filepath, mime=True) + relative_path = os.path.relpath(filepath, os.path.dirname(directory)) + + if url_path_prefix is not None and url_path_prefix != "": + neofs_path_attr = os.path.join(url_path_prefix, relative_path) + else: + neofs_path_attr = relative_path + + file_infos.append({ + 'filepath': filepath, + 'mime_type': mime_type, + 'neofs_path_attr': neofs_path_attr, + }) + + return file_infos + + +def push_file( + endpoint: str, + wallet: str, + password: str, + cid: str, + file_info: dict, + attributes: str, + put_timeout: int, + expiration_epoch: int = None, +) -> None: + filepath = file_info['filepath'] + mime_type = file_info['mime_type'] + neofs_path_attr = file_info['neofs_path_attr'] + + attrs = compile_attributes(neofs_path_attr, mime_type, attributes) + + base_cmd = ( + f"NEOFS_CLI_PASSWORD={password} neofs-cli --rpc-endpoint {endpoint} " + f"--wallet {wallet} object put --cid {cid} --timeout {put_timeout}s" + ) + if expiration_epoch: + base_cmd += f" --expire-at {expiration_epoch}" + + cmd = f"{base_cmd} --file {filepath} --attributes {attrs}" + print(f"Neofs cli cmd is: {cmd}") + + neofs_cli_execute(cmd, timeout=put_timeout) + + def push_files_to_neofs( directory: str, endpoint: str, @@ -148,33 +273,49 @@ def push_files_to_neofs( lifetime: int, put_timeout: int, password: str, + replace_objects: bool, + replace_container_contents: bool ) -> None: if not os.path.exists(directory): raise Exception(f"Directory '{directory}' does not exist.") if not os.listdir(directory): raise Exception(f"Directory '{directory}' is empty.") - base_cmd = ( - f"NEOFS_CLI_PASSWORD={password} neofs-cli --rpc-endpoint {endpoint} " - f"--wallet {wallet} object put --cid {cid} --timeout {put_timeout}s" - ) if lifetime is not None and lifetime > 0: current_epoch = get_current_epoch(endpoint) expiration_epoch = current_epoch + lifetime - base_cmd += f" --expire-at {expiration_epoch}" - base_path = os.path.abspath(directory) - for subdir, dirs, files in os.walk(base_path): - for filename in files: - push_file( - base_path, - subdir, - url_path_prefix, - filename, - attributes, - base_cmd, - put_timeout, + files = get_file_info(directory, url_path_prefix) + flat_existing_objects = [] + if replace_container_contents: + flat_existing_objects = list_objects_in_container(endpoint, wallet, password, cid) + elif replace_objects: + existing_objects = [] + for file in files: + search_attrs = compile_attributes( + file['neofs_path_attr'], output_format="filter_str" ) + obj_to_delete = search_objects_in_container( + endpoint, wallet, password, cid, search_attrs + ) + existing_objects.append(obj_to_delete) + flat_existing_objects = [obj for sublist in existing_objects for obj in + (sublist if isinstance(sublist, list) else [sublist])] + + for file in files: + push_file( + endpoint, + wallet, + password, + cid, + file, + attributes, + put_timeout, + expiration_epoch, + ) + + if flat_existing_objects: + delete_objects(endpoint, wallet, password, cid, flat_existing_objects) if __name__ == "__main__": @@ -192,4 +333,6 @@ def push_files_to_neofs( args.lifetime, args.put_timeout, neofs_password, + args.replace_objects, + args.replace_container_contents, ) From 323706b32399f2f585cd2eada99b2d841fd12e2a Mon Sep 17 00:00:00 2001 From: MaxGelbakhiani Date: Tue, 19 Mar 2024 10:21:35 +0100 Subject: [PATCH 2/3] Tests for replace_objects option Signed-off-by: MaxGelbakhiani --- .github/workflows/tests.yml | 132 ++++++++++++++++++++++++++++++++++++ tests/conftest.py | 11 +++ tests/test_downloads.py | 7 +- 3 files changed, 147 insertions(+), 3 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 53b1544..5ee4b4f 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -76,6 +76,8 @@ jobs: PATH_TO_FILES_DIR: ${{ env.TESTS_DATA_DIR }} NEOFS_ATTRIBUTES: ${{ env.NEOFS_ATTRIBUTES }} URL_PREFIX: ${{ env.URL_PREFIX }} + REPLACE_OBJECTS: False + REPLACE_CONTAINER_CONTENTS: False - name: Run tests env: @@ -96,6 +98,8 @@ jobs: LIFETIME: ${{ vars.LIFETIME }} PATH_TO_FILES_DIR: ${{ env.TESTS_DATA_DIR }} URL_PREFIX: ${{ env.URL_PREFIX }} + REPLACE_OBJECTS: False + REPLACE_CONTAINER_CONTENTS: False - name: Run tests env: @@ -136,6 +140,8 @@ jobs: LIFETIME: ${{ vars.LIFETIME }} PATH_TO_FILES_DIR: ${{ env.PREFIX_DIR }} NEOFS_ATTRIBUTES: ${{ env.NEOFS_ATTRIBUTES }} + REPLACE_OBJECTS: False + REPLACE_CONTAINER_CONTENTS: False - name: Run tests env: @@ -144,3 +150,129 @@ jobs: run: | source venv/bin/activate && pytest test_downloads.py --base_url="$OUTPUT_CONTAINER_URL" --report_dir="$REPORT_DIR" working-directory: ./tests + + - name: Prepare directory name for tests with object by object replacement + shell: bash + env: + TIMESTAMP: ${{ steps.date.outputs.timestamp }} + run: | + echo "REPL_DATA_DIR=${{ github.run_number }}-$TIMESTAMP-$(uuidgen)-obj-by-obj-replacement" >> $GITHUB_ENV + + - name: Create a directory tree for tests with objects replacement + shell: bash + run: | + mkdir "$REPL_DATA_DIR" + + - name: Move files to the directory tree for tests with url prefix + shell: bash + env: + SOURCE_DIR: ${{ env.TESTS_DATA_DIR }} + DEST_DIR: ${{ env.REPL_DATA_DIR }} + run: | + rsync -av "$SOURCE_DIR" "$DEST_DIR" + + - name: Run gh-push-to-neofs with pre objects replacement + id: gh_push_to_neofs_with_pre_replace_objects + uses: ./ + with: + NEOFS_WALLET: ${{ secrets.NEOFS_WALLET }} + NEOFS_WALLET_PASSWORD: ${{ secrets.NEOFS_WALLET_PASSWORD }} + NEOFS_NETWORK_DOMAIN: ${{ vars.NEOFS_NETWORK_DOMAIN }} + NEOFS_HTTP_GATE: ${{ vars.NEOFS_HTTP_GATE }} + STORE_OBJECTS_CID: ${{ vars.STORE_OBJECTS_CID }} + LIFETIME: ${{ vars.LIFETIME }} + PATH_TO_FILES_DIR: ${{ env.REPL_DATA_DIR }} + REPLACE_OBJECTS: False + REPLACE_CONTAINER_CONTENTS: False + + - name: Modify test data + shell: bash + env: + DATA_DIR: ${{ env.REPL_DATA_DIR }} + run: | + find $DATA_DIR -type f -name '*.txt' -exec sed -i '$ s/$/_replaced_obj_by_obj/' {} + + + - name: Run gh-push-to-neofs with objects replacement + id: gh_push_to_neofs_with_replace_objects + uses: ./ + with: + NEOFS_WALLET: ${{ secrets.NEOFS_WALLET }} + NEOFS_WALLET_PASSWORD: ${{ secrets.NEOFS_WALLET_PASSWORD }} + NEOFS_NETWORK_DOMAIN: ${{ vars.NEOFS_NETWORK_DOMAIN }} + NEOFS_HTTP_GATE: ${{ vars.NEOFS_HTTP_GATE }} + STORE_OBJECTS_CID: ${{ vars.STORE_OBJECTS_CID }} + LIFETIME: ${{ vars.LIFETIME }} + PATH_TO_FILES_DIR: ${{ env.REPL_DATA_DIR }} + REPLACE_OBJECTS: True + REPLACE_CONTAINER_CONTENTS: False + + - name: Run tests + env: + OUTPUT_CONTAINER_URL: ${{ steps.gh_push_to_neofs_with_replace_objects.outputs.OUTPUT_CONTAINER_URL }} + REPORT_DIR: ${{ env.REPL_DATA_DIR }} + run: | + source venv/bin/activate && pytest test_downloads.py --base_url="$OUTPUT_CONTAINER_URL" --report_dir="$REPORT_DIR" --data_dir_prefix=../$REPORT_DIR + working-directory: ./tests + + - name: Prepare directory name for tests with entire container rewrite + shell: bash + env: + TIMESTAMP: ${{ steps.date.outputs.timestamp }} + run: | + echo "REWRITE_CONT_DIR=${{ github.run_number }}-$TIMESTAMP-$(uuidgen)-cont-rewrite" >> $GITHUB_ENV + + - name: Create a directory tree for tests with objects replacement + shell: bash + run: | + mkdir "$REWRITE_CONT_DIR" + + - name: Move files to the directory tree for tests with url prefix + shell: bash + env: + SOURCE_DIR: ${{ env.TESTS_DATA_DIR }} + DEST_DIR: ${{ env.REWRITE_CONT_DIR }} + run: | + rsync -av "$SOURCE_DIR" "$DEST_DIR" + + - name: Run gh-push-to-neofs with pre objects replacement + id: gh_push_to_neofs_with_pre_cont_rewrite + uses: ./ + with: + NEOFS_WALLET: ${{ secrets.NEOFS_WALLET }} + NEOFS_WALLET_PASSWORD: ${{ secrets.NEOFS_WALLET_PASSWORD }} + NEOFS_NETWORK_DOMAIN: ${{ vars.NEOFS_NETWORK_DOMAIN }} + NEOFS_HTTP_GATE: ${{ vars.NEOFS_HTTP_GATE }} + STORE_OBJECTS_CID: ${{ vars.STORE_OBJECTS_CID }} + LIFETIME: ${{ vars.LIFETIME }} + PATH_TO_FILES_DIR: ${{ env.REWRITE_CONT_DIR }} + REPLACE_OBJECTS: False + REPLACE_CONTAINER_CONTENTS: False + + - name: Modify test data + shell: bash + env: + DATA_DIR: ${{ env.REWRITE_CONT_DIR }} + run: | + find $DATA_DIR -type f -name '*.txt' -exec sed -i '$ s/$/_cont_rewrite/' {} + + + - name: Run gh-push-to-neofs with objects replacement + id: gh_push_to_neofs_with_cont_rewrite + uses: ./ + with: + NEOFS_WALLET: ${{ secrets.NEOFS_WALLET }} + NEOFS_WALLET_PASSWORD: ${{ secrets.NEOFS_WALLET_PASSWORD }} + NEOFS_NETWORK_DOMAIN: ${{ vars.NEOFS_NETWORK_DOMAIN }} + NEOFS_HTTP_GATE: ${{ vars.NEOFS_HTTP_GATE }} + STORE_OBJECTS_CID: ${{ vars.STORE_OBJECTS_CID }} + LIFETIME: ${{ vars.LIFETIME }} + PATH_TO_FILES_DIR: ${{ env.REWRITE_CONT_DIR }} + REPLACE_OBJECTS: True + REPLACE_CONTAINER_CONTENTS: True + + - name: Run tests + env: + OUTPUT_CONTAINER_URL: ${{ steps.gh_push_to_neofs_with_cont_rewrite.outputs.OUTPUT_CONTAINER_URL }} + REPORT_DIR: ${{ env.REWRITE_CONT_DIR }} + run: | + source venv/bin/activate && pytest test_downloads.py --base_url="$OUTPUT_CONTAINER_URL" --report_dir="$REPORT_DIR" --data_dir_prefix=../$REPORT_DIR + working-directory: ./tests diff --git a/tests/conftest.py b/tests/conftest.py index c1c8209..f1a3d98 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -11,6 +11,12 @@ def pytest_addoption(parser): default=None, help="Directory combine report and zip files", ) + parser.addoption( + "--data_dir_prefix", + action="store", + default=None, + help="Prefix dir to add to modified test data", + ) @pytest.fixture @@ -21,3 +27,8 @@ def base_url(request): @pytest.fixture def report_dir(request): return request.config.getoption("--report_dir") + + +@pytest.fixture +def data_dir_prefix(request): + return request.config.getoption("--data_dir_prefix") diff --git a/tests/test_downloads.py b/tests/test_downloads.py index c84d1e9..c0c3bda 100644 --- a/tests/test_downloads.py +++ b/tests/test_downloads.py @@ -20,9 +20,10 @@ def download_file(url: str) -> str: "data/dir/subdir/subdir_2/5.txt", ], ) -def test_file_content(base_url, report_dir, path): +def test_file_content(base_url, report_dir, data_dir_prefix, path): if base_url is None: pytest.fail("base_url is not provided. Provide it using --base_url option.") + full_path = os.path.join(data_dir_prefix or "", path) if not base_url.endswith("/"): base_url += "/" @@ -35,9 +36,9 @@ def test_file_content(base_url, report_dir, path): print(f"full_url: {full_url}") remote_content = download_file(full_url) - with open(path, "r") as local_file: + with open(full_path, "r") as local_file: local_content = local_file.read() assert ( remote_content == local_content - ), f"Contents of {full_url} and {path} do not match." + ), f"Contents of {full_url} and {full_path} do not match." From 688f6e21e14880ed3265f51009811d407217845c Mon Sep 17 00:00:00 2001 From: MaxGelbakhiani Date: Tue, 26 Mar 2024 07:31:01 +0100 Subject: [PATCH 3/3] Added test_objects_number Signed-off-by: MaxGelbakhiani --- .github/workflows/tests.yml | 14 ++++++++++-- helpers/neofs.py | 45 +++++++++++++++++++++++++++++++++++++ push-to-neofs.py | 41 +-------------------------------- tests/conftest.py | 40 ++++++++++++++++++++++++++++++--- tests/test_downloads.py | 12 ++-------- tests/test_objects.py | 21 +++++++++++++++++ 6 files changed, 118 insertions(+), 55 deletions(-) create mode 100644 helpers/neofs.py create mode 100644 tests/test_objects.py diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 5ee4b4f..67632e0 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -169,7 +169,7 @@ jobs: SOURCE_DIR: ${{ env.TESTS_DATA_DIR }} DEST_DIR: ${{ env.REPL_DATA_DIR }} run: | - rsync -av "$SOURCE_DIR" "$DEST_DIR" + cp -a "$SOURCE_DIR" "$DEST_DIR" - name: Run gh-push-to-neofs with pre objects replacement id: gh_push_to_neofs_with_pre_replace_objects @@ -232,7 +232,7 @@ jobs: SOURCE_DIR: ${{ env.TESTS_DATA_DIR }} DEST_DIR: ${{ env.REWRITE_CONT_DIR }} run: | - rsync -av "$SOURCE_DIR" "$DEST_DIR" + cp -a "$SOURCE_DIR" "$DEST_DIR" - name: Run gh-push-to-neofs with pre objects replacement id: gh_push_to_neofs_with_pre_cont_rewrite @@ -276,3 +276,13 @@ jobs: run: | source venv/bin/activate && pytest test_downloads.py --base_url="$OUTPUT_CONTAINER_URL" --report_dir="$REPORT_DIR" --data_dir_prefix=../$REPORT_DIR working-directory: ./tests + + - name: Run obj count test + env: + NEOFS_WALLET: "../wallet.json" + NEOFS_WALLET_PASSWORD: ${{ secrets.NEOFS_WALLET_PASSWORD }} + NEOFS_NETWORK_DOMAIN: ${{ vars.NEOFS_NETWORK_DOMAIN }} + STORE_OBJECTS_CID: ${{ vars.STORE_OBJECTS_CID }} + run: | + source venv/bin/activate && pytest test_objects.py + working-directory: ./tests diff --git a/helpers/neofs.py b/helpers/neofs.py new file mode 100644 index 0000000..92baa09 --- /dev/null +++ b/helpers/neofs.py @@ -0,0 +1,45 @@ +import subprocess +import json + + +def neofs_cli_execute(cmd: str, json_output: bool = False, timeout: int = None): + """ + Executes a given command and returns its output. + + :param cmd: Command to execute. + :param json_output: Specifies if the command output is JSON. + :param timeout: Optional timeout for command execution. + :return: Command output as a string or a JSON object. + """ + + try: + compl_proc = subprocess.run( + cmd, + check=True, + text=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + timeout=timeout, + shell=True, + ) + + print(f"RC: {compl_proc.returncode}") + print(f"Output: {compl_proc.stdout}") + + if json_output: + try: + return json.loads(compl_proc.stdout) + except json.JSONDecodeError: + output_list = compl_proc.stdout.splitlines() + return json.dumps(output_list) + else: + return compl_proc.stdout.splitlines() + + except subprocess.CalledProcessError as e: + raise Exception( + f"Command failed: {e.cmd}\n" + f"Error code: {e.returncode}\n" + f"Output: {e.output}\n" + f"Stdout: {e.stdout}\n" + f"Stderr: {e.stderr}\n" + ) diff --git a/push-to-neofs.py b/push-to-neofs.py index 0c75de8..eecbbd4 100644 --- a/push-to-neofs.py +++ b/push-to-neofs.py @@ -3,8 +3,8 @@ import subprocess import argparse import magic -import json import distutils.util +from helpers.neofs import neofs_cli_execute FILE_PATH = "FilePath" # the key for the attribute, is the path for the static page and allure report zip files CONTENT_TYPE = "ContentType" @@ -113,45 +113,6 @@ def get_rpc_endpoint(neofs_domain: str) -> str: return f"{neofs_domain}:{PORT_8080}" -def neofs_cli_execute(cmd: str, json_output: bool = False, timeout: int = None): - """ - Executes a given command and returns its output. - - :param cmd: Command to execute. - :param json_output: Specifies if the command output is JSON. - :param timeout: Optional timeout for command execution. - :return: Command output as a string or a JSON object. - """ - - try: - compl_proc = subprocess.run( - cmd, - check=True, - text=True, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - timeout=timeout, - shell=True, - ) - - print(f"RC: {compl_proc.returncode}") - print(f"Output: {compl_proc.stdout}") - - if json_output: - return json.loads(compl_proc.stdout) - else: - return compl_proc.stdout.splitlines() - - except subprocess.CalledProcessError as e: - raise Exception( - f"Command failed: {e.cmd}\n" - f"Error code: {e.returncode}\n" - f"Output: {e.output}\n" - f"Stdout: {e.stdout}\n" - f"Stderr: {e.stderr}\n" - ) - - def search_objects_in_container(endpoint: str, wallet: str, password: str, diff --git a/tests/conftest.py b/tests/conftest.py index f1a3d98..831c13f 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,6 +1,16 @@ +import os import pytest +DATA_PATH_LIST = [ + "data/1.txt", + "data/2.txt", + "data/dir/3.txt", + "data/dir/subdir/4.txt", + "data/dir/subdir/subdir_2/5.txt", + ] + + def pytest_addoption(parser): parser.addoption( "--base_url", action="store", default=None, help="Base URL to test against" @@ -17,18 +27,42 @@ def pytest_addoption(parser): default=None, help="Prefix dir to add to modified test data", ) + parser.addoption("--wallet", action="store", help="NeoFS wallet") + parser.addoption("--wallet-password", action="store", help="NeoFS wallet password") + parser.addoption("--network-domain", action="store", help="NeoFS network domain") + parser.addoption("--cid", action="store", help="NeoFS Container ID") @pytest.fixture def base_url(request): - return request.config.getoption("--base_url") + return os.environ.get('BASE_URL') or request.config.getoption("--base_url") @pytest.fixture def report_dir(request): - return request.config.getoption("--report_dir") + return os.environ.get('REPORT_DIR') or request.config.getoption("--report_dir") @pytest.fixture def data_dir_prefix(request): - return request.config.getoption("--data_dir_prefix") + return os.environ.get('DATA_DIR_PREFIX') or request.config.getoption("--data_dir_prefix") + + +@pytest.fixture +def wallet(request): + return os.environ.get('NEOFS_WALLET') or request.config.getoption("--wallet") + + +@pytest.fixture +def wallet_password(request): + return os.environ.get('NEOFS_WALLET_PASSWORD') or request.config.getoption("--wallet-password") + + +@pytest.fixture +def network_domain(request): + return os.environ.get('NEOFS_NETWORK_DOMAIN') or request.config.getoption("--network-domain") + + +@pytest.fixture +def cid(request): + return os.environ.get('STORE_OBJECTS_CID') or request.config.getoption("--cid") diff --git a/tests/test_downloads.py b/tests/test_downloads.py index c0c3bda..46f68b2 100644 --- a/tests/test_downloads.py +++ b/tests/test_downloads.py @@ -2,6 +2,7 @@ import requests import pytest from urllib.parse import urljoin +from conftest import DATA_PATH_LIST def download_file(url: str) -> str: @@ -10,16 +11,7 @@ def download_file(url: str) -> str: return response.text -@pytest.mark.parametrize( - "path", - [ - "data/1.txt", - "data/2.txt", - "data/dir/3.txt", - "data/dir/subdir/4.txt", - "data/dir/subdir/subdir_2/5.txt", - ], -) +@pytest.mark.parametrize("path", DATA_PATH_LIST) def test_file_content(base_url, report_dir, data_dir_prefix, path): if base_url is None: pytest.fail("base_url is not provided. Provide it using --base_url option.") diff --git a/tests/test_objects.py b/tests/test_objects.py new file mode 100644 index 0000000..477b388 --- /dev/null +++ b/tests/test_objects.py @@ -0,0 +1,21 @@ +import sys +import json +import pytest +from conftest import DATA_PATH_LIST + +sys.path.insert(0, '..') +from helpers.neofs import neofs_cli_execute + + +def test_objects_number(wallet, wallet_password, network_domain, cid): + cmd = ( + f"NEOFS_CLI_PASSWORD={wallet_password} neofs-cli --rpc-endpoint {network_domain}:8080 " + f"--wallet {wallet} container list-objects --cid {cid}" + ) + objects_json = neofs_cli_execute(cmd, json_output=True) + objects_list = json.loads(objects_json) + obj_number = len(objects_list) + files_num = len(DATA_PATH_LIST) + assert ( + obj_number == files_num + ), f"Objects number of {obj_number} in the container and the number {files_num} of uploaded files doesn't match"