Merge pull request #16 from nspcc-dev/added_replace_existing_flag

.github/workflows/tests.yml

@@ -76,6 +76,8 @@ jobs:
           PATH_TO_FILES_DIR: ${{ env.TESTS_DATA_DIR }}
           NEOFS_ATTRIBUTES: ${{ env.NEOFS_ATTRIBUTES }}
           URL_PREFIX: ${{ env.URL_PREFIX }}
+          REPLACE_OBJECTS: False
+          REPLACE_CONTAINER_CONTENTS: False
 
       - name: Run tests
         env:
@@ -96,6 +98,8 @@ jobs:
           LIFETIME: ${{ vars.LIFETIME }}
           PATH_TO_FILES_DIR: ${{ env.TESTS_DATA_DIR }}
           URL_PREFIX: ${{ env.URL_PREFIX }}
+          REPLACE_OBJECTS: False
+          REPLACE_CONTAINER_CONTENTS: False
 
       - name: Run tests
         env:
@@ -136,6 +140,8 @@ jobs:
           LIFETIME: ${{ vars.LIFETIME }}
           PATH_TO_FILES_DIR: ${{ env.PREFIX_DIR }}
           NEOFS_ATTRIBUTES: ${{ env.NEOFS_ATTRIBUTES }}
+          REPLACE_OBJECTS: False
+          REPLACE_CONTAINER_CONTENTS: False
 
       - name: Run tests
         env:
@@ -144,3 +150,139 @@ jobs:
         run: |
           source venv/bin/activate && pytest test_downloads.py --base_url="$OUTPUT_CONTAINER_URL" --report_dir="$REPORT_DIR"
         working-directory: ./tests
+
+      - name: Prepare directory name for tests with object by object replacement
+        shell: bash
+        env:
+          TIMESTAMP: ${{ steps.date.outputs.timestamp }}
+        run: |
+          echo "REPL_DATA_DIR=${{ github.run_number }}-$TIMESTAMP-$(uuidgen)-obj-by-obj-replacement" >> $GITHUB_ENV
+
+      - name: Create a directory tree for tests with objects replacement
+        shell: bash
+        run: |
+          mkdir "$REPL_DATA_DIR"
+
+      - name: Move files to the directory tree for tests with url prefix
+        shell: bash
+        env:
+          SOURCE_DIR: ${{ env.TESTS_DATA_DIR }}
+          DEST_DIR: ${{ env.REPL_DATA_DIR }}
+        run: |
+          cp -a "$SOURCE_DIR" "$DEST_DIR"
+
+      - name: Run gh-push-to-neofs with pre objects replacement
+        id: gh_push_to_neofs_with_pre_replace_objects
+        uses: ./
+        with:
+          NEOFS_WALLET: ${{ secrets.NEOFS_WALLET }}
+          NEOFS_WALLET_PASSWORD: ${{ secrets.NEOFS_WALLET_PASSWORD }}
+          NEOFS_NETWORK_DOMAIN: ${{ vars.NEOFS_NETWORK_DOMAIN }}
+          NEOFS_HTTP_GATE: ${{ vars.NEOFS_HTTP_GATE }}
+          STORE_OBJECTS_CID: ${{ vars.STORE_OBJECTS_CID }}
+          LIFETIME: ${{ vars.LIFETIME }}
+          PATH_TO_FILES_DIR: ${{ env.REPL_DATA_DIR }}
+          REPLACE_OBJECTS: False
+          REPLACE_CONTAINER_CONTENTS: False
+
+      - name: Modify test data
+        shell: bash
+        env:
+          DATA_DIR: ${{ env.REPL_DATA_DIR }}
+        run: |
+          find $DATA_DIR -type f -name '*.txt' -exec sed -i '$ s/$/_replaced_obj_by_obj/' {} +
+
+      - name: Run gh-push-to-neofs with objects replacement
+        id: gh_push_to_neofs_with_replace_objects
+        uses: ./
+        with:
+          NEOFS_WALLET: ${{ secrets.NEOFS_WALLET }}
+          NEOFS_WALLET_PASSWORD: ${{ secrets.NEOFS_WALLET_PASSWORD }}
+          NEOFS_NETWORK_DOMAIN: ${{ vars.NEOFS_NETWORK_DOMAIN }}
+          NEOFS_HTTP_GATE: ${{ vars.NEOFS_HTTP_GATE }}
+          STORE_OBJECTS_CID: ${{ vars.STORE_OBJECTS_CID }}
+          LIFETIME: ${{ vars.LIFETIME }}
+          PATH_TO_FILES_DIR: ${{ env.REPL_DATA_DIR }}
+          REPLACE_OBJECTS: True
+          REPLACE_CONTAINER_CONTENTS: False
+
+      - name: Run tests
+        env:
+          OUTPUT_CONTAINER_URL: ${{ steps.gh_push_to_neofs_with_replace_objects.outputs.OUTPUT_CONTAINER_URL }}
+          REPORT_DIR: ${{ env.REPL_DATA_DIR }}
+        run: |
+          source venv/bin/activate && pytest test_downloads.py --base_url="$OUTPUT_CONTAINER_URL" --report_dir="$REPORT_DIR" --data_dir_prefix=../$REPORT_DIR
+        working-directory: ./tests
+
+      - name: Prepare directory name for tests with entire container rewrite
+        shell: bash
+        env:
+          TIMESTAMP: ${{ steps.date.outputs.timestamp }}
+        run: |
+          echo "REWRITE_CONT_DIR=${{ github.run_number }}-$TIMESTAMP-$(uuidgen)-cont-rewrite" >> $GITHUB_ENV
+
+      - name: Create a directory tree for tests with objects replacement
+        shell: bash
+        run: |
+          mkdir "$REWRITE_CONT_DIR"
+
+      - name: Move files to the directory tree for tests with url prefix
+        shell: bash
+        env:
+          SOURCE_DIR: ${{ env.TESTS_DATA_DIR }}
+          DEST_DIR: ${{ env.REWRITE_CONT_DIR }}
+        run: |
+          cp -a "$SOURCE_DIR" "$DEST_DIR"
+
+      - name: Run gh-push-to-neofs with pre objects replacement
+        id: gh_push_to_neofs_with_pre_cont_rewrite
+        uses: ./
+        with:
+          NEOFS_WALLET: ${{ secrets.NEOFS_WALLET }}
+          NEOFS_WALLET_PASSWORD: ${{ secrets.NEOFS_WALLET_PASSWORD }}
+          NEOFS_NETWORK_DOMAIN: ${{ vars.NEOFS_NETWORK_DOMAIN }}
+          NEOFS_HTTP_GATE: ${{ vars.NEOFS_HTTP_GATE }}
+          STORE_OBJECTS_CID: ${{ vars.STORE_OBJECTS_CID }}
+          LIFETIME: ${{ vars.LIFETIME }}
+          PATH_TO_FILES_DIR: ${{ env.REWRITE_CONT_DIR }}
+          REPLACE_OBJECTS: False
+          REPLACE_CONTAINER_CONTENTS: False
+
+      - name: Modify test data
+        shell: bash
+        env:
+          DATA_DIR: ${{ env.REWRITE_CONT_DIR }}
+        run: |
+          find $DATA_DIR -type f -name '*.txt' -exec sed -i '$ s/$/_cont_rewrite/' {} +
+
+      - name: Run gh-push-to-neofs with objects replacement
+        id: gh_push_to_neofs_with_cont_rewrite
+        uses: ./
+        with:
+          NEOFS_WALLET: ${{ secrets.NEOFS_WALLET }}
+          NEOFS_WALLET_PASSWORD: ${{ secrets.NEOFS_WALLET_PASSWORD }}
+          NEOFS_NETWORK_DOMAIN: ${{ vars.NEOFS_NETWORK_DOMAIN }}
+          NEOFS_HTTP_GATE: ${{ vars.NEOFS_HTTP_GATE }}
+          STORE_OBJECTS_CID: ${{ vars.STORE_OBJECTS_CID }}
+          LIFETIME: ${{ vars.LIFETIME }}
+          PATH_TO_FILES_DIR: ${{ env.REWRITE_CONT_DIR }}
+          REPLACE_OBJECTS: True
+          REPLACE_CONTAINER_CONTENTS: True
+
+      - name: Run tests
+        env:
+          OUTPUT_CONTAINER_URL: ${{ steps.gh_push_to_neofs_with_cont_rewrite.outputs.OUTPUT_CONTAINER_URL }}
+          REPORT_DIR: ${{ env.REWRITE_CONT_DIR }}
+        run: |
+          source venv/bin/activate && pytest test_downloads.py --base_url="$OUTPUT_CONTAINER_URL" --report_dir="$REPORT_DIR" --data_dir_prefix=../$REPORT_DIR
+        working-directory: ./tests
+
+      - name: Run obj count test
+        env:
+          NEOFS_WALLET: "../wallet.json"
+          NEOFS_WALLET_PASSWORD: ${{ secrets.NEOFS_WALLET_PASSWORD }}
+          NEOFS_NETWORK_DOMAIN: ${{ vars.NEOFS_NETWORK_DOMAIN }}
+          STORE_OBJECTS_CID: ${{ vars.STORE_OBJECTS_CID }}
+        run: |
+          source venv/bin/activate && pytest test_objects.py
+        working-directory: ./tests

action.yml

@@ -35,6 +35,14 @@ inputs:
     description: Number of epochs for object to stay valid
     required: false
     default: 0
+  REPLACE_OBJECTS:
+    description: Replace existing objects with the same attributes in the container
+    required: false
+    default: 'True'
+  REPLACE_CONTAINER_CONTENTS:
+    description: Remove all the old existing objects in the container after the new objects are uploaded
+    required: false
+    default: 'False'
 
 outputs:
   OUTPUT_CONTAINER_URL:
@@ -93,11 +101,14 @@ runs:
         NEOFS_ATTRIBUTES: ${{ inputs.NEOFS_ATTRIBUTES }}
         URL_PREFIX: ${{ inputs.URL_PREFIX }}
         LIFETIME: ${{ inputs.LIFETIME }}
+        REPLACE_OBJECTS: ${{ inputs.REPLACE_OBJECTS }}
+        REPLACE_CONTAINER_CONTENTS: ${{ inputs.REPLACE_CONTAINER_CONTENTS }}
         GITHUB_ACTION_PATH: ${{ github.action_path }}
       run: |
         source "$GITHUB_ACTION_PATH/venv/bin/activate" && NEOFS_CLI_PASSWORD=$NEOFS_WALLET_PASSWORD python "$GITHUB_ACTION_PATH/push-to-neofs.py" \
         --lifetime "$LIFETIME" --neofs_domain "$NEOFS_NETWORK_DOMAIN" --attributes "$NEOFS_ATTRIBUTES" \
         --cid "$STORE_OBJECTS_CID" --files-dir "$PATH_TO_FILES_DIR" --url_path_prefix "$URL_PREFIX" \
+        --replace-objects "$REPLACE_OBJECTS" --replace-container-contents "$REPLACE_CONTAINER_CONTENTS" \
         --wallet "$GITHUB_ACTION_PATH/wallet.json"
         BASE_URL="https://$NEOFS_HTTP_GATE/$STORE_OBJECTS_CID"
         if [ -z "$URL_PREFIX" ]; then

helpers/neofs.py

@@ -0,0 +1,45 @@
+import subprocess
+import json
+
+
+def neofs_cli_execute(cmd: str, json_output: bool = False, timeout: int = None):
+    """
+    Executes a given command and returns its output.
+
+    :param cmd: Command to execute.
+    :param json_output: Specifies if the command output is JSON.
+    :param timeout: Optional timeout for command execution.
+    :return: Command output as a string or a JSON object.
+    """
+
+    try:
+        compl_proc = subprocess.run(
+            cmd,
+            check=True,
+            text=True,
+            stdout=subprocess.PIPE,
+            stderr=subprocess.PIPE,
+            timeout=timeout,
+            shell=True,
+        )
+
+        print(f"RC: {compl_proc.returncode}")
+        print(f"Output: {compl_proc.stdout}")
+
+        if json_output:
+            try:
+                return json.loads(compl_proc.stdout)
+            except json.JSONDecodeError:
+                output_list = compl_proc.stdout.splitlines()
+                return json.dumps(output_list)
+        else:
+            return compl_proc.stdout.splitlines()
+
+    except subprocess.CalledProcessError as e:
+        raise Exception(
+            f"Command failed: {e.cmd}\n"
+            f"Error code: {e.returncode}\n"
+            f"Output: {e.output}\n"
+            f"Stdout: {e.stdout}\n"
+            f"Stderr: {e.stderr}\n"
+        )