Merge pull request #288 from esmero/ISSUE-286

ISSUE-286: Break the 5 Gbytes barrier without breaking the bank

config/schema/strawberryfield.schema.yml

@@ -79,6 +79,10 @@ strawberryfield.storage_settings:
     file_scheme:
       type: string
       label: 'Storage Scheme for Persisting Files'
+    multipart_upload_threshold:
+      type: integer
+      label: 'At what file size (in bytes) Multipart copy/put should be used when the destination of a file is S3'
+      default: 5368709120
     file_path:
       type: string
       label: 'Storage Path under the Storage Scheme, for Persisting Files'

src/Form/FilePersisterServiceSettingsForm.php

@@ -81,6 +81,16 @@ public function buildForm(array $form, FormStateInterface $form_state) {
       '#options' => $scheme_options,
       '#required' => TRUE
     ];
+    $form['multipart_upload_threshold'] = [
+      '#type' => 'number',
+      '#title' => $this->t('Specify file size (in bytes) when Multipart Upload/Copy should be attempted.'),
+      '#description' => $this->t('Only applies when the Storage Scheme used is S3 driven (e.g s3://). The default is 5Gbytes. The lowest threshold possible is 100 Mbytes'),
+      '#default_value' => $config_storage->get('multipart_upload_threshold') ?? 5368709120,
+      '#min' => 104857600,
+      '#step' => 1048576,
+      '#max' => 5368709120,
+      '#required' => TRUE
+    ];
     $form['file_path'] = [
       '#type' => 'textfield',
       '#title' => $this->t('Relative Path for Persisting Files'),
@@ -593,6 +603,7 @@ public function submitForm(array &$form, FormStateInterface $form_state) {
       ->save();
     $this->config('strawberryfield.storage_settings')
       ->set('file_scheme', $form_state->getValue('file_scheme'))
+      ->set('multipart_upload_threshold', (int) $form_state->getValue('multipart_upload_threshold'))
       ->set('file_path', trim($form_state->getValue('file_path')," \n\r\t\v\0/"))
       ->set('object_file_scheme', $form_state->getValue('object_file_scheme'))
       ->set('object_file_strategy', $form_state->getValue('object_file_strategy'))

src/Plugin/QueueWorker/CompostQueueWorker.php

@@ -243,21 +243,10 @@ function str_ends_with(string $haystack, string $needle): bool {
 
 
     if (!$safe) {
-      $this->loggerFactory->get('archipelago')->warning('Attempt to compost an unsafe File with path <em>@path</em> was made. Please manually delete it or lower/configure your security settings and code overrides defining what a Safe Path is.',[
+      $this->loggerFactory->get('archipelago')->warning('Attempt to compost an unsafe File with path <em>@path</em> was made. If you need to remove it, please do it so manually. If not, you can ignore this warning.',[
         '@path' => $data->uri
       ]);
-      if (class_exists('\Drupal\Core\Queue\DelayedRequeueException')) {
-        throw new \Drupal\Core\Queue\DelayedRequeueException(
-          0, 'Unsafe File found. Check your logs and deal with it. We will still try again later.'
-        );
-      }
-      else {
-        // Drupal 8 Compat. This might make the queue run over this until
-        // Lease time expires. Not optimal.
-        throw new RequeueException(
-          'Unsafe File found. Check your logs and deal with it. We will still try again later.'
-        );
-      }
+      return;
     }
     // Now check if the file is attached to an entity or not
     /** @var \Drupal\file\FileInterface[] $files */