Initial changes

Author: jayamala17

Gemfile

@@ -1,13 +1,13 @@
 source "https://rubygems.org"
 
 gem "aws-sdk-s3", "~> 1.136"
-gem "bagit", "~> 0.4.6"
+gem "bagit", "~> 0.5.0"
 gem "canister", "~> 0.9.2"
 gem "faraday", "~> 2.9"
 gem "faraday-retry", "~> 2.2"
 gem "mysql2", "~> 0.5.6"
 gem "prometheus-client", "~> 4.2.2"
-gem "rexml", "~> 3.2.7"
+gem "rexml", "~> 3.3.2"
 gem "semantic_logger", "~> 4.15"
 gem "sequel", "~> 5.77"
 gem "tty-command", "~> 0.10.1"

Gemfile.lock

@@ -25,7 +25,7 @@ GEM
       aws-sigv4 (~> 1.8)
     aws-sigv4 (1.8.0)
       aws-eventstream (~> 1, >= 1.0.2)
-    bagit (0.4.6)
+    bagit (0.5.0)
       docopt (~> 0.5.0)
       validatable (~> 1.6)
     base64 (0.2.0)
@@ -49,8 +49,8 @@ GEM
       tty-color (~> 0.5)
     prometheus-client (4.2.2)
     rake (13.1.0)
-    rexml (3.2.7)
-      strscan (>= 3.0.9)
+    rexml (3.3.4)
+      strscan
     semantic_logger (4.15.0)
       concurrent-ruby (~> 1.0)
     sequel (5.77.0)
@@ -69,7 +69,7 @@ PLATFORMS
 
 DEPENDENCIES
   aws-sdk-s3 (~> 1.136)
-  bagit (~> 0.4.6)
+  bagit (~> 0.5.0)
   canister (~> 0.9.2)
   dotenv (~> 3.1.0)
   faraday (~> 2.9)
@@ -78,7 +78,7 @@ DEPENDENCIES
   mysql2 (~> 0.5.6)
   prometheus-client (~> 4.2.2)
   rake (~> 13.1)
-  rexml (~> 3.2.7)
+  rexml (~> 3.3.2)
   semantic_logger (~> 4.15)
   sequel (~> 5.77)
   sftp!

example.env

@@ -18,6 +18,8 @@ SETTINGS_WORKFLOW_EXPORT_DIR=./export
 SETTINGS_WORKFLOW_REMOVE_EXPORT=true
 # Determines whether the process skips sending bag(s)
 SETTINGS_WORKFLOW_DRY_RUN=false
+# Set the detect hidden flag
+SETTINGS_DETECT_HIDDEN=false
 
 # Repository
 # Name that will be used for the repository at the beginning of bag identifiers

lib/bag_adapter.rb

@@ -8,8 +8,9 @@ class BagAdapter
 
     attr_reader :additional_tag_files
 
-    def initialize(target_dir)
-      @bag = BagIt::Bag.new(target_dir)
+    def initialize(target_dir, detect_hidden)
+      @detect_hidden = detect_hidden
+      @bag = BagIt::Bag.new(target_dir, {}, false, @detect_hidden)
       @additional_tag_files = []
     end
 

lib/bag_courier.rb

@@ -39,7 +39,8 @@ def initialize(
       working_dir:,
       export_dir:,
       dry_run:,
-      remove_export:
+      remove_export:,
+      detect_hidden:
     )
       @bag_id = bag_id
       @bag_info = bag_info
@@ -54,6 +55,7 @@ def initialize(
       @remove_export = remove_export
       @dry_run = dry_run
       @validator = validator
+      @detect_hidden = detect_hidden
     end
 
     def track!(status:, note: nil)
@@ -106,7 +108,7 @@ def deliver
       begin
         track!(status: BagStatus::BAGGING)
         bag_path = File.join(@working_dir, @bag_id.to_s)
-        bag = BagAdapter::BagAdapter.new(bag_path)
+        bag = BagAdapter::BagAdapter.new(bag_path, @detect_hidden)
 
         track!(status: BagStatus::COPYING)
         logger.measure_info("Copied data for bag #{@bag_id} in #{@working_dir}.") do

lib/bag_validator.rb

@@ -7,8 +7,9 @@ def validate(path)
 end
 
 class InnerBagValidator < BagValidator
-  def initialize(inner_bag_name)
+  def initialize(inner_bag_name, detect_hidden)
     @inner_bag_name = inner_bag_name
+    @detect_hidden = detect_hidden
   end
 
   def validate(data_path)
@@ -17,7 +18,7 @@ def validate(data_path)
       raise BagValidationError, "Inner bag path does not exist: #{path}"
     end
 
-    @bag = BagIt::Bag.new(path)
+    @bag = BagIt::Bag.new(path, {}, false, @detect_hidden)
     validity = @bag.valid?
 
     if !validity

lib/config.rb

@@ -117,6 +117,7 @@ def get_subset_by_key_stem(stem)
     :num_objects_per_repo,
     :restore_dir,
     :workflow,
+    :detect_hidden,
     keyword_init: true
   )
 
@@ -320,7 +321,8 @@ def self.create_config(data)
             export_dir: settings_workflow_data.get_value(key: "EXPORT_DIR"),
             dry_run: settings_workflow_data.get_value(key: "DRY_RUN", checks: [BOOLEAN_CHECK]) == "true",
             remove_export: settings_workflow_data.get_value(key: "REMOVE_EXPORT", checks: [BOOLEAN_CHECK]) == "true"
-          )
+          ),
+          detect_hidden: data.get_value(key: "SETTINGS_DETECT_HIDDEN", checks:[BOOLEAN_CHECK]) == "true"
         ),
         repository: RepositoryConfig.new(
           name: data.get_value(key: "REPOSITORY_NAME"),

lib/dispatcher.rb

@@ -27,7 +27,8 @@ def initialize(
       context: nil,
       extra_bag_info_data: nil,
       status_event_repo: StatusEventRepository::StatusEventInMemoryRepository.new,
-      bag_repo: BagRepository::BagInMemoryRepository.new
+      bag_repo: BagRepository::BagInMemoryRepository.new,
+      detect_hidden:
     )
       @settings = settings
       @repository = repository
@@ -36,6 +37,7 @@ def initialize(
       @target_client = target_client
       @status_event_repo = status_event_repo
       @bag_repo = bag_repo
+      @detect_hidden = detect_hidden
     end
 
     def dispatch(
@@ -79,7 +81,8 @@ def dispatch(
         working_dir: @settings.working_dir,
         export_dir: @settings.export_dir,
         dry_run: @settings.dry_run,
-        remove_export: @settings.remove_export
+        remove_export: @settings.remove_export,
+        detect_hidden: @detect_hidden
       )
     end
   end

run_dark_blue.rb

@@ -37,6 +37,7 @@ def initialize(config)
 
     @arch_configs = config.dark_blue.archivematicas
     @object_size_limit = config.settings.object_size_limit
+    @detect_hidden = config.settings.detect_hidden
   end
 
   def create_dispatcher(context:, extra_bag_info_data:)
@@ -50,7 +51,8 @@ def create_dispatcher(context:, extra_bag_info_data:)
       status_event_repo: @status_event_repo,
       bag_repo: @bag_repo,
       context: context,
-      extra_bag_info_data: extra_bag_info_data
+      extra_bag_info_data: extra_bag_info_data,
+      detect_hidden: @detect_hidden
     )
   end
 
@@ -81,7 +83,7 @@ def deliver_package(dispatcher:, package_data:, remote_client:)
         remote_client: remote_client,
         remote_path: package_data.remote_path
       ),
-      validator: InnerBagValidator.new(package_data.dir_name)
+      validator: InnerBagValidator.new(package_data.dir_name, @detect_hidden)
     )
     logger.measure_info("Delivered package #{package_data.metadata.id}.") do
       courier.deliver

test/test_bag_adapter.rb

@@ -14,6 +14,8 @@ def setup
     # Reset test directory
     FileUtils.rm_r(@test_dir_path) if File.exist?(@test_dir_path)
     Dir.mkdir @test_dir_path
+    @detect_hidden_yes = true
+    @detect_hidden_no = false
   end
 
   def add_data_file
@@ -25,14 +27,14 @@ def add_data_file
 
   def test_bag_dir
     assert_equal(
-      BagAdapter::BagAdapter.new(@test_dir_path).bag_dir,
+      BagAdapter::BagAdapter.new(@test_dir_path, @detect_hidden_no).bag_dir,
       @test_dir_path
     )
   end
 
   def test_data_dir
     assert_equal(
-      BagAdapter::BagAdapter.new(@test_dir_path).data_dir,
+      BagAdapter::BagAdapter.new(@test_dir_path, @detect_hidden_no).data_dir,
       @data_dir_path
     )
   end
@@ -46,7 +48,7 @@ def test_add_bag_info
     TEXT
 
     Date.stub :today, Date.new(2023, 12, 22) do
-      bag = BagAdapter::BagAdapter.new(@test_dir_path)
+      bag = BagAdapter::BagAdapter.new(@test_dir_path, @detect_hidden_no)
       add_data_file
       bag.add_bag_info(@test_bag_info_data)
     end
@@ -56,7 +58,7 @@ def test_add_bag_info
   end
 
   def test_add_tag_file
-    bag = BagAdapter::BagAdapter.new(@test_dir_path)
+    bag = BagAdapter::BagAdapter.new(@test_dir_path, @detect_hidden_no)
     bag.add_tag_file!(tag_file_text: @test_tag_file_text, file_name: @test_tag_file_name)
 
     expected_file_path = File.join(@test_dir_path, @test_tag_file_name)
@@ -68,7 +70,7 @@ def test_add_tag_file
   end
 
   def test_add_manifests
-    bag = BagAdapter::BagAdapter.new(@test_dir_path)
+    bag = BagAdapter::BagAdapter.new(@test_dir_path, @detect_hidden_no)
     add_data_file
     bag.add_tag_file!(tag_file_text: @test_tag_file_text, file_name: @test_tag_file_name)
     bag.add_manifests

test/test_bag_courier.rb

@@ -120,7 +120,7 @@ def setup
     )
   end
 
-  def create_courier(dry_run:, target_client:, validator: @validator, remove_export: false)
+  def create_courier(dry_run:, target_client:, validator: @validator, remove_export: false, detect_hidden:)
     BagCourier::BagCourier.new(
       bag_id: @bag_id,
       bag_info: @bag_info,
@@ -132,12 +132,13 @@ def create_courier(dry_run:, target_client:, validator: @validator, remove_expor
       dry_run: dry_run,
       status_event_repo: @status_event_repo,
       target_client: target_client,
-      validator: validator
+      validator: validator,
+      detect_hidden: detect_hidden
     )
   end
 
   def test_deliver_with_dry_run_false
-    courier = create_courier(dry_run: false, target_client: @mock_target_client)
+    courier = create_courier(dry_run: false, target_client: @mock_target_client, detect_hidden: false)
     expected_tar_file_path = File.join(@export_path, @bag_id.to_s + ".tar")
     @mock_target_client.expect(:remote_text, "AWS S3 remote location in bucket fake")
     @mock_target_client.expect(:send_file, nil, local_file_path: expected_tar_file_path)
@@ -163,7 +164,7 @@ def test_deliver_with_dry_run_false
   end
 
   def test_deliver_with_dry_run
-    courier = create_courier(dry_run: true, target_client: @mock_target_client)
+    courier = create_courier(dry_run: true, target_client: @mock_target_client, detect_hidden: false)
     courier.deliver
     @mock_target_client.verify
 
@@ -179,7 +180,7 @@ def test_deliver_with_dry_run
   end
 
   def test_deliver_when_deposit_raises_error
-    courier = create_courier(dry_run: false, target_client: @aptrust_target_client)
+    courier = create_courier(dry_run: false, target_client: @aptrust_target_client, detect_hidden: false)
     raise_error = proc { raise RemoteClient::RemoteClientError, "specific details" }
     @aptrust_target_client.stub :send_file, raise_error do
       courier.deliver
@@ -193,7 +194,7 @@ def test_deliver_when_deposit_raises_error
   end
 
   def test_deliver_with_remove_export
-    courier = create_courier(dry_run: false, target_client: @mock_target_client, remove_export: true)
+    courier = create_courier(dry_run: false, target_client: @mock_target_client, remove_export: true, detect_hidden: false)
     expected_tar_file_path = File.join(@export_path, @bag_id.to_s + ".tar")
     @mock_target_client.expect(:remote_text, "AWS S3 remote location in bucket fake")
     @mock_target_client.expect(:send_file, nil, local_file_path: expected_tar_file_path)