diff --git a/.rubocop.yml b/.rubocop.yml index 7f122ca..d2ac346 100644 --- a/.rubocop.yml +++ b/.rubocop.yml @@ -20,10 +20,15 @@ AllCops: Lint/MissingCopEnableDirective: Enabled: false +Metrics/AbcSize: + Exclude: + - app/jobs/aws_check_fixity_job.rb + Metrics/MethodLength: Exclude: - lib/check_please/aws/object_fixity_checker.rb - app/controllers/fixity_checks_controller.rb + - app/jobs/aws_check_fixity_job.rb RSpec/VerifiedDoubles: Exclude: diff --git a/app/channels/fixity_check_channel.rb b/app/channels/fixity_check_channel.rb index ae847ce..16b5417 100644 --- a/app/channels/fixity_check_channel.rb +++ b/app/channels/fixity_check_channel.rb @@ -41,6 +41,12 @@ def run_fixity_check_for_s3_object(data) object_path = data['object_path'] checksum_algorithm_name = data['checksum_algorithm_name'] - AwsCheckFixityJob.perform_later(job_identifier, bucket_name, object_path, checksum_algorithm_name) + fixity_check = FixityCheck.create!( + job_identifier: job_identifier, + bucket_name: bucket_name, + object_path: object_path, + checksum_algorithm_name: checksum_algorithm_name + ) + AwsCheckFixityJob.perform_later(fixity_check.id) end end diff --git a/app/controllers/fixity_checks_controller.rb b/app/controllers/fixity_checks_controller.rb index 0c98cbc..1991b2c 100644 --- a/app/controllers/fixity_checks_controller.rb +++ b/app/controllers/fixity_checks_controller.rb @@ -13,15 +13,40 @@ def run_fixity_check_for_s3_object bucket_name, object_path, checksum_algorithm_name ) - render plain: { + render json: { bucket_name: bucket_name, object_path: object_path, checksum_algorithm_name: checksum_algorithm_name, checksum_hexdigest: checksum_hexdigest, object_size: object_size - }.to_json + } rescue StandardError => e - render plain: { + render json: { error_message: e.message, bucket_name: bucket_name, object_path: object_path, checksum_algorithm_name: checksum_algorithm_name - }.to_json, status: :bad_request + }, status: :bad_request + end + + def create + bucket_name = fixity_check_params['bucket_name'] + object_path = fixity_check_params['object_path'] + checksum_algorithm_name = fixity_check_params['checksum_algorithm_name'] + fixity_check = FixityCheck.create!( + # User does not need to supply a job_identifier param when using the create endpoint. + # We'll just use a ranom UUID here. + job_identifier: SecureRandom.uuid, + bucket_name: bucket_name, + object_path: object_path, + checksum_algorithm_name: checksum_algorithm_name + ) + AwsCheckFixityJob.perform_later(fixity_check.id) + render json: fixity_check + rescue StandardError => e + render json: { + error_message: e.message + }, status: :bad_request + end + + # GET /fixity_checks/1 + def show + render json: FixityCheck.find(params[:id]) end private diff --git a/app/jobs/aws_check_fixity_job.rb b/app/jobs/aws_check_fixity_job.rb index 5b31887..7a40c2c 100644 --- a/app/jobs/aws_check_fixity_job.rb +++ b/app/jobs/aws_check_fixity_job.rb @@ -3,22 +3,41 @@ class AwsCheckFixityJob < ApplicationJob queue_as CheckPlease::Queues::CHECK_FIXITY - def perform(job_identifier, bucket_name, object_path, checksum_algorithm_name) - response_stream_name = "#{FixityCheckChannel::FIXITY_CHECK_STREAM_PREFIX}#{job_identifier}" + def perform(fixity_check_id) + fixity_check = FixityCheck.find(fixity_check_id) + response_stream_name = "#{FixityCheckChannel::FIXITY_CHECK_STREAM_PREFIX}#{fixity_check.job_identifier}" + # Begin calculating checksum and file size + fixity_check.in_progress! checksum_hexdigest, object_size = CheckPlease::Aws::ObjectFixityChecker.check( - bucket_name, - object_path, - checksum_algorithm_name, - on_chunk: progress_report_lambda(response_stream_name) + fixity_check.bucket_name, + fixity_check.object_path, + fixity_check.checksum_algorithm_name, + on_chunk: progress_report_lambda(fixity_check, response_stream_name) + ) + + fixity_check.update!( + checksum_hexdigest: checksum_hexdigest, + object_size: object_size, + status: :success ) # Broadcast message when job is complete broadcast_fixity_check_complete( - response_stream_name, bucket_name, object_path, checksum_algorithm_name, checksum_hexdigest, object_size + response_stream_name, fixity_check.bucket_name, fixity_check.object_path, + fixity_check.checksum_algorithm_name, checksum_hexdigest, object_size ) rescue StandardError => e - broadcast_fixity_check_error(response_stream_name, e.message, bucket_name, object_path, checksum_algorithm_name) + fixity_check.update!( + checksum_hexdigest: checksum_hexdigest, + object_size: object_size, + status: :failure, + error_message: "An unexpected error occurred: #{e.class.name} -> #{e.message}" + ) + broadcast_fixity_check_error( + response_stream_name, e.message, fixity_check.bucket_name, + fixity_check.object_path, fixity_check.checksum_algorithm_name + ) end def broadcast_fixity_check_complete( @@ -52,10 +71,14 @@ def broadcast_fixity_check_error( ) end - def progress_report_lambda(response_stream_name) + def progress_report_lambda(fixity_check, response_stream_name) lambda do |_chunk, _bytes_read, chunk_counter| + # Only provide an update once per 100 chunks processed return unless (chunk_counter % 100).zero? + # Update the updated_at attribute for this FixityCheck record + fixity_check.touch # rubocop:disable Rails/SkipsModelValidations + # We periodically broadcast a message to indicate that the processing is still happening. # This is so that a client can check whether a job has stalled. ActionCable.server.broadcast( diff --git a/app/models/fixity_check.rb b/app/models/fixity_check.rb new file mode 100644 index 0000000..8dcbe85 --- /dev/null +++ b/app/models/fixity_check.rb @@ -0,0 +1,5 @@ +# frozen_string_literal: true + +class FixityCheck < ApplicationRecord + enum status: { pending: 0, in_progress: 1, success: 2, failure: 3 } +end diff --git a/config/routes.rb b/config/routes.rb index 7830e8d..0d5af1f 100644 --- a/config/routes.rb +++ b/config/routes.rb @@ -26,7 +26,11 @@ # Defines the root path route ("/") root 'pages#home' - post '/fixity_checks/run_fixity_check_for_s3_object', to: 'fixity_checks#run_fixity_check_for_s3_object' + resources :fixity_checks, only: [:create, :show], defaults: { format: 'json' } do + collection do + post 'run_fixity_check_for_s3_object' + end + end # Mount ActionCable Websocket route mount ActionCable.server => '/cable' diff --git a/db/migrate/20241005213933_create_fixity_checks.rb b/db/migrate/20241005213933_create_fixity_checks.rb new file mode 100644 index 0000000..36f2b96 --- /dev/null +++ b/db/migrate/20241005213933_create_fixity_checks.rb @@ -0,0 +1,17 @@ +class CreateFixityChecks < ActiveRecord::Migration[7.1] + def change + create_table :fixity_checks do |t| + t.string :job_identifier, null: false, limit: 255 + t.string :bucket_name, null: false + t.string :object_path, null: false + t.string :checksum_algorithm_name, null: false + t.string :checksum_hexdigest, null: true + t.bigint :object_size, null: true + t.integer :status, null: false, default: 0 + t.text :error_message, null: true + t.timestamps + end + add_index :fixity_checks, :job_identifier, unique: true + add_index :fixity_checks, :updated_at + end +end diff --git a/db/schema.rb b/db/schema.rb index 4121a90..3d116ad 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -10,7 +10,22 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema[7.1].define(version: 2024_04_19_183323) do +ActiveRecord::Schema[7.1].define(version: 2024_10_05_213933) do + create_table "fixity_checks", force: :cascade do |t| + t.string "job_identifier", limit: 255, null: false + t.string "bucket_name", null: false + t.string "object_path", null: false + t.string "checksum_algorithm_name", null: false + t.string "checksum_hexdigest" + t.bigint "object_size" + t.integer "status", default: 0, null: false + t.text "error_message" + t.datetime "created_at", null: false + t.datetime "updated_at", null: false + t.index ["job_identifier"], name: "index_fixity_checks_on_job_identifier", unique: true + t.index ["updated_at"], name: "index_fixity_checks_on_updated_at" + end + create_table "users", force: :cascade do |t| t.string "email", default: "", null: false t.string "encrypted_password", default: "", null: false diff --git a/spec/aws_check_fixity_job_spec.rb b/spec/aws_check_fixity_job_spec.rb index 621c0c9..55093c9 100644 --- a/spec/aws_check_fixity_job_spec.rb +++ b/spec/aws_check_fixity_job_spec.rb @@ -15,24 +15,46 @@ let(:object_size) { example_content.bytesize } let(:stream_name) { "#{FixityCheckChannel::FIXITY_CHECK_STREAM_PREFIX}#{job_identifier}" } let(:error_message) { 'oh no!' } + let(:fixity_check) do + FactoryBot.create( + :fixity_check, + job_identifier: job_identifier, + bucket_name: bucket_name, + object_path: object_path, + checksum_algorithm_name: checksum_algorithm_name + ) + end describe '#perform' do - it 'works as expected' do - allow(CheckPlease::Aws::ObjectFixityChecker).to receive(:check).with( - bucket_name, - object_path, - checksum_algorithm_name, - on_chunk: Proc - ).and_return([checksum_hexdigest, object_size]) - expect(aws_check_fixity_job).to receive(:broadcast_fixity_check_complete).with( - stream_name, - bucket_name, - object_path, - checksum_algorithm_name, - checksum_hexdigest, - object_size - ) - aws_check_fixity_job.perform(job_identifier, bucket_name, object_path, checksum_algorithm_name) + context 'a successful run' do + before do + allow(CheckPlease::Aws::ObjectFixityChecker).to receive(:check).with( + bucket_name, + object_path, + checksum_algorithm_name, + on_chunk: Proc + ).and_return([checksum_hexdigest, object_size]) + end + + it 'broadcasts a fixity check complete message' do + expect(aws_check_fixity_job).to receive(:broadcast_fixity_check_complete).with( + stream_name, + bucket_name, + object_path, + checksum_algorithm_name, + checksum_hexdigest, + object_size + ) + aws_check_fixity_job.perform(fixity_check.id) + end + + it 'saves the FixityCheck result in the database' do + aws_check_fixity_job.perform(fixity_check.id) + FixityCheck.first.tap do |fixity_check| + expect(fixity_check.checksum_hexdigest).to eq(checksum_hexdigest) + expect(fixity_check.object_size).to eq(object_size) + end + end end it 'broadcasts a fixity check error message when an error occurs during processing' do @@ -45,7 +67,7 @@ object_path, checksum_algorithm_name ) - aws_check_fixity_job.perform(job_identifier, bucket_name, object_path, checksum_algorithm_name) + aws_check_fixity_job.perform(fixity_check.id) end end @@ -53,8 +75,9 @@ let(:chunk) { 'a chunk of content' } let(:bytes_read) { 12_345 } - it 'broadcasts an Action Cable message at the expected interval' do - progress_report_lambda = aws_check_fixity_job.progress_report_lambda(stream_name) + it 'broadcasts an Action Cable message at the expected interval, and touches the FixityCheck record' do + progress_report_lambda = aws_check_fixity_job.progress_report_lambda(fixity_check, stream_name) + expect(fixity_check).to receive(:touch).exactly(10).times expect(ActionCable.server).to receive(:broadcast).exactly(10).times (1..1000).each do |i| progress_report_lambda.call(chunk, bytes_read, i) diff --git a/spec/channels/fixity_check_channel_spec.rb b/spec/channels/fixity_check_channel_spec.rb index fcd4619..488c5e3 100644 --- a/spec/channels/fixity_check_channel_spec.rb +++ b/spec/channels/fixity_check_channel_spec.rb @@ -37,6 +37,15 @@ let(:file_content) { 'A' * 1024 } let(:checksum_hexdigest) { Digest::SHA256.hexdigest(file_content) } let(:object_size) { file_content.bytesize } + let(:fixity_check) do + FactoryBot.create( + :fixity_check, + job_identifier: job_identifier, + bucket_name: bucket_name, + object_path: object_path, + checksum_algorithm_name: checksum_algorithm_name + ) + end before do allow(CheckPlease::Aws::ObjectFixityChecker).to receive(:check).with( @@ -50,7 +59,7 @@ it 'initiates a checksum calculation, which queues a background job and '\ 'responds with a fixity_check_complete broadcast' do expect(AwsCheckFixityJob).to receive(:perform_later).with( - job_identifier, bucket_name, object_path, checksum_algorithm_name + FixityCheck.count + 1 ).and_call_original expect { diff --git a/spec/factories/fixity_checks.rb b/spec/factories/fixity_checks.rb new file mode 100644 index 0000000..491ac16 --- /dev/null +++ b/spec/factories/fixity_checks.rb @@ -0,0 +1,26 @@ +# frozen_string_literal: true + +FactoryBot.define do + factory :fixity_check do + job_identifier { SecureRandom.uuid } + bucket_name { 'sample_bucket' } + object_path { 'some/object/path' } + checksum_algorithm_name { 'sha256' } + + trait :in_progress do + status { 'in_progress' } + end + + trait :failure do + status { 'failure' } + error_message { 'An error occurred and this is the message associated with it.' } + end + + trait :success do + status { 'success' } + example_content = 'example' + checksum_hexdigest { Digest::SHA256.hexdigest(example_content) } + object_size { example_content.bytesize } + end + end +end diff --git a/spec/requests/fixity_checks/create_spec.rb b/spec/requests/fixity_checks/create_spec.rb new file mode 100644 index 0000000..92a8610 --- /dev/null +++ b/spec/requests/fixity_checks/create_spec.rb @@ -0,0 +1,93 @@ +# frozen_string_literal: true + +require 'rails_helper' + +endpoint = '/fixity_checks' + +RSpec.describe endpoint, type: :request do + describe "POST #{endpoint}" do + context 'when unauthenticated request' do + it 'returns a 401 (unauthorized) status when no auth token is provided' do + post endpoint + expect(response.status).to eq(401) + end + + it 'returns a 401 (unauthorized) status when an incorrect auth token is provided' do + post endpoint, headers: { 'Authorization' => 'Token NOTVALID' } + expect(response.status).to eq(401) + end + end + + context 'when authenticated request' do + let(:bucket_name) { 'cul-dlstor-digital-testing1' } + let(:object_path) { 'test-909kb-file.jpg' } + let(:checksum_algorithm_name) { 'sha256' } + + let(:example_content) { 'example' } + let(:checksum_hexdigest) { Digest::SHA256.hexdigest(example_content) } + let(:object_size) { example_content.bytesize } + + let(:fixity_check_params) do + { + fixity_check: { + bucket_name: bucket_name, + object_path: object_path, + checksum_algorithm_name: checksum_algorithm_name + } + } + end + + context 'when valid params are given' do + it 'returns a 200 (ok) status ' do + post_with_auth endpoint, params: fixity_check_params + expect(response.status).to eq(200) + end + + it 'creates the expected FixityCheck record' do + expect(FixityCheck).to receive(:create!).with({ + bucket_name: bucket_name, + object_path: object_path, + checksum_algorithm_name: checksum_algorithm_name, + job_identifier: String # We expect a random UUID string here + }).and_call_original + post_with_auth endpoint, params: fixity_check_params + end + + it 'returns the expected response body' do + post_with_auth endpoint, params: fixity_check_params + expect(response.body).to be_json_eql(%( + { + "bucket_name": "#{bucket_name}", + "checksum_algorithm_name": "#{checksum_algorithm_name}", + "checksum_hexdigest": null, "error_message": null, + "job_identifier": "#{FixityCheck.first.job_identifier}", + "object_path": "#{object_path}", + "object_size": null, "status": "pending" + } + )) + end + end + + context 'when a required param is missing' do + [:bucket_name, :object_path, :checksum_algorithm_name].each do |required_param| + context "when required param #{required_param} is missing" do + before do + fixity_check_params[:fixity_check].delete(required_param) + post_with_auth endpoint, params: fixity_check_params + end + + it 'returns a 400 (bad request) status ' do + expect(response.status).to eq(400) + end + + it 'returns the expected error' do + expect(response.body).to be_json_eql(%({ + "error_message" : "param is missing or the value is empty: #{required_param}" + })) + end + end + end + end + end + end +end diff --git a/spec/requests/fixity_checks/show_spec.rb b/spec/requests/fixity_checks/show_spec.rb new file mode 100644 index 0000000..0858617 --- /dev/null +++ b/spec/requests/fixity_checks/show_spec.rb @@ -0,0 +1,69 @@ +# frozen_string_literal: true + +require 'rails_helper' + +fixity_check_id = 1 +endpoint = "/fixity_checks/#{fixity_check_id}" + +RSpec.describe endpoint, type: :request do + describe "GET #{endpoint}" do + context 'when unauthenticated request' do + it 'returns a 401 (unauthorized) status when no auth token is provided' do + get endpoint + expect(response.status).to eq(401) + end + + it 'returns a 401 (unauthorized) status when an incorrect auth token is provided' do + get endpoint, headers: { 'Authorization' => 'Token NOTVALID' } + expect(response.status).to eq(401) + end + end + + context 'when authenticated request' do + let(:job_identifier) { SecureRandom.uuid } + let(:bucket_name) { 'cul-dlstor-digital-testing1' } + let(:object_path) { 'test-909kb-file.jpg' } + let(:checksum_algorithm_name) { 'sha256' } + + let(:example_content) { 'example' } + let(:checksum_hexdigest) { Digest::SHA256.hexdigest(example_content) } + let(:object_size) { example_content.bytesize } + + context 'when a resource exists at the requested path' do + before do + FactoryBot.create( + :fixity_check, + :success, + id: fixity_check_id, + job_identifier: job_identifier, + bucket_name: bucket_name, + object_path: object_path, + checksum_algorithm_name: checksum_algorithm_name, + checksum_hexdigest: checksum_hexdigest, + object_size: object_size + ) + end + + it 'returns a 200 (ok) status ' do + get_with_auth endpoint + expect(response.status).to eq(200) + end + + it 'returns the expected response body' do + get_with_auth endpoint + expect(response.body).to be_json_eql(%( + { + "bucket_name": "#{bucket_name}", + "checksum_algorithm_name": "#{checksum_algorithm_name}", + "checksum_hexdigest": "#{checksum_hexdigest}", "error_message": null, + "job_identifier": "#{job_identifier}", + "object_path": "#{object_path}", + "object_size": #{object_size}, + "status": "success" + } + )) + end + end + end + end +end