From bff0629a2aba895c93e80ac322e47405b68cf439 Mon Sep 17 00:00:00 2001 From: Dean Lofts Date: Wed, 18 Sep 2024 01:37:01 +1000 Subject: [PATCH] I can't believe I ended up here --- app/models/user.rb | 153 +++++++++++-------- app/services/digital_ocean_spaces_service.rb | 91 +++++++++++ lib/tasks/migrate_images_to_spaces.rake | 100 ++++++++++++ lib/tasks/update_image_urls.rake | 76 +++++++++ spec/requests/spaces_spec.rb | 71 +++++++++ 5 files changed, 431 insertions(+), 60 deletions(-) create mode 100644 app/services/digital_ocean_spaces_service.rb create mode 100644 lib/tasks/migrate_images_to_spaces.rake create mode 100644 lib/tasks/update_image_urls.rake create mode 100644 spec/requests/spaces_spec.rb diff --git a/app/models/user.rb b/app/models/user.rb index 8fd04cd..4f950e9 100644 --- a/app/models/user.rb +++ b/app/models/user.rb @@ -1,6 +1,6 @@ class User < ApplicationRecord - FALLBACK_AVATAR_URL = '/avatars/default_avatar.jpg' - FALLBACK_BANNER_URL = '/banners/default_banner.jpg' + FALLBACK_AVATAR_URL = 'https://linkarooie.syd1.digitaloceanspaces.com/defaults/default_avatar.jpg' + FALLBACK_BANNER_URL = 'https://linkarooie.syd1.digitaloceanspaces.com/defaults/default_banner.jpg' attr_accessor :invite_code devise :database_authenticatable, :registerable, @@ -24,8 +24,8 @@ class User < ApplicationRecord before_validation :ensure_username_presence before_create :set_default_images after_create :generate_open_graph_image, unless: -> { Rails.env.test? } - after_save :download_and_store_avatar, if: -> { saved_change_to_avatar? && avatar.present? } - after_save :download_and_store_banner, if: -> { saved_change_to_banner? && banner.present? } + before_save :process_avatar, if: :will_save_change_to_avatar? + before_save :process_banner, if: :will_save_change_to_banner? serialize :tags, coder: JSON @@ -45,20 +45,12 @@ def generate_open_graph_image OpenGraphImageGenerator.new(self).generate end - def download_and_store_avatar - download_and_store_image(:avatar, FALLBACK_AVATAR_URL) - end - - def download_and_store_banner - download_and_store_image(:banner, FALLBACK_BANNER_URL) - end - def avatar_url - avatar_local_path.present? ? "/#{avatar_local_path}" : (avatar.presence || FALLBACK_AVATAR_URL) + avatar.presence || FALLBACK_AVATAR_URL end def banner_url - banner_local_path.present? ? "/#{banner_local_path}" : (banner.presence || FALLBACK_BANNER_URL) + banner.presence || FALLBACK_BANNER_URL end def valid_url?(url) @@ -81,58 +73,99 @@ def set_default_images self.banner ||= FALLBACK_BANNER_URL end - def download_and_store_image(type, fallback_url) - url = send(type) - - Rails.logger.info "Downloading #{type} from #{url}" - - if url.blank? || !valid_url?(url) - Rails.logger.warn "#{type.capitalize} URL invalid or blank. Using fallback." - update_column("#{type}_local_path", fallback_url) + def process_avatar + process_image(:avatar) + end + + def process_banner + process_image(:banner) + end + + def process_image(type) + url = self[type] + return if url.blank? + + if url.start_with?('https://linkarooie.syd1.digitaloceanspaces.com/') + # URL is already a Spaces URL, no need to process return end - + begin - uri = URI.parse(url) - Rails.logger.info "Attempting to download #{type} from #{uri}" - Net::HTTP.start(uri.host, uri.port, use_ssl: uri.scheme == 'https') do |http| - request = Net::HTTP::Get.new(uri) - response = http.request(request) + response = fetch_image(url) + + case response + when Net::HTTPSuccess + content_type = response['Content-Type'] - if response.is_a?(Net::HTTPSuccess) - content_type = response['Content-Type'] - Rails.logger.info "Downloaded #{type}, content type: #{content_type}" - - unless content_type.start_with?('image/') - raise "Invalid content type: #{content_type}" - end - - extension = case content_type - when 'image/jpeg' then '.jpg' - when 'image/png' then '.png' - when 'image/gif' then '.gif' - else '' - end - - image_dir = Rails.root.join('public', type.to_s.pluralize) - FileUtils.mkdir_p(image_dir) unless File.directory?(image_dir) - - filename = "#{username}_#{type}#{extension}" - filepath = File.join(image_dir, filename) - - File.open(filepath, 'wb') { |file| file.write(response.body) } - - update_column("#{type}_local_path", "#{type.to_s.pluralize}/#{filename}") - Rails.logger.info "#{type.capitalize} successfully downloaded for user #{username}" - + if content_type.start_with?('image/') + upload_image_to_spaces(type, response.body, content_type) else - Rails.logger.warn "Failed to download #{type} for user #{username}: HTTP Error: #{response.code} #{response.message}. Using local fallback." - update_column(type, fallback_url) + handle_non_image_content(type) end + when Net::HTTPNotFound + handle_404_error(type) + else + handle_other_error(type, response) end - rescue StandardError => e - Rails.logger.error "Failed to download #{type} for user #{username}: #{e.message}. Using fallback." - update_column(type, fallback_url) + rescue SocketError, URI::InvalidURIError => e + handle_invalid_url(type, e) + end + end + + def fetch_image(url) + uri = URI.parse(url) + Net::HTTP.start(uri.host, uri.port, use_ssl: uri.scheme == 'https') do |http| + http.request(Net::HTTP::Get.new(uri)) end - end + end + + def upload_image_to_spaces(type, content, content_type) + extension = extract_extension(content_type) + filename = "#{id}_#{type}#{extension}" + key = "#{type.to_s.pluralize}/#{filename}" + + spaces_url = upload_to_spaces(content, key) + self[type] = spaces_url if spaces_url + end + + def upload_to_spaces(content, key) + bucket = S3_CLIENT.bucket(ENV['SPACES_BUCKET_IMAGES']) + obj = bucket.object(key) + + obj.put(body: content, acl: 'public-read') + + "https://#{ENV['SPACES_BUCKET_IMAGES']}.syd1.digitaloceanspaces.com/#{key}" + rescue Aws::S3::Errors::ServiceError => e + Rails.logger.error "Failed to upload to Spaces: #{e.message}" + nil + end + + def extract_extension(content_type) + case content_type + when 'image/jpeg' then '.jpg' + when 'image/png' then '.png' + when 'image/gif' then '.gif' + else '.jpg' # Default to jpg if unknown + end + end + + def handle_non_image_content(type) + Rails.logger.warn "Invalid content type for #{type} upload: #{self[type]}" + self[type] = type == :avatar ? FALLBACK_AVATAR_URL : FALLBACK_BANNER_URL + end + + def handle_404_error(type) + Rails.logger.warn "404 error for #{type} upload: #{self[type]}" + self[type] = type == :avatar ? FALLBACK_AVATAR_URL : FALLBACK_BANNER_URL + end + + def handle_other_error(type, response) + Rails.logger.error "Error downloading #{type}: #{response.code} #{response.message}" + self[type] = type == :avatar ? FALLBACK_AVATAR_URL : FALLBACK_BANNER_URL + end + + def handle_invalid_url(type, error) + Rails.logger.error "Invalid URL for #{type}: #{error.message}" + self[type] = type == :avatar ? FALLBACK_AVATAR_URL : FALLBACK_BANNER_URL + end end \ No newline at end of file diff --git a/app/services/digital_ocean_spaces_service.rb b/app/services/digital_ocean_spaces_service.rb new file mode 100644 index 0000000..57b5d06 --- /dev/null +++ b/app/services/digital_ocean_spaces_service.rb @@ -0,0 +1,91 @@ +require 'rails_helper' +require 'aws-sdk-s3' + +RSpec.describe DigitalOceanSpacesService do + let(:bucket_name) { 'linkarooie' } + let(:service) { described_class.new(bucket_name) } + let(:s3_bucket) { instance_double(Aws::S3::Bucket) } + let(:s3_object) { instance_double(Aws::S3::Object) } + let(:file_content) { 'image_content' } + let(:content_type) { 'image/jpeg' } + let(:key) { 'avatars/default_avatar.jpg' } + + before do + # Mock S3 Client initialization and bucket + allow(S3_CLIENT).to receive(:bucket).with(bucket_name).and_return(s3_bucket) + allow(s3_bucket).to receive(:object).with(key).and_return(s3_object) + end + + describe '#upload_file' do + it 'uploads the file to DigitalOcean Spaces and returns the public URL' do + # Mock successful upload + allow(s3_object).to receive(:put).and_return(true) + allow(s3_object).to receive(:public_url).and_return("https://#{bucket_name}.syd1.digitaloceanspaces.com/#{key}") + + result = service.upload_file(key, file_content, content_type) + + # Verify that the correct methods are called + expect(s3_bucket).to have_received(:object).with(key) + expect(s3_object).to have_received(:put).with(body: file_content, acl: 'public-read', content_type: content_type) + # Ensure the public URL is returned + expect(result).to eq("https://#{bucket_name}.syd1.digitaloceanspaces.com/#{key}") + end + + it 'returns nil if the upload fails' do + # Mock an upload failure + allow(s3_object).to receive(:put).and_raise(Aws::S3::Errors::ServiceError.new(nil, 'Upload failed')) + + result = service.upload_file(key, file_content, content_type) + + # Verify that nil is returned on failure + expect(result).to be_nil + end + end + + describe '#delete_file' do + it 'deletes the file from DigitalOcean Spaces' do + # Mock successful deletion + allow(s3_object).to receive(:delete).and_return(true) + + result = service.delete_file(key) + + # Verify that the correct delete method is called + expect(s3_bucket).to have_received(:object).with(key) + expect(s3_object).to have_received(:delete) + # Ensure the result is nil since the method doesn't return anything on success + expect(result).to be_nil + end + + it 'returns nil and logs an error if deletion fails' do + # Mock a deletion failure + allow(s3_object).to receive(:delete).and_raise(Aws::S3::Errors::ServiceError.new(nil, 'Delete failed')) + + result = service.delete_file(key) + + # Verify that nil is returned on failure + expect(result).to be_nil + end + end + + describe '#download_file' do + it 'downloads the file from DigitalOcean Spaces' do + # Mock successful download + allow(s3_object).to receive(:get).and_return(double(body: double(read: 'file_content'))) + + result = service.download_file(key) + + # Verify that the file content is returned + expect(result).to eq('file_content') + end + + it 'returns nil if the download fails' do + # Mock a download failure + allow(s3_object).to receive(:get).and_raise(Aws::S3::Errors::ServiceError.new(nil, 'Download failed')) + + result = service.download_file(key) + + # Verify that nil is returned on failure + expect(result).to be_nil + end + end +end diff --git a/lib/tasks/migrate_images_to_spaces.rake b/lib/tasks/migrate_images_to_spaces.rake new file mode 100644 index 0000000..effd05a --- /dev/null +++ b/lib/tasks/migrate_images_to_spaces.rake @@ -0,0 +1,100 @@ +# lib/tasks/migrate_images_to_spaces.rake + +namespace :images do + desc 'Migrate user avatars and banners to DigitalOcean Spaces (use dry_run=true for testing)' + task :migrate_to_spaces, [:dry_run] => :environment do |t, args| + dry_run = args[:dry_run] == 'true' + puts "\n#{'DRY RUN: ' if dry_run}Starting image migration to DigitalOcean Spaces" + puts "=========================================================\n\n" + + unless ENV['SPACES_BUCKET_IMAGES'] + puts "Error: SPACES_BUCKET_IMAGES environment variable is not set." + exit + end + + def upload_to_spaces(file_path, key, dry_run) + if dry_run + puts " Would upload: #{file_path}" + puts " To: #{key}" + "https://#{ENV['SPACES_BUCKET_IMAGES']}.syd1.digitaloceanspaces.com/#{key}" + else + bucket = S3_CLIENT.bucket(ENV['SPACES_BUCKET_IMAGES']) + obj = bucket.object(key) + + begin + File.open(file_path, 'rb') do |file| + obj.put(body: file, acl: 'public-read') + end + puts " Uploaded: #{key}" + "https://#{ENV['SPACES_BUCKET_IMAGES']}.syd1.digitaloceanspaces.com/#{key}" + rescue Aws::S3::Errors::ServiceError => e + puts " Failed to upload #{key}: #{e.message}" + nil + end + end + end + + def migrate_image(user, attribute, folder, dry_run) + local_path = user.send("#{attribute}_local_path") || user.send(attribute) + if local_path.present? + file_path = local_path.start_with?('/') ? Rails.root.join('public', local_path.sub(/\A\//, '')) : Rails.root.join('public', local_path) + if File.exist?(file_path) + key = "#{folder}/#{user.id}_#{File.basename(file_path)}" + spaces_url = upload_to_spaces(file_path, key, dry_run) + if spaces_url + if dry_run + puts " Would update #{attribute} URL to: #{spaces_url}" + else + user.update(attribute => spaces_url) + puts " Updated #{attribute} URL to: #{spaces_url}" + end + end + else + puts " File not found: #{file_path}" + end + else + puts " No #{attribute} found" + end + end + + puts "Migrating user images:" + User.find_each do |user| + puts "\nProcessing user: #{user.email}" + puts " Avatar:" + migrate_image(user, :avatar, 'avatars', dry_run) + puts " Banner:" + migrate_image(user, :banner, 'banners', dry_run) + end + + def handle_default_image(image_type, dry_run) + puts "\nProcessing default #{image_type}:" + fallback_constant = "User::FALLBACK_#{image_type.upcase}_URL" + fallback_url = User.const_get(fallback_constant) + file_path = Rails.root.join('public', fallback_url.sub(/\A\//, '')) + + if File.exist?(file_path) + key = "defaults/default_#{image_type}#{File.extname(file_path)}" + spaces_url = upload_to_spaces(file_path, key, dry_run) + if spaces_url + affected_users = User.where(image_type => fallback_url) + if dry_run + puts " Would update #{affected_users.count} users with default #{image_type}" + puts " New URL would be: #{spaces_url}" + else + affected_users.update_all(image_type => spaces_url) + puts " Updated #{affected_users.count} users' default #{image_type}" + puts " New URL is: #{spaces_url}" + end + end + else + puts " Default #{image_type} file not found: #{file_path}" + end + end + + handle_default_image('avatar', dry_run) + handle_default_image('banner', dry_run) + + puts "\n#{'DRY RUN: ' if dry_run}Image migration completed!" + puts "=========================================================\n\n" + end +end \ No newline at end of file diff --git a/lib/tasks/update_image_urls.rake b/lib/tasks/update_image_urls.rake new file mode 100644 index 0000000..5752cb5 --- /dev/null +++ b/lib/tasks/update_image_urls.rake @@ -0,0 +1,76 @@ +namespace :users do + desc 'Update user avatar and banner URLs to use correct DigitalOcean Spaces URLs with proper extensions' + task update_image_urls: :environment do + puts "Updating user image URLs..." + + def fetch_content_type(url) + uri = URI.parse(url) + Net::HTTP.start(uri.host, uri.port, use_ssl: uri.scheme == 'https') do |http| + response = http.head(uri.path) + return response['Content-Type'] + end + rescue StandardError => e + puts "Error fetching content type for #{url}: #{e.message}" + nil + end + + def get_extension_from_content_type(content_type) + case content_type + when 'image/jpeg' then '.jpg' + when 'image/png' then '.png' + when 'image/gif' then '.gif' + else '.jpg' # Default to jpg if unknown + end + end + + User.find_each do |user| + puts "\nProcessing user: #{user.email} (ID: #{user.id})" + + # Update avatar URL + if user.avatar.present? + new_avatar_url = "https://linkarooie.syd1.digitaloceanspaces.com/avatars/#{user.id}_#{user.username}_avatar.jpg" + if user.avatar != new_avatar_url + puts " Old avatar URL: #{user.avatar}" + puts " New avatar URL: #{new_avatar_url}" + user.update_column(:avatar, new_avatar_url) + puts " Avatar URL updated successfully" + else + puts " Avatar URL already correct: #{user.avatar}" + end + else + puts " No avatar URL present" + end + + # Update banner URL + if user.banner.present? + if user.banner.start_with?('https://linkarooie.syd1.digitaloceanspaces.com/banners/') && !File.extname(user.banner).present? + content_type = fetch_content_type(user.banner) + if content_type + extension = get_extension_from_content_type(content_type) + new_banner_url = "https://linkarooie.syd1.digitaloceanspaces.com/banners/#{user.id}_#{user.username}_banner#{extension}" + puts " Old banner URL: #{user.banner}" + puts " New banner URL: #{new_banner_url}" + user.update_column(:banner, new_banner_url) + puts " Banner URL updated successfully" + else + puts " Couldn't determine file type for banner: #{user.banner}" + end + else + new_banner_url = "https://linkarooie.syd1.digitaloceanspaces.com/banners/#{user.id}_#{user.username}_banner.jpg" + if user.banner != new_banner_url + puts " Old banner URL: #{user.banner}" + puts " New banner URL: #{new_banner_url}" + user.update_column(:banner, new_banner_url) + puts " Banner URL updated successfully" + else + puts " Banner URL already correct: #{user.banner}" + end + end + else + puts " No banner URL present" + end + end + + puts "\nUser image URL update completed!" + end +end diff --git a/spec/requests/spaces_spec.rb b/spec/requests/spaces_spec.rb new file mode 100644 index 0000000..0d88631 --- /dev/null +++ b/spec/requests/spaces_spec.rb @@ -0,0 +1,71 @@ +require 'rails_helper' +require 'aws-sdk-s3' + +RSpec.describe DigitalOceanSpacesService do + let(:bucket_name) { 'linkarooie' } + let(:service) { described_class.new(bucket_name) } + let(:s3_resource) { instance_double(Aws::S3::Resource) } + let(:s3_bucket) { instance_double(Aws::S3::Bucket) } + let(:s3_object) { instance_double(Aws::S3::Object) } + let(:file_content) { 'image_content' } + let(:content_type) { 'image/jpeg' } + let(:key) { 'avatars/test_image.jpg' } + + before do + # Correctly mock the S3 Resource and Bucket + allow(Aws::S3::Resource).to receive(:new).and_return(s3_resource) + allow(s3_resource).to receive(:bucket).with(bucket_name).and_return(s3_bucket) + allow(s3_bucket).to receive(:object).with(key).and_return(s3_object) + end + + describe '#upload_file' do + it 'uploads the file to DigitalOcean Spaces and returns the public URL' do + # Mock successful upload + allow(s3_object).to receive(:put).and_return(true) + allow(s3_object).to receive(:public_url).and_return("https://#{bucket_name}.syd1.digitaloceanspaces.com/#{key}") + + result = service.upload_file(key, file_content, content_type) + + # Expect the correct methods to be called + expect(s3_bucket).to have_received(:object).with(key) + expect(s3_object).to have_received(:put).with(body: file_content, acl: 'public-read', content_type: content_type) + # Check the returned URL + expect(result).to eq("https://#{bucket_name}.syd1.digitaloceanspaces.com/#{key}") + end + + it 'returns nil if the upload fails' do + # Mock an upload failure + allow(s3_object).to receive(:put).and_raise(Aws::S3::Errors::ServiceError.new(nil, 'Upload failed')) + + result = service.upload_file(key, file_content, content_type) + + # Expect result to be nil + expect(result).to be_nil + end + end + + describe '#delete_file' do + it 'deletes the file from DigitalOcean Spaces' do + # Mock successful deletion + allow(s3_object).to receive(:delete).and_return(true) + + result = service.delete_file(key) + + # Expect the object to be deleted + expect(s3_bucket).to have_received(:object).with(key) + expect(s3_object).to have_received(:delete) + # Expect the result to be nil since it doesn’t return anything on success + expect(result).to be_nil + end + + it 'returns nil and logs an error if deletion fails' do + # Mock a deletion failure + allow(s3_object).to receive(:delete).and_raise(Aws::S3::Errors::ServiceError.new(nil, 'Delete failed')) + + result = service.delete_file(key) + + # Expect result to be nil + expect(result).to be_nil + end + end +end