diff --git a/lib/cloud_controller/blobstore/client_provider.rb b/lib/cloud_controller/blobstore/client_provider.rb index 6ba786ee474..1f622e31ea3 100644 --- a/lib/cloud_controller/blobstore/client_provider.rb +++ b/lib/cloud_controller/blobstore/client_provider.rb @@ -6,6 +6,7 @@ require 'cloud_controller/blobstore/safe_delete_client' require 'cloud_controller/blobstore/storage_cli/storage_cli_client' require 'cloud_controller/blobstore/storage_cli/azure_storage_cli_client' +require 'cloud_controller/blobstore/storage_cli/alioss_storage_cli_client' require 'google/apis/errors' module CloudController diff --git a/lib/cloud_controller/blobstore/storage_cli/alioss_storage_cli_client.rb b/lib/cloud_controller/blobstore/storage_cli/alioss_storage_cli_client.rb new file mode 100644 index 00000000000..47a3943f4de --- /dev/null +++ b/lib/cloud_controller/blobstore/storage_cli/alioss_storage_cli_client.rb @@ -0,0 +1,11 @@ +module CloudController + module Blobstore + class AliStorageCliClient < StorageCliClient + def cli_path + ENV['ALI_STORAGE_CLI_PATH'] || '/var/vcap/packages/ali-storage-cli/bin/ali-storage-cli' + end + + CloudController::Blobstore::StorageCliClient.register('aliyun', AliStorageCliClient) + end + end +end diff --git a/lib/cloud_controller/blobstore/storage_cli/storage_cli_client.rb b/lib/cloud_controller/blobstore/storage_cli/storage_cli_client.rb index 3ded78213ba..eeb3e8303df 100644 --- a/lib/cloud_controller/blobstore/storage_cli/storage_cli_client.rb +++ b/lib/cloud_controller/blobstore/storage_cli/storage_cli_client.rb @@ -80,7 +80,15 @@ def validate_required_keys!(json, path) provider = json['provider'].to_s.strip raise BlobstoreError.new("No provider specified in config file: #{path.inspect}") if provider.empty? - required = %w[account_key account_name container_name environment] + required = + case provider + when 'AzureRM' + %w[account_key account_name container_name environment] + when 'aliyun' + %w[access_key_id access_key_secret endpoint bucket_name] + else + [] + end missing = required.reject { |k| json.key?(k) && !json[k].to_s.strip.empty? } return if missing.empty? diff --git a/spec/unit/lib/cloud_controller/blobstore/storage_cli/alioss_storage_cli_client_spec.rb b/spec/unit/lib/cloud_controller/blobstore/storage_cli/alioss_storage_cli_client_spec.rb new file mode 100644 index 00000000000..be168268b88 --- /dev/null +++ b/spec/unit/lib/cloud_controller/blobstore/storage_cli/alioss_storage_cli_client_spec.rb @@ -0,0 +1,215 @@ +require 'spec_helper' +require 'tempfile' +require 'json' +require_relative '../client_shared' +require 'cloud_controller/blobstore/storage_cli/azure_storage_cli_client' +require 'cloud_controller/blobstore/storage_cli/storage_cli_blob' + +module CloudController + module Blobstore + RSpec.describe AzureStorageCliClient do + let!(:tmp_cfg) do + f = Tempfile.new(['storage_cli_config', '.json']) + f.write({ provider: 'AzureRM', + account_name: 'some-account-name', + account_key: 'some-access-key', + container_name: directory_key, + environment: 'AzureCloud' }.to_json) + f.flush + f + end + + before do + cc_cfg = instance_double(VCAP::CloudController::Config) + allow(VCAP::CloudController::Config).to receive(:config).and_return(cc_cfg) + + allow(cc_cfg).to receive(:get) do |key, *_| + case key + when :storage_cli_config_file_droplets, + :storage_cli_config_file_buildpacks, + :storage_cli_config_file_packages, + :storage_cli_config_file_resource_pool + tmp_cfg.path + end + end + allow(Steno).to receive(:logger).and_return(double(info: nil, error: nil)) + end + + after { tmp_cfg.close! } + + subject(:client) { AzureStorageCliClient.new(provider: 'AzureRM', directory_key: directory_key, resource_type: resource_type, root_dir: 'bommel', config_path: 'path') } + let(:directory_key) { 'my-bucket' } + let(:resource_type) { 'resource_pool' } + let(:downloaded_file) do + Tempfile.open('') do |tmpfile| + tmpfile.write('downloaded file content') + tmpfile + end + end + + let(:deletable_blob) { StorageCliBlob.new('deletable-blob') } + let(:dest_path) { File.join(Dir.mktmpdir, SecureRandom.uuid) } + + describe 'conforms to the blobstore client interface' do + before do + allow(client).to receive(:run_cli).with('exists', anything, allow_exit_code_three: true).and_return([nil, instance_double(Process::Status, exitstatus: 0)]) + allow(client).to receive(:run_cli).with('get', anything, anything).and_wrap_original do |_original_method, _cmd, _source, dest_path| + File.write(dest_path, 'downloaded content') + [nil, instance_double(Process::Status, exitstatus: 0)] + end + allow(client).to receive(:run_cli).with('put', anything, anything).and_return([nil, instance_double(Process::Status, exitstatus: 0)]) + allow(client).to receive(:run_cli).with('copy', anything, anything).and_return([nil, instance_double(Process::Status, exitstatus: 0)]) + allow(client).to receive(:run_cli).with('delete', anything).and_return([nil, instance_double(Process::Status, exitstatus: 0)]) + allow(client).to receive(:run_cli).with('delete-recursive', anything).and_return([nil, instance_double(Process::Status, exitstatus: 0)]) + allow(client).to receive(:run_cli).with('list', anything).and_return(["aa/bb/blob1\ncc/dd/blob2\n", instance_double(Process::Status, exitstatus: 0)]) + allow(client).to receive(:run_cli).with('ensure-bucket-exists').and_return([nil, instance_double(Process::Status, exitstatus: 0)]) + allow(client).to receive(:run_cli).with('properties', anything).and_return(['{"dummy": "json"}', instance_double(Process::Status, exitstatus: 0)]) + allow(client).to receive(:run_cli).with('sign', anything, 'get', '3600s').and_return(['some-url', instance_double(Process::Status, exitstatus: 0)]) + end + + it_behaves_like 'a blobstore client' + end + + describe '#local?' do + it 'returns false' do + expect(client.local?).to be false + end + end + + describe '#cli_path' do + it 'returns the default CLI path' do + expect(client.cli_path).to eq('/var/vcap/packages/azure-storage-cli/bin/azure-storage-cli') + end + + it 'can be overridden by an environment variable' do + allow(ENV).to receive(:[]).with('AZURE_STORAGE_CLI_PATH').and_return('/custom/path/to/AzureRM-storage-cli') + expect(client.cli_path).to eq('/custom/path/to/AzureRM-storage-cli') + end + end + + describe '#exists?' do + context 'when the blob exists' do + before { allow(client).to receive(:run_cli).with('exists', any_args).and_return([nil, instance_double(Process::Status, exitstatus: 0)]) } + + it('returns true') { expect(client.exists?('some-blob-key')).to be true } + end + + context 'when the blob does not exist' do + before { allow(client).to receive(:run_cli).with('exists', any_args).and_return([nil, instance_double(Process::Status, exitstatus: 3)]) } + + it('returns false') { expect(client.exists?('some-blob-key')).to be false } + end + end + + describe '#files_for' do + context 'when CLI returns multiple files' do + let(:cli_output) { "aa/bb/blob1\ncc/dd/blob2\n" } + + before do + allow(client).to receive(:run_cli). + with('list', 'some-prefix'). + and_return([cli_output, instance_double(Process::Status, success?: true)]) + end + + it 'returns StorageCliBlob instances for each file' do + blobs = client.files_for('some-prefix') + expect(blobs.map(&:key)).to eq(['aa/bb/blob1', 'cc/dd/blob2']) + expect(blobs).to all(be_a(StorageCliBlob)) + end + end + + context 'when CLI returns empty output' do + before do + allow(client).to receive(:run_cli). + with('list', 'some-prefix'). + and_return(["\n", instance_double(Process::Status, success?: true)]) + end + + it 'returns an empty array' do + expect(client.files_for('some-prefix')).to eq([]) + end + end + + context 'when CLI output has extra whitespace' do + let(:cli_output) { "aa/bb/blob1 \n \ncc/dd/blob2\n" } + + before do + allow(client).to receive(:run_cli). + with('list', 'some-prefix'). + and_return([cli_output, instance_double(Process::Status, success?: true)]) + end + + it 'strips and rejects empty lines' do + blobs = client.files_for('some-prefix') + expect(blobs.map(&:key)).to eq(['aa/bb/blob1', 'cc/dd/blob2']) + end + end + end + + describe '#blob' do + let(:properties_json) { '{"etag": "test-etag", "last_modified": "2024-10-01T00:00:00Z", "content_length": 1024}' } + + it 'returns a list of StorageCliBlob instances for a given key' do + allow(client).to receive(:run_cli).with('properties', 'bommel/va/li/valid-blob').and_return([properties_json, instance_double(Process::Status, exitstatus: 0)]) + allow(client).to receive(:run_cli).with('sign', 'bommel/va/li/valid-blob', 'get', '3600s').and_return(['some-url', instance_double(Process::Status, exitstatus: 0)]) + + blob = client.blob('valid-blob') + expect(blob).to be_a(StorageCliBlob) + expect(blob.key).to eq('valid-blob') + expect(blob.attributes(:etag, :last_modified, :content_length)).to eq({ + etag: 'test-etag', + last_modified: '2024-10-01T00:00:00Z', + content_length: 1024 + }) + expect(blob.internal_download_url).to eq('some-url') + expect(blob.public_download_url).to eq('some-url') + end + + it 'raises an error if the cli output is empty' do + allow(client).to receive(:run_cli).with('properties', 'bommel/no/ne/nonexistent-blob').and_return([nil, instance_double(Process::Status, exitstatus: 0)]) + expect { client.blob('nonexistent-blob') }.to raise_error(BlobstoreError, /Properties command returned empty output/) + end + + it 'raises an error if the cli output is not valid JSON' do + allow(client).to receive(:run_cli).with('properties', 'bommel/in/va/invalid-json').and_return(['not a json', instance_double(Process::Status, exitstatus: 0)]) + expect { client.blob('invalid-json') }.to raise_error(BlobstoreError, /Failed to parse json properties/) + end + end + + describe '#run_cli' do + it 'returns output and status on success' do + status = instance_double(Process::Status, success?: true, exitstatus: 0) + allow(Open3).to receive(:capture3).with(anything, '-c', anything, 'list', 'arg1').and_return(['ok', '', status]) + + output, returned_status = client.send(:run_cli, 'list', 'arg1') + expect(output).to eq('ok') + expect(returned_status).to eq(status) + end + + it 'raises an error on failure' do + status = instance_double(Process::Status, success?: false, exitstatus: 1) + allow(Open3).to receive(:capture3).with(anything, '-c', anything, 'list', 'arg1').and_return(['', 'error message', status]) + + expect do + client.send(:run_cli, 'list', 'arg1') + end.to raise_error(RuntimeError, /storage-cli list failed with exit code 1/) + end + + it 'allows exit code 3 if specified' do + status = instance_double(Process::Status, success?: false, exitstatus: 3) + allow(Open3).to receive(:capture3).with(anything, '-c', anything, 'list', 'arg1').and_return(['', 'error message', status]) + + output, returned_status = client.send(:run_cli, 'list', 'arg1', allow_exit_code_three: true) + expect(output).to eq('') + expect(returned_status).to eq(status) + end + + it 'raises BlobstoreError on Open3 failure' do + allow(Open3).to receive(:capture3).and_raise(StandardError.new('Open3 error')) + + expect { client.send(:run_cli, 'list', 'arg1') }.to raise_error(BlobstoreError, /Open3 error/) + end + end + end + end +end diff --git a/spec/unit/lib/cloud_controller/blobstore/storage_cli/storage_cli_client_spec.rb b/spec/unit/lib/cloud_controller/blobstore/storage_cli/storage_cli_client_spec.rb index ad76e4a64a6..0bdab42816b 100644 --- a/spec/unit/lib/cloud_controller/blobstore/storage_cli/storage_cli_client_spec.rb +++ b/spec/unit/lib/cloud_controller/blobstore/storage_cli/storage_cli_client_spec.rb @@ -1,5 +1,6 @@ require 'spec_helper' require 'cloud_controller/blobstore/storage_cli/azure_storage_cli_client' +require 'cloud_controller/blobstore/storage_cli/alioss_storage_cli_client' module CloudController module Blobstore @@ -28,6 +29,31 @@ module Blobstore droplets_cfg.close! end + it 'builds the correct client when JSON has provider aliyun' do + droplets_cfg = Tempfile.new(['droplets', '.json']) + droplets_cfg.write({ + provider: 'aliyun', + access_key_id: 'ali-id', + access_key_secret: 'ali-secret', + endpoint: 'oss-example.aliyuncs.com', + bucket_name: 'ali-bucket' + }.to_json) + droplets_cfg.flush + + config_double = instance_double(VCAP::CloudController::Config) + allow(VCAP::CloudController::Config).to receive(:config).and_return(config_double) + allow(config_double).to receive(:get).with(:storage_cli_config_file_droplets).and_return(droplets_cfg.path) + + client_from_registry = StorageCliClient.build( + directory_key: 'dummy-key', + root_dir: 'dummy-root', + resource_type: 'droplets' + ) + expect(client_from_registry).to be_a(AliStorageCliClient) + + droplets_cfg.close! + end + it 'raises an error for an unregistered provider' do droplets_cfg = Tempfile.new(['droplets', '.json']) droplets_cfg.write( @@ -195,6 +221,27 @@ def build_client(resource_type) end.to raise_error(CloudController::Blobstore::BlobstoreError, /Missing required keys.*#{k}/) end end + + %w[access_key_id access_key_secret endpoint bucket_name].each do |k| + it "raises when #{k} missing for aliyun" do + cfg = { + 'provider' => 'aliyun', + 'access_key_id' => 'ali-id', + 'access_key_secret' => '[redacted]', + 'endpoint' => 'oss-example.aliyuncs.com', + 'bucket_name' => 'ali-bucket' + } + cfg.delete(k) + File.write(droplets_cfg.path, cfg.to_json) + + expect do + StorageCliClient.build(directory_key: 'dir', root_dir: 'root', resource_type: 'droplets') + end.to raise_error( + CloudController::Blobstore::BlobstoreError, + /Missing required keys.*#{k}/ + ) + end + end end describe '#exists? exit code handling' do