From ff4f6eb24ba5649990c557640128f425d4b60294 Mon Sep 17 00:00:00 2001 From: KrzyczaK Date: Mon, 28 Nov 2016 12:46:02 +0100 Subject: [PATCH] Added mocks for Fog::Storage::OpenStack 1. The class `Mock` was added in this module where needed 2. Added `fail "Mock Not Implemented (#method_name) in: #{__FILE__}:#{__LINE__}"` where mock was not implemented Code is mostly coppied from `Fog::Storage::Rackspace::Mock` with some additional changes. --- lib/fog/storage/openstack.rb | 232 ++++++++++++++++++ .../storage/openstack/requests/copy_object.rb | 21 ++ .../openstack/requests/delete_container.rb | 6 + .../requests/delete_multiple_objects.rb | 6 + .../openstack/requests/delete_object.rb | 12 + .../requests/delete_static_large_object.rb | 6 + .../openstack/requests/get_container.rb | 23 ++ .../openstack/requests/get_containers.rb | 16 ++ .../storage/openstack/requests/get_object.rb | 24 ++ .../openstack/requests/get_object_http_url.rb | 6 + .../requests/get_object_https_url.rb | 13 +- .../openstack/requests/head_container.rb | 6 + .../openstack/requests/head_containers.rb | 6 + .../storage/openstack/requests/head_object.rb | 24 ++ .../requests/post_set_meta_temp_url_key.rb | 6 + .../storage/openstack/requests/public_url.rb | 6 + .../openstack/requests/put_container.rb | 14 ++ .../requests/put_dynamic_obj_manifest.rb | 6 + .../storage/openstack/requests/put_object.rb | 37 +++ .../openstack/requests/put_object_manifest.rb | 6 + .../requests/put_static_obj_manifest.rb | 6 + 21 files changed, 480 insertions(+), 2 deletions(-) diff --git a/lib/fog/storage/openstack.rb b/lib/fog/storage/openstack.rb index 77d87892d..73f82aa01 100644 --- a/lib/fog/storage/openstack.rb +++ b/lib/fog/storage/openstack.rb @@ -44,6 +44,201 @@ class OpenStack < Fog::Service request :public_url class Mock + class MockContainer + attr_reader :objects, :meta, :service + + # Create a new container. Generally, you should call + # {Fog::Storage::OpenStack#add_container} instead. + def initialize(service) + @service = service + @objects, @meta = {}, {} + end + + # Determine if this container contains any MockObjects or not. + # + # @return [Boolean] + def empty? + @objects.empty? + end + + # Total sizes of all objects added to this container. + # + # @return [Integer] The number of bytes occupied by each contained + # object. + def bytes_used + @objects.values.map { |o| o.bytes_used }.reduce(0) { |a, b| a + b } + end + + # Render the HTTP headers that would be associated with this + # container. + # + # @return [Hash] Any metadata supplied to this + # container, plus additional headers indicating the container's + # size. + def to_headers + @meta.merge({ + 'X-Container-Object-Count' => @objects.size, + 'X-Container-Bytes-Used' => bytes_used + }) + end + + # Access a MockObject within this container by (unescaped) name. + # + # @return [MockObject, nil] Return the MockObject at this name if + # one exists; otherwise, `nil`. + def mock_object(name) + @objects[(name)] + end + + # Access a MockObject with a specific name, raising a + # ` Fog::Storage::OpenStack::NotFound` exception if none are present. + # + # @param name [String] (Unescaped) object name. + # @return [MockObject] The object within this container with the + # specified name. + def mock_object!(name) + mock_object(name) or raise Fog::Storage::OpenStack::NotFound.new + end + + # Add a new MockObject to this container. An existing object with + # the same name will be overwritten. + # + # @param name [String] The object's name, unescaped. + # @param data [String, #read] The contents of the object. + def add_object(name, data) + @objects[(name)] = MockObject.new(data, service) + end + + # Remove a MockObject from the container by name. No effect if the + # object is not present beforehand. + # + # @param name [String] The (unescaped) object name to remove. + def remove_object(name) + @objects.delete (name) + end + end + + class MockObject + attr_reader :hash, :bytes_used, :content_type, :last_modified + attr_reader :body, :meta, :service + attr_accessor :static_manifest + + # Construct a new object. Generally, you should call + # {MockContainer#add_object} instead of instantiating these directly. + def initialize(data, service) + data = Fog::Storage.parse_data(data) + @service = service + + @bytes_used = data[:headers]['Content-Length'] + @content_type = data[:headers]['Content-Type'] + if data[:body].respond_to? :read + @body = data[:body].read + elsif data[:body].respond_to? :body + @body = data[:body].body + else + @body = data[:body] + end + @last_modified = Time.now.utc + @hash = Digest::MD5.hexdigest(@body) + @meta = {} + @static_manifest = false + end + + # Determine if this object was created as a static large object + # manifest. + # + # @return [Boolean] + def static_manifest? + @static_manifest + end + + # Determine if this object has the metadata header that marks it as a + # dynamic large object manifest. + # + # @return [Boolean] + def dynamic_manifest? + ! large_object_prefix.nil? + end + + # Iterate through each MockObject that contains a part of the data for + # this logical object. In the normal case, this will only yield the + # receiver directly. For dynamic and static large object manifests, + # however, this call will yield each MockObject that contains a part + # of the whole, in sequence. + # + # Manifests that refer to containers or objects that don't exist will + # skip those sections and log a warning, instead. + # + # @yield [MockObject] Each object that holds a part of this logical + # object. + def each_part + case + when dynamic_manifest? + # Concatenate the contents and sizes of each matching object. + # Note that cname and oprefix are already escaped. + cname, oprefix = large_object_prefix.split('/', 2) + + target_container = service.data[cname] + if target_container + all = target_container.objects.keys + matching = all.select { |name| name.start_with? oprefix } + keys = matching.sort + + keys.each do |name| + yield target_container.objects[name] + end + else + Fog::Logger.warning "Invalid container in dynamic object manifest: #{cname}" + yield self + end + when static_manifest? + Fog::JSON.decode(body).each do |segment| + cname, oname = segment['path'].split('/', 2) + + cont = service.mock_container cname + unless cont + Fog::Logger.warning "Invalid container in static object manifest: #{cname}" + next + end + + obj = cont.mock_object oname + unless obj + Fog::Logger.warning "Invalid object in static object manifest: #{oname}" + next + end + + yield obj + end + else + yield self + end + end + + # Access the object name prefix that controls which other objects + # comprise a dynamic large object. + # + # @return [String, nil] The object name prefix, or `nil` if none is + # present. + def large_object_prefix + @meta['X-Object-Manifest'] + end + + # Construct the fake HTTP headers that should be returned on requests + # targetting this object. Includes computed `Content-Type`, + # `Content-Length`, `Last-Modified` and `ETag` headers in addition to + # whatever metadata has been associated with this object manually. + # + # @return [Hash] Header values stored in a Hash. + def to_headers + { + 'Content-Type' => @content_type, + 'Content-Length' => @bytes_used, + 'Last-Modified' => @last_modified.strftime('%a, %b %d %Y %H:%M:%S %Z'), + 'ETag' => @hash + }.merge(@meta) + end + end + def self.data @data ||= Hash.new do |hash, key| hash[key] = {} @@ -57,6 +252,18 @@ def self.reset def initialize(options = {}) @openstack_api_key = options[:openstack_api_key] @openstack_username = options[:openstack_username] + @openstack_project_id = options[:openstack_project_id] + @openstack_domain_name = options[:openstack_domain_name] + @openstack_temp_url_key = options[:openstack_temp_url_key] + + uri = URI.parse(options[:openstack_management_url]) + @host = uri.host + @port = uri.port + @path = uri.path + @scheme = uri.scheme + rescue URI::InvalidURIError => _ex + @scheme = 'https' + @host = 'www.opensctak.url' @path = '/v1/AUTH_1234' end @@ -77,6 +284,31 @@ def change_account(account) def reset_account_name @path = @original_path end + + # Access a MockContainer with the specified name, if one exists. + # + # @param cname [String] The (unescaped) container name. + # @return [MockContainer, nil] The named MockContainer, or `nil` if + # none exist. + def mock_container(cname) + data[(cname)] + end + + # Access a MockContainer with the specified name, raising a + # { Fog::Storage::OpenStack::NotFound} exception if none exist. + # + # @param cname [String] The (unescaped) container name. + # @throws [ Fog::Storage::OpenStack::NotFound] If no container with the + # given name exists. + # @return [MockContainer] The existing MockContainer. + def mock_container!(cname) + mock_container(cname) or raise Fog::Storage::OpenStack::NotFound.new + end + + def add_container(name) + data[(name)] = MockContainer.new(self) + end + end class Real diff --git a/lib/fog/storage/openstack/requests/copy_object.rb b/lib/fog/storage/openstack/requests/copy_object.rb index 2e98e523f..3412ac241 100644 --- a/lib/fog/storage/openstack/requests/copy_object.rb +++ b/lib/fog/storage/openstack/requests/copy_object.rb @@ -1,6 +1,27 @@ module Fog module Storage class OpenStack + class Mock + def copy_object(source_container_name, source_object_name, target_container_name, target_object_name, options = {}) + source_container = mock_container!(source_container_name) + source_object = source_container.mock_object!(source_object_name) + + target_container = mock_container!(target_container_name) + target_object = target_container.add_object(target_object_name, source_object.dup) + + response = Excon::Response.new + + if source_object && target_container + response.status = 200 + else + response.status = 404 + raise(Excon::Errors.status_error({:expects => 200}, response)) + end + + response + end + end + class Real # Copy object # diff --git a/lib/fog/storage/openstack/requests/delete_container.rb b/lib/fog/storage/openstack/requests/delete_container.rb index 5023110f5..2fbdfc35d 100644 --- a/lib/fog/storage/openstack/requests/delete_container.rb +++ b/lib/fog/storage/openstack/requests/delete_container.rb @@ -1,6 +1,12 @@ module Fog module Storage class OpenStack + class Mock + def delete_container(name) + fail "Mock Not Implemented (#delete_container) in: #{__FILE__}:#{__LINE__}" + end + end + class Real # Delete an existing container # diff --git a/lib/fog/storage/openstack/requests/delete_multiple_objects.rb b/lib/fog/storage/openstack/requests/delete_multiple_objects.rb index a91d5a245..ac7a57796 100644 --- a/lib/fog/storage/openstack/requests/delete_multiple_objects.rb +++ b/lib/fog/storage/openstack/requests/delete_multiple_objects.rb @@ -1,6 +1,12 @@ module Fog module Storage class OpenStack + class Mock + def delete_multiple_objects(container, object_names, options = {}) + fail "Mock Not Implemented (#delete_multiple_objects) in #{__FILE__}:#{__LINE__}" + end + end + class Real # Deletes multiple objects or containers with a single request. # diff --git a/lib/fog/storage/openstack/requests/delete_object.rb b/lib/fog/storage/openstack/requests/delete_object.rb index bbc344cda..5d72a2e90 100644 --- a/lib/fog/storage/openstack/requests/delete_object.rb +++ b/lib/fog/storage/openstack/requests/delete_object.rb @@ -1,6 +1,18 @@ module Fog module Storage class OpenStack + class Mock + def delete_object(container, object) + cc = mock_container!(container) + cc.mock_object!(object) + cc.remove_object(object) + + response = Excon::Response.new + response.status = 204 + response + end + end + class Real # Delete an existing object # diff --git a/lib/fog/storage/openstack/requests/delete_static_large_object.rb b/lib/fog/storage/openstack/requests/delete_static_large_object.rb index 4acddd449..fadb19080 100644 --- a/lib/fog/storage/openstack/requests/delete_static_large_object.rb +++ b/lib/fog/storage/openstack/requests/delete_static_large_object.rb @@ -1,6 +1,12 @@ module Fog module Storage class OpenStack + class Mock + def delete_static_large_object(container, object, options = {}) + fail "Mock Not Implemented (#delete_static_large_object) in: #{__FILE__}:#{__LINE__}" + end + end + class Real # Delete a static large object. # diff --git a/lib/fog/storage/openstack/requests/get_container.rb b/lib/fog/storage/openstack/requests/get_container.rb index 27b93da95..88dd363b7 100644 --- a/lib/fog/storage/openstack/requests/get_container.rb +++ b/lib/fog/storage/openstack/requests/get_container.rb @@ -1,6 +1,29 @@ module Fog module Storage class OpenStack + class Mock + def get_container(container, options = {}) + c = mock_container! container + + results = [] + c.objects.each do |key, mock_file| + results << { + "hash" => mock_file.hash, + "last_modified" => mock_file.last_modified.strftime('%Y-%m-%dT%H:%M:%S.%L'), + "bytes" => mock_file.bytes_used, + "name" => key, + "content_type" => mock_file.content_type + } + end + + response = Excon::Response.new + response.status = 200 + response.headers = c.to_headers + response.body = results + response + end + end + class Real # Get details for container and total bytes stored # diff --git a/lib/fog/storage/openstack/requests/get_containers.rb b/lib/fog/storage/openstack/requests/get_containers.rb index 254cd0d9a..8f9830dab 100644 --- a/lib/fog/storage/openstack/requests/get_containers.rb +++ b/lib/fog/storage/openstack/requests/get_containers.rb @@ -1,6 +1,22 @@ module Fog module Storage class OpenStack + class Mock + def get_containers(options = {}) + results = data.map do |name, container| + { + "name" => name, + "count" => container.objects.size, + "bytes" => container.bytes_used + } + end + response = Excon::Response.new + response.status = 200 + response.body = results + response + end + end + class Real # List existing storage containers # diff --git a/lib/fog/storage/openstack/requests/get_object.rb b/lib/fog/storage/openstack/requests/get_object.rb index bdfa1b02a..026bf58ba 100644 --- a/lib/fog/storage/openstack/requests/get_object.rb +++ b/lib/fog/storage/openstack/requests/get_object.rb @@ -1,6 +1,30 @@ module Fog module Storage class OpenStack + class Mock + def get_object(container, object, &block) + cc = mock_container!(container) + obj = cc.mock_object!(object) + + body, size = "", 0 + + obj.each_part do |part| + body << part.body + size += part.bytes_used + end + + if block_given? + # Just send it all in one chunk. + block.call(body, 0, size) + end + + response = Excon::Response.new + response.body = body + response.headers = obj.to_headers + response + end + end + class Real # Get details for object # diff --git a/lib/fog/storage/openstack/requests/get_object_http_url.rb b/lib/fog/storage/openstack/requests/get_object_http_url.rb index 0e8582417..bdd9b3276 100644 --- a/lib/fog/storage/openstack/requests/get_object_http_url.rb +++ b/lib/fog/storage/openstack/requests/get_object_http_url.rb @@ -1,6 +1,12 @@ module Fog module Storage class OpenStack + class Mock + def get_object_http_url(container, object, expires, options = {}) + fail "Mock Not Implemented (#get_object_http_url) in: #{__FILE__}:#{__LINE__}" + end + end + class Real # Get an expiring object http url # diff --git a/lib/fog/storage/openstack/requests/get_object_https_url.rb b/lib/fog/storage/openstack/requests/get_object_https_url.rb index 79aa509be..7df985057 100644 --- a/lib/fog/storage/openstack/requests/get_object_https_url.rb +++ b/lib/fog/storage/openstack/requests/get_object_https_url.rb @@ -1,7 +1,7 @@ module Fog module Storage class OpenStack - class Real + module Common # Get an expiring object https url from Cloud Files # # ==== Parameters @@ -63,7 +63,8 @@ def create_temp_url(container, object, expires, method, options = {}) :path => object_path_escaped, :query => "temp_url_sig=#{sig}&temp_url_expires=#{expires}" } - URI::Generic.build(temp_url_options).to_s + + CGI.unescape(URI::Generic.build(temp_url_options).to_s) end private @@ -76,6 +77,14 @@ def sig_to_hex(str) end.join end end + + class Mock + include Common + end + + class Real + include Common + end end end end diff --git a/lib/fog/storage/openstack/requests/head_container.rb b/lib/fog/storage/openstack/requests/head_container.rb index c42b83ca9..b2661d1f7 100644 --- a/lib/fog/storage/openstack/requests/head_container.rb +++ b/lib/fog/storage/openstack/requests/head_container.rb @@ -1,6 +1,12 @@ module Fog module Storage class OpenStack + class Mock + def head_container(container) + fail "Mock Not Implemented (#head_container) in: #{__FILE__}:#{__LINE__}" + end + end + class Real # List number of objects and total bytes stored # diff --git a/lib/fog/storage/openstack/requests/head_containers.rb b/lib/fog/storage/openstack/requests/head_containers.rb index d4dea719a..3eb4fc512 100644 --- a/lib/fog/storage/openstack/requests/head_containers.rb +++ b/lib/fog/storage/openstack/requests/head_containers.rb @@ -1,6 +1,12 @@ module Fog module Storage class OpenStack + class Mock + def head_containers + fail "Mock Not Implemented (#head_containers) in: #{__FILE__}:#{__LINE__}" + end + end + class Real # List number of containers and total bytes stored # diff --git a/lib/fog/storage/openstack/requests/head_object.rb b/lib/fog/storage/openstack/requests/head_object.rb index c38095991..8d34b0b61 100644 --- a/lib/fog/storage/openstack/requests/head_object.rb +++ b/lib/fog/storage/openstack/requests/head_object.rb @@ -1,6 +1,30 @@ module Fog module Storage class OpenStack + class Mock + def head_object(container, object) + c = mock_container! container + o = c.mock_object! object + + headers = o.to_headers + + hashes, length = [], 0 + o.each_part do |part| + hashes << part.hash + length += part.bytes_used + end + + headers['Etag'] = "\"#{Digest::MD5.hexdigest(hashes.join)}\"" + headers['Content-Length'] = length.to_s + headers['X-Static-Large-Object'] = "True" if o.static_manifest? + + response = Excon::Response.new + response.status = 200 + response.headers = headers + response + end + end + class Real # Get headers for object # diff --git a/lib/fog/storage/openstack/requests/post_set_meta_temp_url_key.rb b/lib/fog/storage/openstack/requests/post_set_meta_temp_url_key.rb index 45cc2dc8b..1c7f7beb5 100644 --- a/lib/fog/storage/openstack/requests/post_set_meta_temp_url_key.rb +++ b/lib/fog/storage/openstack/requests/post_set_meta_temp_url_key.rb @@ -1,6 +1,12 @@ module Fog module Storage class OpenStack + class Mock + def post_set_meta_temp_url_key(key) + fail "Mock Not Implemented (#post_set_meta_temp_url_key) in: #{__FILE__}:#{__LINE__}" + end + end + class Real # Set the account wide Temp URL Key. This is a secret key that's # used to generate signed expiring URLs. diff --git a/lib/fog/storage/openstack/requests/public_url.rb b/lib/fog/storage/openstack/requests/public_url.rb index 63a1e3948..ce682fafc 100644 --- a/lib/fog/storage/openstack/requests/public_url.rb +++ b/lib/fog/storage/openstack/requests/public_url.rb @@ -1,6 +1,12 @@ module Fog module Storage class OpenStack + class Mock + def public_url(container = nil, object = nil) + fail "Mock Not Implemented (#public_url) in: #{__FILE__}:#{__LINE__}" + end + end + class Real # Get public_url for an object # diff --git a/lib/fog/storage/openstack/requests/put_container.rb b/lib/fog/storage/openstack/requests/put_container.rb index 4b3c976d2..4e6c17c09 100644 --- a/lib/fog/storage/openstack/requests/put_container.rb +++ b/lib/fog/storage/openstack/requests/put_container.rb @@ -1,6 +1,20 @@ module Fog module Storage class OpenStack + class Mock + def put_container(name, options = {}) + existed = ! mock_container(name).nil? + container = add_container(name) + options.keys.each do |k| + container.meta[k] = options[k].to_s if k =~ /^X-Container-Meta/ + end + + response = Excon::Response.new + response.status = existed ? 202 : 201 + response + end + end + class Real # Create a new container # diff --git a/lib/fog/storage/openstack/requests/put_dynamic_obj_manifest.rb b/lib/fog/storage/openstack/requests/put_dynamic_obj_manifest.rb index b35f20e5a..6d6bbbae1 100644 --- a/lib/fog/storage/openstack/requests/put_dynamic_obj_manifest.rb +++ b/lib/fog/storage/openstack/requests/put_dynamic_obj_manifest.rb @@ -1,6 +1,12 @@ module Fog module Storage class OpenStack + class Mock + def put_dynamic_obj_manifest(container, object, options = {}) + fail "Mock Not Implemented (#put_dynamic_obj_manifest) in: #{__FILE__}:#{__LINE__}" + end + end + class Real # Create a new dynamic large object manifest # diff --git a/lib/fog/storage/openstack/requests/put_object.rb b/lib/fog/storage/openstack/requests/put_object.rb index 51615c53f..e9d6495f8 100644 --- a/lib/fog/storage/openstack/requests/put_object.rb +++ b/lib/fog/storage/openstack/requests/put_object.rb @@ -1,6 +1,43 @@ module Fog module Storage class OpenStack + class Mock + HeaderOptions = %w{ + Content-Type Access-Control-Allow-Origin Origin Content-Disposition + Etag Content-Encoding + }.freeze + + def put_object(container, object, data, options = {}, &block) + c = mock_container! container + + if block_given? + data = "" + loop do + chunk = yield + break if chunk.empty? + data << chunk + end + end + + o = c.add_object object, data + options.keys.each do |k| + o.meta[k] = options[k].to_s if k =~ /^X-Object-Meta/ + o.meta[k] = options[k] if HeaderOptions.include? k + end + + # Validate the provided Etag + etag = o.meta['Etag'] + if etag && etag != o.hash + c.remove_object object + raise Fog::Storage::Rackspace::ServiceError.new + end + + response = Excon::Response.new + response.status = 201 + response + end + end + class Real # Create a new object # diff --git a/lib/fog/storage/openstack/requests/put_object_manifest.rb b/lib/fog/storage/openstack/requests/put_object_manifest.rb index 1fe01c6d8..3a99e251e 100644 --- a/lib/fog/storage/openstack/requests/put_object_manifest.rb +++ b/lib/fog/storage/openstack/requests/put_object_manifest.rb @@ -1,6 +1,12 @@ module Fog module Storage class OpenStack + class Mock + def put_object_manifest(container, object, options = {}) + fail "Mock Not Implemented (#put_object_manifest) in: #{__FILE__}:#{__LINE__}" + end + end + class Real # Create a new dynamic large object manifest # diff --git a/lib/fog/storage/openstack/requests/put_static_obj_manifest.rb b/lib/fog/storage/openstack/requests/put_static_obj_manifest.rb index 23b18a260..cbb1df97e 100644 --- a/lib/fog/storage/openstack/requests/put_static_obj_manifest.rb +++ b/lib/fog/storage/openstack/requests/put_static_obj_manifest.rb @@ -1,6 +1,12 @@ module Fog module Storage class OpenStack + class Mock + def put_static_obj_manifest(container, object, segments, options = {}) + fail "Mock Not Implemented (#put_static_obj_manifest) in: #{__FILE__}:#{__LINE__}" + end + end + class Real # Create a new static large object manifest. #