diff --git a/Rakefile b/Rakefile index 57814301a06..f389dbe8dba 100644 --- a/Rakefile +++ b/Rakefile @@ -6,6 +6,7 @@ $:.unshift("#{$REPO_ROOT}/build_tools") $:.unshift("#{$REPO_ROOT}/build_tools/aws-sdk-code-generator/lib") $:.unshift("#{$GEMS_DIR}/aws-sdk-core/lib") $:.unshift("#{$GEMS_DIR}/aws-partitions/lib") +$:.unshift("#{$GEMS_DIR}/aws-eventstream/lib") $:.unshift("#{$GEMS_DIR}/aws-sigv4/lib") require 'build_tools' diff --git a/build_tools/aws-sdk-code-generator/lib/aws-sdk-code-generator.rb b/build_tools/aws-sdk-code-generator/lib/aws-sdk-code-generator.rb index 9a695324713..86a1388298e 100644 --- a/build_tools/aws-sdk-code-generator/lib/aws-sdk-code-generator.rb +++ b/build_tools/aws-sdk-code-generator/lib/aws-sdk-code-generator.rb @@ -34,6 +34,7 @@ require_relative 'aws-sdk-code-generator/resource_waiter' require_relative 'aws-sdk-code-generator/service' require_relative 'aws-sdk-code-generator/shared_example' +require_relative 'aws-sdk-code-generator/eventstream_example' require_relative 'aws-sdk-code-generator/syntax_example' require_relative 'aws-sdk-code-generator/syntax_example_hash' require_relative 'aws-sdk-code-generator/underscore' @@ -50,6 +51,7 @@ require_relative 'aws-sdk-code-generator/views/service_module' require_relative 'aws-sdk-code-generator/views/spec/spec_helper' require_relative 'aws-sdk-code-generator/views/types_module' +require_relative 'aws-sdk-code-generator/views/event_streams_module' require_relative 'aws-sdk-code-generator/views/authorizer_class' require_relative 'aws-sdk-code-generator/views/apig_endpoint_class' require_relative 'aws-sdk-code-generator/views/apig_readme' diff --git a/build_tools/aws-sdk-code-generator/lib/aws-sdk-code-generator/api.rb b/build_tools/aws-sdk-code-generator/lib/aws-sdk-code-generator/api.rb index c8d4ab96ad9..912ec6c8938 100644 --- a/build_tools/aws-sdk-code-generator/lib/aws-sdk-code-generator/api.rb +++ b/build_tools/aws-sdk-code-generator/lib/aws-sdk-code-generator/api.rb @@ -101,7 +101,14 @@ def ruby_type(shape_ref, api) # @return [Boolean] def streaming?(shape_or_shape_ref, api) ref, shape = resolve(shape_or_shape_ref, api) - ref['streaming'] || shape['streaming'] + ref['streaming'] || shape['streaming'] || + ref['eventstream'] || shape['eventstream'] + end + + # @return [Boolean] + def eventstream?(shape_or_shape_ref, api) + ref, shape = resolve(shape_or_shape_ref, api) + ref['eventstream'] || shape['eventstream'] end def plural?(resource) diff --git a/build_tools/aws-sdk-code-generator/lib/aws-sdk-code-generator/client_operation_documentation.rb b/build_tools/aws-sdk-code-generator/lib/aws-sdk-code-generator/client_operation_documentation.rb index 67763461ef3..27da99ca26a 100644 --- a/build_tools/aws-sdk-code-generator/lib/aws-sdk-code-generator/client_operation_documentation.rb +++ b/build_tools/aws-sdk-code-generator/lib/aws-sdk-code-generator/client_operation_documentation.rb @@ -12,11 +12,15 @@ def initialize(options) @api = options.fetch(:api) @client_examples = options.fetch(:client_examples, []) @examples = options.fetch(:examples) + @module_name = options.fetch(:module_name) end # @return [String] attr_reader :method_name + # @return [String] + attr_reader :module_name + # @return [Hash] attr_reader :operation @@ -37,6 +41,7 @@ def to_str option_tags(operation, api), return_tag(operation, api), generated_examples(operation, api), + eventstream_examples(module_name, method_name, operation, api), shared_examples(examples, operation, api), given_examples(client_examples), request_syntax_example(method_name, operation, api), @@ -161,6 +166,18 @@ def generated_examples(operation, api) nil end + def eventstream_examples(module_name, method_name, operation, api) + return unless !!Helper.eventstream_output?(operation, api) + EventStreamExample.new( + api: api, + operation: operation, + method_name: method_name, + module_name: module_name, + receiver: 'client', + resp_var: 'resp' + ).format + end + def given_examples(client_examples) client_examples.map do |example| name = example[:name] diff --git a/build_tools/aws-sdk-code-generator/lib/aws-sdk-code-generator/client_operation_list.rb b/build_tools/aws-sdk-code-generator/lib/aws-sdk-code-generator/client_operation_list.rb index 9dfeabf78a5..de8a10266cc 100644 --- a/build_tools/aws-sdk-code-generator/lib/aws-sdk-code-generator/client_operation_list.rb +++ b/build_tools/aws-sdk-code-generator/lib/aws-sdk-code-generator/client_operation_list.rb @@ -6,20 +6,26 @@ class ClientOperationList def initialize(options) api = options.fetch(:api) examples = options.fetch(:examples, {}) + module_name = options.fetch(:module_name) client_examples = options.fetch(:client_examples, {}) @operations = api['operations'].map do |name, operation| + if AwsSdkCodeGenerator::Helper.eventstream_input?(operation, api) + raise 'eventstream at operation input is not supported' + end method_name = Underscore.underscore(name) Operation.new( name: method_name, documentation: ClientOperationDocumentation.new( name: name, + module_name: module_name, method_name: method_name, operation: operation, api: api, examples: examples, client_examples: client_examples[method_name] || [] ).to_s, - streaming: AwsSdkCodeGenerator::Helper.operation_streaming?(operation, api) + streaming: AwsSdkCodeGenerator::Helper.operation_streaming?(operation, api), + eventstream_output: AwsSdkCodeGenerator::Helper.eventstream_output?(operation, api) ) end end @@ -35,6 +41,9 @@ def initialize(options) @name = options.fetch(:name) @documentation = options.fetch(:documentation) @streaming = options.fetch(:streaming) + @eventstream_output = !!options.fetch(:eventstream_output) + @eventstream_member = @eventstream_output ? + options.fetch(:eventstream_output) : nil end # @return [String] @@ -43,11 +52,18 @@ def initialize(options) # @return [String, nil] attr_reader :documentation + # @return [Boolean] + attr_reader :eventstream_output + + # @return [String] + attr_reader :eventstream_member + def block_option if @streaming ", &block" end end + end end end diff --git a/build_tools/aws-sdk-code-generator/lib/aws-sdk-code-generator/client_response_structure_example.rb b/build_tools/aws-sdk-code-generator/lib/aws-sdk-code-generator/client_response_structure_example.rb index 16c083767a1..084ef093b75 100644 --- a/build_tools/aws-sdk-code-generator/lib/aws-sdk-code-generator/client_response_structure_example.rb +++ b/build_tools/aws-sdk-code-generator/lib/aws-sdk-code-generator/client_response_structure_example.rb @@ -19,8 +19,27 @@ def to_str def structure(ref, context, visited) lines = [] - shape(ref)['members'].each_pair do |member_name, member_ref| - lines += entry(member_ref, "#{context}.#{underscore(member_name)}", visited) + shape = shape(ref) + if shape['eventstream'] + event_types = [] + # Add event entry + event_ctx = shape['members'].each.inject([]) do |ctx, (member_name, member_ref)| + event_type = Underscore.underscore(member_name).to_sym + event_types << event_type + ctx << "For #{event_type.inspect} event available at #on_#{event_type}_event callback"\ + " and response eventstream enumerator:" + event_entry = entry(member_ref, "event", Set.new).join("\n ") + ctx << (event_entry.empty? ? " #=> EmptyStruct" : event_entry + "\n") + end + # Add eventstream entry + event_ctx.unshift("#{context}.event_types #=> #{event_types.inspect}\n") + event_ctx.unshift("#{context} #=> Enumerator") + event_ctx.unshift("All events are available at #{context}:") + return event_ctx + else + shape['members'].each_pair do |member_name, member_ref| + lines += entry(member_ref, "#{context}.#{underscore(member_name)}", visited) + end end lines end diff --git a/build_tools/aws-sdk-code-generator/lib/aws-sdk-code-generator/code_builder.rb b/build_tools/aws-sdk-code-generator/lib/aws-sdk-code-generator/code_builder.rb index 4e7b55af9fd..3593c8b0a9e 100644 --- a/build_tools/aws-sdk-code-generator/lib/aws-sdk-code-generator/code_builder.rb +++ b/build_tools/aws-sdk-code-generator/lib/aws-sdk-code-generator/code_builder.rb @@ -59,6 +59,7 @@ def source_files(options = {}) y.yield("#{prefix}.rb", service_module(prefix)) y.yield("#{prefix}/customizations.rb", '') y.yield("#{prefix}/types.rb", types_module) + y.yield("#{prefix}/event_streams.rb", event_streams_module) if has_eventstream y.yield("#{prefix}/client_api.rb", client_api_module) y.yield("#{prefix}/client.rb", client_class) y.yield("#{prefix}/errors.rb", errors_module) @@ -84,6 +85,10 @@ def types_module Views::TypesModule.new(service: @service).render end + def event_streams_module + Views::EventStreamsModule.new(service: @service).render + end + def client_api_module Views::ClientApiModule.new(service: @service).render end @@ -163,6 +168,16 @@ def apig_readme module_name: @service.module_name ).render end + + private + + def has_eventstream + @service.api['shapes'].each do |_, ref| + return true if ref['eventstream'] + end + false + end + end end diff --git a/build_tools/aws-sdk-code-generator/lib/aws-sdk-code-generator/eventstream_example.rb b/build_tools/aws-sdk-code-generator/lib/aws-sdk-code-generator/eventstream_example.rb new file mode 100644 index 00000000000..6d40ef9eb72 --- /dev/null +++ b/build_tools/aws-sdk-code-generator/lib/aws-sdk-code-generator/eventstream_example.rb @@ -0,0 +1,117 @@ +module AwsSdkCodeGenerator + class EventStreamExample + + def initialize(options = {}) + @api = options.fetch(:api) + @method_name = options.fetch(:method_name) + @module_name = options.fetch(:module_name) + @receiver = options.fetch(:receiver) + @resp_var = options.fetch(:resp_var) + @operation = options.fetch(:operation) + @eventstream_member, @eventstream_shape = eventstream_shape + end + + def format + <<-EXAMPLE.strip +# @example EventStream Operation Example +# +# You can process event once it arrives immediately, or wait until +# full response complete and iterate through eventstream enumerator. +# +# To interact with event immediately, you need to register ##{@method_name} +# with callbacks, callbacks can be register for specifc events or for all events, +# callback for errors in the event stream is also available for register. +# +# Callbacks can be passed in by `:event_stream_handler` option or within block +# statement attached to ##{@method_name} call directly. Hybrid pattern of both +# is also supported. +# +# `:event_stream_handler` option takes in either Proc object or +# EventStreams::#{@eventstream_shape} object. +# +# Usage pattern a): callbacks with a block attached to ##{@method_name} +# Example for registering callbacks for all event types and error event +# +# #{@receiver}.#{@method_name}( # params input# ) do |stream| +# +# stream.on_error_event do |event| +# # catch unmodeled error event in the stream +# raise event +# # => Aws::Errors::EventError +# # event.event_type => :error +# # event.error_code => String +# # event.error_message => String +# end +# +# stream.on_event do |event| +# # process all events arrive +# puts event.event_type +# ... +# end +# +# end +# +# Usage pattern b): pass in `:event_stream_handler` for ##{@method_name} +# +# 1) create a EventStreams::#{@eventstream_shape} object +# Example for registering callbacks with specific events +# +# handler = #{@module_name}::EventStreams::#{@eventstream_shape}.new +#{event_entry('handler')} +# +# #{@receiver}.#{@method_name}( # params input #, event_stream_handler: handler) +# +# 2) use a Ruby Proc object +# Example for registering callbacks with specific events +# +# handler = Proc.new do |stream| +#{event_entry('stream')} +# end +# +# #{@receiver}.#{@method_name}( # params input #, event_stream_handler: handler) +# +# Usage pattern c): hybird pattern of a) and b) +# +# handler = #{@module_name}::EventStreams::#{@eventstream_shape}.new +#{event_entry('handler')} +# +# #{@receiver}.#{@method_name}( # params input #, event_stream_handler: handler) do |stream| +# stream.on_error_event do |event| +# # catch unmodeled error event in the stream +# raise event +# # => Aws::Errors::EventError +# # event.event_type => :error +# # event.error_code => String +# # event.error_message => String +# end +# end +# +# Besides above usage patterns for process events when they arrive immediately, you can also +# iterate through events after response complete. +# +# Events are available at #{@resp_var}.#{@eventstream_member} # => Enumerator +# For parameter input example, please refer to following request syntax + EXAMPLE + end + + private + + def event_entry(ctx) + @api['shapes'][@eventstream_shape]['members'].keys.each.inject([]) do |entry, name| + event_type = Underscore.underscore(name) + entry << "# #{ctx}.on_#{event_type}_event do |event|" + entry << "# event # => #{@module_name}::Types::#{name}" + entry << "# end" + end.join("\n") + end + + def eventstream_shape + output_shape = @api['shapes'][@operation['output']['shape']] + output_shape['members'].each do |name, ref| + return [Underscore.underscore(name), ref['shape']] if Api.eventstream?(ref, @api) + end + raise 'Cannot find eventstream member at eventstream operation' + end + + end +end diff --git a/build_tools/aws-sdk-code-generator/lib/aws-sdk-code-generator/helper.rb b/build_tools/aws-sdk-code-generator/lib/aws-sdk-code-generator/helper.rb index ba3130d0353..dda2c6dd840 100644 --- a/build_tools/aws-sdk-code-generator/lib/aws-sdk-code-generator/helper.rb +++ b/build_tools/aws-sdk-code-generator/lib/aws-sdk-code-generator/helper.rb @@ -175,6 +175,27 @@ def operation_streaming?(operation, api) end end + def eventstream_output?(operation, api) + return false unless operation.key? 'output' + output_shape = api['shapes'][operation['output']['shape']] + return false unless output_shape.key? 'members' + output_shape['members'].each do |name, ref| + return ref['shape'] if Api.eventstream?(ref, api) + end + return false + end + + # currently not support eventstream input + def eventstream_input?(operation, api) + return false unless operation.key? 'input' + input_shape = api['shapes'][operation['input']['shape']] + return false unless input_shape.key? 'members' + input_shape['members'].each do |name, ref| + return true if Api.eventstream?(ref, api) + end + return false + end + def deep_copy(obj) case obj when nil then nil @@ -197,7 +218,8 @@ def wrap_string(str, width, indent = '') str.gsub(/(.{1,#{width}})(\s+|\Z)/, "#{indent}\\1\n").chomp end - module_function :deep_copy, :operation_streaming?, :downcase_first, :wrap_string, :apig_prefix + module_function :deep_copy, :operation_streaming?, :downcase_first, :wrap_string, :apig_prefix, + :eventstream_output?, :eventstream_input? end end diff --git a/build_tools/aws-sdk-code-generator/lib/aws-sdk-code-generator/views/client_api_module.rb b/build_tools/aws-sdk-code-generator/lib/aws-sdk-code-generator/views/client_api_module.rb index 0985b139816..861a5ea5150 100644 --- a/build_tools/aws-sdk-code-generator/lib/aws-sdk-code-generator/views/client_api_module.rb +++ b/build_tools/aws-sdk-code-generator/lib/aws-sdk-code-generator/views/client_api_module.rb @@ -27,9 +27,11 @@ class ClientApiModule < View 'flattened' => true, 'timestampFormat' => true, # glacier api customization 'xmlNamespace' => true, - # ignore event stream traits + # event stream modeling 'event' => false, 'eventstream' => false, + 'eventheader' => false, + 'eventpayload' => false, # ignore 'box' => false, 'fault' => false, @@ -105,14 +107,12 @@ def shapes if @service.protocol == 'api-gateway' shape_name = lstrip_prefix(upcase_first(shape_name)) end - # exclude event stream/event shapes - next if shape['eventstream'] || shape['event'] Shape.new.tap do |s| s.name = shape_name s.class_name, shape = shape_class_name(shape) s.constructor_args = shape_constructor_args(shape_name, shape) end - end.compact + end end def shape_definitions @@ -122,10 +122,7 @@ def shape_definitions shape_name = lstrip_prefix(upcase_first(shape_name)) end lines = [] - # exclude event stream/event shapes - if shape['eventstream'] || shape['event'] - groups - elsif non_error_struct?(shape) + if non_error_struct?(shape) required = Set.new(shape['required'] || []) unless shape['members'].nil? shape['members'].each do |member_name, member_ref| @@ -267,6 +264,10 @@ def shape_ref(ref, member_name = nil, required = Set.new) line = "Shapes::ShapeRef.new(shape: #{ref_name}" line += shape_ref_required(required, member_name) line += shape_ref_deprecated(ref) + line += shape_ref_event(ref) + line += shape_ref_eventstream(ref) + line += shape_ref_eventpayload(ref) + line += shape_ref_eventheader(ref) line += shape_ref_location(ref) line += shape_ref_location_name(member_name, ref) line += shape_ref_metadata(ref) @@ -290,6 +291,40 @@ def shape_ref_deprecated(ref) end end + def shape_ref_eventstream(ref) + if @service.api['shapes'][ref['shape']]['eventstream'] + ", eventstream: true" + else + '' + end + end + + def shape_ref_event(ref) + if @service.api['shapes'][ref['shape']]['event'] + ", event: true" + else + '' + end + end + + def shape_ref_eventpayload(ref) + if ref['eventpayload'] + type = @service.api['shapes'][ref['shape']]['type'] + ", eventpayload: true, eventpayload_type: '#{type}'" + else + '' + end + end + + def shape_ref_eventheader(ref) + if ref['eventheader'] + type = @service.api['shapes'][ref['shape']]['type'] + ", eventheader: true, eventheader_type: '#{type}'" + else + '' + end + end + def shape_ref_location(ref) if ref['location'] ", location: #{ref['location'].inspect}" diff --git a/build_tools/aws-sdk-code-generator/lib/aws-sdk-code-generator/views/event_streams_module.rb b/build_tools/aws-sdk-code-generator/lib/aws-sdk-code-generator/views/event_streams_module.rb new file mode 100644 index 00000000000..4caae89eca2 --- /dev/null +++ b/build_tools/aws-sdk-code-generator/lib/aws-sdk-code-generator/views/event_streams_module.rb @@ -0,0 +1,60 @@ +module AwsSdkCodeGenerator + module Views + class EventStreamsModule < View + + include Helper + + def initialize(options) + @service = options.fetch(:service) + end + + # @return [String|nil] + def generated_src_warning + return if @service.protocol == 'api-gateway' + GENERATED_SRC_WARNING + end + + def module_name + @service.module_name + end + + # @return [Array] + def eventstreams + @service.api['shapes'].inject([]) do |list, (name, shape)| + if shape['eventstream'] + list << EventStreamClass.new( + class_name: name, + types: eventstream_members(shape) + ) + else + list + end + end + end + + def eventstream_members(shape) + raise 'no event members for an eventstream' if shape['members'].nil? + shape['members'].keys.map {|m| underscore(m)} + end + + private + + class EventStreamClass + + def initialize(options) + @class_name = options.fetch(:class_name) + @types = options.fetch(:types) + end + + # @return [String] + attr_accessor :class_name + + # @return [Array] + attr_accessor :types + + end + + end + end +end + diff --git a/build_tools/aws-sdk-code-generator/lib/aws-sdk-code-generator/views/service_module.rb b/build_tools/aws-sdk-code-generator/lib/aws-sdk-code-generator/views/service_module.rb index 61ff08d5108..823c6722a11 100644 --- a/build_tools/aws-sdk-code-generator/lib/aws-sdk-code-generator/views/service_module.rb +++ b/build_tools/aws-sdk-code-generator/lib/aws-sdk-code-generator/views/service_module.rb @@ -69,6 +69,7 @@ def relative_requires end end paths << "#{@prefix}/customizations" + paths << "#{@prefix}/event_streams" if eventstream_shape? paths.to_a end @@ -82,6 +83,12 @@ def example_operation_name nil end + def eventstream_shape? + @service.api['shapes'].each do |_, shape_ref| + return true if shape_ref['eventstream'] + end + false + end end end end diff --git a/build_tools/aws-sdk-code-generator/lib/aws-sdk-code-generator/views/types_module.rb b/build_tools/aws-sdk-code-generator/lib/aws-sdk-code-generator/views/types_module.rb index 6f5f99d35c9..2cc2cdb26ca 100644 --- a/build_tools/aws-sdk-code-generator/lib/aws-sdk-code-generator/views/types_module.rb +++ b/build_tools/aws-sdk-code-generator/lib/aws-sdk-code-generator/views/types_module.rb @@ -30,8 +30,8 @@ def structures if @service.protocol == 'api-gateway' shape_name = lstrip_prefix(upcase_first(shape_name)) end - # exclude eventstream and event shapes - if shape['eventstream'] || shape['event'] + # eventstream shape will be inheriting from enumerator + if shape['eventstream'] list elsif struct_type?(shape) list << StructClass.new( @@ -45,13 +45,30 @@ def structures end end + # return [Array] + def eventstreams + @service.api['shapes'].inject([]) do |list, (shape_name, shape)| + if shape['eventstream'] + list << EventStreamClass.new( + class_name: shape_name, + types: struct_members(shape), + documentation: eventstream_class_docs(shape_name) + ) + else + list + end + end + end + private def struct_members(shape) return if shape['members'].nil? - shape['members'].map do |member_name, _| + members = shape['members'].map do |member_name, _| StructMember.new(member_name: underscore(member_name)) end + members << StructMember.new(member_name: "event_type") if shape['event'] + members end def struct_class_docs(shape_name) @@ -63,6 +80,20 @@ def struct_class_docs(shape_name) ]) end + def eventstream_class_docs(shape_name) + join_docstrings([ + html_to_markdown(Api.docstring(shape_name, @api)), + input_example_docs(shape_name), + eventstream_docs(shape_name), + see_also_tag(shape_name), + ]) + end + + def eventstream_docs(shape_name) + "EventStream is an Enumerator of Events.\n"\ + " #event_types #=> Array, returns all modeled event types in the stream" + end + def input_example_docs(shape_name) if @input_shapes.include?(shape_name) return if shape(shape_name)['members'].nil? @@ -156,6 +187,36 @@ def shape(shape_ref) Api.resolve(shape_ref, @api)[1] end + class EventStreamClass + + def initialize(options) + @class_name = options.fetch(:class_name) + @types = options.fetch(:types) + @documentation = options.fetch(:documentation) + if @types.nil? || @types.empty? + @empty = true + else + @empty = false + @types.last.last = true + end + end + + # @return [String] + attr_accessor :class_name + + # @return [Array] + attr_accessor :types + + # @return [String, nil] + attr_accessor :documentation + + # @return [Boolean] + def empty? + @empty + end + + end + class StructClass def initialize(options) diff --git a/build_tools/aws-sdk-code-generator/spec/fixtures/interfaces/events/api.json b/build_tools/aws-sdk-code-generator/spec/fixtures/interfaces/events/api.json new file mode 100644 index 00000000000..b3a5299c9ad --- /dev/null +++ b/build_tools/aws-sdk-code-generator/spec/fixtures/interfaces/events/api.json @@ -0,0 +1,89 @@ +{ + "version": "2.0", + "metadata": { + "endpointPrefix": "svc", + "protocol": "rest-xml", + "signatureVersion": "v4" + }, + "operations": { + "Foo": { + "name": "Foo", + "http": { + "method": "GET", + "requestUri": "/" + }, + "output": {"shape": "FooOutput"} + } + }, + "shapes": { + "FooOutput": { + "type": "structure", + "members": { + "Payload": { + "shape": "BarPayload" + } + }, + "payload": "Payload" + }, + "BarPayload": { + "type": "structure", + "members": { + "A": {"shape": "AEventShape"}, + "B": {"shape": "BEventShape"}, + "C": {"shape": "CEventShape"} + }, + "eventstream": true + }, + "AEventShape": { + "type": "structure", + "members": { + "MemberA": { + "shape": "StringShape", + "eventheader": true + }, + "MemberB": { + "shape": "BlobShape", + "eventpayload": true + } + }, + "event": true + }, + "BEventShape": { + "type": "structure", + "members": { + "MemberC": { + "shape": "StringShape", + "eventpayload": true + } + }, + "event": true + }, + "CEventShape": { + "type": "structure", + "members": { + "MemberD": { + "shape": "StructShape", + "eventpayload": true + } + }, + "event": true + }, + "StructShape": { + "type": "structure", + "members": { + "StructMemberA": { + "shape": "StringShape" + }, + "StructMemberB": { + "shape": "StringShape" + } + } + }, + "StringShape": { + "type": "string" + }, + "BlobShape": { + "type": "blob" + } + } +} diff --git a/build_tools/aws-sdk-code-generator/spec/interfaces/client/event_stream_spec.rb b/build_tools/aws-sdk-code-generator/spec/interfaces/client/event_stream_spec.rb new file mode 100644 index 00000000000..ba21d1da2b9 --- /dev/null +++ b/build_tools/aws-sdk-code-generator/spec/interfaces/client/event_stream_spec.rb @@ -0,0 +1,134 @@ +require_relative '../../spec_helper' + +describe 'Client Interface:' do + describe 'Support Event Streaming Operations' do + + before(:all) do + SpecHelper.generate_service(['Events'], multiple_files: false) + end + + let(:stream) { + [ + # eventheader & eventpayload(blob) + { message_type: 'event', event_type: :a, member_a: 'foo', member_b: StringIO.new('bar') }, + # eventpayload(string) + { message_type: 'event', event_type: :b, member_c: 'baz' }, + # eventpayload(structure) + { message_type: 'event', event_type: :c, member_d: {struct_member_a: 'foo', struct_member_b: 'bar'} }, + # an unmodeled error event + { message_type: 'error', error_code: 'InternalError', error_message: 'An internal server error occurred'} + ].each + } + let(:client) { + Events::Client.new( + region: 'us-west-2', + credentials: Aws::Credentials.new('akid', 'secret'), + stub_responses: {foo: { + payload: stream + }} + ) + } + + it 'supports eventstream object for `event_stream_handler` option' do + handler = Events::EventStreams::BarPayload.new + tracker = {} + handler.on_a_event {|e| tracker[:a] = e} + handler.on_b_event {|e| tracker[:b] = e} + handler.on_c_event {|e| tracker[:c] = e} + resp = client.foo(event_stream_handler: handler) + + expect(tracker[:a].member_a).to eq('foo') + expect(tracker[:a].member_b.read).to eq('bar') + expect(tracker[:b].member_c.read).to eq('baz') + expect(tracker[:c].member_d.struct_member_a).to eq('foo') + expect(tracker[:c].member_d.struct_member_b).to eq('bar') + expect(resp.payload).to be_a(Enumerator) + end + + it 'support Proc object for `event_stream_handler` option' do + tracker = {} + handler = Proc.new do |stream| + stream.on_a_event {|e| tracker[:a] = e} + stream.on_b_event {|e| tracker[:b] = e} + stream.on_c_event {|e| tracker[:c] = e} + end + resp = client.foo(event_stream_handler: handler) + + expect(tracker[:a].member_a).to eq('foo') + expect(tracker[:a].member_b.read).to eq('bar') + expect(tracker[:b].member_c.read).to eq('baz') + expect(tracker[:c].member_d.struct_member_a).to eq('foo') + expect(tracker[:c].member_d.struct_member_b).to eq('bar') + expect(resp.payload).to be_a(Enumerator) + end + + it 'supports no `event_stream_handler` option input' do + resp = client.foo + + expect(resp.payload).to be_a(Enumerator) + resp.payload.each do |event| + case event.event_type + when :a + expect(event.member_a).to eq('foo') + expect(event.member_b.read).to eq('bar') + when :b + expect(event.member_c.read).to eq('baz') + when :c + expect(event.member_d.struct_member_a).to eq('foo') + expect(event.member_d.struct_member_b).to eq('bar') + end + end + end + + it 'supports block streaming' do + tracker = {} + resp = client.foo do |stream| + stream.on_event {|event| tracker[event.event_type] = event} + end + + expect(tracker[:a].member_a).to eq('foo') + expect(tracker[:a].member_b.read).to eq('bar') + expect(tracker[:b].member_c.read).to eq('baz') + expect(tracker[:c].member_d.struct_member_a).to eq('foo') + expect(tracker[:c].member_d.struct_member_b).to eq('bar') + expect(resp.payload).to be_a(Enumerator) + end + + it 'supports `event_stream_handler` and block streaming at same time' do + tracker = {later: []} + handler = Events::EventStreams::BarPayload.new + handler.on_a_event {|e| tracker[:a] = e} + handler.on_b_event {|e| tracker[:b] = e} + handler.on_c_event {|e| tracker[:c] = e} + resp = client.foo(event_stream_handler: handler) do |stream| + stream.on_event {|e| tracker[:later] << e} + end + + expect(tracker[:a].member_a).to eq('foo') + expect(tracker[:a].member_b.read).to eq('bar') + expect(tracker[:b].member_c.read).to eq('baz') + expect(tracker[:c].member_d.struct_member_a).to eq('foo') + expect(tracker[:c].member_d.struct_member_b).to eq('bar') + expect(tracker[:later].size).to eq(3) + expect(tracker[:later][0]).to eq(tracker[:a]) + expect(tracker[:later][1]).to eq(tracker[:b]) + expect(tracker[:later][2]).to eq(tracker[:c]) + expect(resp.payload).to be_a(Enumerator) + end + + it 'supports error event' do + error = nil + handler = Events::EventStreams::BarPayload.new + handler.on_error_event do |e| + error = e + raise e + end + expect { + client.foo(event_stream_handler: handler) + }.to raise_error(Aws::Errors::EventError) + expect(error.error_code).to eq('InternalError') + expect(error.error_message).to eq('An internal server error occurred') + end + + end +end diff --git a/build_tools/aws-sdk-code-generator/spec/protocols/output/rest-xml.json b/build_tools/aws-sdk-code-generator/spec/protocols/output/rest-xml.json index 8e8fbc4e587..de94f6340d5 100644 --- a/build_tools/aws-sdk-code-generator/spec/protocols/output/rest-xml.json +++ b/build_tools/aws-sdk-code-generator/spec/protocols/output/rest-xml.json @@ -778,5 +778,379 @@ } } ] + }, + { + "description": "Scalar eventheader members", + "metadata": { + "protocol": "rest-xml" + }, + "shapes": { + "OutputShape": { + "type": "structure", + "members": { + "Payload": { + "shape": "EventStreamShape" + } + }, + "payload": "Payload" + }, + "EventStreamShape": { + "type": "structure", + "members": { + "EventMember": { + "shape": "EventShape" + } + }, + "eventstream": true + }, + "EventShape": { + "type": "structure", + "members": { + "Str": { + "shape": "StringType", + "eventheader": true + }, + "Num": { + "shape": "IntegerType", + "eventheader": true + }, + "FalseBool": { + "shape": "BooleanType", + "eventheader": true + }, + "TrueBool": { + "shape": "BooleanType", + "eventheader": true + }, + "Long": { + "shape": "LongType", + "eventheader": true + }, + "Byte": { + "shape": "ByteType", + "eventheader": true + }, + "Timestamp": { + "shape": "TimestampType", + "eventheader": true + } + }, + "event": true + }, + "StringType": { + "type": "string" + }, + "IntegerType": { + "type": "integer" + }, + "BooleanType": { + "type": "boolean" + }, + "LongType": { + "type": "long" + }, + "ByteType": { + "type": "byte" + }, + "TimestampType": { + "type": "timestamp" + } + }, + "cases": [ + { + "given": { + "output": { + "shape": "OutputShape" + }, + "name": "OperationName" + }, + "result": { + "Payload": { + "EventMember": { + "Str": "myname", + "Num": 123, + "FalseBool": false, + "TrueBool": true, + "Long": 250, + "Byte": 5, + "Timestamp": 8675.309 + } + } + }, + "response": { + "eventstream": true, + "status_code": 200, + "headers": {}, + "body":"AAAAkwAAAIOHNVAzCzpldmVudC10eXBlBwALRXZlbnRNZW1iZXINOm1lc3NhZ2UtdHlwZQcABWV2ZW50A1N0cgcABm15bmFtZQNOdW0EAAAAewlGYWxzZUJvb2wBCFRydWVCb29sAARMb25nBQAAAAAAAAD6BEJ5dGUCBQlUaW1lc3RhbXAIAAAAAACEX+1DFJaG" + } + } + ] + }, + { + "description": "String eventpayload member", + "metadata": { + "protocol": "rest-xml" + }, + "shapes": { + "OutputShape": { + "type": "structure", + "members": { + "Payload": { + "shape": "EventStreamShape" + } + }, + "payload": "Payload" + }, + "EventStreamShape": { + "type": "structure", + "members": { + "EventMember": { + "shape": "EventShape" + } + }, + "eventstream": true + }, + "EventShape": { + "type": "structure", + "members": { + "Str": { + "shape": "StringType", + "eventpayload": true + } + }, + "event": true + }, + "StringType": { + "type": "string" + } + }, + "cases": [ + { + "given": { + "output": { + "shape": "OutputShape" + }, + "name": "OperationName" + }, + "result": { + "Payload": { + "EventMember": { + "Str": "foo" + } + } + }, + "response": { + "eventstream": true, + "status_code": 200, + "headers": {}, + "body": "AAAAQwAAADBcqM1cCzpldmVudC10eXBlBwALRXZlbnRNZW1iZXINOm1lc3NhZ2UtdHlwZQcABWV2ZW50Zm9vwg+otA==" + } + } + ] + }, + { + "description": "Blob eventpayload members", + "metadata": { + "protocol": "rest-xml" + }, + "shapes": { + "OutputShape": { + "type": "structure", + "members": { + "Payload": { + "shape": "EventStreamShape" + } + }, + "payload": "Payload" + }, + "EventStreamShape": { + "type": "structure", + "members": { + "EventMember": { + "shape": "EventShape" + } + }, + "eventstream": true + }, + "EventShape": { + "type": "structure", + "members": { + "Blob": { + "shape": "BlobType", + "eventpayload": true + } + }, + "event": true + }, + "BlobType": { + "type": "blob" + } + }, + "cases": [ + { + "given": { + "output": { + "shape": "OutputShape" + }, + "name": "OperationName" + }, + "result": { + "Payload": { + "EventMember": { + "Blob": "value" + } + } + }, + "response": { + "eventstream": true, + "status_code": 200, + "headers": {}, + "body": "AAAARQAAADDT6Dj8CzpldmVudC10eXBlBwALRXZlbnRNZW1iZXINOm1lc3NhZ2UtdHlwZQcABWV2ZW50dmFsdWXkSTLo" + } + } + ] + }, + { + "description": "Structure eventpayload members", + "metadata": { + "protocol": "rest-xml" + }, + "shapes": { + "OutputShape": { + "type": "structure", + "members": { + "Payload": { + "shape": "EventStreamShape" + } + }, + "payload": "Payload" + }, + "EventStreamShape": { + "type": "structure", + "members": { + "EventMember": { + "shape": "EventShape" + } + }, + "eventstream": true + }, + "EventShape": { + "type": "structure", + "members": { + "Struct": { + "shape": "StructType", + "eventpayload": true + } + }, + "event": true + }, + "StructType": { + "type": "structure", + "members": { + "Foo": { + "shape": "StringType" + }, + "Bar": { + "shape": "StringType" + } + } + }, + "StringType": { + "type": "string" + } + }, + "cases": [ + { + "given": { + "output": { + "shape": "OutputShape" + }, + "name": "OperationName" + }, + "result": { + "Payload": { + "EventMember": { + "Struct": { + "Foo": "foo", + "Bar": "bar" + } + } + } + }, + "response": { + "eventstream": true, + "status_code": 200, + "headers": {}, + "body": "AAAAVwAAABoSczXICzpldmVudC10eXBlBwALRXZlbnRNZW1iZXI8U3RydWN0PjxGb28+Zm9vPC9Gb28+PEJhcj5iYXI8L0Jhcj48L1N0cnVjdD6FqRTC" + } + } + ] + }, + { + "description": "event with both eventheader and eventpayload", + "metadata": { + "protocol": "rest-xml" + }, + "shapes": { + "OutputShape": { + "type": "structure", + "members": { + "Payload": { + "shape": "EventStreamShape" + } + }, + "payload": "Payload" + }, + "EventStreamShape": { + "type": "structure", + "members": { + "EventMember": { + "shape": "EventShape" + } + }, + "eventstream": true + }, + "EventShape": { + "type": "structure", + "members": { + "Str": { + "shape": "StringType", + "eventpayload": true + }, + "HeaderStr": { + "shape": "StringType", + "eventheader": true + } + }, + "event": true + }, + "StringType": { + "type": "string" + } + }, + "cases": [ + { + "given": { + "output": { + "shape": "OutputShape" + }, + "name": "OperationName" + }, + "result": { + "Payload": { + "EventMember": { + "Str": "foo", + "HeaderStr": "bar" + } + } + }, + "response": { + "eventstream": true, + "status_code": 200, + "headers": {}, + "body": "AAAAUwAAAEBsTSviCzpldmVudC10eXBlBwALRXZlbnRNZW1iZXINOm1lc3NhZ2UtdHlwZQcABWV2ZW50CUhlYWRlclN0cgcAA2JhcmZvb2nR2pg=" + } + } + ] } ] diff --git a/build_tools/aws-sdk-code-generator/spec/protocols_spec.rb b/build_tools/aws-sdk-code-generator/spec/protocols_spec.rb index 46f57d64a94..c3e8e4dbbec 100644 --- a/build_tools/aws-sdk-code-generator/spec/protocols_spec.rb +++ b/build_tools/aws-sdk-code-generator/spec/protocols_spec.rb @@ -211,7 +211,11 @@ def normalize_xml(xml) # temporary work-around for header case-sensitive test context.http_response.headers = test_case['response']['headers'] - context.http_response.signal_data(test_case['response']['body']) + # Base64 encoded binary body is provided for eventstream + body = test_case['response']['eventstream'] ? + Base64.decode64(test_case['response']['body']) : + test_case['response']['body'] + context.http_response.signal_data(body) context.http_response.signal_done Seahorse::Client::Response.new(context:context) end @@ -219,8 +223,24 @@ def normalize_xml(xml) data = data_to_hash(resp.data) expected_data = format_data(resp.context.operation.output, test_case['result'] || {}) - - expect(data).to eq(expected_data) + if test_case['response']['eventstream'] + data.each do |member_name, value| + if value.respond_to?(:each) + # event stream member + value.each do |event_struct| + # verify each event + event = event_struct.to_h + expect_event = expected_data[member_name][event.delete(:event_type)] + expect(data_to_hash(event)).to eq(expect_event) + end + else + # non event stream member + expect(value).to eq(expected_data[member_name]) + end + end + else + expect(data).to eq(expected_data) + end end end diff --git a/build_tools/aws-sdk-code-generator/templates/client_class.mustache b/build_tools/aws-sdk-code-generator/templates/client_class.mustache index eb210b693e1..9a20babc742 100644 --- a/build_tools/aws-sdk-code-generator/templates/client_class.mustache +++ b/build_tools/aws-sdk-code-generator/templates/client_class.mustache @@ -36,7 +36,29 @@ module {{module_name}} # @overload {{name}}(params = {}) # @param [Hash] params ({}) def {{name}}(params = {}, options = {}{{{block_option}}}) + {{#eventstream_output}} + params = params.dup + event_stream_handler = case handler = params.delete(:event_stream_handler) + when EventStreams::{{eventstream_member}} then handler + when Proc then EventStreams::{{eventstream_member}}.new.tap(&handler) + when nil then EventStreams::{{eventstream_member}}.new + else + msg = "expected :event_stream_handler to be a block or "\ + "instance of {{module_name}}::EventStreams::{{eventstream_member}}"\ + ", got `#{handler.inspect}` instead" + raise ArgumentError, msg + end + + yield(event_stream_handler) if block_given? + + {{/eventstream_output}} req = build_request(:{{name}}, params) + {{#eventstream_output}} + + req.context[:event_stream_handler] = event_stream_handler + req.handlers.add(Aws::Binary::DecodeHandler, priority: 95) + + {{/eventstream_output}} req.send_request(options{{{block_option}}}) end {{/operations}} diff --git a/build_tools/aws-sdk-code-generator/templates/event_streams_module.mustache b/build_tools/aws-sdk-code-generator/templates/event_streams_module.mustache new file mode 100644 index 00000000000..e265776c247 --- /dev/null +++ b/build_tools/aws-sdk-code-generator/templates/event_streams_module.mustache @@ -0,0 +1,38 @@ +{{#generated_src_warning}} +{{generated_src_warning}} +{{/generated_src_warning}} +module {{module_name}} + module EventStreams + {{#eventstreams}} + class {{class_name}} + + def initialize + @event_emitter = EventEmitter.new + end + + {{#types}} + def on_{{.}}_event(&block) + @event_emitter.on(:{{.}}, Proc.new) + end + + {{/types}} + def on_error_event(&block) + @event_emitter.on(:error, Proc.new) + end + + def on_event(&block) + {{#types}} + on_{{.}}_event(&block) + {{/types}} + end + + # @api private + # @return EventEmitter + attr_reader :event_emitter + + end + {{/eventstreams}} + + end +end + diff --git a/build_tools/aws-sdk-code-generator/templates/types_module.mustache b/build_tools/aws-sdk-code-generator/templates/types_module.mustache index 0368548aa97..f9f8fb323fa 100644 --- a/build_tools/aws-sdk-code-generator/templates/types_module.mustache +++ b/build_tools/aws-sdk-code-generator/templates/types_module.mustache @@ -18,6 +18,26 @@ module {{module_name}} end {{/empty?}} {{/structures}} + {{#eventstreams}} + + {{> documentation}} + class {{class_name}} < Enumerator + + def event_types + {{#empty?}} + [] + {{/empty?}} + {{^empty?}} + [ + {{#types}} + :{{member_name}}{{^last}},{{/last}} + {{/types}} + ] + {{/empty?}} + end + + end + {{/eventstreams}} end end diff --git a/build_tools/customizations.rb b/build_tools/customizations.rb index 4ee9cfc2432..5ce1ce3a41b 100644 --- a/build_tools/customizations.rb +++ b/build_tools/customizations.rb @@ -15,8 +15,6 @@ def doc(svc_name, &block) end def apply_api_customizations(svc_name, api) - # delay eventstream support in V3 - api = exclude_eventstream(api) if api['operations'] @api_customizations[svc_name].call(api) if @api_customizations[svc_name] end @@ -24,33 +22,6 @@ def apply_doc_customizations(svc_name, docs) @doc_customizations[svc_name].call(docs) if @doc_customizations[svc_name] end - private - - def exclude_eventstream(api) - api['operations'].each do |name, ref| - inbound = ref['input'] && is_eventstream?(api, ref['input']['shape']) - outbound = ref['output'] && is_eventstream?(api, ref['output']['shape']) - # for eventstream operations, avoid operation, input and output shapes - if !!inbound || !!outbound - api['shapes'].delete(ref['input']['shape']) - api['shapes'].delete(ref['output']['shape']) - api['operations'].delete(name) - end - end - api - end - - def is_eventstream?(api, shape_name) - shape = api['shapes'][shape_name] - if shape['type'] == 'structure' && shape['payload'] - payload_ref = shape['members'][shape['payload']] - api['shapes'][payload_ref['shape']]['eventstream'] - else - # non structure request/response shape - # check if it's eventstream itself - shape['eventstream'] - end - end end api('CloudFront') do |api| diff --git a/build_tools/services.rb b/build_tools/services.rb index d3c1336125b..cc8c720b906 100644 --- a/build_tools/services.rb +++ b/build_tools/services.rb @@ -7,6 +7,10 @@ class ServiceEnumerator MANIFEST_PATH = File.expand_path('../../services.json', __FILE__) + # Minimum `aws-sdk-core` version for eventstream support + EVENTSTREAM_CORE_VERSION = "3.21.0" + EVENTSTREAM_PLUGIN = "Aws::Plugins::EventStreamConfiguration" + # @option options [String] :manifest_path (MANIFEST_PATH) def initialize(options = {}) @manifest_path = options.fetch(:manifest_path, MANIFEST_PATH) @@ -57,7 +61,7 @@ def build_service(svc_name, config) resources: model_path('resources-1.json', config['models']), examples: model_path('examples-1.json', config['models']), gem_dependencies: gem_dependencies(api, config['dependencies'] || {}), - add_plugins: add_plugins(config['addPlugins'] || []), + add_plugins: add_plugins(api, config['addPlugins'] || []), remove_plugins: config['removePlugins'] || [] ) end @@ -74,7 +78,8 @@ def load_docs(svc_name, models_dir) docs end - def add_plugins(plugins) + def add_plugins(api, plugins) + plugins << EVENTSTREAM_PLUGIN if eventstream?(api) plugins.inject({}) do |hash, plugin| hash[plugin] = plugin_path(plugin) hash @@ -106,10 +111,8 @@ def gem_version(gem_name) def gem_dependencies(api, dependencies) version_file = File.read("#{$GEMS_DIR}/aws-sdk-core/VERSION").rstrip - core_version = version_file.match(/^\d+\.\d+\.\d+$/) ? - "#{version_file.split('.')[0]}" : - version_file - dependencies['aws-sdk-core'] = "~> #{core_version}" + eventstream_version_string = eventstream?(api) ? "', '>= #{EVENTSTREAM_CORE_VERSION}" : '' + dependencies['aws-sdk-core'] = "~> #{version_file.split('.')[0]}#{eventstream_version_string}" case api['metadata']['signatureVersion'] when 'v4' then dependencies['aws-sigv4'] = '~> 1.0' @@ -123,6 +126,13 @@ def model_path(model_name, models_dir) File.exists?(path) ? path : nil end + def eventstream?(api) + api['shapes'].each do |_, ref| + return true if ref['eventstream'] || ref['event'] + end + false + end + end Services = ServiceEnumerator.new diff --git a/build_tools/spec/region_spec.rb b/build_tools/spec/region_spec.rb index 6c82bfdc84f..6dc72fb488c 100644 --- a/build_tools/spec/region_spec.rb +++ b/build_tools/spec/region_spec.rb @@ -9,7 +9,7 @@ def gem_lib_paths def whitelist { "core" => { - "errors.rb" => 102, + "errors.rb" => 137, "signature_v4.rb" => 35, "stub_responses.rb" => 19 }, diff --git a/gems/aws-sdk-core/CHANGELOG.md b/gems/aws-sdk-core/CHANGELOG.md index 32c4774de31..124971e7435 100644 --- a/gems/aws-sdk-core/CHANGELOG.md +++ b/gems/aws-sdk-core/CHANGELOG.md @@ -1,6 +1,8 @@ Unreleased Changes ------------------ +* Feature - Support `vnd.amazon.event-stream` binary stream protocol over HTTP1.1 + 3.20.2 (2018-04-26) ------------------ diff --git a/gems/aws-sdk-core/aws-sdk-core.gemspec b/gems/aws-sdk-core/aws-sdk-core.gemspec index 0ab637797df..3145bebe0e0 100644 --- a/gems/aws-sdk-core/aws-sdk-core.gemspec +++ b/gems/aws-sdk-core/aws-sdk-core.gemspec @@ -14,6 +14,7 @@ Gem::Specification.new do |spec| spec.add_dependency('jmespath', '~> 1.0') spec.add_dependency('aws-partitions', '~> 1.0') spec.add_dependency('aws-sigv4', '~> 1.0') # necessary for making Aws::STS API calls + spec.add_dependency('aws-eventstream', '~> 1.0') # necessary for binary eventstream spec.metadata = { 'source_code_uri' => 'https://github.com/aws/aws-sdk-ruby/tree/master/gems/aws-sdk-core', diff --git a/gems/aws-sdk-core/features/features_helper.rb b/gems/aws-sdk-core/features/features_helper.rb index bcbbc6b473c..0b0754ca6a5 100644 --- a/gems/aws-sdk-core/features/features_helper.rb +++ b/gems/aws-sdk-core/features/features_helper.rb @@ -1,5 +1,6 @@ $LOAD_PATH.unshift(File.expand_path('../../lib', __FILE__)) $LOAD_PATH.unshift(File.expand_path('../../../aws-sigv4/lib', __FILE__)) +$LOAD_PATH.unshift(File.expand_path('../../../aws-eventstream/lib', __FILE__)) $LOAD_PATH.unshift(File.expand_path('../../../aws-partitions/lib', __FILE__)) $LOAD_PATH.unshift(File.expand_path('../../../../build_tools/aws-sdk-code-generator/lib', __FILE__)) diff --git a/gems/aws-sdk-core/lib/aws-sdk-core.rb b/gems/aws-sdk-core/lib/aws-sdk-core.rb index 5e351ecb05f..1edbcd72418 100644 --- a/gems/aws-sdk-core/lib/aws-sdk-core.rb +++ b/gems/aws-sdk-core/lib/aws-sdk-core.rb @@ -62,6 +62,10 @@ require_relative 'aws-sdk-core/xml' require_relative 'aws-sdk-core/json' +# event stream +require_relative 'aws-sdk-core/binary' +require_relative 'aws-sdk-core/event_emitter' + # aws-sdk-sts is vendored to support Aws::AssumeRoleCredentials require 'aws-sdk-sts' diff --git a/gems/aws-sdk-core/lib/aws-sdk-core/binary.rb b/gems/aws-sdk-core/lib/aws-sdk-core/binary.rb new file mode 100644 index 00000000000..b772d588545 --- /dev/null +++ b/gems/aws-sdk-core/lib/aws-sdk-core/binary.rb @@ -0,0 +1,3 @@ +require_relative 'binary/decode_handler' +require_relative 'binary/event_stream_decoder' +require_relative 'binary/event_parser' diff --git a/gems/aws-sdk-core/lib/aws-sdk-core/binary/decode_handler.rb b/gems/aws-sdk-core/lib/aws-sdk-core/binary/decode_handler.rb new file mode 100644 index 00000000000..0adde90bf40 --- /dev/null +++ b/gems/aws-sdk-core/lib/aws-sdk-core/binary/decode_handler.rb @@ -0,0 +1,46 @@ +module Aws + module Binary + + # @api private + class DecodeHandler < Seahorse::Client::Handler + + def call(context) + if eventstream_member = eventstream?(context) + attach_eventstream_listeners(context, eventstream_member) + end + @handler.call(context) + end + + private + + def eventstream?(ctx) + ctx.operation.output.shape.members.each do |_, ref| + return ref if ref.eventstream + end + end + + def attach_eventstream_listeners(context, rules) + + context.http_response.on_headers(200) do + protocol = context.config.api.metadata['protocol'] + context.http_response.body = EventStreamDecoder.new( + protocol, + rules, + context.http_response.body, + context[:event_stream_handler]) + end + + context.http_response.on_success(200) do + context.http_response.body = context.http_response.body.events + end + + context.http_response.on_error do + context.http_response.body = context.http_response.raw_stream + end + + end + + end + + end +end diff --git a/gems/aws-sdk-core/lib/aws-sdk-core/binary/event_parser.rb b/gems/aws-sdk-core/lib/aws-sdk-core/binary/event_parser.rb new file mode 100644 index 00000000000..688c9a2162d --- /dev/null +++ b/gems/aws-sdk-core/lib/aws-sdk-core/binary/event_parser.rb @@ -0,0 +1,104 @@ +module Aws + module Binary + # @api private + class EventParser + + include Seahorse::Model::Shapes + + # @param [Class] parser_class + # @param [Seahorse::Model::ShapeRef] rules + def initialize(parser_class, rules) + @parser_class = parser_class + @rules = rules + end + + # Parse raw event message into event struct + # based on its ShapeRef + # + # @return [Struct] Event Struct + def apply(raw_event) + parse(raw_event) + end + + private + + def parse(raw_event) + message_type = raw_event.headers.delete(":message-type") + if message_type + case message_type.value + when 'error' + parse_error_event(raw_event) + when 'event' + parse_event(raw_event) + when 'exception' + # Pending + raise Aws::Errors::EventStreamParserError.new( + ':exception event parsing is not supported') + else + raise Aws::Errors::EventStreamParserError.new( + 'Unrecognized :message-type value for the event') + end + else + # no :message-type header, regular event by default + parse_event(raw_event) + end + end + + def parse_error_event(raw_event) + error_code = raw_event.headers.delete(":error-code") + error_message = raw_event.headers.delete(":error-message") + Aws::Errors::EventError.new( + :error, + error_code ? error_code.value : error_code, + error_message ? error_message.value : error_message + ) + end + + def parse_event(raw_event) + event_type = raw_event.headers.delete(":event-type").value + # content_type = raw_event.headers.delete(":content-type").value + + # Pending + if event_type == 'initial-response' + raise Aws::Errors::EventStreamParserError.new( + 'non eventstream member at response is not supported yet' + ) + end + + # locate event from eventstream + name, ref = @rules.shape.member_by_location_name(event_type) + raise "Non event member found at eventstream" unless ref.event + + event = ref.shape.struct_class.new + event.event_type = name + # locate payload and headers in the event + ref.shape.members.each do |member_name, member_ref| + if member_ref.eventpayload + eventpayload_streaming?(member_ref) ? + event.send("#{member_name}=", raw_event.payload) : + event.send("#{member_name}=", parse_payload(raw_event.payload.read, member_ref)) + elsif member_ref.eventheader + # allow incomplete event members in response + if raw_event.headers.key?(member_ref.location_name) + event.send("#{member_name}=", raw_event.headers[member_ref.location_name].value) + end + else + raise "Non eventpayload or eventheader member found at event" + end + end + + event + end + + def eventpayload_streaming?(ref) + BlobShape === ref.shape || StringShape === ref.shape + end + + def parse_payload(body, rules) + @parser_class.new(rules).parse(body) if body.size > 0 + end + + end + + end +end diff --git a/gems/aws-sdk-core/lib/aws-sdk-core/binary/event_stream_decoder.rb b/gems/aws-sdk-core/lib/aws-sdk-core/binary/event_stream_decoder.rb new file mode 100644 index 00000000000..05ced115087 --- /dev/null +++ b/gems/aws-sdk-core/lib/aws-sdk-core/binary/event_stream_decoder.rb @@ -0,0 +1,59 @@ +require 'aws-eventstream' + +module Aws + module Binary + # @api private + class EventStreamDecoder + + # @param [String] protocol + # @param [ShapeRef] rules ShapeRef of the eventstream member + # @param [EventStream|nil] event_stream_handler A Service EventStream object + # that registered with callbacks for processing events when they arrive + def initialize(protocol, rules, io, event_stream_handler = nil) + @decoder = Aws::EventStream::Decoder.new + @event_parser = EventParser.new(parser_class(protocol), rules) + @stream_class = extract_stream_class(rules.shape.struct_class) + @emitter = event_stream_handler.event_emitter + @events = [] + end + + # @return [Array] events Array of arrived event objects + attr_reader :events + + def write(chunk) + raw_event, eof = @decoder.decode_chunk(chunk) + emit_event(raw_event) if raw_event + while !eof + # exhaust message_buffer data + raw_event, eof = @decoder.decode_chunk + emit_event(raw_event) if raw_event + end + end + + private + + def emit_event(raw_event) + event = @event_parser.apply(raw_event) + @events << event + @emitter.signal(event.event_type, event) unless @emitter.nil? + end + + def parser_class(protocol) + case protocol + when 'rest-xml' then Aws::Xml::Parser + when 'rest-json' then Aws::Json::Parser + else raise "unsupported protocol #{protocol} for event stream" + end + end + + def extract_stream_class(type_class) + parts = type_class.to_s.split('::') + parts.inject(Kernel) do |const, part_name| + part_name == 'Types' ? const.const_get('EventStreams') + : const.const_get(part_name) + end + end + end + + end +end diff --git a/gems/aws-sdk-core/lib/aws-sdk-core/errors.rb b/gems/aws-sdk-core/lib/aws-sdk-core/errors.rb index d7ed8e49f2c..051048b2a41 100644 --- a/gems/aws-sdk-core/lib/aws-sdk-core/errors.rb +++ b/gems/aws-sdk-core/lib/aws-sdk-core/errors.rb @@ -34,6 +34,41 @@ class << self end end + # Raised when EventStream Parser failed to parse + # a raw event message + class EventStreamParserError < RuntimeError; end + + # Error event in an event stream which has event_type :error + # error code and error message can be retrieved when available. + # + # example usage: + # + # client.stream_foo(name: 'bar') do |event| + # stream.on_error_event do |event| + # puts "Error #{event.error_code}: #{event.error_message}" + # raise event + # end + # end + # + class EventError < RuntimeError + + def initialize(event_type, code, message) + @event_type = event_type + @error_code = code + @error_message = message + end + + # @return [Symbol] + attr_reader :event_type + + # @return [String] + attr_reader :error_code + + # @return [String] + attr_reader :error_message + + end + # Various plugins perform client-side checksums of responses. # This error indicates a checksum failed. class ChecksumError < RuntimeError; end diff --git a/gems/aws-sdk-core/lib/aws-sdk-core/event_emitter.rb b/gems/aws-sdk-core/lib/aws-sdk-core/event_emitter.rb new file mode 100644 index 00000000000..c70b3b8a32b --- /dev/null +++ b/gems/aws-sdk-core/lib/aws-sdk-core/event_emitter.rb @@ -0,0 +1,18 @@ +class EventEmitter + + def initialize + @listeners = {} + end + + def on(type, callback) + (@listeners[type] ||= []) << callback + end + + def signal(type, event) + return unless @listeners[type] + @listeners[type].each do |listener| + listener.call(event) if event.event_type == type + end + end + +end diff --git a/gems/aws-sdk-core/lib/aws-sdk-core/param_validator.rb b/gems/aws-sdk-core/lib/aws-sdk-core/param_validator.rb index b8b3b1c9345..8d1bd884f2f 100644 --- a/gems/aws-sdk-core/lib/aws-sdk-core/param_validator.rb +++ b/gems/aws-sdk-core/lib/aws-sdk-core/param_validator.rb @@ -38,28 +38,48 @@ def structure(ref, values, errors, context) # ensure the value is hash like return unless correct_type?(ref, values, errors, context) - shape = ref.shape - - # ensure required members are present - if @validate_required - shape.required.each do |member_name| - if values[member_name].nil? - param = "#{context}[#{member_name.inspect}]" - errors << "missing required parameter #{param}" + if ref.eventstream + values.each do |value| + # each event is structure type + case value[:message_type] + when 'event' + val = value.dup + val.delete(:message_type) + structure(ref.shape.member(val[:event_type]), val, errors, context) + when 'error' # Error is unmodeled + when 'exception' # Pending + raise Aws::Errors::EventStreamParserError.new( + ':exception event validation is not supported') + end + end + else + shape = ref.shape + + # ensure required members are present + if @validate_required + shape.required.each do |member_name| + if values[member_name].nil? + param = "#{context}[#{member_name.inspect}]" + errors << "missing required parameter #{param}" + end end end - end - # validate non-nil members - values.each_pair do |name, value| - unless value.nil? - if shape.member?(name) - member_ref = shape.member(name) - shape(member_ref, value, errors, context + "[#{name.inspect}]") - else - errors << "unexpected value at #{context}[#{name.inspect}]" + # validate non-nil members + values.each_pair do |name, value| + unless value.nil? + # :event_type is not modeled + # and also needed when construct body + next if name == :event_type + if shape.member?(name) + member_ref = shape.member(name) + shape(member_ref, value, errors, context + "[#{name.inspect}]") + else + errors << "unexpected value at #{context}[#{name.inspect}]" + end end end + end end @@ -130,6 +150,7 @@ def correct_type?(ref, value, errors, context) case value when Hash then true when ref.shape.struct_class then true + when Enumerator then ref.eventstream && value.respond_to?(:event_types) else errors << expected_got(context, "a hash", value) false diff --git a/gems/aws-sdk-core/lib/aws-sdk-core/plugins/event_stream_configuration.rb b/gems/aws-sdk-core/lib/aws-sdk-core/plugins/event_stream_configuration.rb new file mode 100644 index 00000000000..94964b4200b --- /dev/null +++ b/gems/aws-sdk-core/lib/aws-sdk-core/plugins/event_stream_configuration.rb @@ -0,0 +1,16 @@ +module Aws + module Plugins + + class EventStreamConfiguration < Seahorse::Client::Plugin + + option(:event_stream_handler, + default: nil, + doc_type: 'Proc', + docstring: <<-DOCS) +When an EventStream or Proc object is provided, it will be used as callback for each chunk of event stream response received along the way. + DOCS + + end + + end +end diff --git a/gems/aws-sdk-core/lib/aws-sdk-core/rest/response/body.rb b/gems/aws-sdk-core/lib/aws-sdk-core/rest/response/body.rb index f43b5894fa1..1673f7515e8 100644 --- a/gems/aws-sdk-core/lib/aws-sdk-core/rest/response/body.rb +++ b/gems/aws-sdk-core/lib/aws-sdk-core/rest/response/body.rb @@ -15,7 +15,9 @@ def initialize(parser_class, rules) # @param [IO] body # @param [Hash, Struct] data def apply(body, data) - if streaming? + if event_stream? + data[@rules[:payload]] = parse_eventstream(body) + elsif streaming? data[@rules[:payload]] = body elsif @rules[:payload] data[@rules[:payload]] = parse(body.read, @rules[:payload_member]) @@ -26,6 +28,10 @@ def apply(body, data) private + def event_stream? + @rules[:payload] && @rules[:payload_member].eventstream + end + def streaming? @rules[:payload] && ( BlobShape === @rules[:payload_member].shape || @@ -37,6 +43,13 @@ def parse(body, rules, target = nil) @parser_class.new(rules).parse(body, target) if body.size > 0 end + def parse_eventstream(body) + # body contains an array of parsed event when they arrive + @rules[:payload_member].shape.struct_class.new do |payload| + body.each { |event| payload << event } + end + end + end end end diff --git a/gems/aws-sdk-core/lib/aws-sdk-core/rest/response/parser.rb b/gems/aws-sdk-core/lib/aws-sdk-core/rest/response/parser.rb index 7be042acd4e..129dcf0628e 100644 --- a/gems/aws-sdk-core/lib/aws-sdk-core/rest/response/parser.rb +++ b/gems/aws-sdk-core/lib/aws-sdk-core/rest/response/parser.rb @@ -29,7 +29,10 @@ def extract_headers(rules, response) def extract_body(rules, response) Body.new(parser_class(response), rules). - apply(response.context.http_response.body, response.data) + apply( + response.context.http_response.body, + response.data + ) end def parser_class(response) diff --git a/gems/aws-sdk-core/lib/aws-sdk-core/stubbing/empty_stub.rb b/gems/aws-sdk-core/lib/aws-sdk-core/stubbing/empty_stub.rb index f33f5d94a83..50fcc889842 100644 --- a/gems/aws-sdk-core/lib/aws-sdk-core/stubbing/empty_stub.rb +++ b/gems/aws-sdk-core/lib/aws-sdk-core/stubbing/empty_stub.rb @@ -36,7 +36,10 @@ def stub_ref(ref, visited = []) def stub_structure(ref, visited) ref.shape.members.inject(ref.shape.struct_class.new) do |struct, (mname, mref)| - struct[mname] = stub_ref(mref, visited) + # For eventstream shape, it returns an Enumerator + unless mref.eventstream + struct[mname] = stub_ref(mref, visited) + end struct end end diff --git a/gems/aws-sdk-core/lib/aws-sdk-core/stubbing/protocols/rest.rb b/gems/aws-sdk-core/lib/aws-sdk-core/stubbing/protocols/rest.rb index 2fbac357451..10c2f0354c4 100644 --- a/gems/aws-sdk-core/lib/aws-sdk-core/stubbing/protocols/rest.rb +++ b/gems/aws-sdk-core/lib/aws-sdk-core/stubbing/protocols/rest.rb @@ -1,3 +1,5 @@ +require 'aws-eventstream' + module Aws module Stubbing module Protocols @@ -74,6 +76,77 @@ def head_operation(operation) operation.http_method == "HEAD" end + def eventstream?(rules) + rules.eventstream + end + + def encode_eventstream_response(rules, data, builder) + data.inject('') do |stream, event_data| + # construct message headers and payload + opts = {headers: {}} + case event_data.delete(:message_type) + when 'event' + encode_event(opts, rules, event_data, builder) + when 'error' + # errors are unmodeled + encode_error(opts, event_data) + when 'exception' + # Pending + raise 'Stubbing :exception event is not supported' + end + stream << Aws::EventStream::Encoder.new.encode( + Aws::EventStream::Message.new(opts)) + stream + end + end + + def encode_error(opts, event_data) + opts[:headers][':error-message'] = Aws::EventStream::HeaderValue.new( + value: event_data[:error_message], + type: 'string' + ) + opts[:headers][':error-code'] = Aws::EventStream::HeaderValue.new( + value: event_data[:error_code], + type: 'string' + ) + opts[:headers][':message-type'] = Aws::EventStream::HeaderValue.new( + value: 'error', + type: 'string' + ) + opts + end + + def encode_event(opts, rules, event_data, builder) + event_ref = rules.shape.member(event_data.delete(:event_type)) + event_data.each do |k, v| + member_ref = event_ref.shape.member(k) + if member_ref.eventheader + opts[:headers][member_ref.location_name] = Aws::EventStream::HeaderValue.new( + value: v, + type: member_ref.eventheader_type + ) + elsif member_ref.eventpayload + case member_ref.eventpayload_type + when 'string' + opts[:payload] = StringIO.new(v) + when 'blob' + opts[:payload] = v + when 'structure' + opts[:payload] = StringIO.new(builder.new(member_ref).serialize(v)) + end + end + end + opts[:headers][':event-type'] = Aws::EventStream::HeaderValue.new( + value: event_ref.location_name, + type: 'string' + ) + opts[:headers][':message-type'] = Aws::EventStream::HeaderValue.new( + value: 'event', + type: 'string' + ) + opts + end + end end end diff --git a/gems/aws-sdk-core/lib/aws-sdk-core/stubbing/protocols/rest_json.rb b/gems/aws-sdk-core/lib/aws-sdk-core/stubbing/protocols/rest_json.rb index 25fc7833326..d8c2e8d0a42 100644 --- a/gems/aws-sdk-core/lib/aws-sdk-core/stubbing/protocols/rest_json.rb +++ b/gems/aws-sdk-core/lib/aws-sdk-core/stubbing/protocols/rest_json.rb @@ -4,7 +4,11 @@ module Protocols class RestJson < Rest def body_for(_, _, rules, data) - Aws::Json::Builder.new(rules).serialize(data) + if eventstream?(rules) + encode_eventstream_response(rules, data, Aws::Json::Builder) + else + Aws::Json::Builder.new(rules).serialize(data) + end end def stub_error(error_code) diff --git a/gems/aws-sdk-core/lib/aws-sdk-core/stubbing/protocols/rest_xml.rb b/gems/aws-sdk-core/lib/aws-sdk-core/stubbing/protocols/rest_xml.rb index a96d3f53ffa..2456e888abb 100644 --- a/gems/aws-sdk-core/lib/aws-sdk-core/stubbing/protocols/rest_xml.rb +++ b/gems/aws-sdk-core/lib/aws-sdk-core/stubbing/protocols/rest_xml.rb @@ -6,11 +6,15 @@ class RestXml < Rest include Seahorse::Model::Shapes def body_for(api, operation, rules, data) - xml = [] - rules.location_name = operation.name + 'Result' - rules['xmlNamespace'] = { 'uri' => api.metadata['xmlNamespace'] } - Xml::Builder.new(rules, target:xml).to_xml(data) - xml.join + if eventstream?(rules) + encode_eventstream_response(rules, data, Xml::Builder) + else + xml = [] + rules.location_name = operation.name + 'Result' + rules['xmlNamespace'] = { 'uri' => api.metadata['xmlNamespace'] } + Xml::Builder.new(rules, target:xml).to_xml(data) + xml.join + end end def stub_error(error_code) diff --git a/gems/aws-sdk-core/lib/seahorse/client/http/response.rb b/gems/aws-sdk-core/lib/seahorse/client/http/response.rb index e39a68e1ffa..17e8f234e2c 100644 --- a/gems/aws-sdk-core/lib/seahorse/client/http/response.rb +++ b/gems/aws-sdk-core/lib/seahorse/client/http/response.rb @@ -10,6 +10,7 @@ def initialize(options = {}) @status_code = options[:status_code] || 0 @headers = options[:headers] || Headers.new @body = options[:body] || StringIO.new + @raw_stream = options[:raw_stream] || StringIO.new @listeners = Hash.new { |h,k| h[k] = [] } @complete = false @done = nil @@ -26,6 +27,12 @@ def initialize(options = {}) # @return [StandardError, nil] attr_reader :error + # Raw binary stream data used for eventstream only + # body is concatenated parsed event structs in StringIO + # + # @return [IO] + attr_accessor :raw_stream + # @return [IO] def body @body @@ -59,6 +66,9 @@ def signal_headers(status_code, headers) # @param [string] chunk def signal_data(chunk) unless chunk == '' + # record raw binary stream for eventstream + # in case error happens, body is used for emit/track events + @raw_stream.write(chunk) @body.write(chunk) emit(:data, chunk) end diff --git a/gems/aws-sdk-core/lib/seahorse/model/shapes.rb b/gems/aws-sdk-core/lib/seahorse/model/shapes.rb index 0cb8fdf2165..de366bc72a5 100644 --- a/gems/aws-sdk-core/lib/seahorse/model/shapes.rb +++ b/gems/aws-sdk-core/lib/seahorse/model/shapes.rb @@ -10,6 +10,12 @@ def initialize(options = {}) @metadata = {} @required = false @deprecated = false + @event = false + @eventstream = false + @eventpayload = false + @eventpayload_type = '' + @eventheader = false + @eventheader_type = '' options.each do |key, value| if key == :metadata value.each do |k,v| @@ -33,6 +39,24 @@ def initialize(options = {}) # @return [Boolean] attr_accessor :deprecated + # @return [Boolean] + attr_accessor :event + + # @return [Boolean] + attr_accessor :eventstream + + # @return [Boolean] + attr_accessor :eventpayload + + # @return [Boolean] + attr_accessor :eventheader + + # @return [String] + attr_accessor :eventpayload_type + + # @return [Boolean] + attr_accessor :eventheader_type + # @return [String, nil] def location @location || (shape && shape[:location]) diff --git a/gems/aws-sdk-core/spec/shared_spec_helper.rb b/gems/aws-sdk-core/spec/shared_spec_helper.rb index 014c74211bb..533dad4c6a3 100644 --- a/gems/aws-sdk-core/spec/shared_spec_helper.rb +++ b/gems/aws-sdk-core/spec/shared_spec_helper.rb @@ -1,6 +1,7 @@ $LOAD_PATH.unshift(File.expand_path('../../lib', __FILE__)) $LOAD_PATH.unshift(File.expand_path('../../../aws-sigv2/lib', __FILE__)) $LOAD_PATH.unshift(File.expand_path('../../../aws-sigv4/lib', __FILE__)) +$LOAD_PATH.unshift(File.expand_path('../../../aws-eventstream/lib', __FILE__)) $LOAD_PATH.unshift(File.expand_path('../../../aws-partitions/lib', __FILE__)) require 'webmock/rspec' diff --git a/gems/aws-sdk-resources/bin/aws-v3.rb b/gems/aws-sdk-resources/bin/aws-v3.rb index cbf934abf8d..65c21e1a526 100755 --- a/gems/aws-sdk-resources/bin/aws-v3.rb +++ b/gems/aws-sdk-resources/bin/aws-v3.rb @@ -98,7 +98,7 @@ def env_bool key, default # when running the REPL locally, we want to load all of the gems from source if File.directory?(File.expand_path('../../../../build_tools', __FILE__)) - gems = %w(aws-sdk-core aws-sigv4 aws-sigv2 aws-partitions) + gems = %w(aws-sdk-core aws-sigv4 aws-sigv2 aws-partitions aws-eventstream) Aws.constants.each do |const_name| if Aws.autoload?(const_name) gems << "aws-sdk-#{const_name.downcase}" diff --git a/gems/aws-sdk-s3/CHANGELOG.md b/gems/aws-sdk-s3/CHANGELOG.md index 045b3d01211..01e84171737 100644 --- a/gems/aws-sdk-s3/CHANGELOG.md +++ b/gems/aws-sdk-s3/CHANGELOG.md @@ -1,6 +1,8 @@ Unreleased Changes ------------------ +* Feature - Support S3 `SelectObjectContent` API + 1.10.0 (2018-05-07) ------------------ diff --git a/gems/aws-sdk-s3/aws-sdk-s3.gemspec b/gems/aws-sdk-s3/aws-sdk-s3.gemspec index 8cff4d37b8e..2188671f72c 100644 --- a/gems/aws-sdk-s3/aws-sdk-s3.gemspec +++ b/gems/aws-sdk-s3/aws-sdk-s3.gemspec @@ -25,6 +25,6 @@ Gem::Specification.new do |spec| spec.add_dependency('aws-sdk-kms', '~> 1') spec.add_dependency('aws-sigv4', '~> 1.0') - spec.add_dependency('aws-sdk-core', '~> 3') + spec.add_dependency('aws-sdk-core', '~> 3', '>= 3.21.0') end diff --git a/gems/aws-sdk-s3/features/client/eventstream.feature b/gems/aws-sdk-s3/features/client/eventstream.feature new file mode 100644 index 00000000000..a983782ecc6 --- /dev/null +++ b/gems/aws-sdk-s3/features/client/eventstream.feature @@ -0,0 +1,40 @@ +# language: en +@s3 @client @eventstream +Feature: S3 EventStream Operation + + Background: + Given I create a bucket + Given I put a file with content: + |user |age | + |foo |12 | + |bar |15 | + + Scenario: Select Object Content Sync + When I select it with query "SELECT * FROM S3Object WHERE cast(age as int) > 12" + Then response should contain "records" event + And the event should have payload member with content "bar,15" + + Scenario: Select Object Content with a block + When I select it with query "SELECT * FROM S3Object WHERE cast(age as int) > 12" with block + Then "records" event should be processed "1" times when it arrives + And the event should have payload member with content "bar,15" + Then response should contain "records" event + + Scenario: Select Object Content with a handler + When I select it with query "SELECT * FROM S3Object WHERE cast(age as int) > 12" with event stream handler + Then "records" event should be processed "1" times when it arrives + And the event should have payload member with content "bar,15" + Then response should contain "records" event + + Scenario: Select Object Content with a Proc + When I select it with query "SELECT * FROM S3Object WHERE cast(age as int) > 12" with Proc Object + Then "records" event should be processed "1" times when it arrives + And the event should have payload member with content "bar,15" + Then response should contain "records" event + + Scenario: Select Object Content with handler and block + When I select it with query "SELECT * FROM S3Object WHERE cast(age as int) > 12" with handler and block + Then "records" event should be processed "2" times when it arrives + And the event should have payload member with content "bar,15" + Then response should contain "records" event + diff --git a/gems/aws-sdk-s3/features/client/step_definitions.rb b/gems/aws-sdk-s3/features/client/step_definitions.rb index b66e79c88c4..7886c326caa 100644 --- a/gems/aws-sdk-s3/features/client/step_definitions.rb +++ b/gems/aws-sdk-s3/features/client/step_definitions.rb @@ -1,4 +1,5 @@ require 'openssl' +require "csv" Before("@s3", "@client") do @client = Aws::S3::Client.new @@ -294,3 +295,142 @@ def create_bucket(options = {}) expect(resp.body).to be_a(Seahorse::Client::BlockIO) expect(resp.context[:response_target]).to be_a(Proc) end + +Given(/^I put a file with content:$/) do |table| + @select_file_name = "test.csv" + csv = Tempfile.new("file.csv") + CSV.open(csv.path, "wb") do |f| + table.raw.each {|row| f << row} + end + @client.put_object( + bucket: @bucket_name, + key: @select_file_name, + body: File.read(csv.path) + ) + csv.unlink +end + +When(/^I select it with query "([^"]*)"$/) do |query| + @select_resp = @client.select_object_content( + bucket: @bucket_name, + key: @select_file_name, + expression_type: "SQL", + expression: query, + input_serialization: { + csv: { + file_header_info: "USE" + } + }, + output_serialization: {csv: {}} + ) + @tracker = Hash.new([]) +end + +Then(/^response should contain "([^"]*)" event$/) do |type| + @select_resp.payload.each do |event| + next unless event.event_type == type.to_sym + @tracker[type.to_sym] << event + end + expect(@tracker[:records]).not_to be_nil +end + +Then(/^the event should have payload member with content "([^"]*)"$/) do |payload| + @tracker[:records].each do |e| + # same event process twice, same string IO + e.payload.rewind + expect(e.payload.read.strip).to eq(payload) + end +end + +When(/^I select it with query "([^"]*)" with block$/) do |query| + @tracker = Hash.new([]) + @select_resp = @client.select_object_content( + bucket: @bucket_name, + key: @select_file_name, + expression_type: "SQL", + expression: query, + input_serialization: { + csv: { + file_header_info: "USE" + } + }, + output_serialization: {csv: {}} + ) do |stream| + stream.on_records_event do |e| + @tracker[e.event_type] << e + end + end +end + +Then(/^"([^"]*)" event should be processed "(\d+)" times when it arrives$/) do |type, times| + expect(@tracker[type.to_sym].size).to eq(times.to_i) +end + +When(/^I select it with query "([^"]*)" with event stream handler$/) do |string| + @tracker = Hash.new([]) + handler = Aws::S3::EventStreams::SelectObjectContentEventStream.new + handler.on_records_event do |e| + @tracker[:records] << e + end + @select_resp = @client.select_object_content( + bucket: @bucket_name, + key: @select_file_name, + expression_type: "SQL", + expression: string, + input_serialization: { + csv: { + file_header_info: "USE" + } + }, + output_serialization: {csv: {}}, + event_stream_handler: handler + ) +end + +When(/^I select it with query "([^"]*)" with Proc Object$/) do |query| + @tracker = Hash.new([]) + handler = Proc.new do |stream| + stream.on_records_event do |e| + @tracker[:records] << e + end + end + @select_resp = @client.select_object_content( + bucket: @bucket_name, + key: @select_file_name, + expression_type: "SQL", + expression: query, + input_serialization: { + csv: { + file_header_info: "USE" + } + }, + output_serialization: {csv: {}}, + event_stream_handler: handler + ) +end + +When(/^I select it with query "([^"]*)" with handler and block$/) do |query| + @tracker = Hash.new([]) + handler = Aws::S3::EventStreams::SelectObjectContentEventStream.new + handler.on_records_event do |e| + @tracker[:records] << e + end + @select_resp = @client.select_object_content( + bucket: @bucket_name, + key: @select_file_name, + expression_type: "SQL", + expression: query, + input_serialization: { + csv: { + file_header_info: "USE" + } + }, + output_serialization: {csv: {}}, + event_stream_handler: handler + ) do |stream| + stream.on_records_event do |e| + @tracker[:records] << e + end + end + +end diff --git a/gems/aws-sdk-s3/lib/aws-sdk-s3.rb b/gems/aws-sdk-s3/lib/aws-sdk-s3.rb index fa49e620728..b0dbfb90453 100644 --- a/gems/aws-sdk-s3/lib/aws-sdk-s3.rb +++ b/gems/aws-sdk-s3/lib/aws-sdk-s3.rb @@ -34,6 +34,7 @@ require_relative 'aws-sdk-s3/object_summary' require_relative 'aws-sdk-s3/object_version' require_relative 'aws-sdk-s3/customizations' +require_relative 'aws-sdk-s3/event_streams' # This module provides support for Amazon Simple Storage Service. This module is available in the # `aws-sdk-s3` gem. diff --git a/gems/aws-sdk-s3/lib/aws-sdk-s3/client.rb b/gems/aws-sdk-s3/lib/aws-sdk-s3/client.rb index c62f96ea1c4..c6eae37b569 100644 --- a/gems/aws-sdk-s3/lib/aws-sdk-s3/client.rb +++ b/gems/aws-sdk-s3/lib/aws-sdk-s3/client.rb @@ -34,6 +34,7 @@ require 'aws-sdk-s3/plugins/url_encoded_keys.rb' require 'aws-sdk-s3/plugins/s3_signer.rb' require 'aws-sdk-s3/plugins/bucket_name_restrictions.rb' +require 'aws-sdk-core/plugins/event_stream_configuration.rb' Aws::Plugins::GlobalConfiguration.add_identifier(:s3) @@ -75,6 +76,7 @@ class Client < Seahorse::Client::Base add_plugin(Aws::S3::Plugins::UrlEncodedKeys) add_plugin(Aws::S3::Plugins::S3Signer) add_plugin(Aws::S3::Plugins::BucketNameRestrictions) + add_plugin(Aws::Plugins::EventStreamConfiguration) # @option options [required, Aws::CredentialProvider] :credentials # Your AWS credentials. This can be an instance of any one of the @@ -133,6 +135,9 @@ class Client < Seahorse::Client::Base # option. You should only configure an `:endpoint` when connecting # to test endpoints. This should be avalid HTTP(S) URI. # + # @option options [Proc] :event_stream_handler + # When an EventStream or Proc object is provided, it will be used as callback for each chunk of event stream response received along the way. + # # @option options [Boolean] :follow_redirects (true) # When `true`, this client will follow 307 redirects returned # by Amazon S3. @@ -5790,6 +5795,278 @@ def restore_object(params = {}, options = {}) req.send_request(options) end + # This operation filters the contents of an Amazon S3 object based on a + # simple Structured Query Language (SQL) statement. In the request, + # along with the SQL expression, you must also specify a data + # serialization format (JSON or CSV) of the object. Amazon S3 uses this + # to parse object data into records, and returns only records that match + # the specified SQL expression. You must also specify the data + # serialization format for the response. + # + # @option params [required, String] :bucket + # The S3 Bucket. + # + # @option params [required, String] :key + # The Object Key. + # + # @option params [String] :sse_customer_algorithm + # The SSE Algorithm used to encrypt the object. For more information, go + # to [ Server-Side Encryption (Using Customer-Provided Encryption + # Keys][1]. + # + # + # + # [1]: https://docs.aws.amazon.com/AmazonS3/latest/dev/ServerSideEncryptionCustomerKeys.html + # + # @option params [String] :sse_customer_key + # The SSE Customer Key. For more information, go to [ Server-Side + # Encryption (Using Customer-Provided Encryption Keys][1]. + # + # + # + # [1]: https://docs.aws.amazon.com/AmazonS3/latest/dev/ServerSideEncryptionCustomerKeys.html + # + # @option params [String] :sse_customer_key_md5 + # The SSE Customer Key MD5. For more information, go to [ Server-Side + # Encryption (Using Customer-Provided Encryption Keys][1]. + # + # + # + # [1]: https://docs.aws.amazon.com/AmazonS3/latest/dev/ServerSideEncryptionCustomerKeys.html + # + # @option params [required, String] :expression + # The expression that is used to query the object. + # + # @option params [required, String] :expression_type + # The type of the provided expression (e.g., SQL). + # + # @option params [Types::RequestProgress] :request_progress + # Specifies if periodic request progress information should be enabled. + # + # @option params [required, Types::InputSerialization] :input_serialization + # Describes the format of the data in the object that is being queried. + # + # @option params [required, Types::OutputSerialization] :output_serialization + # Describes the format of the data that you want Amazon S3 to return in + # response. + # + # @return [Types::SelectObjectContentOutput] Returns a {Seahorse::Client::Response response} object which responds to the following methods: + # + # * {Types::SelectObjectContentOutput#payload #payload} => Types::SelectObjectContentEventStream + # + # @example EventStream Operation Example + # + # You can process event once it arrives immediately, or wait until + # full response complete and iterate through eventstream enumerator. + # + # To interact with event immediately, you need to register #select_object_content + # with callbacks, callbacks can be register for specifc events or for all events, + # callback for errors in the event stream is also available for register. + # + # Callbacks can be passed in by `:event_stream_handler` option or within block + # statement attached to #select_object_content call directly. Hybrid pattern of both + # is also supported. + # + # `:event_stream_handler` option takes in either Proc object or + # EventStreams::SelectObjectContentEventStream object. + # + # Usage pattern a): callbacks with a block attached to #select_object_content + # Example for registering callbacks for all event types and error event + # + # client.select_object_content( # params input# ) do |stream| + # + # stream.on_error_event do |event| + # # catch unmodeled error event in the stream + # raise event + # # => Aws::Errors::EventError + # # event.event_type => :error + # # event.error_code => String + # # event.error_message => String + # end + # + # stream.on_event do |event| + # # process all events arrive + # puts event.event_type + # ... + # end + # + # end + # + # Usage pattern b): pass in `:event_stream_handler` for #select_object_content + # + # 1) create a EventStreams::SelectObjectContentEventStream object + # Example for registering callbacks with specific events + # + # handler = Aws::S3::EventStreams::SelectObjectContentEventStream.new + # handler.on_records_event do |event| + # event # => Aws::S3::Types::Records + # end + # handler.on_stats_event do |event| + # event # => Aws::S3::Types::Stats + # end + # handler.on_progress_event do |event| + # event # => Aws::S3::Types::Progress + # end + # handler.on_cont_event do |event| + # event # => Aws::S3::Types::Cont + # end + # handler.on_end_event do |event| + # event # => Aws::S3::Types::End + # end + # + # client.select_object_content( # params input #, event_stream_handler: handler) + # + # 2) use a Ruby Proc object + # Example for registering callbacks with specific events + # + # handler = Proc.new do |stream| + # stream.on_records_event do |event| + # event # => Aws::S3::Types::Records + # end + # stream.on_stats_event do |event| + # event # => Aws::S3::Types::Stats + # end + # stream.on_progress_event do |event| + # event # => Aws::S3::Types::Progress + # end + # stream.on_cont_event do |event| + # event # => Aws::S3::Types::Cont + # end + # stream.on_end_event do |event| + # event # => Aws::S3::Types::End + # end + # end + # + # client.select_object_content( # params input #, event_stream_handler: handler) + # + # Usage pattern c): hybird pattern of a) and b) + # + # handler = Aws::S3::EventStreams::SelectObjectContentEventStream.new + # handler.on_records_event do |event| + # event # => Aws::S3::Types::Records + # end + # handler.on_stats_event do |event| + # event # => Aws::S3::Types::Stats + # end + # handler.on_progress_event do |event| + # event # => Aws::S3::Types::Progress + # end + # handler.on_cont_event do |event| + # event # => Aws::S3::Types::Cont + # end + # handler.on_end_event do |event| + # event # => Aws::S3::Types::End + # end + # + # client.select_object_content( # params input #, event_stream_handler: handler) do |stream| + # stream.on_error_event do |event| + # # catch unmodeled error event in the stream + # raise event + # # => Aws::Errors::EventError + # # event.event_type => :error + # # event.error_code => String + # # event.error_message => String + # end + # end + # + # Besides above usage patterns for process events when they arrive immediately, you can also + # iterate through events after response complete. + # + # Events are available at resp.payload # => Enumerator + # For parameter input example, please refer to following request syntax + # + # @example Request syntax with placeholder values + # + # resp = client.select_object_content({ + # bucket: "BucketName", # required + # key: "ObjectKey", # required + # sse_customer_algorithm: "SSECustomerAlgorithm", + # sse_customer_key: "SSECustomerKey", + # sse_customer_key_md5: "SSECustomerKeyMD5", + # expression: "Expression", # required + # expression_type: "SQL", # required, accepts SQL + # request_progress: { + # enabled: false, + # }, + # input_serialization: { # required + # csv: { + # file_header_info: "USE", # accepts USE, IGNORE, NONE + # comments: "Comments", + # quote_escape_character: "QuoteEscapeCharacter", + # record_delimiter: "RecordDelimiter", + # field_delimiter: "FieldDelimiter", + # quote_character: "QuoteCharacter", + # }, + # compression_type: "NONE", # accepts NONE, GZIP + # json: { + # type: "DOCUMENT", # accepts DOCUMENT, LINES + # }, + # }, + # output_serialization: { # required + # csv: { + # quote_fields: "ALWAYS", # accepts ALWAYS, ASNEEDED + # quote_escape_character: "QuoteEscapeCharacter", + # record_delimiter: "RecordDelimiter", + # field_delimiter: "FieldDelimiter", + # quote_character: "QuoteCharacter", + # }, + # json: { + # record_delimiter: "RecordDelimiter", + # }, + # }, + # }) + # + # @example Response structure + # + # All events are available at resp.payload: + # resp.payload #=> Enumerator + # resp.payload.event_types #=> [:records, :stats, :progress, :cont, :end] + # + # For :records event available at #on_records_event callback and response eventstream enumerator: + # event.payload #=> IO + # + # For :stats event available at #on_stats_event callback and response eventstream enumerator: + # event.details.bytes_scanned #=> Integer + # event.details.bytes_processed #=> Integer + # event.details.bytes_returned #=> Integer + # + # For :progress event available at #on_progress_event callback and response eventstream enumerator: + # event.details.bytes_scanned #=> Integer + # event.details.bytes_processed #=> Integer + # event.details.bytes_returned #=> Integer + # + # For :cont event available at #on_cont_event callback and response eventstream enumerator: + # #=> EmptyStruct + # For :end event available at #on_end_event callback and response eventstream enumerator: + # #=> EmptyStruct + # + # @see http://docs.aws.amazon.com/goto/WebAPI/s3-2006-03-01/SelectObjectContent AWS API Documentation + # + # @overload select_object_content(params = {}) + # @param [Hash] params ({}) + def select_object_content(params = {}, options = {}, &block) + params = params.dup + event_stream_handler = case handler = params.delete(:event_stream_handler) + when EventStreams::SelectObjectContentEventStream then handler + when Proc then EventStreams::SelectObjectContentEventStream.new.tap(&handler) + when nil then EventStreams::SelectObjectContentEventStream.new + else + msg = "expected :event_stream_handler to be a block or "\ + "instance of Aws::S3::EventStreams::SelectObjectContentEventStream"\ + ", got `#{handler.inspect}` instead" + raise ArgumentError, msg + end + + yield(event_stream_handler) if block_given? + + req = build_request(:select_object_content, params) + + req.context[:event_stream_handler] = event_stream_handler + req.handlers.add(Aws::Binary::DecodeHandler, priority: 95) + + req.send_request(options, &block) + end + # Uploads a part in a multipart upload. # # **Note:** After you initiate multipart upload and upload one or more diff --git a/gems/aws-sdk-s3/lib/aws-sdk-s3/client_api.rb b/gems/aws-sdk-s3/lib/aws-sdk-s3/client_api.rb index 723db3ce835..2697546fa99 100644 --- a/gems/aws-sdk-s3/lib/aws-sdk-s3/client_api.rb +++ b/gems/aws-sdk-s3/lib/aws-sdk-s3/client_api.rb @@ -79,6 +79,7 @@ module ClientApi ContentMD5 = Shapes::StringShape.new(name: 'ContentMD5') ContentRange = Shapes::StringShape.new(name: 'ContentRange') ContentType = Shapes::StringShape.new(name: 'ContentType') + ContinuationEvent = Shapes::StructureShape.new(name: 'ContinuationEvent') CopyObjectOutput = Shapes::StructureShape.new(name: 'CopyObjectOutput') CopyObjectRequest = Shapes::StructureShape.new(name: 'CopyObjectRequest') CopyObjectResult = Shapes::StructureShape.new(name: 'CopyObjectResult') @@ -136,6 +137,7 @@ module ClientApi EncodingType = Shapes::StringShape.new(name: 'EncodingType') Encryption = Shapes::StructureShape.new(name: 'Encryption') EncryptionConfiguration = Shapes::StructureShape.new(name: 'EncryptionConfiguration') + EndEvent = Shapes::StructureShape.new(name: 'EndEvent') Error = Shapes::StructureShape.new(name: 'Error') ErrorDocument = Shapes::StructureShape.new(name: 'ErrorDocument') Errors = Shapes::ListShape.new(name: 'Errors', flattened: true) @@ -343,6 +345,7 @@ module ClientApi Policy = Shapes::StringShape.new(name: 'Policy') Prefix = Shapes::StringShape.new(name: 'Prefix') Progress = Shapes::StructureShape.new(name: 'Progress') + ProgressEvent = Shapes::StructureShape.new(name: 'ProgressEvent') Protocol = Shapes::StringShape.new(name: 'Protocol') PutBucketAccelerateConfigurationRequest = Shapes::StructureShape.new(name: 'PutBucketAccelerateConfigurationRequest') PutBucketAclRequest = Shapes::StructureShape.new(name: 'PutBucketAclRequest') @@ -378,6 +381,7 @@ module ClientApi QuoteFields = Shapes::StringShape.new(name: 'QuoteFields') Range = Shapes::StringShape.new(name: 'Range') RecordDelimiter = Shapes::StringShape.new(name: 'RecordDelimiter') + RecordsEvent = Shapes::StructureShape.new(name: 'RecordsEvent') Redirect = Shapes::StructureShape.new(name: 'Redirect') RedirectAllRequestsTo = Shapes::StructureShape.new(name: 'RedirectAllRequestsTo') ReplaceKeyPrefixWith = Shapes::StringShape.new(name: 'ReplaceKeyPrefixWith') @@ -417,6 +421,9 @@ module ClientApi SSEKMS = Shapes::StructureShape.new(name: 'SSEKMS') SSEKMSKeyId = Shapes::StringShape.new(name: 'SSEKMSKeyId') SSES3 = Shapes::StructureShape.new(name: 'SSES3') + SelectObjectContentEventStream = Shapes::StructureShape.new(name: 'SelectObjectContentEventStream') + SelectObjectContentOutput = Shapes::StructureShape.new(name: 'SelectObjectContentOutput') + SelectObjectContentRequest = Shapes::StructureShape.new(name: 'SelectObjectContentRequest') SelectParameters = Shapes::StructureShape.new(name: 'SelectParameters') ServerSideEncryption = Shapes::StringShape.new(name: 'ServerSideEncryption') ServerSideEncryptionByDefault = Shapes::StructureShape.new(name: 'ServerSideEncryptionByDefault') @@ -429,6 +436,7 @@ module ClientApi SseKmsEncryptedObjectsStatus = Shapes::StringShape.new(name: 'SseKmsEncryptedObjectsStatus') StartAfter = Shapes::StringShape.new(name: 'StartAfter') Stats = Shapes::StructureShape.new(name: 'Stats') + StatsEvent = Shapes::StructureShape.new(name: 'StatsEvent') StorageClass = Shapes::StringShape.new(name: 'StorageClass') StorageClassAnalysis = Shapes::StructureShape.new(name: 'StorageClassAnalysis') StorageClassAnalysisDataExport = Shapes::StructureShape.new(name: 'StorageClassAnalysisDataExport') @@ -604,6 +612,8 @@ module ClientApi Condition.add_member(:key_prefix_equals, Shapes::ShapeRef.new(shape: KeyPrefixEquals, location_name: "KeyPrefixEquals")) Condition.struct_class = Types::Condition + ContinuationEvent.struct_class = Types::ContinuationEvent + CopyObjectOutput.add_member(:copy_object_result, Shapes::ShapeRef.new(shape: CopyObjectResult, location_name: "CopyObjectResult")) CopyObjectOutput.add_member(:expiration, Shapes::ShapeRef.new(shape: Expiration, location: "header", location_name: "x-amz-expiration")) CopyObjectOutput.add_member(:copy_source_version_id, Shapes::ShapeRef.new(shape: CopySourceVersionId, location: "header", location_name: "x-amz-copy-source-version-id")) @@ -820,6 +830,8 @@ module ClientApi EncryptionConfiguration.add_member(:replica_kms_key_id, Shapes::ShapeRef.new(shape: ReplicaKmsKeyID, location_name: "ReplicaKmsKeyID")) EncryptionConfiguration.struct_class = Types::EncryptionConfiguration + EndEvent.struct_class = Types::EndEvent + Error.add_member(:key, Shapes::ShapeRef.new(shape: ObjectKey, location_name: "Key")) Error.add_member(:version_id, Shapes::ShapeRef.new(shape: ObjectVersionId, location_name: "VersionId")) Error.add_member(:code, Shapes::ShapeRef.new(shape: Code, location_name: "Code")) @@ -1464,6 +1476,9 @@ module ClientApi Progress.add_member(:bytes_returned, Shapes::ShapeRef.new(shape: BytesReturned, location_name: "BytesReturned")) Progress.struct_class = Types::Progress + ProgressEvent.add_member(:details, Shapes::ShapeRef.new(shape: Progress, eventpayload: true, eventpayload_type: 'structure', location_name: "Details", metadata: {"eventpayload"=>true})) + ProgressEvent.struct_class = Types::ProgressEvent + PutBucketAccelerateConfigurationRequest.add_member(:bucket, Shapes::ShapeRef.new(shape: BucketName, required: true, location: "uri", location_name: "Bucket")) PutBucketAccelerateConfigurationRequest.add_member(:accelerate_configuration, Shapes::ShapeRef.new(shape: AccelerateConfiguration, required: true, location_name: "AccelerateConfiguration", metadata: {"xmlNamespace"=>{"uri"=>"http://s3.amazonaws.com/doc/2006-03-01/"}})) PutBucketAccelerateConfigurationRequest.struct_class = Types::PutBucketAccelerateConfigurationRequest @@ -1680,6 +1695,9 @@ module ClientApi QueueConfigurationList.member = Shapes::ShapeRef.new(shape: QueueConfiguration) + RecordsEvent.add_member(:payload, Shapes::ShapeRef.new(shape: Body, eventpayload: true, eventpayload_type: 'blob', location_name: "Payload", metadata: {"eventpayload"=>true})) + RecordsEvent.struct_class = Types::RecordsEvent + Redirect.add_member(:host_name, Shapes::ShapeRef.new(shape: HostName, location_name: "HostName")) Redirect.add_member(:http_redirect_code, Shapes::ShapeRef.new(shape: HttpRedirectCode, location_name: "HttpRedirectCode")) Redirect.add_member(:protocol, Shapes::ShapeRef.new(shape: Protocol, location_name: "Protocol")) @@ -1768,6 +1786,30 @@ module ClientApi SSES3.struct_class = Types::SSES3 + SelectObjectContentEventStream.add_member(:records, Shapes::ShapeRef.new(shape: RecordsEvent, event: true, location_name: "Records")) + SelectObjectContentEventStream.add_member(:stats, Shapes::ShapeRef.new(shape: StatsEvent, event: true, location_name: "Stats")) + SelectObjectContentEventStream.add_member(:progress, Shapes::ShapeRef.new(shape: ProgressEvent, event: true, location_name: "Progress")) + SelectObjectContentEventStream.add_member(:cont, Shapes::ShapeRef.new(shape: ContinuationEvent, event: true, location_name: "Cont")) + SelectObjectContentEventStream.add_member(:end, Shapes::ShapeRef.new(shape: EndEvent, event: true, location_name: "End")) + SelectObjectContentEventStream.struct_class = Types::SelectObjectContentEventStream + + SelectObjectContentOutput.add_member(:payload, Shapes::ShapeRef.new(shape: SelectObjectContentEventStream, eventstream: true, location_name: "Payload")) + SelectObjectContentOutput.struct_class = Types::SelectObjectContentOutput + SelectObjectContentOutput[:payload] = :payload + SelectObjectContentOutput[:payload_member] = SelectObjectContentOutput.member(:payload) + + SelectObjectContentRequest.add_member(:bucket, Shapes::ShapeRef.new(shape: BucketName, required: true, location: "uri", location_name: "Bucket")) + SelectObjectContentRequest.add_member(:key, Shapes::ShapeRef.new(shape: ObjectKey, required: true, location: "uri", location_name: "Key")) + SelectObjectContentRequest.add_member(:sse_customer_algorithm, Shapes::ShapeRef.new(shape: SSECustomerAlgorithm, location: "header", location_name: "x-amz-server-side-encryption-customer-algorithm")) + SelectObjectContentRequest.add_member(:sse_customer_key, Shapes::ShapeRef.new(shape: SSECustomerKey, location: "header", location_name: "x-amz-server-side-encryption-customer-key")) + SelectObjectContentRequest.add_member(:sse_customer_key_md5, Shapes::ShapeRef.new(shape: SSECustomerKeyMD5, location: "header", location_name: "x-amz-server-side-encryption-customer-key-MD5")) + SelectObjectContentRequest.add_member(:expression, Shapes::ShapeRef.new(shape: Expression, required: true, location_name: "Expression")) + SelectObjectContentRequest.add_member(:expression_type, Shapes::ShapeRef.new(shape: ExpressionType, required: true, location_name: "ExpressionType")) + SelectObjectContentRequest.add_member(:request_progress, Shapes::ShapeRef.new(shape: RequestProgress, location_name: "RequestProgress")) + SelectObjectContentRequest.add_member(:input_serialization, Shapes::ShapeRef.new(shape: InputSerialization, required: true, location_name: "InputSerialization")) + SelectObjectContentRequest.add_member(:output_serialization, Shapes::ShapeRef.new(shape: OutputSerialization, required: true, location_name: "OutputSerialization")) + SelectObjectContentRequest.struct_class = Types::SelectObjectContentRequest + SelectParameters.add_member(:input_serialization, Shapes::ShapeRef.new(shape: InputSerialization, required: true, location_name: "InputSerialization")) SelectParameters.add_member(:expression_type, Shapes::ShapeRef.new(shape: ExpressionType, required: true, location_name: "ExpressionType")) SelectParameters.add_member(:expression, Shapes::ShapeRef.new(shape: Expression, required: true, location_name: "Expression")) @@ -1797,6 +1839,9 @@ module ClientApi Stats.add_member(:bytes_returned, Shapes::ShapeRef.new(shape: BytesReturned, location_name: "BytesReturned")) Stats.struct_class = Types::Stats + StatsEvent.add_member(:details, Shapes::ShapeRef.new(shape: Stats, eventpayload: true, eventpayload_type: 'structure', location_name: "Details", metadata: {"eventpayload"=>true})) + StatsEvent.struct_class = Types::StatsEvent + StorageClassAnalysis.add_member(:data_export, Shapes::ShapeRef.new(shape: StorageClassAnalysisDataExport, location_name: "DataExport")) StorageClassAnalysis.struct_class = Types::StorageClassAnalysis @@ -2570,6 +2615,19 @@ module ClientApi o.errors << Shapes::ShapeRef.new(shape: ObjectAlreadyInActiveTierError) end) + api.add_operation(:select_object_content, Seahorse::Model::Operation.new.tap do |o| + o.name = "SelectObjectContent" + o.http_method = "POST" + o.http_request_uri = "/{Bucket}/{Key+}?select&select-type=2" + o.input = Shapes::ShapeRef.new(shape: SelectObjectContentRequest, + location_name: "SelectObjectContentRequest", + metadata: { + "xmlNamespace" => {"uri"=>"http://s3.amazonaws.com/doc/2006-03-01/"} + } + ) + o.output = Shapes::ShapeRef.new(shape: SelectObjectContentOutput) + end) + api.add_operation(:upload_part, Seahorse::Model::Operation.new.tap do |o| o.name = "UploadPart" o.http_method = "PUT" diff --git a/gems/aws-sdk-s3/lib/aws-sdk-s3/event_streams.rb b/gems/aws-sdk-s3/lib/aws-sdk-s3/event_streams.rb new file mode 100644 index 00000000000..ed22b75832f --- /dev/null +++ b/gems/aws-sdk-s3/lib/aws-sdk-s3/event_streams.rb @@ -0,0 +1,56 @@ +# WARNING ABOUT GENERATED CODE +# +# This file is generated. See the contributing guide for more information: +# https://github.com/aws/aws-sdk-ruby/blob/master/CONTRIBUTING.md +# +# WARNING ABOUT GENERATED CODE + +module Aws::S3 + module EventStreams + class SelectObjectContentEventStream + + def initialize + @event_emitter = EventEmitter.new + end + + def on_records_event(&block) + @event_emitter.on(:records, Proc.new) + end + + def on_stats_event(&block) + @event_emitter.on(:stats, Proc.new) + end + + def on_progress_event(&block) + @event_emitter.on(:progress, Proc.new) + end + + def on_cont_event(&block) + @event_emitter.on(:cont, Proc.new) + end + + def on_end_event(&block) + @event_emitter.on(:end, Proc.new) + end + + def on_error_event(&block) + @event_emitter.on(:error, Proc.new) + end + + def on_event(&block) + on_records_event(&block) + on_stats_event(&block) + on_progress_event(&block) + on_cont_event(&block) + on_end_event(&block) + end + + # @api private + # @return EventEmitter + attr_reader :event_emitter + + end + + end +end + diff --git a/gems/aws-sdk-s3/lib/aws-sdk-s3/types.rb b/gems/aws-sdk-s3/lib/aws-sdk-s3/types.rb index 3f97f6e0538..88c158d2cf4 100644 --- a/gems/aws-sdk-s3/lib/aws-sdk-s3/types.rb +++ b/gems/aws-sdk-s3/lib/aws-sdk-s3/types.rb @@ -871,6 +871,13 @@ class Condition < Struct.new( include Aws::Structure end + # @see http://docs.aws.amazon.com/goto/WebAPI/s3-2006-03-01/ContinuationEvent AWS API Documentation + # + class ContinuationEvent < Struct.new( + :event_type) + include Aws::Structure + end + # @!attribute [rw] copy_object_result # @return [Types::CopyObjectResult] # @@ -2114,6 +2121,13 @@ class EncryptionConfiguration < Struct.new( include Aws::Structure end + # @see http://docs.aws.amazon.com/goto/WebAPI/s3-2006-03-01/EndEvent AWS API Documentation + # + class EndEvent < Struct.new( + :event_type) + include Aws::Structure + end + # @!attribute [rw] key # @return [String] # @@ -5807,6 +5821,18 @@ class Progress < Struct.new( include Aws::Structure end + # @!attribute [rw] details + # The Progress event details. + # @return [Types::Progress] + # + # @see http://docs.aws.amazon.com/goto/WebAPI/s3-2006-03-01/ProgressEvent AWS API Documentation + # + class ProgressEvent < Struct.new( + :details, + :event_type) + include Aws::Structure + end + # @note When making an API call, you may pass PutBucketAccelerateConfigurationRequest # data as a hash: # @@ -7209,6 +7235,18 @@ class QueueConfigurationDeprecated < Struct.new( include Aws::Structure end + # @!attribute [rw] payload + # The byte array of partial, one or more result records. + # @return [String] + # + # @see http://docs.aws.amazon.com/goto/WebAPI/s3-2006-03-01/RecordsEvent AWS API Documentation + # + class RecordsEvent < Struct.new( + :payload, + :event_type) + include Aws::Structure + end + # @note When making an API call, you may pass Redirect # data as a hash: # @@ -7419,6 +7457,13 @@ class RequestPaymentConfiguration < Struct.new( include Aws::Structure end + # @note When making an API call, you may pass RequestProgress + # data as a hash: + # + # { + # enabled: false, + # } + # # @!attribute [rw] enabled # Specifies whether periodic QueryProgress frames should be sent. # Valid values: TRUE, FALSE. Default value: FALSE. @@ -7971,6 +8016,146 @@ class SSEKMS < Struct.new( # class SSES3 < Aws::EmptyStructure; end + # @!attribute [rw] payload + # @return [Types::SelectObjectContentEventStream] + # + # @see http://docs.aws.amazon.com/goto/WebAPI/s3-2006-03-01/SelectObjectContentOutput AWS API Documentation + # + class SelectObjectContentOutput < Struct.new( + :payload) + include Aws::Structure + end + + # Request to filter the contents of an Amazon S3 object based on a + # simple Structured Query Language (SQL) statement. In the request, + # along with the SQL expression, you must also specify a data + # serialization format (JSON or CSV) of the object. Amazon S3 uses this + # to parse object data into records, and returns only records that match + # the specified SQL expression. You must also specify the data + # serialization format for the response. For more information, go to + # [S3Select API Documentation][1]. + # + # + # + # [1]: https://docs.aws.amazon.com/AmazonS3/latest/API/RESTObjectSELECTContent.html + # + # @note When making an API call, you may pass SelectObjectContentRequest + # data as a hash: + # + # { + # bucket: "BucketName", # required + # key: "ObjectKey", # required + # sse_customer_algorithm: "SSECustomerAlgorithm", + # sse_customer_key: "SSECustomerKey", + # sse_customer_key_md5: "SSECustomerKeyMD5", + # expression: "Expression", # required + # expression_type: "SQL", # required, accepts SQL + # request_progress: { + # enabled: false, + # }, + # input_serialization: { # required + # csv: { + # file_header_info: "USE", # accepts USE, IGNORE, NONE + # comments: "Comments", + # quote_escape_character: "QuoteEscapeCharacter", + # record_delimiter: "RecordDelimiter", + # field_delimiter: "FieldDelimiter", + # quote_character: "QuoteCharacter", + # }, + # compression_type: "NONE", # accepts NONE, GZIP + # json: { + # type: "DOCUMENT", # accepts DOCUMENT, LINES + # }, + # }, + # output_serialization: { # required + # csv: { + # quote_fields: "ALWAYS", # accepts ALWAYS, ASNEEDED + # quote_escape_character: "QuoteEscapeCharacter", + # record_delimiter: "RecordDelimiter", + # field_delimiter: "FieldDelimiter", + # quote_character: "QuoteCharacter", + # }, + # json: { + # record_delimiter: "RecordDelimiter", + # }, + # }, + # } + # + # @!attribute [rw] bucket + # The S3 Bucket. + # @return [String] + # + # @!attribute [rw] key + # The Object Key. + # @return [String] + # + # @!attribute [rw] sse_customer_algorithm + # The SSE Algorithm used to encrypt the object. For more information, + # go to [ Server-Side Encryption (Using Customer-Provided Encryption + # Keys][1]. + # + # + # + # [1]: https://docs.aws.amazon.com/AmazonS3/latest/dev/ServerSideEncryptionCustomerKeys.html + # @return [String] + # + # @!attribute [rw] sse_customer_key + # The SSE Customer Key. For more information, go to [ Server-Side + # Encryption (Using Customer-Provided Encryption Keys][1]. + # + # + # + # [1]: https://docs.aws.amazon.com/AmazonS3/latest/dev/ServerSideEncryptionCustomerKeys.html + # @return [String] + # + # @!attribute [rw] sse_customer_key_md5 + # The SSE Customer Key MD5. For more information, go to [ Server-Side + # Encryption (Using Customer-Provided Encryption Keys][1]. + # + # + # + # [1]: https://docs.aws.amazon.com/AmazonS3/latest/dev/ServerSideEncryptionCustomerKeys.html + # @return [String] + # + # @!attribute [rw] expression + # The expression that is used to query the object. + # @return [String] + # + # @!attribute [rw] expression_type + # The type of the provided expression (e.g., SQL). + # @return [String] + # + # @!attribute [rw] request_progress + # Specifies if periodic request progress information should be + # enabled. + # @return [Types::RequestProgress] + # + # @!attribute [rw] input_serialization + # Describes the format of the data in the object that is being + # queried. + # @return [Types::InputSerialization] + # + # @!attribute [rw] output_serialization + # Describes the format of the data that you want Amazon S3 to return + # in response. + # @return [Types::OutputSerialization] + # + # @see http://docs.aws.amazon.com/goto/WebAPI/s3-2006-03-01/SelectObjectContentRequest AWS API Documentation + # + class SelectObjectContentRequest < Struct.new( + :bucket, + :key, + :sse_customer_algorithm, + :sse_customer_key, + :sse_customer_key_md5, + :expression, + :expression_type, + :request_progress, + :input_serialization, + :output_serialization) + include Aws::Structure + end + # Describes the parameters for Select job types. # # @note When making an API call, you may pass SelectParameters @@ -8184,6 +8369,18 @@ class Stats < Struct.new( include Aws::Structure end + # @!attribute [rw] details + # The Stats event details. + # @return [Types::Stats] + # + # @see http://docs.aws.amazon.com/goto/WebAPI/s3-2006-03-01/StatsEvent AWS API Documentation + # + class StatsEvent < Struct.new( + :details, + :event_type) + include Aws::Structure + end + # @note When making an API call, you may pass StorageClassAnalysis # data as a hash: # @@ -8854,5 +9051,24 @@ class WebsiteConfiguration < Struct.new( include Aws::Structure end + # EventStream is an Enumerator of Events. + # #event_types #=> Array, returns all modeled event types in the stream + # + # @see http://docs.aws.amazon.com/goto/WebAPI/s3-2006-03-01/SelectObjectContentEventStream AWS API Documentation + # + class SelectObjectContentEventStream < Enumerator + + def event_types + [ + :records, + :stats, + :progress, + :cont, + :end + ] + end + + end + end end