Skip to content

Commit

Permalink
Adding support for S3 Select [eventstream] (#1753)
Browse files Browse the repository at this point in the history
* Adding support for S3 Select
  • Loading branch information
cjyclaire committed May 17, 2018
1 parent 35a6a52 commit 571c2d0
Show file tree
Hide file tree
Showing 54 changed files with 2,375 additions and 83 deletions.
1 change: 1 addition & 0 deletions Rakefile
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ $:.unshift("#{$REPO_ROOT}/build_tools")
$:.unshift("#{$REPO_ROOT}/build_tools/aws-sdk-code-generator/lib")
$:.unshift("#{$GEMS_DIR}/aws-sdk-core/lib")
$:.unshift("#{$GEMS_DIR}/aws-partitions/lib")
$:.unshift("#{$GEMS_DIR}/aws-eventstream/lib")
$:.unshift("#{$GEMS_DIR}/aws-sigv4/lib")

require 'build_tools'
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,7 @@
require_relative 'aws-sdk-code-generator/resource_waiter'
require_relative 'aws-sdk-code-generator/service'
require_relative 'aws-sdk-code-generator/shared_example'
require_relative 'aws-sdk-code-generator/eventstream_example'
require_relative 'aws-sdk-code-generator/syntax_example'
require_relative 'aws-sdk-code-generator/syntax_example_hash'
require_relative 'aws-sdk-code-generator/underscore'
Expand All @@ -50,6 +51,7 @@
require_relative 'aws-sdk-code-generator/views/service_module'
require_relative 'aws-sdk-code-generator/views/spec/spec_helper'
require_relative 'aws-sdk-code-generator/views/types_module'
require_relative 'aws-sdk-code-generator/views/event_streams_module'
require_relative 'aws-sdk-code-generator/views/authorizer_class'
require_relative 'aws-sdk-code-generator/views/apig_endpoint_class'
require_relative 'aws-sdk-code-generator/views/apig_readme'
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -101,7 +101,14 @@ def ruby_type(shape_ref, api)
# @return [Boolean]
def streaming?(shape_or_shape_ref, api)
ref, shape = resolve(shape_or_shape_ref, api)
ref['streaming'] || shape['streaming']
ref['streaming'] || shape['streaming'] ||
ref['eventstream'] || shape['eventstream']
end

# @return [Boolean]
def eventstream?(shape_or_shape_ref, api)
ref, shape = resolve(shape_or_shape_ref, api)
ref['eventstream'] || shape['eventstream']
end

def plural?(resource)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -12,11 +12,15 @@ def initialize(options)
@api = options.fetch(:api)
@client_examples = options.fetch(:client_examples, [])
@examples = options.fetch(:examples)
@module_name = options.fetch(:module_name)
end

# @return [String]
attr_reader :method_name

# @return [String]
attr_reader :module_name

# @return [Hash]
attr_reader :operation

Expand All @@ -37,6 +41,7 @@ def to_str
option_tags(operation, api),
return_tag(operation, api),
generated_examples(operation, api),
eventstream_examples(module_name, method_name, operation, api),
shared_examples(examples, operation, api),
given_examples(client_examples),
request_syntax_example(method_name, operation, api),
Expand Down Expand Up @@ -161,6 +166,18 @@ def generated_examples(operation, api)
nil
end

def eventstream_examples(module_name, method_name, operation, api)
return unless !!Helper.eventstream_output?(operation, api)
EventStreamExample.new(
api: api,
operation: operation,
method_name: method_name,
module_name: module_name,
receiver: 'client',
resp_var: 'resp'
).format
end

def given_examples(client_examples)
client_examples.map do |example|
name = example[:name]
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -6,20 +6,26 @@ class ClientOperationList
def initialize(options)
api = options.fetch(:api)
examples = options.fetch(:examples, {})
module_name = options.fetch(:module_name)
client_examples = options.fetch(:client_examples, {})
@operations = api['operations'].map do |name, operation|
if AwsSdkCodeGenerator::Helper.eventstream_input?(operation, api)
raise 'eventstream at operation input is not supported'
end
method_name = Underscore.underscore(name)
Operation.new(
name: method_name,
documentation: ClientOperationDocumentation.new(
name: name,
module_name: module_name,
method_name: method_name,
operation: operation,
api: api,
examples: examples,
client_examples: client_examples[method_name] || []
).to_s,
streaming: AwsSdkCodeGenerator::Helper.operation_streaming?(operation, api)
streaming: AwsSdkCodeGenerator::Helper.operation_streaming?(operation, api),
eventstream_output: AwsSdkCodeGenerator::Helper.eventstream_output?(operation, api)
)
end
end
Expand All @@ -35,6 +41,9 @@ def initialize(options)
@name = options.fetch(:name)
@documentation = options.fetch(:documentation)
@streaming = options.fetch(:streaming)
@eventstream_output = !!options.fetch(:eventstream_output)
@eventstream_member = @eventstream_output ?
options.fetch(:eventstream_output) : nil
end

# @return [String]
Expand All @@ -43,11 +52,18 @@ def initialize(options)
# @return [String, nil]
attr_reader :documentation

# @return [Boolean]
attr_reader :eventstream_output

# @return [String]
attr_reader :eventstream_member

def block_option
if @streaming
", &block"
end
end

end
end
end
Original file line number Diff line number Diff line change
Expand Up @@ -19,8 +19,27 @@ def to_str

def structure(ref, context, visited)
lines = []
shape(ref)['members'].each_pair do |member_name, member_ref|
lines += entry(member_ref, "#{context}.#{underscore(member_name)}", visited)
shape = shape(ref)
if shape['eventstream']
event_types = []
# Add event entry
event_ctx = shape['members'].each.inject([]) do |ctx, (member_name, member_ref)|
event_type = Underscore.underscore(member_name).to_sym
event_types << event_type
ctx << "For #{event_type.inspect} event available at #on_#{event_type}_event callback"\
" and response eventstream enumerator:"
event_entry = entry(member_ref, "event", Set.new).join("\n ")
ctx << (event_entry.empty? ? " #=> EmptyStruct" : event_entry + "\n")
end
# Add eventstream entry
event_ctx.unshift("#{context}.event_types #=> #{event_types.inspect}\n")
event_ctx.unshift("#{context} #=> Enumerator")
event_ctx.unshift("All events are available at #{context}:")
return event_ctx
else
shape['members'].each_pair do |member_name, member_ref|
lines += entry(member_ref, "#{context}.#{underscore(member_name)}", visited)
end
end
lines
end
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -59,6 +59,7 @@ def source_files(options = {})
y.yield("#{prefix}.rb", service_module(prefix))
y.yield("#{prefix}/customizations.rb", '')
y.yield("#{prefix}/types.rb", types_module)
y.yield("#{prefix}/event_streams.rb", event_streams_module) if has_eventstream
y.yield("#{prefix}/client_api.rb", client_api_module)
y.yield("#{prefix}/client.rb", client_class)
y.yield("#{prefix}/errors.rb", errors_module)
Expand All @@ -84,6 +85,10 @@ def types_module
Views::TypesModule.new(service: @service).render
end

def event_streams_module
Views::EventStreamsModule.new(service: @service).render
end

def client_api_module
Views::ClientApiModule.new(service: @service).render
end
Expand Down Expand Up @@ -163,6 +168,16 @@ def apig_readme
module_name: @service.module_name
).render
end

private

def has_eventstream
@service.api['shapes'].each do |_, ref|
return true if ref['eventstream']
end
false
end

end

end
Original file line number Diff line number Diff line change
@@ -0,0 +1,117 @@
module AwsSdkCodeGenerator
class EventStreamExample

def initialize(options = {})
@api = options.fetch(:api)
@method_name = options.fetch(:method_name)
@module_name = options.fetch(:module_name)
@receiver = options.fetch(:receiver)
@resp_var = options.fetch(:resp_var)
@operation = options.fetch(:operation)
@eventstream_member, @eventstream_shape = eventstream_shape
end

def format
<<-EXAMPLE.strip
# @example EventStream Operation Example
#
# You can process event once it arrives immediately, or wait until
# full response complete and iterate through eventstream enumerator.
#
# To interact with event immediately, you need to register ##{@method_name}
# with callbacks, callbacks can be register for specifc events or for all events,
# callback for errors in the event stream is also available for register.
#
# Callbacks can be passed in by `:event_stream_handler` option or within block
# statement attached to ##{@method_name} call directly. Hybrid pattern of both
# is also supported.
#
# `:event_stream_handler` option takes in either Proc object or
# EventStreams::#{@eventstream_shape} object.
#
# Usage pattern a): callbacks with a block attached to ##{@method_name}
# Example for registering callbacks for all event types and error event
#
# #{@receiver}.#{@method_name}( # params input# ) do |stream|
#
# stream.on_error_event do |event|
# # catch unmodeled error event in the stream
# raise event
# # => Aws::Errors::EventError
# # event.event_type => :error
# # event.error_code => String
# # event.error_message => String
# end
#
# stream.on_event do |event|
# # process all events arrive
# puts event.event_type
# ...
# end
#
# end
#
# Usage pattern b): pass in `:event_stream_handler` for ##{@method_name}
#
# 1) create a EventStreams::#{@eventstream_shape} object
# Example for registering callbacks with specific events
#
# handler = #{@module_name}::EventStreams::#{@eventstream_shape}.new
#{event_entry('handler')}
#
# #{@receiver}.#{@method_name}( # params input #, event_stream_handler: handler)
#
# 2) use a Ruby Proc object
# Example for registering callbacks with specific events
#
# handler = Proc.new do |stream|
#{event_entry('stream')}
# end
#
# #{@receiver}.#{@method_name}( # params input #, event_stream_handler: handler)
#
# Usage pattern c): hybird pattern of a) and b)
#
# handler = #{@module_name}::EventStreams::#{@eventstream_shape}.new
#{event_entry('handler')}
#
# #{@receiver}.#{@method_name}( # params input #, event_stream_handler: handler) do |stream|
# stream.on_error_event do |event|
# # catch unmodeled error event in the stream
# raise event
# # => Aws::Errors::EventError
# # event.event_type => :error
# # event.error_code => String
# # event.error_message => String
# end
# end
#
# Besides above usage patterns for process events when they arrive immediately, you can also
# iterate through events after response complete.
#
# Events are available at #{@resp_var}.#{@eventstream_member} # => Enumerator
# For parameter input example, please refer to following request syntax
EXAMPLE
end

private

def event_entry(ctx)
@api['shapes'][@eventstream_shape]['members'].keys.each.inject([]) do |entry, name|
event_type = Underscore.underscore(name)
entry << "# #{ctx}.on_#{event_type}_event do |event|"
entry << "# event # => #{@module_name}::Types::#{name}"
entry << "# end"
end.join("\n")
end

def eventstream_shape
output_shape = @api['shapes'][@operation['output']['shape']]
output_shape['members'].each do |name, ref|
return [Underscore.underscore(name), ref['shape']] if Api.eventstream?(ref, @api)
end
raise 'Cannot find eventstream member at eventstream operation'
end

end
end
Original file line number Diff line number Diff line change
Expand Up @@ -175,6 +175,27 @@ def operation_streaming?(operation, api)
end
end

def eventstream_output?(operation, api)
return false unless operation.key? 'output'
output_shape = api['shapes'][operation['output']['shape']]
return false unless output_shape.key? 'members'
output_shape['members'].each do |name, ref|
return ref['shape'] if Api.eventstream?(ref, api)
end
return false
end

# currently not support eventstream input
def eventstream_input?(operation, api)
return false unless operation.key? 'input'
input_shape = api['shapes'][operation['input']['shape']]
return false unless input_shape.key? 'members'
input_shape['members'].each do |name, ref|
return true if Api.eventstream?(ref, api)
end
return false
end

def deep_copy(obj)
case obj
when nil then nil
Expand All @@ -197,7 +218,8 @@ def wrap_string(str, width, indent = '')
str.gsub(/(.{1,#{width}})(\s+|\Z)/, "#{indent}\\1\n").chomp
end

module_function :deep_copy, :operation_streaming?, :downcase_first, :wrap_string, :apig_prefix
module_function :deep_copy, :operation_streaming?, :downcase_first, :wrap_string, :apig_prefix,
:eventstream_output?, :eventstream_input?

end
end
Loading

0 comments on commit 571c2d0

Please sign in to comment.