diff --git a/CHANGELOG.md b/CHANGELOG.md index 7b45d96..52679fc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,10 @@ +## 3.2.0 + - Feat: ECS compatibility [#55](https://github.com/logstash-plugins/logstash-input-imap/pull/55) + * added (optional) `headers_target` configuration option + * added (optional) `attachments_target` configuration option + - Fix: plugin should not close `$stdin`, while stoping + - Fix: make sure the 'Date' header is skipped regardless of the `lowercase_headers` setting + ## 3.1.0 - Adds an option to recursively search the message parts for attachment and inline attachment filenames. If the save_attachments option is set to true, the content of attachments is included in the `attachments.data` field. The attachment data can then be used by the Elasticsearch Ingest Attachment Processor Plugin. [#48](https://github.com/logstash-plugins/logstash-input-imap/pull/48) diff --git a/docs/index.asciidoc b/docs/index.asciidoc index 084381e..3916c47 100644 --- a/docs/index.asciidoc +++ b/docs/index.asciidoc @@ -26,6 +26,15 @@ Read mails from IMAP server Periodically scan an IMAP folder (`INBOX` by default) and move any read messages to the trash. +[id="plugins-{type}s-{plugin}-ecs"] +==== Compatibility with the Elastic Common Schema (ECS) + +The plugin includes sensible defaults that change based on <>. +When ECS compatibility is disabled, mail headers and attachments are targeted at the root level. +When targeting an ECS version, headers and attachments target `@metadata` sub-fields unless configured otherwise in order +to avoid conflict with ECS fields. +See <>, and <>. + [id="plugins-{type}s-{plugin}-options"] ==== Imap Input Configuration Options @@ -34,12 +43,15 @@ This plugin supports the following configuration options plus the <> |<>|No | <> |<>|No | <> |<>|No | <> |<>|No +| <> |<>|No | <> |<>|No | <> |<>|No | <> |<>|No +| <> |<>|No | <> |<>|Yes | <> |<>|No | <> |<>|Yes @@ -58,6 +70,16 @@ input plugins.   +[id="plugins-{type}s-{plugin}-attachments_target"] +===== `attachments_target` + + * Value type is <> + * Default value depends on whether <> is enabled: + ** ECS Compatibility disabled: no default value for this setting + ** ECS Compatibility enabled: `"[@metadata][input][imap][attachments]" + +The name of the field under which mail attachments information will be added, if <> is set. + [id="plugins-{type}s-{plugin}-check_interval"] ===== `check_interval` @@ -72,8 +94,7 @@ input plugins. * Value type is <> * Default value is `"text/plain"` -For multipart messages, use the first part that has this -content-type as the event message. +For multipart messages, use the first part that has this content-type as the event message. [id="plugins-{type}s-{plugin}-delete"] ===== `delete` @@ -83,6 +104,21 @@ content-type as the event message. +[id="plugins-{type}s-{plugin}-ecs_compatibility"] +===== `ecs_compatibility` + + * Value type is <> + * Supported values are: + ** `disabled`: does not use ECS-compatible field names (for example, `From` header field is added to the event) + ** `v1`, `v8`: avoids field names that might conflict with Elastic Common Schema (for example, the `From` header is added as metadata) + * Default value depends on which version of Logstash is running: + ** When Logstash provides a `pipeline.ecs_compatibility` setting, its value is used as the default + ** Otherwise, the default value is `disabled`. + +Controls this plugin's compatibility with the {ecs-ref}[Elastic Common Schema (ECS)]. +The value of this setting affects the _default_ value of <> and +<>. + [id="plugins-{type}s-{plugin}-expunge"] ===== `expunge` @@ -107,6 +143,16 @@ content-type as the event message. +[id="plugins-{type}s-{plugin}-headers_target"] +===== `headers_target` + + * Value type is <> + * Default value depends on whether <> is enabled: + ** ECS Compatibility disabled: no default value for this setting + ** ECS Compatibility enabled: `"[@metadata][input][imap][headers]" + +The name of the field under which mail headers will be added. + [id="plugins-{type}s-{plugin}-host"] ===== `host` diff --git a/lib/logstash/inputs/imap.rb b/lib/logstash/inputs/imap.rb index ea6869c..a2ab806 100644 --- a/lib/logstash/inputs/imap.rb +++ b/lib/logstash/inputs/imap.rb @@ -3,13 +3,22 @@ require "logstash/namespace" require "logstash/timestamp" require "stud/interval" -require "socket" # for Socket.gethostname +require 'fileutils' + +require 'logstash/plugin_mixins/ecs_compatibility_support' +require 'logstash/plugin_mixins/ecs_compatibility_support/target_check' +require 'logstash/plugin_mixins/validator_support/field_reference_validation_adapter' # Read mails from IMAP server # # Periodically scan an IMAP folder (`INBOX` by default) and move any read messages # to the trash. class LogStash::Inputs::IMAP < LogStash::Inputs::Base + + include LogStash::PluginMixins::ECSCompatibilitySupport(:disabled, :v1, :v8 => :v1) + + extend LogStash::PluginMixins::ValidatorSupport::FieldReferenceValidationAdapter + config_name "imap" default :codec, "plain" @@ -24,15 +33,23 @@ class LogStash::Inputs::IMAP < LogStash::Inputs::Base config :folder, :validate => :string, :default => 'INBOX' config :fetch_count, :validate => :number, :default => 50 - config :lowercase_headers, :validate => :boolean, :default => true config :check_interval, :validate => :number, :default => 300 + + config :lowercase_headers, :validate => :boolean, :default => true + + config :headers_target, :validate => :field_reference # ECS default: [@metadata][input][imap][headers] + config :delete, :validate => :boolean, :default => false config :expunge, :validate => :boolean, :default => false + config :strip_attachments, :validate => :boolean, :default => false config :save_attachments, :validate => :boolean, :default => false - # For multipart messages, use the first part that has this - # content-type as the event message. + # Legacy default: [attachments] + # ECS default: [@metadata][input][imap][attachments] + config :attachments_target, :validate => :field_reference + + # For multipart messages, use the first part that has this content-type as the event message. config :content_type, :validate => :string, :default => "text/plain" # Whether to use IMAP uid to track last processed message @@ -41,6 +58,40 @@ class LogStash::Inputs::IMAP < LogStash::Inputs::Base # Path to file with last run time metadata config :sincedb_path, :validate => :string, :required => false + # NOTE: when set an extra hash of email information is provided under the target field. + # The hash is based on ECS's email.* fields. + # Due compatibility these fields are only set when target is configured. + config :target, :validate => :field_reference # ECS default: [email], legacy default: nil + + def initialize(*params) + super + + if original_params.include?('headers_target') + @headers_target = normalize_field_ref(headers_target) + else + @headers_target = ecs_compatibility != :disabled ? '[@metadata][input][imap][headers]' : '' + end + + if original_params.include?('attachments_target') + @attachments_target = normalize_field_ref(attachments_target) + else + @attachments_target = ecs_compatibility != :disabled ? '[@metadata][input][imap][attachments]' : '[attachments]' + end + + if original_params.include?('target') + @target = normalize_field_ref(target) + else + @target = '[email]' if ecs_compatibility != :disabled + end + end + + def normalize_field_ref(target) + return nil if target.nil? || target.empty? + # so we can later event.set("#{target}[#{name}]", ...) + target.match?(/\A[^\[\]]+\z/) ? "[#{target}]" : target + end + private :normalize_field_ref + def register require "net/imap" # in stdlib require "mail" # gem 'mail' @@ -63,14 +114,16 @@ def register # Ensure that the filepath exists before writing, since it's deeply nested. FileUtils::mkdir_p datapath @sincedb_path = File.join(datapath, ".sincedb_" + Digest::MD5.hexdigest("#{@user}_#{@host}_#{@port}_#{@folder}")) + @logger.debug? && @logger.debug("Generated sincedb path", sincedb_path: @sincedb_path) end - if File.directory?(@sincedb_path) - raise ArgumentError.new("The \"sincedb_path\" argument must point to a file, received a directory: \"#{@sincedb_path}\"") - end - @logger.info("Using \"sincedb_path\": \"#{@sincedb_path}\"") + if File.exist?(@sincedb_path) + if File.directory?(@sincedb_path) + raise ArgumentError.new("The \"sincedb_path\" argument must point to a file, received a directory: \"#{@sincedb_path}\"") + end + @logger.debug? && @logger.debug("Found existing sincedb path", sincedb_path: @sincedb_path) @uid_last_value = File.read(@sincedb_path).to_i - @logger.info("Loading \"uid_last_value\": \"#{@uid_last_value}\"") + @logger.debug? && @logger.debug("Loaded from sincedb", uid_last_value: @uid_last_value) end @content_type_re = Regexp.new("^" + @content_type) @@ -136,7 +189,6 @@ def check_mail(queue) rescue => e @logger.error("Encountered error #{e.class}", :message => e.message, :backtrace => e.backtrace) # Do not raise error, check_mail will be invoked in the next run time - ensure # Close the connection (and ignore errors) imap.close rescue nil @@ -145,12 +197,12 @@ def check_mail(queue) # Always save @uid_last_value so when tracking is switched from # "NOT SEEN" to "UID" we will continue from first unprocessed message if @uid_last_value - @logger.info("Saving \"uid_last_value\": \"#{@uid_last_value}\"") + @logger.debug? && @logger.debug("Saving to sincedb", uid_last_value: @uid_last_value) File.write(@sincedb_path, @uid_last_value) end end - def parse_attachments(mail) + def legacy_parse_attachments(mail) attachments = [] mail.attachments.each do |attachment| if @save_attachments @@ -164,7 +216,8 @@ def parse_attachments(mail) def parse_mail(mail) # Add a debug message so we can track what message might cause an error later - @logger.debug? && @logger.debug("Working with message_id", :message_id => mail.message_id) + @logger.debug? && @logger.debug("Processing mail", message_id: mail.message_id) + # TODO(sissel): What should a multipart message look like as an event? # For now, just take the plain-text part and set it as the message. if mail.parts.count == 0 @@ -174,44 +227,19 @@ def parse_mail(mail) # Multipart message; use the first text/plain part we find part = mail.parts.find { |p| p.content_type.match @content_type_re } || mail.parts.first message = part.decoded - - # Parse attachments - attachments = parse_attachments(mail) end @codec.decode(message) do |event| # Use the 'Date' field as the timestamp - event.timestamp = LogStash::Timestamp.new(mail.date.to_time) - - # Add fields: Add message.header_fields { |h| h.name=> h.value } - mail.header_fields.each do |header| - # 'header.name' can sometimes be a Mail::Multibyte::Chars, get it in String form - name = @lowercase_headers ? header.name.to_s.downcase : header.name.to_s - # Call .decoded on the header in case it's in encoded-word form. - # Details at: - # https://github.com/mikel/mail/blob/master/README.md#encodings - # http://tools.ietf.org/html/rfc2047#section-2 - value = transcode_to_utf8(header.decoded.to_s) - - # Assume we already processed the 'date' above. - next if name == "Date" - - case (field = event.get(name)) - when String - # promote string to array if a header appears multiple times - # (like 'received') - event.set(name, [field, value]) - when Array - field << value - event.set(name, field) - when nil - event.set(name, value) - end - end + event.timestamp = LogStash::Timestamp.new(mail.date.to_time) if mail.date + + set_target_fields(event, mail) if @target + + process_headers(mail, event) if @headers_target # Add attachments - if attachments && attachments.length > 0 - event.set('attachments', attachments) + if @attachments_target && mail.has_attachments? + event.set(@attachments_target, legacy_parse_attachments(mail)) end decorate(event) @@ -219,9 +247,63 @@ def parse_mail(mail) end end + def set_target_fields(event, mail) + event.set("#{@target}[direction]", 'inbound') # we're reading mails from IMAP + event.set("#{@target}[subject]", mail.subject) + event.set("#{@target}[from]", mail.from) # Array + event.set("#{@target}[to]", mail.to) if mail.to + event.set("#{@target}[cc]", mail.cc) if mail.cc + event.set("#{@target}[bcc]", mail.bcc) if mail.bcc + event.set("#{@target}[content_type]", mail.mime_type) if mail.mime_type + event.set("#{@target}[message_id]", mail.message_id) if mail.has_message_id? + event.set("#{@target}[reply_to]", mail.reply_to) if mail.reply_to + if mail.has_attachments? + attachments = mail.attachments.map do |attachment| + { + "file" => { + 'name' => attachment.filename, + 'mime_type' => attachment.mime_type, + 'size' => attachment.body.to_s.size + } + } + end + event.set("#{@target}[attachments]", attachments) + end + end + + def process_headers(mail, event) + # Add fields: Add message.header_fields { |h| h.name=> h.value } + mail.header_fields.each do |header| + # 'header.name' can sometimes be a Mail::Multibyte::Chars, get it in String form + name = header.name.to_s + + # assume we already processed the 'date' into event.timestamp + next if name == "Date" + + name = name.downcase if @lowercase_headers + + # Call .decoded on the header in case it's in encoded-word form. + # Details at: + # https://github.com/mikel/mail/blob/master/README.md#encodings + # http://tools.ietf.org/html/rfc2047#section-2 + value = transcode_to_utf8(header.decoded) + + targeted_name = "#{@headers_target}[#{name}]" + case (field = event.get(targeted_name)) + when String + # promote string to array if a header appears multiple times (like 'received') + event.set(targeted_name, [field, value]) + when Array + field << value + event.set(targeted_name, field) + when nil + event.set(targeted_name, value) + end + end + end + def stop Stud.stop!(@run_thread) - $stdin.close end private @@ -230,8 +312,7 @@ def stop # the mail gem will set the correct encoding on header strings decoding # and we want to transcode it to utf8 def transcode_to_utf8(s) - unless s.nil? - s.encode(Encoding::UTF_8, :invalid => :replace, :undef => :replace) - end + return nil if s.nil? + s.encode(Encoding::UTF_8, :invalid => :replace, :undef => :replace) end end diff --git a/logstash-input-imap.gemspec b/logstash-input-imap.gemspec index 8234078..5faef12 100644 --- a/logstash-input-imap.gemspec +++ b/logstash-input-imap.gemspec @@ -1,7 +1,7 @@ Gem::Specification.new do |s| s.name = 'logstash-input-imap' - s.version = '3.1.0' + s.version = '3.2.0' s.licenses = ['Apache License (2.0)'] s.summary = "Reads mail from an IMAP server" s.description = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program" @@ -21,11 +21,12 @@ Gem::Specification.new do |s| # Gem dependencies s.add_runtime_dependency "logstash-core-plugin-api", ">= 1.60", "<= 2.99" + s.add_runtime_dependency 'logstash-mixin-ecs_compatibility_support', '~> 1.3' + s.add_runtime_dependency 'logstash-mixin-validator_support', '~> 1.0' s.add_runtime_dependency 'logstash-codec-plain' s.add_runtime_dependency 'mail', '~> 2.6.3' s.add_runtime_dependency 'mime-types', '2.6.2' s.add_runtime_dependency 'stud', '~> 0.0.22' s.add_development_dependency 'logstash-devutils' - s.add_development_dependency 'insist' end diff --git a/spec/inputs/imap_spec.rb b/spec/inputs/imap_spec.rb index 2108df3..a9c38ee 100644 --- a/spec/inputs/imap_spec.rb +++ b/spec/inputs/imap_spec.rb @@ -1,13 +1,12 @@ # encoding: utf-8 require "logstash/devutils/rspec/spec_helper" -require "insist" require "logstash/devutils/rspec/shared_examples" +require 'logstash/plugin_mixins/ecs_compatibility_support/spec_helper' require "logstash/inputs/imap" require "mail" require "net/imap" require "base64" - describe LogStash::Inputs::IMAP do context "when interrupting the plugin" do @@ -36,7 +35,7 @@ end -describe LogStash::Inputs::IMAP do +describe LogStash::Inputs::IMAP, :ecs_compatibility_support do user = "logstash" password = "secret" msg_time = Time.new @@ -45,129 +44,246 @@ msg_binary = "\x42\x43\x44" msg_unencoded = "raw text 🐐" - subject do + let(:config) do + { "host" => "localhost", "user" => "#{user}", "password" => "#{password}" } + end + + subject(:input) do + LogStash::Inputs::IMAP.new config + end + + let(:mail) do Mail.new do from "me@example.com" to "you@example.com" subject "logstash imap input test" date msg_time body msg_text + message_id '<123@message.id>' # 'Message-ID' header + # let's have some headers: + header['X-Priority'] = '3' + header['X-Bot-ID'] = '111' + header['X-AES-Category'] = 'LEGIT' + header['X-Spam-Category'] = 'LEGIT' + header['Spam-Stopper-Id'] = '464bbb1a-1b86-4006-8a09-ce797fb56346' + header['Spam-Stopper-v2'] = 'Yes' + header['X-Mailer'] = 'Microsoft Outlook Express 6.00.2800.1106' + header['X-MimeOLE'] = 'Produced By Microsoft MimeOLE V6.00.2800.1106' add_file :filename => "some.html", :content => msg_html add_file :filename => "image.png", :content => msg_binary add_file :filename => "unencoded.data", :content => msg_unencoded, :content_transfer_encoding => "7bit" end end - context "with both text and html parts" do + before do + input.register + end + + ecs_compatibility_matrix(:disabled, :v1, :v8) do |ecs_select| + + let(:ecs_compatibility?) { ecs_select.active_mode != :disabled } + + let (:config) { super().merge('ecs_compatibility' => ecs_select.active_mode) } + + context "mail fields" do + + before { @event = input.parse_mail(mail) } + + it "sets email fields (in ECS mode)" do + expect( @event.get("[email][subject]") ).to eql 'logstash imap input test' + expect( @event.get("[email][from]") ).to eql ['me@example.com'] + expect( @event.get("[email][to]") ).to eql ['you@example.com'] + expect( @event.include?("[email][content_type]") ).to be true + expect( @event.include?("[email][cc]") ).to be false + expect( @event.include?("[email][bcc]") ).to be false + expect( @event.get("[email][message_id]") ).to eql '123@message.id' + + attachments = @event.get("[email][attachments]") + expect( attachments ).to_not be_empty + expect( attachments ).to include "file" => { "name" => "image.png", "size" => msg_binary.size, "mime_type" => "image/png" } + end if ecs_select.active_mode != :disabled + + it "does not set email field (in legacy mode)" do + expect( @event.include?("[email]") ).to be false + end if ecs_select.active_mode == :disabled + + context 'with target' do + + let (:config) { super().merge('target' => '[foo]') } + + it "sets email fields" do + expect( @event.get("[foo][subject]") ).to eql 'logstash imap input test' + expect( @event.get("[foo][from]") ).to eql ['me@example.com'] + expect( @event.get("[foo][message_id]") ).to eql '123@message.id' + end + + end + end + context "when no content-type selected" do it "should select text/plain part" do - config = {"type" => "imap", "host" => "localhost", - "user" => "#{user}", "password" => "#{password}"} - - input = LogStash::Inputs::IMAP.new config - input.register - event = input.parse_mail(subject) - insist { event.get("message") } == msg_text + event = input.parse_mail(mail) + expect( event.get("message") ).to eql msg_text end end context "when text/html content-type selected" do - it "should select text/html part" do - config = {"type" => "imap", "host" => "localhost", - "user" => "#{user}", "password" => "#{password}", - "content_type" => "text/html"} + let(:config) { super().merge("content_type" => "text/html") } - input = LogStash::Inputs::IMAP.new config - input.register - event = input.parse_mail(subject) - insist { event.get("message") } == msg_html + it "should select text/html part" do + event = input.parse_mail(mail) + expect( event.get("message") ).to eql msg_html end end - end - context "when subject is in RFC 2047 encoded-word format" do - it "should be decoded" do - subject.subject = "=?iso-8859-1?Q?foo_:_bar?=" - config = {"type" => "imap", "host" => "localhost", - "user" => "#{user}", "password" => "#{password}"} + context "mail headers" do + let(:config) { super().merge("lowercase_headers" => true) } # default + + before { @event = input.parse_mail(mail) } + + it "sets all header fields" do + if ecs_compatibility? + expect( @event.get("[@metadata][input][imap][headers][x-spam-category]") ).to eql 'LEGIT' + expect( @event.get("[@metadata][input][imap][headers][x-aes-category]") ).to eql 'LEGIT' + expect( @event.get("[@metadata][input][imap][headers][x-bot-id]") ).to eql '111' + ['spam-stopper-id', 'spam-stopper-v2', 'x-mimeole', 'message-id', 'x-priority'].each do |name| + expect( @event.include?("[@metadata][input][imap][headers][#{name}]") ).to be true + end + expect( @event.get("[@metadata][input][imap][headers][from]") ).to eql 'me@example.com' + expect( @event.get("[@metadata][input][imap][headers][to]") ).to eql 'you@example.com' + expect( @event.get("[@metadata][input][imap][headers][subject]") ).to eql 'logstash imap input test' + else + expect( @event.get("x-spam-category") ).to eql 'LEGIT' + expect( @event.get("x-aes-category") ).to eql 'LEGIT' + expect( @event.get("x-bot-id") ).to eql '111' + ['spam-stopper-id', 'spam-stopper-v2', 'x-mimeole', 'message-id', 'x-priority'].each do |name| + expect( @event.include?(name) ).to be true + end + expect( @event.get("from") ).to eql 'me@example.com' + expect( @event.get("to") ).to eql 'you@example.com' + expect( @event.get("subject") ).to eql 'logstash imap input test' + end + end - input = LogStash::Inputs::IMAP.new config - input.register - event = input.parse_mail(subject) - insist { event.get("subject") } == "foo : bar" + it 'does not set date header' do + expect( @event.include?('date') ).to be false + expect( @event.include?('Date') ).to be false + end end - end - context "with multiple values for same header" do - it "should add 2 values as array in event" do - subject.received = "test1" - subject.received = "test2" + context "mail headers (not lower-cased)" do + let(:config) { super().merge("lowercase_headers" => false) } + + before { @event = input.parse_mail(mail) } + + it "sets all header fields" do + if ecs_compatibility? + expect( @event.get("[@metadata][input][imap][headers][X-Spam-Category]") ).to eql 'LEGIT' + expect( @event.get("[@metadata][input][imap][headers][X-AES-Category]") ).to eql 'LEGIT' + expect( @event.get("[@metadata][input][imap][headers][X-Bot-ID]") ).to eql '111' + ['Spam-Stopper-Id', 'Spam-Stopper-v2', 'X-MimeOLE', 'Message-ID', 'X-Priority'].each do |name| + expect( @event.include?("[@metadata][input][imap][headers][#{name}]") ).to be true + end + expect( @event.get("[@metadata][input][imap][headers][From]") ).to eql 'me@example.com' + expect( @event.get("[@metadata][input][imap][headers][To]") ).to eql 'you@example.com' + expect( @event.get("[@metadata][input][imap][headers][Subject]") ).to eql 'logstash imap input test' + else + expect( @event.get("X-Spam-Category") ).to eql 'LEGIT' + expect( @event.get("X-AES-Category") ).to eql 'LEGIT' + expect( @event.get("X-Bot-ID") ).to eql '111' + ['Spam-Stopper-Id', 'Spam-Stopper-v2', 'X-MimeOLE', 'Message-ID', 'X-Priority'].each do |name| + expect( @event.include?(name) ).to be true + end + expect( @event.get("From") ).to eql 'me@example.com' + expect( @event.get("To") ).to eql 'you@example.com' + expect( @event.get("Subject") ).to eql 'logstash imap input test' + end + end + + it 'does not set date header' do + expect( @event.include?('Date') ).to be false + end + end - config = {"type" => "imap", "host" => "localhost", - "user" => "#{user}", "password" => "#{password}"} + context "when subject is in RFC 2047 encoded-word format" do + before do + mail.subject = "=?iso-8859-1?Q?foo_:_bar?=" + end - input = LogStash::Inputs::IMAP.new config - input.register - event = input.parse_mail(subject) - insist { event.get("received") } == ["test1", "test2"] + it "should be decoded" do + event = input.parse_mail(mail) + if ecs_compatibility? + expect( event.get("[@metadata][input][imap][headers][subject]") ).to eql "foo : bar" + else + expect( event.get("subject") ).to eql "foo : bar" + end + end end - it "should add more than 2 values as array in event" do - subject.received = "test1" - subject.received = "test2" - subject.received = "test3" + context "with multiple values for same header" do + it "should add 2 values as array in event" do + mail.received = "test1" + mail.received = "test2" + + event = input.parse_mail(mail) + expected_value = ["test1", "test2"] + if ecs_compatibility? + expect( event.get("[@metadata][input][imap][headers][received]") ).to eql expected_value + else + expect( event.get("received") ).to eql expected_value + end + end - config = {"type" => "imap", "host" => "localhost", - "user" => "#{user}", "password" => "#{password}"} + it "should add more than 2 values as array in event" do + mail.received = "test1" + mail.received = "test2" + mail.received = "test3" - input = LogStash::Inputs::IMAP.new config - input.register - event = input.parse_mail(subject) - insist { event.get("received") } == ["test1", "test2", "test3"] + event = input.parse_mail(mail) + expected_value = ["test1", "test2", "test3"] + if ecs_compatibility? + expect( event.get("[@metadata][input][imap][headers][received]") ).to eql expected_value + else + expect( event.get("received") ).to eql expected_value + end + end end - end - context "when a header field is nil" do - it "should parse mail" do - subject.header['X-Custom-Header'] = nil - config = {"type" => "imap", "host" => "localhost", - "user" => "#{user}", "password" => "#{password}"} + context "when a header field is nil" do + it "should parse mail" do + mail.header['X-Custom-Header'] = nil - input = LogStash::Inputs::IMAP.new config - input.register - event = input.parse_mail(subject) - insist { event.get("message") } == msg_text + event = input.parse_mail(mail) + expect( event.get("message") ).to eql msg_text + end end - end - context "with attachments" do - it "should extract filenames" do - config = {"type" => "imap", "host" => "localhost", - "user" => "#{user}", "password" => "#{password}"} - - input = LogStash::Inputs::IMAP.new config - input.register - event = input.parse_mail(subject) - insist { event.get("attachments") } == [ - {"filename"=>"some.html"}, - {"filename"=>"image.png"}, - {"filename"=>"unencoded.data"} - ] + context "attachments" do + it "should extract filenames" do + event = input.parse_mail(mail) + target = ecs_compatibility? ? '[@metadata][input][imap][attachments]' : 'attachments' + expect( event.get(target) ).to eql [ + {"filename"=>"some.html"}, + {"filename"=>"image.png"}, + {"filename"=>"unencoded.data"} + ] + end end - it "should extract the encoded content" do - config = {"type" => "imap", "host" => "localhost", - "user" => "#{user}", "password" => "#{password}", - "save_attachments" => true} + context "with attachments saving" do + let(:config) { super().merge("save_attachments" => true) } - input = LogStash::Inputs::IMAP.new config - input.register - event = input.parse_mail(subject) - insist { event.get("attachments") } == [ - {"data"=> Base64.encode64(msg_html).encode(crlf_newline: true), "filename"=>"some.html"}, - {"data"=> Base64.encode64(msg_binary).encode(crlf_newline: true), "filename"=>"image.png"}, - {"data"=> msg_unencoded, "filename"=>"unencoded.data"} - ] + it "should extract the encoded content" do + event = input.parse_mail(mail) + target = ecs_compatibility? ? '[@metadata][input][imap][attachments]' : 'attachments' + expect( event.get(target) ).to eql [ + {"data"=> Base64.encode64(msg_html).encode(crlf_newline: true), "filename"=>"some.html"}, + {"data"=> Base64.encode64(msg_binary).encode(crlf_newline: true), "filename"=>"image.png"}, + {"data"=> msg_unencoded, "filename"=>"unencoded.data"} + ] end + end + end + end