Feat: ECS compatibility (RFC email fields)

lib/logstash/inputs/imap.rb

@@ -58,23 +58,35 @@ class LogStash::Inputs::IMAP < LogStash::Inputs::Base
   # Path to file with last run time metadata
   config :sincedb_path, :validate => :string, :required => false
 
+  # NOTE: when set an extra hash of email information is provided under the target field.
+  # The hash is based on ECS's email.* fields.
+  # Due compatibility these fields are only set when target is configured.
+  config :target, :validate => :field_reference # ECS default: [email], legacy default: nil
+
   def initialize(*params)
     super
 
     if original_params.include?('headers_target')
-      @headers_target = normalize_field_ref(@headers_target)
+      @headers_target = normalize_field_ref(headers_target)
     else
-      @headers_target = '[@metadata][input][imap][headers]' if ecs_compatibility != :disabled
+      @headers_target = ecs_compatibility != :disabled ? '[@metadata][input][imap][headers]' : ''
     end
 
     if original_params.include?('attachments_target')
-      @attachments_target = normalize_field_ref(@attachments_target)
+      @attachments_target = normalize_field_ref(attachments_target)
     else
       @attachments_target = ecs_compatibility != :disabled ? '[@metadata][input][imap][attachments]' : '[attachments]'
     end
+
+    if original_params.include?('target')
+      @target = normalize_field_ref(target)
+    else
+      @target = '[email]' if ecs_compatibility != :disabled
+    end
   end
 
   def normalize_field_ref(target)
+    return nil if target.nil? || target.empty?
     # so we can later event.set("#{target}[#{name}]", ...)
     target.match?(/\A[^\[\]]+\z/) ? "[#{target}]" : target
   end
@@ -190,7 +202,7 @@ def check_mail(queue)
     end
   end
 
-  def parse_attachments(mail)
+  def legacy_parse_attachments(mail)
     attachments = []
     mail.attachments.each do |attachment|
       if @save_attachments
@@ -215,27 +227,50 @@ def parse_mail(mail)
       # Multipart message; use the first text/plain part we find
       part = mail.parts.find { |p| p.content_type.match @content_type_re } || mail.parts.first
       message = part.decoded
-
-      # Parse attachments
-      attachments = parse_attachments(mail)
     end
 
     @codec.decode(message) do |event|
       # Use the 'Date' field as the timestamp
-      event.timestamp = LogStash::Timestamp.new(mail.date.to_time)
+      event.timestamp = LogStash::Timestamp.new(mail.date.to_time) if mail.date
 
-      process_headers(mail, event)
+      set_target_fields(event, mail) if @target
+
+      process_headers(mail, event) if @headers_target
 
       # Add attachments
-      if attachments && attachments.length > 0
-        event.set(@attachments_target, attachments)
+      if @attachments_target && mail.has_attachments?
+        event.set(@attachments_target, legacy_parse_attachments(mail))
       end
 
       decorate(event)
       event
     end
   end
 
+  def set_target_fields(event, mail)
+    event.set("#{@target}[direction]", 'inbound') # we're reading mails from IMAP
+    event.set("#{@target}[subject]", mail.subject)
+    event.set("#{@target}[from]", mail.from) # Array<String>
+    event.set("#{@target}[to]", mail.to) if mail.to
+    event.set("#{@target}[cc]", mail.cc) if mail.cc
+    event.set("#{@target}[bcc]", mail.bcc) if mail.bcc
+    event.set("#{@target}[content_type]", mail.mime_type) if mail.mime_type
+    event.set("#{@target}[message_id]", mail.message_id) if mail.has_message_id?
+    event.set("#{@target}[reply_to]", mail.reply_to) if mail.reply_to
+    if mail.has_attachments?
+      attachments = mail.attachments.map do |attachment|
+        {
+            "file" => {
+                'name' => attachment.filename,
+                'mime_type' => attachment.mime_type,
+                'size' => attachment.body.to_s.size
+            }
+        }
+      end
+      event.set("#{@target}[attachments]", attachments)
+    end
+  end
+
   def process_headers(mail, event)
     # Add fields: Add message.header_fields { |h| h.name=> h.value }
     mail.header_fields.each do |header|

spec/inputs/imap_spec.rb

@@ -85,6 +85,41 @@
 
     let (:config) { super().merge('ecs_compatibility' => ecs_select.active_mode) }
 
+    context "mail fields" do
+
+      before { @event = input.parse_mail(mail) }
+
+      it "sets email fields (in ECS mode)" do
+        expect( @event.get("[email][subject]") ).to eql 'logstash imap input test'
+        expect( @event.get("[email][from]") ).to eql ['me@example.com']
+        expect( @event.get("[email][to]") ).to eql ['you@example.com']
+        expect( @event.include?("[email][content_type]") ).to be true
+        expect( @event.include?("[email][cc]") ).to be false
+        expect( @event.include?("[email][bcc]") ).to be false
+        expect( @event.get("[email][message_id]") ).to eql '123@message.id'
+
+        attachments = @event.get("[email][attachments]")
+        expect( attachments ).to_not be_empty
+        expect( attachments ).to include "file" => { "name" => "image.png", "size" => msg_binary.size, "mime_type" => "image/png" }
+      end if ecs_select.active_mode != :disabled
+
+      it "does not set email field (in legacy mode)" do
+        expect( @event.include?("[email]") ).to be false
+      end if ecs_select.active_mode == :disabled
+
+      context 'with target' do
+
+        let (:config) { super().merge('target' => '[foo]') }
+
+        it "sets email fields" do
+          expect( @event.get("[foo][subject]") ).to eql 'logstash imap input test'
+          expect( @event.get("[foo][from]") ).to eql ['me@example.com']
+          expect( @event.get("[foo][message_id]") ).to eql '123@message.id'
+        end
+
+      end
+    end
+
     context "when no content-type selected" do
       it "should select text/plain part" do
         event = input.parse_mail(mail)