filter_logs.conf

#Tail and parse arbitrary text/log file

<source>
  @type tail
  <parse> #Parse timestamp, everything else to be stored in message field
    @type regexp
    expression /^\[(?<logtime>[^\]]*)\] (?<message>.*)$/
    time_key logtime
    time_format %Y-%m-%d %H:%M:%S %z
  </parse>
  path /var/log/backend-app*.log
  pos_file /var/log/backend.application.pos
  path_key filename # Add watched file path to path_key field for every event/record.
  tag backend.application
</source>

#Add hostname and service_name fields to all events ("records") with a Fluentd tag of backend.application
 
<filter backend.application>
  @type record_transformer
  <record>
    hostname "#{Socket.gethostname}"
    service_name ${tag}
  </record>
</filter>

# For all events with a tag of backend.application:
# Keep ONLY events where service_name field contains a value matching /backend.application/ AND where message field contains a value matching /Cannot connect to/
# Discard any events where value of hostname field matches /staging/

<filter backend.application>
  @type grep
  <regexp>
    key service_name
    pattern /backend.application/
  </regexp>
   <regexp>
    key message
    pattern /Cannot connect to/
  </regexp>
  <exclude>
    key hostname
    pattern /staging/
  </exclude>
</filter>

#Write events with backend.application tag to New Relic

<match backend.application>
  @type newrelic
  api_key <YOUR INSERT KEY>
</match>