parse_fiscal_pipeline.json

{
  "description": "Parsing the state of california all fiscal years file",
  "processors": [
    {
      "grok": {
        "field": "budget",
        "patterns": [
          "(\"%{DATA:business_unit}\"|%{DATA:business_unit})?,(\"%{DATA:agency_name}\"|%{DATA:agency_name})?,(\"%{DATA:department_name}\"|%{DATA:department_name})?,(\"%{DATA:document_id}\"|%{DATA:document_id})?,(\"%{DATA:related_document}\"|%{DATA:related_document})?,(\"%{DATA:accounting_date}\"|%{DATA:accounting_date})?,(\"%{DATA:fiscal_year_begin}\"|%{DATA:fiscal_year_begin})?,(\"%{DATA:accounting_period}\"|%{DATA:accounting_period})?,(\"%{DATA:account}\"|%{DATA:account})?,(\"%{DATA:account_type}\"|%{DATA:account_type})?,(\"%{DATA:account_category}\"|%{DATA:account_category})?,(\"%{DATA:account_sub_category}\"|%{DATA:account_sub_category})?,(\"%{DATA:account_description}\"|%{DATA:account_description})?,(\"%{DATA:fund_code}\"|%{DATA:fund_code})?,(\"%{DATA:fund_group}\"|%{DATA:fund_group})?,(\"%{DATA:fund_description}\"|%{DATA:fund_description})?,(\"%{DATA:program_code}\"|%{DATA:program_code})?,(\"%{DATA:program_description}\"|%{DATA:program_description})?,(\"%{DATA:sub_program_description}\"|%{DATA:sub_program_description})?,(\"%{DATA:budget_reference}\"|%{DATA:budget_reference})?,(\"%{DATA:budget_reference_category}\"|%{DATA:budget_reference_category})?,(\"%{DATA:budget_reference_sub_category}\"|%{DATA:budget_reference_sub_category})?,(\"%{DATA:budget_reference_description}\"|%{DATA:budget_reference_description})?,(\"%{DATA:year_of_enactment}\"|%{DATA:year_of_enactment})?,(\"%{NUMBER:monetary_amount}\"|%{NUMBER:monetary_amount})"
        ]
      }
    },
    {
      "remove": {
        "field": "budget"
      }
    }
  ]
}