diff --git a/_meta/ecs-migration.yml b/_meta/ecs-migration.yml new file mode 100644 index 00000000000..2ba4eccf029 --- /dev/null +++ b/_meta/ecs-migration.yml @@ -0,0 +1,115 @@ +# The ECS migration file contains the information about all the fields which are migrated to ECS in 7.0. +# The goal of the file is to potentially have scripts on top of this information to convert visualisations and templates +# based on this information in an automated way and to keep track of all changes which were applied. +# +# The format of the file is as following: +# +# - from: source-field-in-6.x +# to: target-filed-in-ECS +# # Alias field is useful for fields where there is a 1-1 mapping from old to new +# alias: true-if-alias-is-required-in-6x (default is true) +# # Copy to is useful for fields where multiple fields map to the same ECS field +# copy_to: true-if-field-should-be-copied-to-target-in-6x (default is false) + +- from: context.service.agent.name + to: agent.name + +- from: context.service.agent.version + to: agent.version + +- from: context.system.architecture + to: host.architecture + +- from: context.system.ip + to: host.ip + +- from: context.system.hostname + to: host.name + +- from: context.system.platform + to: host.os.platform + +- from: context.request.method + to: http.method + +- from: context.request.http_version + to: http.version + +- from: context.tags + to: labels + alias: false + copy_to: true + +- from: context.process.pid + to: process.pid + +- from: context.process.ppid + to: process.ppid + +- from: context.process.title + to: process.title + + # not in ECS +- from: context.service.environment + to: service.environment + + # not in ECS +- from: context.service.framework.name + to: service.framework.name + + # not in ECS +- from: context.service.framework.version + to: service.framework.version + + # not in ECS +- from: context.service.language.name + to: service.language.name + + # not in ECS +- from: context.service.language.version + to: service.language.version + +- from: context.service.name + to: service.name + + # not in ECS +- from: context.service.runtime.name + to: service.runtime.name + + # not in ECS +- from: context.service.runtime.version + to: service.runtime.version + +- from: context.request.url.full + to: url.original + +- from: context.request.url.hash + to: url.fragment + +- from: context.request.url.hostname + to: url.domain + +- from: context.request.url.pathname + to: url.path + +- from: context.request.url.port + to: url.port + alias: false + copy_to: true + +- from: context.request.url.search + to: url.query + +- from: context.request.url.protocol + to: url.scheme + alias: false + copy_to: true + +- from: context.user.email + to: user.email + +- from: context.user.id + to: user.id + +- from: context.user.username + to: user.name diff --git a/_meta/fields.common.yml b/_meta/fields.common.yml index fbc624bf788..c0c9315e521 100644 --- a/_meta/fields.common.yml +++ b/_meta/fields.common.yml @@ -316,3 +316,181 @@ type: keyword description: > The ID of the parent event. + + # ECS + - name: agent + type: group + dynamic: false + fields: + - name: name + type: alias + path: context.service.agent.name + + - name: version + type: alias + path: context.service.agent.version + + - name: host + type: group + dynamic: false + fields: + - name: architecture + type: alias + path: context.system.architecture + + - name: ip + type: alias + path: context.system.ip + + - name: name + type: alias + path: context.system.hostname + + - name: os + type: group + fields: + - name: platform + type: alias + path: context.system.platform + + - name: http + type: group + dynamic: false + fields: + - name: method + type: alias + path: context.request.method + + - name: version + type: alias + path: context.request.http_version + + # context.tags copied here (can't alias objects) + - name: labels + type: object + object_type: keyword + dynamic: true + description: > + A flat mapping of user-defined labels with string values. + + - name: process + type: group + dynamic: false + fields: + - name: pid + type: alias + path: context.process.pid + + - name: ppid + type: alias + path: context.process.ppid + + - name: title + type: alias + path: context.process.title + + - name: service + type: group + dynamic: false + fields: + # not in ECS + - name: environment + type: alias + path: context.service.environment + + # not in ECS + - name: framework + type: group + fields: + - name: name + type: alias + path: context.service.framework.name + + - name: version + type: alias + path: context.service.framework.version + + # not in ECS + - name: language + type: group + fields: + - name: name + type: alias + path: context.service.language.name + + - name: version + type: alias + path: context.service.language.version + + - name: name + type: alias + path: context.service.name + + # not in ECS + - name: runtime + type: group + fields: + - name: name + type: alias + path: context.service.runtime.name + + - name: version + type: alias + path: context.service.runtime.version + + - name: version + type: alias + path: context.service.version + + - name: url + type: group + dynamic: false + fields: + - name: domain + type: alias + path: context.request.url.hostname + + - name: fragment + type: alias + path: context.request.url.hash + + - name: original + type: alias + path: context.request.url.full + + - name: path + type: alias + path: context.request.url.pathname + + # context.request.url.port keyword -> long + - name: port + type: long + description: > + The port of the request, e.g. 443. + + - name: query + type: alias + path: context.request.url.search + # TODO: multifield original.text + + # context.request.url.protocol minus the ":" + - name: scheme + type: keyword + description: > + The scheme of the request, e.g. "https". + + - name: user + type: group + dynamic: false + fields: + - name: email + type: alias + path: context.user.email + + - name: id + type: alias + path: context.user.id + + - name: name + type: alias + path: context.user.username diff --git a/docs/fields.asciidoc b/docs/fields.asciidoc index 36e9f94f993..6c907eea0fa 100644 --- a/docs/fields.asciidoc +++ b/docs/fields.asciidoc @@ -523,6 +523,243 @@ type: keyword The ID of the parent event. +-- + + +*`context.service.agent.name`*:: ++ +-- +type: alias + +-- + +*`context.service.agent.version`*:: ++ +-- +type: alias + +-- + + +*`context.system.architecture`*:: ++ +-- +type: alias + +-- + +*`context.system.ip`*:: ++ +-- +type: alias + +-- + +*`context.system.hostname`*:: ++ +-- +type: alias + +-- + + +*`context.system.platform`*:: ++ +-- +type: alias + +-- + + +*`context.request.method`*:: ++ +-- +type: alias + +-- + +*`context.request.http_version`*:: ++ +-- +type: alias + +-- + +*`labels`*:: ++ +-- +type: object + +A flat mapping of user-defined labels with string values. + + +-- + + +*`context.process.pid`*:: ++ +-- +type: alias + +-- + +*`context.process.ppid`*:: ++ +-- +type: alias + +-- + +*`context.process.title`*:: ++ +-- +type: alias + +-- + + +*`context.service.environment`*:: ++ +-- +type: alias + +-- + + +*`context.service.framework.name`*:: ++ +-- +type: alias + +-- + +*`context.service.framework.version`*:: ++ +-- +type: alias + +-- + + +*`context.service.language.name`*:: ++ +-- +type: alias + +-- + +*`context.service.language.version`*:: ++ +-- +type: alias + +-- + +*`context.service.name`*:: ++ +-- +type: alias + +-- + + +*`context.service.runtime.name`*:: ++ +-- +type: alias + +-- + +*`context.service.runtime.version`*:: ++ +-- +type: alias + +-- + +*`context.service.version`*:: ++ +-- +type: alias + +-- + + +*`context.request.url.hostname`*:: ++ +-- +type: alias + +-- + +*`context.request.url.hash`*:: ++ +-- +type: alias + +-- + +*`context.request.url.full`*:: ++ +-- +type: alias + +-- + +*`context.request.url.pathname`*:: ++ +-- +type: alias + +-- + +*`url.port`*:: ++ +-- +type: long + +The port of the request, e.g. 443. + + +-- + +*`context.request.url.search`*:: ++ +-- +type: alias + +-- + +*`url.scheme`*:: ++ +-- +type: keyword + +The scheme of the request, e.g. "https". + + +-- + + +*`context.user.email`*:: ++ +-- +type: alias + +-- + +*`context.user.id`*:: ++ +-- +type: alias + +-- + +*`context.user.username`*:: ++ +-- +type: alias + -- [[exported-fields-apm-error]] diff --git a/include/fields.go b/include/fields.go index e4f20826a5f..13a97f6d279 100644 --- a/include/fields.go +++ b/include/fields.go @@ -31,5 +31,5 @@ func init() { // Asset returns asset data func Asset() string { - return "eJzcXP9z3Lax/91/xT51MnJmdLQkW3Z6M2mfXpymmsS1J7b7OtN2Ljhw7w4VCNAAqPO50//9Db6QBEnwvkjn1H36IfHxy+5nF4vFYnfBCdziZgqkLB4BGGY4TuEHFKgIh+s3r2DBkOf6EUCOmipWGibF9BGE6/ZfABMQpMApcKYNCiaW7iqA2ZQ4tfTXUuXhWkwGfhcuAlznuUKtwawQNKo7VMB0SxCkyDqsSiUpai1VZn/vy+9N/Zajkj0aIYl3KMzhNN1rPaKGFagNKcoOuaWSVZkgFi7Fuo2JVbq5VFPistG2/aOyEmYKF9GlEYXbv3c1OJALp3knATABBaNKaqRS5Bo0ExThvWAfAUtJVz0RqRQGP5q9BIxHXGyAqDkziqhNTaQiHJhYSFUQ+zwoXBKVWwNo4J0BJaWpFOYw37jLZOkuS8eCcL6xQ3nH8vaJSqPKajgbQQpGp7AgXGNP4wOVG7IcKl3O/4HURJf9hVnKVjosjapwv6G5hgUnBgpSls78F06GSY4LJjB3sGDNzAq0UfaBO8Ir1NlQgJUx5UCAeHiS4rfva0NMpWdU5tgBmDS/rSIBvLOT29EDS6+2OgsRFOpSCo0JEazkDxHBvh95iZjKcKz2EKGmV+PncrnE3E4bb2cJCCw/BvObHIVhC4bqANZYEMaPwf17S+gQmcsE197F7eK+qblZHrBeocLYTWlQSKXKMT+z1Bl1c5/AGucwV3IdTfp2+Ji2L8q5IW4iLZQsHM2/TP4g1ZpYavZfsEKSozqzCNYrRlfuoQVT2gAKozaWir3UgpSKLZkgHChnbiFIsHbTFFZyjXaFK9hyZUBIA3MEgXYZIYrxjfUT2lixiAZmgBJhn1hItfQ+jUBBOKNMVnpc/85hON+YGIfIY+8xEm/lwqyJCq4WCDXW5zCLakX4wmqAOH5ngMusOwTwBF6/BSqLORPOqSdmuMIPFWrzoEmuUkbeJ7BDzmuLs+RoEN4rfhY8LF1hgWewktoAETmUxKy6o5vA1RGPrHt3tk3CnSgh+CFF1mdQiZIojTm8//mn2hKDOs8As6Xzrnr65Al+JFa0jMpi+uzZ0ycaiaKr33/4FjnRhlH/+zdGltmYHKWSRlLJP4cwNe2UDBmceClORqEtKv5ZYFm6Z1BKrdncOhdr/xOiNRZz/uso3ZrdYPU6knQ17RGlR+jHNV9KZT6LQUhl0riePXs6joaY1efSlqU9oqkwsONa8vc/BypPOTw7R78qfahQberYMA25Z4Pj0FdEfxbglm4Pm51PNT4jy5Pkumbn1uwOlWZSHCukcyFooNnHxJH4HYgEY9fweJvXRVagWcmjRHoNJk/yUEjtuuqD6i9pA+DESuwCaqw+hDi9PD8/TSp5wQTTK0ypeS4lRyIOWe/DK8BEzihxYc16hWaFqgMK1kQ3nEEqG7Ql9K032mCxU9tbML0Om9hAKgxMtn2odi4W22brjrmaWiYCtjYUt9qpY3HoRHgxOOtmmEFqN+/HBRhTfiDIkhOzkKo4LsCa6gPB9XZPI5uqnYjaPYuLac2KmMBct5AS5h0yZA+x75souVOisruwxpEhqEq4n4FRtOXLiSFOSVRyjtR05n8/X9bBzPrDNeK4dmrtT1WBitEG3M3LZjRR3TGK9Z0xyzo+lAGEUw0lUdagtmNxid7jmfjbrgI8+ZR/9M89xIBqVuOOMSXvcX3iTVFUhsw5QiXYhwqh4xwDQCyYMT5z2a7QXTp+Kgz3rgCczJHPDBbWceAUTv75T5c6+Ne/TnpPyhLFjDNxO2NiRitlB39myHyQbLR/lYqIDsSaQMFEHVlN4eQqO8/O+/zsn4MyhZMse0LK8sktmxNBfvMkJ3o1l0TlT55dzK/y316eT158c3kxubjAF5Nv6LMXk+dX82+eXc2v6GL+9Pcz8u1jF6dO/f9mPlydPiaC8M0nnK0ZzylR+fS/zJl/8DSkibOgZJf8n351edmo56vLy9Ovv/56iLov3HMr3ITwckUufhUZORHLiixxyiuKArdK9Ld2vP92cmrFSVp1Kgh+kGH/uRsBbzXlJCIUd0xJUfSTTkfxLhHxEfa1jpO8h7mgnYmbz7GT/FPkK0oll4oUhdVtjR0qjfnoZiw95A8G1Rv4fXE1+4xKGDbiYb9AtQe4X4Kmt0BpNjyKFLiW6vY/Rb0N4C9BwVvBNDuTQZb8C1avz8F/AaodAGmiS0WEJtS0EBJV4W1l2L48ncB5tMq6pdC9whgU3LzsBI7JMGwQgv2Z4RrelkToOF7YO/4aj716oYkLu3ra3xmNPMWnOb44P5+8yPHcRyPzi4urSb74Lf3teU4u88XFvSKuSG0Zy3fHWj1h4jDrc8s0EmH1JGjgt3FVZLcU4cswWWez7f7OQzMyKkb6xMEc7fZRg5H9dha/D/wipQnY6kgy9CDNkZi2Cel//K8E0e+kMIQJDVQWhRTuvYAcyB1h3G3NmADCedCShew2i6m2Jft+ZlfhT1LUihiRxTmS8CS4lI2plGgbTWSJymcRQ4ZHimjErEjA2vxJSHf0Bu7kv5umoZPO6OXE1PAUfqiYwrzjZWo/Fj0XKjhTuK6WlTZw+dys4PL84vkZXFxOn15Nr55mT59e7ie0bxVar1BE9sflMuSOXHpm6VrHzHBJaPtoxvqBml4g19ziklKh8K3RQInK689VYFElFhmvpx7jpn8N0u07W1p3RoA25ue6EqgUC7aslLO5OinRQYBKSTU+DdNMvrcv1UZNPUdrViTPWcgQM7GQ1sop0S42cHyajMjYTLYIHjaXPTSv6jBzKZdV3k7d7+zPuglLQYGG5MSQ9Gx+Fe76VgzaeVVbeaNGrzyfuQdmNcmoae9RLzCLijIkc29lNdn+5EC6YwbE4VcXYQZvfG0YQwsWEIWW4BksKZ6BVJCzJTOES4pEZKPYmNCGCIoztsP8bsKDkTt1WdyC0BUTffNPcYgC1F084sTWflzCA7PIzho9m8uswJxVxXburzwJZ2KHMQ/en3FmNrPImzcIKj1Bos3kgu5wRhEhcM6etY6caQ+H6REP3jU551+aUW2ghDuTj/ubXnjFYvlByiVHP9PGuStctu5xhMHP7pld8oWJnkt66+ZPmOkv698J4v6eK/VFaXs/zf09O2f1Sioz8160jUWIoCupan6TZpaPrOANrLSPHfOFwa+iyh4a37z3eeCGILA828au6Cap7sExtgtHzjunBoBdjOcV46ZpnU5D6e1W74Hku4anb6oe5+W2VEdop91SXnKa8Hwao7XG3JrsH/2vBJEbu6BGhipVwvW0tmmv77TMwPswu5Q6u0UlkD9819sPSE81eNJ1jqDR0m01tzcM6lZXP8bXEpza+80i312xW6IQa2r7pG9f2qneDujDlFzK/AjGH2mglLnP/SRZVQ91MRGnNzKH9zcvh4zsf3VJEhW2+7JqKQ6ZyRyPq0FLcUSF+7qO/Rh5alCQcsiJCCGNK1AfjV1EMs3zmO444ks7nnkb2yMsSEm+nm57xmhSb4aCg7l+88rvJ/r+xV2c6BIpWzDqy//WIV+/eTXiCu4YrsMe6NG4EJ7FFE5ySf+aLL6d/j1zkXy9926ySlASJnjb/zBMG6ZThk6UJme4V74wnSvckSfcnU97sXi+IPR88oI+Jz6fRsjV1eTp/Dy/uqQvLujz81+pKrtflvDoEh2hBtvZ1QPLgdFelnubwbm3Mrc8MbGc3eLmmOY2SWE8NPPw0k61+IgVESHTr7BUqFG4cIKIkACS1FpxOA9AoJCCGWlfrbVZszvo3NVeq2U47fZs/3Co3TP7kUj4xIqXig2PIvQOTATWl2Os/1AJX2qghPOws7Ihud/GsYKoDZSoSjSKGBnO82zp5IxN5mF++odA6Ufc9A7zeJO2XrbSbsNbM71vscRnil6iIYx/iUWTq8X5N+SbF8d3iMNp/qsWTg6Va8QtJqRIFE9iK8WPFEvTrTw+qEWQzGVlOqeruN1YKLkW9QyOTHNbZ+ygf/mAlpR3tb/wDcsaTZQFd9dXpCxRYB6a1224Mic6fmuk8lyg1mMNK4MzWkm3k0TbnEXzAAKXMQwyr+7fDWjZeQrtZmFwJK7vbWE8I72LeeP0k8RWROQc032Wqfbw/VR647vDpeo0h3vduqZUUi1XBrQs0LesMhHSMTm23eLDTY1cPmSiXHdrAc2cqTfg1q03R6cPmiwc77DfkniQQWi8Q8XMpm3up1KNdV+45UfNHtKjaXkOjuQqqFPe23o/PtMMbAs1fNNdYLdPxpIoUsy2gbpXD921J4xGsU+YNxjge+uuTr+TFc/dQVQqhUBqwEj4Sp/2G1abo3olKrOpqQDToA3jvClLnrkqnV45snME/FARXjd5dyQ8g3ll/DnLkhOKK8ld3Ueh+5kPEdwIN89AM1OFPfWAqkVkh9yydBMqbAzByKWbv03WqzmxETakJ3ZH+tbXbl9ZbVEXuCS0Gx4KJ1kK/+wZcHaL8N2b904DBRZSbaDykrqSJlHYT4anCsfD1HiIjFJb386xk70D/V/8a7/U23UNXNL25EmQqF9PfNQ1V1pWj7rm2XVgo2b5Cy2rAWurN1cvaId9tI4pDeGZkKrISjoM1zUlHPPZgktiEtFriYp2W752bBzCC9a25MLhdG17unQ9GJu6POO63/33MpwbItqkWr+js+BMgxWCcDcvAyVRFXN/yN9yolK5JFIOzIAiYom6Q80Z0bm19Yvz86+y/hB5I7znKPmXBwMVDPuQsRoMUe/YQz00843pyLdtYCzdgGVYdyDUVAm2hyywjkI7izF3o7BQiNsPHrStm7jXobkx2XcdX/P4LJcaJBOeSAY3roBHCaeV3T3lYAPSHKSPTF6/zeC1gJ+YqD66TwxIoZk2um2bbGj2mJa8smTpKtjkvFosUGlH7vXbv1hiTAMBXbnTTjE4+7irVwlCDburr7tX/5eJXK71WXjfrRj95U/WPisLL1rivwwMvn9M6TCLD29HJl/P67rEceZmZePxI0ff85lbKo+R27yHYe7rPLfZ5qgD3eFCtznRPY5/HteRHtuVDp1pX22DObHH4L1y77TZbDtKzB29dp1MzVGt9vVS4YJ9nMLJX536/36y15Bq9ulzuhvXB+Zc7h1TsWeMx2xFOoI0JwS0zobsjo/vZ9Qst6b0Fg28ZZ8w89F4YcN2awUJyJLSqmT+azYFsf/xzzz++frV11lcvdCyUhQLUnYrGG+jyx2EzQ1YMI4aUChGV5j7gDfqi/I5HD+mnslswbhB1YzzBBrmWQwjGQ9G92Es9bp/pfT/yam9/h6xPt3k8q6NyjoNtSlQRz90VeumKcgPBmBeiZw7i8CSmNVgIA7K//4kqc8T1GpoJFfIiVuWw0bN8qs/OeDy+6Y7GUqXS4nngb/SnQIlEek63lg7rivkWeKHFtpCb353KpTknmcQBqOQKElYS4H91P4SS4XUBWKPf/ctPM+uvs7g2i/dfFMfJI0O9JYkZQor/Dgbr43siaVbCbGczoAJo2Re0RhfusFgD/M77DOE7nOT1jhQs6VojJM4ZHUqTVPpuwFJ3PmbgLhng+thEH8M+pULaGxZIcc7l12qITZnsHPp1pHHuJzCaT7PSqnNUqH+wDOXWz89g9PaejNUc/vbhdSnZ4CGpjSvDVG7P5CV+Jgk7HCyVT8CGj2UnlDZDrXZv9eLhUYzcC7RIJ7qqL2c+W/cberEpZO7b6qd71MmlJVXihxchkjo69+isJcB/A6huy81vf9Dye0fE2VlZvVDMaHeg7Iy8ZNEv2Kcs63PWr/GfNHqPNH75I567Nr8H+o437kc74jrDKdLOo40Wre6Bwei5etd50Y3o9veOqgp5egn4ZIOuJemLipu2GzbnE/HKfcpJo+57vi03dCDayaWvI3AHrsv/vzw/Tt4UmlU+smU5acpJ/jwwwv3duQZnIZQyDrrOaG3dgxF/g85D077i3VDu+Jit6XLO16H6XgE/3O9j0Jd8aH3ObiN1tOJDuM1AQj88d27N53vWVmPYC9O3JKHefx4arEqiLr9N31juPNN4ehbw84+XS7QgfPD347EmBCZIHds6SzgHSuYGFY3t0s1knGyl5lYzhaEGqmmcHHu/kaFH8ZO7sjFMGoeFoS3mUE0im0p5zTQPoU14xyYoLzK0Z08jI8iNtXZbISOkKaG6SnZCyty59cvDa6LwHepwUtckIob7UIpVSU/tFMSMXPhx07fs22dyJUsy5Fq+raT8dD7GysEdCkmXNdgTNocVJvSCSiDpnzbfteBZY/+LwAA///HkbmF" + return "eJzcXf+T27ax/91/xb7LZM6eOdF3Z5+daqbp87PT1JO49sR2X2fajgKBKxE9EqAB8M5Kp//7G3whCZIgJZ10ifv0Q2JRxO5nF4vFYrHAzeAaN3MgZfEAQDOd4xy+R46S5PDi3RtYMcxT9QAgRUUlKzUTfP4A/HPzL4AZcFLgHHKmNHLG1/YpgN6UODf0b4VM/bOQDHzrHwK8SFOJSoHOEBTKG5TAVEsQBE86rEopKColZGK+78rvXd3KUkkejJDEG+R6f5q2WY+oZgUqTYqyQ24tRVVGiPlHoW5DYpVqHtWUctFo23yoqLiew0XwaETh5vOhBgdiZTVvJQDGoWBUCoVU8FSBYpwifOTsM2ApaNYTkQqu8bPeScCwx/kGiFwyLYnc1EQqkgPjKyELYt4HiWsiU2MADbwzoKTUlcQUlhv7mKztY2FZkDzfmK68YWn7RqVQJjWcDScFo3NYkVxhT+MDlWuyHipdLP+JVAeP3YNFzFY6LLWscLeueQGrnGgoSFla819ZGWYprhjH1MKCW6YzUFqaF25IXqFKhgJkWpcDAcLuiYrftlea6EotqEixAzBqfpMiAXwwg9vSA0OvtjoDESSqUnCFERGM5IeIYNoHXiKkMuyrHUSo6dX4c7FeY2qGjbOzCASWHoP56xS5ZiuGcg/WWBCWH4P7d4bQPjKXEa69h9Pivqu5GR5wm6HE0E0pkEiFTDE9M9QZtWOfwC0uYSnFbTDo2+5jyjQUS03sQFpJUViaf539UchbYqiZf0GGJEV5ZhDcZoxm9qUVk0oDci03hop51IIUkq0ZJznQnNmJIMLaDlPIxC2aGa5g60wDFxqWCBzNNEIkyzfGTyhtxCIKmAZKuHljJeTa+TQCBckZZaJS4/q3DsP6xkg/BB57h554L1b6lkjvaoFQbXwOM6gykq+MBojldwa4TrpdAI/h7Xugolgybp16ZIRL/FSh0gcNchkz8j6BLXK+MDjLHDXCR5mfeQ9LMyzwDDKhNBCeQkl01u3dCK6OeOS298vUINyKErwfkuT2DCpeEqkwhY8//VhbolfnGWCytt5VzR8/xs/EiJZQUcyfPn3yWCGRNPvDp99jTpRm1H3/SosyGZOjlEILKvL7EKamHZMhgRMnxckotFWV3wssQ/cMSqEUWxrnYux/RpTCYpn/Oko3ZjeYvY4kXU17ROkB+nHNl0LqezEIIXUc19OnT8bREJ3dl7YM7RFN+Y4d15L7/T5QOcr+3SW6WelThXJTx4ZxyD0bHIeeEXUvwA3dHjYznmp8WpQn0XnNjK3FDUrFBD9WSGdDUE+zjylH4lYgArSZw8NlXhdZgToTR4n0GkyO5L6Q2nnVBdVf0gLAihVZBdRYXQhxenl+fhpV8opxpjKMqXkpRI6E7zPf+ybAeMoosWHNbYY6Q9kBBbdENZxBSBO0RfStNkpjsVXbE5je+kWsJ+U7Jpnuqq2TxdRo3TJWY9OEx9aG4kY7dSwOnQgvBGfcDNNIzeL9uABDygeCLHOiV0IWxwVYUz0QXG/1NLKo2oqoXbPYmFZnRHvmqoUUMW+fITvEvl8HyZ0SpVmFNY4MQVbcfvWMgiVfSjSxSqIiz5Hqzvjv58s6mFm/u0Yc11at/bkqUDLagHv9qulNlDeMYv3LmGUdH8oAwqmCkkhjUNNYbKL3eCb+vqsARz7mH917hxhQzWrcMcbkPa5PfF0UlSbLHKHi7FOF0HGOHiAWTGuXuWxn6C4dNxSGa1eAnCwxX2gsjOPAOZz86182dfDvf5/03hQl8kXO+PWC8QWtpOn8hSbLQbLRfCoZEB2INYOC8TqymsPJVXKenPf5mY+FMoeTJHlMyvLxNVsSTr56nBKVLQWR6eOnF8ur9HeX57Pn31xezC4u8PnsG/r0+ezZ1fKbp1fLK7paPvnDgvz+oY1T5+5/Cxeuzh8STvLNL7i4ZXlKiUzn/6XP3IunPk2ceCXb5P/868vLRj1fX16ePnr0aIi6L9wzI9yM5GVGLn4VGXPC1xVZ4zyvKHKclOjvbX///eTUiBO16lgQfJBh/6UbAU+achQR8hsmBS/6SaejeJeA+Aj7WsdR3sNc0NbEzX2sJP8c+IpSirUkRWF0W2OHSmE6uhiLd/nBoHodvyuuZp1Rcc1GPOwXqHYP90vQ9ASUZsEjSYG3Ql7/p6i3AfwlKHgSTLMyGWTJv2D1uhz8F6DaAZAmupSEK0J1CyGyKzy1DduXpxM4j+6yTmx0ZxiCgtevOoFjNAwbhGB/YXgL70vCVRgv7Bx/jcdevdDEhl097W+NRp7gkxSfn5/Pnqd47qKR5cXF1Sxd/Y7+7jwll+nq4k4RV6C2hKXbY62eMGGYdd8yjURYPQka+G1cFdgtRfgyTNbabLu+c9C0CDYjXeJgiWb5qECLfjmLWwd+kdJ4bGEk+RV89/J9R4DQKR8Iv+eSHTWSMxLmMkqis6aapYnHLQpXZDQgO3Sz+1OuaXREz4Q6kuQjObedcNocVdKhMDSMYXHJTlRZOaR1l15y1Jq054Cm2J6tmsweTWUD+9BG4TVUup3cVuYc2MmDfYddVOf3EhLf+EjGXVPtbNL4AV6/Y4uXqCgZpmATfA8p4afaUfa1VOpRR1d2MlYdbXWKsCYKsGLFV2MladNFVw7EeNlVPE16YN+WEdc73QV15s+0HJI7gF6UYD+TuA9F17aju26G8I66+8rW1DDeTCmwNUWxj+MOKezGM7aE298ZjSYxd3BEHnoDpDejwY5Zpbtw6niBraqKpHB+E03VOO5dUQ2jgZ6OEL10wU8qfpjD+U307mHcu9prPqNaPyy4i4Z17QLzwEkhFQVhd52aK5lPBEwrSdZ38I0d4kRlkUjMl0ceQNgWeQ2nNKKzA4g2FUPBMIm+J6SuQwuYfdvdtRsphoru723JU+xa91SztKvcA+QfVCZ9BR/evno7h6LKbbFxnjZ9l9iy0W2Kqiv5CsYrt6F8Mj8Z4HZllQPge+d1HJ2pssGTXnwW1JMfOA77tdW76N7W67qGw9XUvrGZJRYLy+4wZ1haTa38A38waIlEtyeD/sd9i3TJS8E1YdwE9kUhuG3n9QbkhrDc7pcyDiTPferCgLI7uLGzRKZ9Yjz0L4JvCdttds+/CbaOQleSt6c/RInSlfb4sgvBgzSKEQlYqxZfg9CzmZP/bk7ynHRsJyW6hmdsj0lMO0uNOrkYvOfLKufwolpXSsPlM53B5fnFszO4uJw/uZpfPUmePLncTWh3fuc2Qx4khXKx9gUdtmZibc9z6WGetj3cMrYiag7o2EWbrRTx1egKNZQonf5sWTTKSObX6anHuDlUBvsv5+JAG/OzRwWo4Cu2rqS1ubpSoIMApRQTTiDO5DvTqDZq6jgasyJpynzZFuMrYaycEmV9kuXTlCmM+RGD4DBP6KA5VfuRS3NRpe3QfWm+1iejJBSoSUo0iY/mN/5Xdz6CdpoqI29w+ipNF/aFRU0yOEn3oLdbEmQsSGJbJTXZ/uBAumUEhHsiXYQJvHMF2+gX6EAkGoJnsKZ4BkJCytZMk1xQJDwZxca40oRTXLAt5vfavxjkOG1pVUFoxnjf/GMcAoe9jUdYbbIbF//CIrCzRs/6MikwZVUxzf2NI2FNbD/m3vuznOnNIvDmDYJKzZAoPbugW5xRQAiss2etI2fKwWFqxIN3Tc76l6ZXGyj+l9nn3U3PNzFYvhdinaMbaePcJa5b9zjC4Cf7zjb5/EBPBb2248eP9Ff19whx95utvw1q6dwwd7+ZMasyIfXCedE2EiKcZkLW/GbNKB+ZwRtYcR875gu9X3VxzUE+8aMrzmoIAkuTKXZFLO2wF8fQLiw555waAGYyXlYs18155jiUaPS2F5KXDU9uTzqP8+qkVuHuZ1wnaj6tJhyfxmj9Foc32T+5bxEir82EGhiqkBHX09qmeb7VMie3V8bsUqjkGiXHYby/95KlH5CeKnCk66RDo6Xraml+0KhaXf0QPotwan9vJvnujN0ShVBT04O+bbRVvR3Q+ym5FOkRjD/QQClSiCc6DKvqUBcTcHonUvj4+lV8KaZKEil7vSurluKQmUjxuBo0FEdUuKvr2I2R314pSGRrkHAutK0aPxq7gGSc5zHdccCXdjzzFNsjTEhRvo5ue/HHrF4MeQfz4t0bt57o+xf7cKZKpGzFqKvJNw75xbs3I67ghuGtXwM9GBfCsZjDSSro36IVsaf/SGwkX6+9m1IPKAnjebvbNqzlidfxWFGaQp6dinjiBTxbine2F7k8Xz1bEXo+e06fEVfkQsjV1ezJ8jy9uqTPL+iz81+pVHq30p2jS3SEwujOqh5YCoz2Ss+mDM62Suz0xPh6cY2bY5rbLIZx38zDKzPUwntPCPfldxJLiQq5DScI9wkgQY0V+0P6BArBmRamaa3Nmt1el6HsNFv6K2ie7h4OtWtm1xMRn1jlpWTDzHrvFgPP+nKM9R8r7ur/KMlzv7IyIblbxrGCyA2UKEvUkmjhL9mYOF4Zmsxhfvp7T+kH3PRu2HAmbbxspeyCt2Z61wpGlyl6hZqw/EusZLxanX9Dvnl+fIc4HOa/ajXjvnKNuMWIFJGKxtBK8TPFUse2Mu94bo8sRaU7V57kZmEhxS2vR3BgmlPHVQeHivc4J/Kh9hfuFLFCHWTB7fOMlCVyTP1mkAlXlkSFrUbKwQtUauwUyeDilKjbiaJtLohxADyXMQwire5+RM+wcxTaxcLgnpq+t4XxjPQ25o3TjxLLCE9zjB9+jJ3Z3k2lr92RbSE7J7adbu1JUVKtMw1KFOjOkTLu0zEptke4h4sasT5koLzo7gU0Y6ZegBu33txnttdgyfEG++cE9zIIhTcomd60e6VUyLEjEXb6kYvRio4defL+PVkS6pT31IGMexqB7UZNvulOsNODsSSSFIspUHc62PbCEUYt2S+YNhjgO+OuTl+KKk9txQ4VnCPVoAV8rU77p0ib+3NKlHpTUwGmQGmW58225JndpVOZJbtEwE8VyeuT1x0Jz2BZaVfnWOaEYiZyu+8j0X5NhwheczvOQDFd+TX1gKpBZLrcsLQDyi8MQYu1Hb9N1qu5RsEvSE/MivS927t9Y7RFbeAS0a5/yV8vUbh3zyBn1wgv3320GiiwEHIDlZPUbmkSif1keGzjeJga95FRbOnbuQti50D/Z9fs53q5riAXtL0OwkvU30980DVXWlYPuubZdWCjZvkzLasBa6M3u1/QdvvoPqbQJE+4kEVS0mG4rijJMV2sckF0JHotUdJu5dG2UhnXwNiWWFmc9iydKu3BiE29PWOPpLtLLK0bIkrHzmMHF7QxBUYIkttx6Snxqli6m/cMJyqkTSKlwDRIwtfYrTuzRnRubP3i/PzrpN9Fzgjv2Euu8aCjvGHv01eDLurVKtVds9zojnxTHWPoeizDfQdCdRVhu88Eaym0oxhT2wsriTh9G0Bb4YY73WQzJvu2O2UcPsOlBsm4I5LAa7uBR0lOK7N6SsEEpCkIF5m8fZ/AWw4/Ml59tvf+Ca6Y0qo9y9jQ7DEt88qQpZm3yWW1WqFUltzb9381xJgCAqqyV5CE4Mzrdr+KE6rZTf3cNv1fxlNxq858eztj9Kc/UfusxDc0xH8eGHz/7pD9LN63Dky+Htf1FseZHZWNxw8cfc9nTuw8Bm7zDoa5q/Ocss1RB7rFhU450R3uZDquIz22Kx06077aBmNih857Y9u02WzTS8zeh2YrmZr7U9rmpcQV+zyHk79Z9f/jZKcuVeyX+3Q3tg7MutwbJkPPGPZZRjqCNCXfSiVDdsfH9xMqlhpTeo8a3rNfMHHReGHCdmMFEciC0sqeEGIcCmL+4955+NOLN4+ScPdCiUpSLEjZ3cF4HzzuIGx+gBXLUQFyyWiGqQt4g7ool8NxfeqYLFYs1yibfp5BwzwJYUTjweB3GEu97r5T+v/kKp3+GrG+csTmXRuVdU65xkAd/SaUWjfNhvygA5YVT3NrERitQN8r//ujoC5PUKuhkVxiTuy07Bdqhl9d5Wzz+7o7GEqbSwnHgXvSHQIl4fF9vLFyXLuRZ4jvu9HmD8x3h0JJ7ngxwKAXIlsSxlJgN7W/wlIitYHYw29/D8+Sq0cJvHBTd76pb3cKbtkqScwUMvy8GN8b2RFLdyfEcDoDxrUUaUVDfDvWeo9uzOz4twHs34AwxoGKrXljnMQiq1NpigpXDUjCyt8IxB0LXPeD+IPXr1hBY8sSc7yx2aUaYnMxmjs0Aw9xPYfTdJmUQum1RPUpT2xu/fQMTmvrTVAuzXcbUp+eAWoa07zSJHLeY8oF7+hkq34ENDIpR1W2RW3m83a1UqgHziXoxFMVlJczd/H8pk5cWrn7ptr5oxERZaWVJHtvQ0T09Zso7JUHv0XobqOm9n8oufkwXlZ6Ub8UEuq9KCodvknUG5bnbPJd49eY27Q6jx2ZkrEjXrsfVIo6zg82xzviOv2VDx1HGsxb3YMDwfT1ofNDN6Pb/rRXUcrRr6eJOuBemtqeplpMjfl4nHKXzeQx1x1egTP04Irxdd5GYA/tNbzff/cBHlcKpXo8Z+lpzAkefnjhzo48gVMfChlnvST02vQhT/8plt5pf7FuaFtcbJd0acfrMBX24H+u95GoqnzoffYuo3V0ghtymgAE/vThw7vOJdPGI5iHMzvlYRq+HpusCiKvf6M//NO5cyK4i8Lap80FWnCu+9ueGBMi4eSGra0FfGAF48PdzWmpRjJO5jHj68WKUC3kHC7O7WdU+GHsZI9cDKPm4YbwlBkEvdhu5Zx62qdwy/IcGKd5laI9eRgeRWx2Z5MROlzoGqajZB5k5MbNXwpsFYGrUoNXuCJVrpUNpWQVvf22JHxhw4+tvmdqnkilKMuR3fSp6+qg9xnbCOhSjLiuQZ+0Oag2peNRek25sv2uA0se/F8AAAD//3KC/1s=" } diff --git a/tests/system/apmserver.py b/tests/system/apmserver.py index 1033215193b..4901ecb6f0e 100644 --- a/tests/system/apmserver.py +++ b/tests/system/apmserver.py @@ -423,3 +423,19 @@ def config(self): def get_debug_vars(self): return requests.get(self.expvar_url) + + +class SubCommandTest(ServerSetUpBaseTest): + def wait_until_started(self): + self.apmserver_proc.check_wait() + + # command and go test output is combined in log, pull out the command output + log = self.get_log() + pos = -1 + for _ in range(2): + # export always uses \n, not os.linesep + pos = log[:pos].rfind("\n") + self.command_output = log[:pos] + for trimmed in log[pos:].strip().splitlines(): + # ensure only skipping expected lines + assert trimmed.split(None, 1)[0] in ("PASS", "coverage:"), trimmed diff --git a/tests/system/test_ecs_mappings.py b/tests/system/test_ecs_mappings.py new file mode 100644 index 00000000000..81ba5f39fc5 --- /dev/null +++ b/tests/system/test_ecs_mappings.py @@ -0,0 +1,86 @@ +import json + +import yaml + +from apmserver import SubCommandTest + + +def flatmap(fields, pfx=None): + if pfx is None: + pfx = [] + for field, attribs in sorted(fields.items()): + if 'properties' in attribs: + for f in flatmap(attribs['properties'], pfx + [field]): + yield f + else: + yield ".".join(pfx + [field]), attribs + + +class ECSTest(SubCommandTest): + """ + Test export template subcommand. + """ + + def start_args(self): + return { + "extra_args": ["export", "template"], + "logging_args": None, + } + + def test_ecs_migration(self): + """ + Test that all fields are aliased or otherwise accounted for in ECS migration. + """ + all_fields = set() + alias_fields = set() + for f, a in flatmap(yaml.load(self.command_output)["mappings"]["doc"]["properties"]): + all_fields.add(f) + if a.get("type") == "alias": + alias_fields.add(a["path"]) + + # fields with special exception, due to mapping type changes, etc + # no comment means unchanged + exception_fields = { + "@timestamp", + "context.request.url.port", # field copy to url.port, keyword -> int + "context.request.url.protocol", # field copy to url.scheme, drop trailing ":" + "context.tags", # field copy, can't alias objects + "labels", # target for context.tags copy + "parent.id", + "processor.event", "processor.name", + "sourcemap.bundle_filepath", "sourcemap.service.name", "sourcemap.service.version", + "span.duration.us", "span.hex_id", "span.id", "span.name", "span.parent", "span.start.us", "span.type", + "tags", + "timestamp.us", + "trace.id", + "transaction.duration.us", "transaction.id", "transaction.marks.navigationTiming", "transaction.name", + "transaction.result", "transaction.sampled", "transaction.span_count.dropped.total", "transaction.type", + "url.port", # field copy from context.request.url.port + "url.scheme", # field copy from context.request.url.protocol + + # scripted fields + "error id icon", + "view errors", + "view spans", + } + + should_not_be_aliased = alias_fields - all_fields + self.assertFalse(should_not_be_aliased, json.dumps(sorted(should_not_be_aliased))) + + # check the migration log too + with open(self._beat_path_join("_meta", "ecs-migration.yml")) as f: + for m in yaml.load(f): + if m.get("alias", True): + self.assertIn(m["from"], alias_fields) + elif m.get("copy_to", False): + self.assertIn(m["from"], all_fields) + self.assertIn(m["to"], all_fields) + + # check that all fields are accounted for + not_aliased = all_fields - alias_fields - exception_fields + self.assertFalse(not_aliased, + "\nall fields ({:d}):\n{}\n\naliased ({:d}):\n{}\n\nunaccounted for ({:d}):\n{}".format( + len(all_fields), json.dumps(sorted(all_fields)), + len(alias_fields), json.dumps(sorted(alias_fields)), + len(not_aliased), json.dumps(sorted(not_aliased)), + )) diff --git a/tests/system/test_export.py b/tests/system/test_export.py index 61030afb1a3..9a60e35cf90 100644 --- a/tests/system/test_export.py +++ b/tests/system/test_export.py @@ -1,23 +1,7 @@ import json import yaml -from apmserver import ServerSetUpBaseTest - - -class SubCommandTest(ServerSetUpBaseTest): - def wait_until_started(self): - self.apmserver_proc.check_wait() - - # command and go test output is combined in log, pull out the command output - log = self.get_log() - pos = -1 - for _ in range(2): - # export always uses \n, not os.linesep - pos = log[:pos].rfind("\n") - self.command_output = log[:pos] - for trimmed in log[pos:].strip().splitlines(): - # ensure only skipping expected lines - assert trimmed.split(None, 1)[0] in ("PASS", "coverage:"), trimmed +from apmserver import ServerSetUpBaseTest, SubCommandTest class ExportConfigDefaultTest(SubCommandTest):