SIGSEGV on 1.9.4 with stack trace

[PettitWesley]

Bug Report

Describe the bug
SIGSEGV and crash/pod restart. We got a stack trace from Valgrind.

{"log":"[2022/07/19 06:35:12] [debug] [input:tail:tail.1] inode=54654517 events: IN_MODIFY \n","stream":"stderr","time":"2022-07-19T06:35:12.801792673Z"}
{"log":"[2022/07/19 06:35:12] [debug] [input:tail:tail.1] inode=54654517 events: IN_MODIFY \n","stream":"stderr","time":"2022-07-19T06:35:12.808897828Z"}
{"log":"[2022/07/19 06:35:12] [debug] [input:tail:tail.1] inode=54654517 events: IN_MODIFY \n","stream":"stderr","time":"2022-07-19T06:35:13.044681592Z"}
{"log":"[2022/07/19 06:35:13] [debug] [input:tail:tail.0] inode=141645686 events: IN_MODIFY \n","stream":"stderr","time":"2022-07-19T06:35:13.04887481Z"}
{"log":"==1== Invalid read of size 8\n","stream":"stderr","time":"2022-07-19T06:35:13.05575357Z"}
{"log":"==1==    at 0x5200A1: input_chunk_get (flb_input_chunk.c:1123)\n","stream":"stderr","time":"2022-07-19T06:35:13.055773607Z"}
{"log":"==1==    by 0x520922: input_chunk_append_raw (flb_input_chunk.c:1408)\n","stream":"stderr","time":"2022-07-19T06:35:13.055798924Z"}
{"log":"==1==    by 0x520F34: flb_input_chunk_append_raw2 (flb_input_chunk.c:1600)\n","stream":"stderr","time":"2022-07-19T06:35:13.055802456Z"}
{"log":"==1==    by 0x551B95: process_content (tail_file.c:536)\n","stream":"stderr","time":"2022-07-19T06:35:13.055814303Z"}
{"log":"==1==    by 0x553EF5: flb_tail_file_chunk (tail_file.c:1315)\n","stream":"stderr","time":"2022-07-19T06:35:13.055817402Z"}
{"log":"==1==    by 0x54B79F: in_tail_collect_event (tail.c:310)\n","stream":"stderr","time":"2022-07-19T06:35:13.055820549Z"}
{"log":"==1==    by 0x54CBCF: tail_fs_event (tail_fs_inotify.c:267)\n","stream":"stderr","time":"2022-07-19T06:35:13.055823481Z"}
{"log":"==1==    by 0x4E0D74: flb_input_collector_fd (flb_input.c:1210)\n","stream":"stderr","time":"2022-07-19T06:35:13.055826561Z"}
{"log":"==1==    by 0x4F64B8: flb_engine_handle_event (flb_engine.c:440)\n","stream":"stderr","time":"2022-07-19T06:35:13.055829567Z"}
{"log":"==1==    by 0x4F64B8: flb_engine_start (flb_engine.c:763)\n","stream":"stderr","time":"2022-07-19T06:35:13.055832418Z"}
{"log":"==1==    by 0x4D3A1D: flb_lib_worker (flb_lib.c:626)\n","stream":"stderr","time":"2022-07-19T06:35:13.055835187Z"}
{"log":"==1==    by 0x4E4444A: start_thread (in /usr/lib64/libpthread-2.26.so)\n","stream":"stderr","time":"2022-07-19T06:35:13.055837982Z"}
{"log":"==1==    by 0x686C56E: clone (in /usr/lib64/libc-2.26.so)\n","stream":"stderr","time":"2022-07-19T06:35:13.055840961Z"}
{"log":"==1==  Address 0xccb5bf0 is 32 bytes inside a block of size 56 free'd\n","stream":"stderr","time":"2022-07-19T06:35:13.055843677Z"}
{"log":"==1==    at 0x4C2F94F: free (vg_replace_malloc.c:872)\n","stream":"stderr","time":"2022-07-19T06:35:13.055846689Z"}
{"log":"==1==    by 0xA2EE02: msgpack_zone_free (zone.c:221)\n","stream":"stderr","time":"2022-07-19T06:35:13.055849549Z"}
{"log":"==1==    by 0xA2C3F9: msgpack_unpacked_destroy (unpack.h:260)\n","stream":"stderr","time":"2022-07-19T06:35:13.055852243Z"}
{"log":"==1==    by 0xA2E763: msgpack_unpack_next (unpack.c:661)\n","stream":"stderr","time":"2022-07-19T06:35:13.055868284Z"}
{"log":"==1==    by 0x60B8C3: cb_kube_filter (kubernetes.c:487)\n","stream":"stderr","time":"2022-07-19T06:35:13.055871266Z"}
{"log":"==1==    by 0x4E14B4: flb_filter_do (flb_filter.c:124)\n","stream":"stderr","time":"2022-07-19T06:35:13.055874083Z"}
{"log":"==1==    by 0x520C09: input_chunk_append_raw (flb_input_chunk.c:1478)\n","stream":"stderr","time":"2022-07-19T06:35:13.055876913Z"}
{"log":"==1==    by 0x520F34: flb_input_chunk_append_raw2 (flb_input_chunk.c:1600)\n","stream":"stderr","time":"2022-07-19T06:35:13.05587978Z"}
{"log":"==1==    by 0x551B95: process_content (tail_file.c:536)\n","stream":"stderr","time":"2022-07-19T06:35:13.055882587Z"}
{"log":"==1==    by 0x553EF5: flb_tail_file_chunk (tail_file.c:1315)\n","stream":"stderr","time":"2022-07-19T06:35:13.055885484Z"}
{"log":"==1==    by 0x54B79F: in_tail_collect_event (tail.c:310)\n","stream":"stderr","time":"2022-07-19T06:35:13.055888358Z"}
{"log":"==1==    by 0x54CBCF: tail_fs_event (tail_fs_inotify.c:267)\n","stream":"stderr","time":"2022-07-19T06:35:13.055891369Z"}
{"log":"==1==  Block was alloc'd at\n","stream":"stderr","time":"2022-07-19T06:35:13.055894462Z"}
{"log":"==1==    at 0x4C2D065: malloc (vg_replace_malloc.c:381)\n","stream":"stderr","time":"2022-07-19T06:35:13.055897358Z"}
{"log":"==1==    by 0xA2ED76: msgpack_zone_new (zone.c:199)\n","stream":"stderr","time":"2022-07-19T06:35:13.055900093Z"}
{"log":"==1==    by 0xA2C77A: template_callback_array (unpack.c:208)\n","stream":"stderr","time":"2022-07-19T06:35:13.055903139Z"}
{"log":"==1==    by 0xA2CFD1: template_execute (unpack_template.h:231)\n","stream":"stderr","time":"2022-07-19T06:35:13.055905929Z"}
{"log":"==1==    by 0xA2E7D7: msgpack_unpack_next (unpack.c:677)\n","stream":"stderr","time":"2022-07-19T06:35:13.055908837Z"}
{"log":"==1==    by 0x60B8C3: cb_kube_filter (kubernetes.c:487)\n","stream":"stderr","time":"2022-07-19T06:35:13.055911465Z"}
{"log":"==1==    by 0x4E14B4: flb_filter_do (flb_filter.c:124)\n","stream":"stderr","time":"2022-07-19T06:35:13.055915039Z"}
{"log":"==1==    by 0x520C09: input_chunk_append_raw (flb_input_chunk.c:1478)\n","stream":"stderr","time":"2022-07-19T06:35:13.055917886Z"}
{"log":"==1==    by 0x520F34: flb_input_chunk_append_raw2 (flb_input_chunk.c:1600)\n","stream":"stderr","time":"2022-07-19T06:35:13.055923108Z"}
{"log":"==1==    by 0x551B95: process_content (tail_file.c:536)\n","stream":"stderr","time":"2022-07-19T06:35:13.055925973Z"}
{"log":"==1==    by 0x553EF5: flb_tail_file_chunk (tail_file.c:1315)\n","stream":"stderr","time":"2022-07-19T06:35:13.055928806Z"}
{"log":"==1==    by 0x54B79F: in_tail_collect_event (tail.c:310)\n","stream":"stderr","time":"2022-07-19T06:35:13.055944328Z"}
{"log":"==1== \n","stream":"stderr","time":"2022-07-19T06:35:13.055947224Z"}
{"log":"==1== Invalid read of size 4\n","stream":"stderr","time":"2022-07-19T06:35:13.144393339Z"}
{"log":"==1==    at 0x85082F: cio_chunk_is_locked (cio_chunk.c:375)\n","stream":"stderr","time":"2022-07-19T06:35:13.144453424Z"}
{"log":"==1==    by 0x5200AC: input_chunk_get (flb_input_chunk.c:1123)\n","stream":"stderr","time":"2022-07-19T06:35:13.144457459Z"}
{"log":"==1==    by 0x520922: input_chunk_append_raw (flb_input_chunk.c:1408)\n","stream":"stderr","time":"2022-07-19T06:35:13.144460659Z"}
{"log":"==1==    by 0x520F34: flb_input_chunk_append_raw2 (flb_input_chunk.c:1600)\n","stream":"stderr","time":"2022-07-19T06:35:13.144463794Z"}
{"log":"==1==    by 0x551B95: process_content (tail_file.c:536)\n","stream":"stderr","time":"2022-07-19T06:35:13.144466771Z"}
{"log":"==1==    by 0x553EF5: flb_tail_file_chunk (tail_file.c:1315)\n","stream":"stderr","time":"2022-07-19T06:35:13.1444699Z"}
{"log":"==1==    by 0x54B79F: in_tail_collect_event (tail.c:310)\n","stream":"stderr","time":"2022-07-19T06:35:13.144473148Z"}
{"log":"==1==    by 0x54CBCF: tail_fs_event (tail_fs_inotify.c:267)\n","stream":"stderr","time":"2022-07-19T06:35:13.14447631Z"}
{"log":"==1==    by 0x4E0D74: flb_input_collector_fd (flb_input.c:1210)\n","stream":"stderr","time":"2022-07-19T06:35:13.144479381Z"}
{"log":"==1==    by 0x4F64B8: flb_engine_handle_event (flb_engine.c:440)\n","stream":"stderr","time":"2022-07-19T06:35:13.144482721Z"}
{"log":"==1==    by 0x4F64B8: flb_engine_start (flb_engine.c:763)\n","stream":"stderr","time":"2022-07-19T06:35:13.144485806Z"}
{"log":"==1==    by 0x4D3A1D: flb_lib_worker (flb_lib.c:626)\n","stream":"stderr","time":"2022-07-19T06:35:13.144488989Z"}
{"log":"==1==    by 0x4E4444A: start_thread (in /usr/lib64/libpthread-2.26.so)\n","stream":"stderr","time":"2022-07-19T06:35:13.144492209Z"}
{"log":"==1==  Address 0x0 is not stack'd, malloc'd or (recently) free'd\n","stream":"stderr","time":"2022-07-19T06:35:13.144495562Z"}
{"log":"==1== \n","stream":"stderr","time":"2022-07-19T06:35:13.144498589Z"}
{"log":"[2022/07/19 06:35:13] [engine] caught signal (SIGSEGV)\n","stream":"stderr","time":"2022-07-19T06:35:13.146001201Z"}
{"log":"[2022/07/19 06:35:18] [debug] [upstream] KA connection #23 to kinesis.ap-south-1.amazonaws.com:443 has been disconnected by the remote service\n","stream":"stderr","time":"2022-07-19T06:35:18.949858473Z"}
{"log":"[2022/07/19 06:35:19] [debug] [socket] could not validate socket status for #23 (don't worry)\n","stream":"stderr","time":"2022-07-19T06:35:19.347963492Z"}
{"log":"#0  0x85082f            in  cio_chunk_is_locked() at lib/chunkio/src/cio_chunk.c:375\n","stream":"stderr","time":"2022-07-19T06:35:21.449837408Z"}
{"log":"#1  0x5200ac            in  input_chunk_get() at src/flb_input_chunk.c:1123\n","stream":"stderr","time":"2022-07-19T06:35:21.551075087Z"}
{"log":"#2  0x520922            in  input_chunk_append_raw() at src/flb_input_chunk.c:1408\n","stream":"stderr","time":"2022-07-19T06:35:21.642916468Z"}
{"log":"#3  0x520f34            in  flb_input_chunk_append_raw2() at src/flb_input_chunk.c:1600\n","stream":"stderr","time":"2022-07-19T06:35:21.642980033Z"}
{"log":"#4  0x551b95            in  process_content() at plugins/in_tail/tail_file.c:536\n","stream":"stderr","time":"2022-07-19T06:35:21.650688891Z"}
{"log":"#5  0x553ef5            in  flb_tail_file_chunk() at plugins/in_tail/tail_file.c:1315\n","stream":"stderr","time":"2022-07-19T06:35:21.650775559Z"}
{"log":"#6  0x54b79f            in  in_tail_collect_event() at plugins/in_tail/tail.c:310\n","stream":"stderr","time":"2022-07-19T06:35:21.745962807Z"}
{"log":"#7  0x54cbcf            in  tail_fs_event() at plugins/in_tail/tail_fs_inotify.c:267\n","stream":"stderr","time":"2022-07-19T06:35:21.750300756Z"}
{"log":"#8  0x4e0d74            in  flb_input_collector_fd() at src/flb_input.c:1210\n","stream":"stderr","time":"2022-07-19T06:35:21.851347422Z"}
{"log":"#9  0x4f64b8            in  flb_engine_handle_event() at src/flb_engine.c:440\n","stream":"stderr","time":"2022-07-19T06:35:21.946636774Z"}
{"log":"#10 0x4f64b8            in  flb_engine_start() at src/flb_engine.c:763\n","stream":"stderr","time":"2022-07-19T06:35:21.946839223Z"}
{"log":"#11 0x4d3a1d            in  flb_lib_worker() at src/flb_lib.c:626\n","stream":"stderr","time":"2022-07-19T06:35:21.951180436Z"}
{"log":"#12 0x4e4444a           in  ???() at ???:0\n","stream":"stderr","time":"2022-07-19T06:35:21.9522142Z"}
{"log":"#13 0x686c56e           in  ???() at ???:0\n","stream":"stderr","time":"2022-07-19T06:35:22.043030087Z"}
{"log":"#14 0xffffffffffffffff  in  ???() at ???:0\n","stream":"stderr","time":"2022-07-19T06:35:22.043334147Z"}
{"log":"==1== \n","stream":"stderr","time":"2022-07-19T06:35:22.045334382Z"}
{"log":"==1== Process terminating with default action of signal 6 (SIGABRT): dumping core\n","stream":"stderr","time":"2022-07-19T06:35:22.04534848Z"}
{"log":"==1==    at 0x67B0CA0: raise (in /usr/lib64/libc-2.26.so)\n","stream":"stderr","time":"2022-07-19T06:35:22.045451907Z"}
{"log":"==1==    by 0x67B2147: abort (in /usr/lib64/libc-2.26.so)\n","stream":"stderr","time":"2022-07-19T06:35:22.045458085Z"}
{"log":"==1==    by 0x455432: flb_signal_handler (fluent-bit.c:581)\n","stream":"stderr","time":"2022-07-19T06:35:22.045461622Z"}
{"log":"==1==    by 0x67B0D0F: ??? (in /usr/lib64/libc-2.26.so)\n","stream":"stderr","time":"2022-07-19T06:35:22.04546512Z"}
{"log":"==1==    by 0x85082E: cio_chunk_is_locked (cio_chunk.c:375)\n","stream":"stderr","time":"2022-07-19T06:35:22.045468677Z"}
{"log":"==1==    by 0x5200AC: input_chunk_get (flb_input_chunk.c:1123)\n","stream":"stderr","time":"2022-07-19T06:35:22.045472835Z"}
{"log":"==1==    by 0x520922: input_chunk_append_raw (flb_input_chunk.c:1408)\n","stream":"stderr","time":"2022-07-19T06:35:22.045476539Z"}
{"log":"==1==    by 0x520F34: flb_input_chunk_append_raw2 (flb_input_chunk.c:1600)\n","stream":"stderr","time":"2022-07-19T06:35:22.045480246Z"}
{"log":"==1==    by 0x551B95: process_content (tail_file.c:536)\n","stream":"stderr","time":"2022-07-19T06:35:22.04548396Z"}
{"log":"==1==    by 0x553EF5: flb_tail_file_chunk (tail_file.c:1315)\n","stream":"stderr","time":"2022-07-19T06:35:22.045487447Z"}
{"log":"==1==    by 0x54B79F: in_tail_collect_event (tail.c:310)\n","stream":"stderr","time":"2022-07-19T06:35:22.045495495Z"}
{"log":"==1==    by 0x54CBCF: tail_fs_event (tail_fs_inotify.c:267)\n","stream":"stderr","time":"2022-07-19T06:35:22.045499104Z"}
{"log":"==1== \n","stream":"stderr","time":"2022-07-19T06:35:23.145947435Z"}
{"log":"==1== HEAP SUMMARY:\n","stream":"stderr","time":"2022-07-19T06:35:23.147551496Z"}
{"log":"==1==     in use at exit: 14,800,062 bytes in 49,247 blocks\n","stream":"stderr","time":"2022-07-19T06:35:23.147563702Z"}
{"log":"==1==   total heap usage: 726,057 allocs, 676,810 frees, 3,613,374,251 bytes allocated\n","stream":"stderr","time":"2022-07-19T06:35:23.147567117Z"}
{"log":"==1== \n","stream":"stderr","time":"2022-07-19T06:35:23.147570439Z"}
{"log":"==1== Thread 1:\n","stream":"stderr","time":"2022-07-19T06:35:25.347256352Z"}
{"log":"==1== 1 bytes in 1 blocks are possibly lost in loss record 2 of 1,985\n","stream":"stderr","time":"2022-07-19T06:35:25.348750799Z"}

Configuration

apiVersion: v1
kind: ConfigMap
metadata:
  name: fluent-bit-config
  namespace: amazon-cloudwatch
  labels:
    k8s-app: fluent-bit-template
data:
  # Configuration files: server, input, filters and output
  # ======================================================
  uniform-time-format.lua: |
    function cb_print(tag, timestamp, record)
        record['time'] = string.format(
            '%s.%sZ',
            os.date('%Y-%m-%dT%H:%M:%S', timestamp['sec']),
            string.sub(string.format('%06d', timestamp['nsec']), 1, 6)
        )
        return 2, timestamp, record
    end
    
  fluent-bit.conf: |
    [SERVICE]
        Flush                       5
        Daemon                      off
        Log_level                   Debug
        Http_server                 On
        Http_listen                 0.0.0.0
        Http_port                   2022
        Parsers_File                parsers.conf
        storage.path                /var/fluent-bit/state/flb-storage/
        storage.sync                normal
        storage.checksum            off
        storage.backlog.mem_limit   5M
    
    [INPUT]
        Name                tail
        Tag                 kube.var.log.containers.*
        Exclude_Path        /var/log/containers/cloudwatch-agent*, /var/log/containers/fluent-bit*, /var/log/containers/aws-node*, /var/log/containers/kube-proxy*
        Path                /var/log/containers/test*default*
        Path_Key            file_name
        Parser              docker
        DB                  /var/fluent-bit/state/flb_container-5aa94922-a0e1-4aaa-9adc-1a3ef6957fb7.cc48429c-2035-471d-a7d4-e033187373b5.db
        DB.locking          True
        Docker_Mode         On
        
        Mem_Buf_Limit       50MB
        Refresh_Interval    10
        Rotate_Wait         30
        storage.type        filesystem
        Read_from_Head      True

    [OUTPUT]
        Name                kinesis_streams
        Match               *
        Region              us-east-1
        Stream              words
        Retry_Limit         False



    [FILTER]
        Name                parser
        Match               kube.var.log.containers.*
        Key_Name            log
        Parser              json_5aa94922-a0e1-4aaa-9adc-1a3ef6957fb7

    [FILTER]
        Name                kubernetes
        Match               kube.var.log.containers.*

        Kube_URL            https://kubernetes.default.svc:443
        Kube_CA_File        /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
        Kube_Token_File     /var/run/secrets/kubernetes.io/serviceaccount/token

        Merge_Log           On
        Merge_Log_Trim      On
        Merge_Log_Key       log_processed

        Buffer_Size         512k
        Use_Kubelet         True
        Kubelet_Port        10250

    [INPUT]
        Name                tail
        Tag                 kube.var.log.containers.*
        Exclude_Path        /var/log/containers/cloudwatch-agent*, /var/log/containers/fluent-bit*, /var/log/containers/aws-node*, /var/log/containers/kube-proxy*
        Path                /var/log/containers/*.log
        Path_Key            file_name
        Parser              docker
        DB                  /var/fluent-bit/state/flb_container-a9622cf2-c350-4796-ad90-7d74963ed565.2a1cd39a-b9b1-40ca-a10d-3d2a74849ce2.db
        DB.locking          True
        Docker_Mode         On
        Mem_Buf_Limit       50MB
        Refresh_Interval    10
        Rotate_Wait         30
        storage.type        filesystem
        Read_from_Head      True

    [OUTPUT]
        Name                kinesis_streams
        Match               *
        Region              us-west-2
        Stream              rayhan-kinesis-stream
        Retry_Limit         False



    [FILTER]
        Name                parser
        Match               kube.var.log.containers.*
        Key_Name            log
        Parser              json_a9622cf2-c350-4796-ad90-7d74963ed565

    [FILTER]
        Name                kubernetes
        Match               kube.var.log.containers.*
        Kube_URL            https://kubernetes.default.svc:443
        Kube_CA_File        /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
        Kube_Token_File     /var/run/secrets/kubernetes.io/serviceaccount/token

        Merge_Log           On
        Merge_Log_Trim      On
        Merge_Log_Key       log_processed

        Buffer_Size         512k
        Use_Kubelet         True
        Kubelet_Port        10250


    [FILTER]
        Name                modify
        Match               *
        Set                 cluster test

    [FILTER]
        Name                lua
        Match               *
        time_as_table       on
        script              uniform-time-format.lua
        call                cb_print
    

  parsers.conf: |
    [PARSER]
        Name   json
        Format json
        Time_Key time
        Time_Format %Y-%m-%dT%H:%M:%S.%LZ

    [PARSER]
        Name         docker
        Format       json
        Time_Key     container_log_time
        Time_Format  %Y-%m-%dT%H:%M:%S.%LZ
        Time_Keep    On

    [PARSER]
        Name        cri_regex
        Format      regex
        Regex       ^(?<container_log_time>[^ ]+) (?<stream>stdout|stderr) (?<logtag>[^ ]*) (?<message>.*)$      
        Time_Key    container_log_time
        Time_Format %Y-%m-%dT%H:%M:%S.%LZ
        Time_Keep    On        


    [PARSER]
        Name        json_5aa94922-a0e1-4aaa-9adc-1a3ef6957fb7
        Format      json
        
        Time_Key    time
        Time_Format ""

    [PARSER]
        Name        json_a9622cf2-c350-4796-ad90-7d74963ed565
        Format      json
        
        Time_Key    time
        Time_Format ""

Your Environment
Amazon EKS on Amazon EC2 on Amazon Linux 2

Additional context

[PettitWesley]

Report from my customer:

Using fluent/fluent-bit:1.9.6-debug to test, also reproduced the bug.

{"log":"[2022/07/20 05:23:41] [debug] [input chunk] update output instances with new chunk size diff=32768\n","stream":"stderr","time":"2022-07-20T05:23:41.186176801Z"}
{"log":"[2022/07/20 05:23:41] [debug] [input:tail:tail.1] inode=24121083 events: IN_MODIFY \n","stream":"stderr","time":"2022-07-20T05:23:41.186263446Z"}
{"log":"[2022/07/20 05:23:41] [engine] caught signal (SIGSEGV)\n","stream":"stderr","time":"2022-07-20T05:23:41.18642497Z"}
{"log":"#0  0x55a67bbdd674      in  cio_chunk_is_locked() at lib/chunkio/src/cio_chunk.c:375\n","stream":"stderr","time":"2022-07-20T05:23:42.383690531Z"}
{"log":"#1  0x55a67b8c5e96      in  input_chunk_get() at src/flb_input_chunk.c:1123\n","stream":"stderr","time":"2022-07-20T05:23:42.383985888Z"}
{"log":"#2  0x55a67b8c670c      in  input_chunk_append_raw() at src/flb_input_chunk.c:1408\n","stream":"stderr","time":"2022-07-20T05:23:42.383993308Z"}
{"log":"#3  0x55a67b8c6d40      in  flb_input_chunk_append_raw2() at src/flb_input_chunk.c:1600\n","stream":"stderr","time":"2022-07-20T05:23:42.38399691Z"}
{"log":"#4  0x55a67b8f14b0      in  process_content() at plugins/in_tail/tail_file.c:536\n","stream":"stderr","time":"2022-07-20T05:23:42.384339208Z"}
{"log":"#5  0x55a67b8f37e3      in  flb_tail_file_chunk() at plugins/in_tail/tail_file.c:1315\n","stream":"stderr","time":"2022-07-20T05:23:42.384345433Z"}
{"log":"#6  0x55a67b8eb8d3      in  in_tail_collect_event() at plugins/in_tail/tail.c:310\n","stream":"stderr","time":"2022-07-20T05:23:42.384492928Z"}
{"log":"#7  0x55a67b8ecae2      in  tail_fs_event() at plugins/in_tail/tail_fs_inotify.c:267\n","stream":"stderr","time":"2022-07-20T05:23:42.384616998Z"}
{"log":"#8  0x55a67b88a260      in  flb_input_collector_fd() at src/flb_input.c:1210\n","stream":"stderr","time":"2022-07-20T05:23:42.384830105Z"}
{"log":"#9  0x55a67b89e332      in  flb_engine_handle_event() at src/flb_engine.c:440\n","stream":"stderr","time":"2022-07-20T05:23:42.385036139Z"}
{"log":"#10 0x55a67b89e332      in  flb_engine_start() at src/flb_engine.c:763\n","stream":"stderr","time":"2022-07-20T05:23:42.385041399Z"}
{"log":"#11 0x55a67b87daa4      in  flb_lib_worker() at src/flb_lib.c:626\n","stream":"stderr","time":"2022-07-20T05:23:42.385207933Z"}
{"log":"#12 0x7f84d7df1ea6      in  start_thread() at reate.c:477\n","stream":"stderr","time":"2022-07-20T05:23:42.386078182Z"}
{"log":"#13 0x7f84d76d0dee      in  ???() at sysv/linux/x86_64/clone.S:95\n","stream":"stderr","time":"2022-07-20T05:23:42.386086777Z"}
{"log":"#14 0xffffffffffffffff  in  ???() at ???:0\n","stream":"stderr","time":"2022-07-20T05:23:42.38609041Z"}

The following content is using fluent/fluent-bit:1.9.6 to test:

{"log":"[2022/07/20 05:15:43] [debug] [input:tail:tail.0] scanning path /var/log/containers/saicappsaibotgatewayidentityserver*default*\n","stream":"stderr","time":"2022-07-20T05:15:43.983466984Z"}
{"log":"[2022/07/20 05:15:43] [debug] [input:tail:tail.0] scan_blog add(): dismissed: /var/log/containers/saicappsaibotgatewayidentityserver-6596dc44d9-9t99g_default_saicappsaibotgatewayidentityserver-8ac181cc0f5612ec9f3fe3445215a0fdfff5b42eb20826e771ce3e86da44bb2a.log, inode 24121106\n","stream":"stderr","time":"2022-07-20T05:15:43.983489172Z"}
{"log":"[2022/07/20 05:15:43] [debug] [input:tail:tail.0] 0 new files found on path '/var/log/containers/saicappsaibotgatewayidentityserver*default*'\n","stream":"stderr","time":"2022-07-20T05:15:43.983492447Z"}
{"log":"[2022/07/20 05:15:43] [debug] [input:tail:tail.1] scanning path /var/log/containers/saicappsaibotgatewayhttpapihost*default*\n","stream":"stderr","time":"2022-07-20T05:15:43.983494829Z"}
{"log":"[2022/07/20 05:15:43] [debug] [input:tail:tail.1] scan_blog add(): dismissed: /var/log/containers/saicappsaibotgatewayhttpapihost-784cf7bc56-4qk8z_default_saicappsaibotgatewayhttpapihost-f3752e7b7eff45466ccc9723407dab2efb6bf2db8cc262a3e056ecd80c04c543.log, inode 24121083\n","stream":"stderr","time":"2022-07-20T05:15:43.983497297Z"}
{"log":"[2022/07/20 05:15:43] [debug] [input:tail:tail.1] 0 new files found on path '/var/log/containers/saicappsaibotgatewayhttpapihost*default*'\n","stream":"stderr","time":"2022-07-20T05:15:43.983500044Z"}
{"log":"[2022/07/20 05:15:50] [debug] [input:tail:tail.0] inode=24121106 events: IN_MODIFY \n","stream":"stderr","time":"2022-07-20T05:15:50.330128635Z"}
{"log":"[2022/07/20 05:15:50] [engine] caught signal (SIGSEGV)\n","stream":"stderr","time":"2022-07-20T05:15:50.330171919Z"}
{"log":"#0  0x555d77031674      in  cio_chunk_is_locked() at lib/chunkio/src/cio_chunk.c:375\n","stream":"stderr","time":"2022-07-20T05:15:50.488559252Z"}
{"log":"#1  0x555d76d19e96      in  input_chunk_get() at src/flb_input_chunk.c:1123\n","stream":"stderr","time":"2022-07-20T05:15:50.488788086Z"}
{"log":"#2  0x555d76d1a70c      in  input_chunk_append_raw() at src/flb_input_chunk.c:1408\n","stream":"stderr","time":"2022-07-20T05:15:50.488821309Z"}
{"log":"#3  0x555d76d1ad40      in  flb_input_chunk_append_raw2() at src/flb_input_chunk.c:1600\n","stream":"stderr","time":"2022-07-20T05:15:50.488825435Z"}
{"log":"#4  0x555d76d454b0      in  process_content() at plugins/in_tail/tail_file.c:536\n","stream":"stderr","time":"2022-07-20T05:15:50.489123194Z"}
{"log":"#5  0x555d76d477e3      in  flb_tail_file_chunk() at plugins/in_tail/tail_file.c:1315\n","stream":"stderr","time":"2022-07-20T05:15:50.489128962Z"}
{"log":"#6  0x555d76d3f8d3      in  in_tail_collect_event() at plugins/in_tail/tail.c:310\n","stream":"stderr","time":"2022-07-20T05:15:50.489257097Z"}
{"log":"#7  0x555d76d40ae2      in  tail_fs_event() at plugins/in_tail/tail_fs_inotify.c:267\n","stream":"stderr","time":"2022-07-20T05:15:50.489369215Z"}
{"log":"#8  0x555d76cde260      in  flb_input_collector_fd() at src/flb_input.c:1210\n","stream":"stderr","time":"2022-07-20T05:15:50.489603773Z"}
{"log":"#9  0x555d76cf2332      in  flb_engine_handle_event() at src/flb_engine.c:440\n","stream":"stderr","time":"2022-07-20T05:15:50.489809312Z"}
{"log":"#10 0x555d76cf2332      in  flb_engine_start() at src/flb_engine.c:763\n","stream":"stderr","time":"2022-07-20T05:15:50.489814998Z"}
{"log":"#11 0x555d76cd1aa4      in  flb_lib_worker() at src/flb_lib.c:626\n","stream":"stderr","time":"2022-07-20T05:15:50.490057025Z"}
{"log":"#12 0x7ff06ac83ea6      in  ???() at ???:0\n","stream":"stderr","time":"2022-07-20T05:15:50.490064162Z"}
{"log":"#13 0x7ff06a562dee      in  ???() at ???:0\n","stream":"stderr","time":"2022-07-20T05:15:50.490067299Z"}
{"log":"#14 0xffffffffffffffff  in  ???() at ???:0\n","stream":"stderr","time":"2022-07-20T05:15:50.490070659Z"}

[ptsneves]

Can you upload the coredump? I would like to see where exactly the 0x0 is passed [as an argument]. That can hint at the issue. Also please edit the traces so it is not in json format but in plain text :)

[PettitWesley]

@ptsneves We have not been able to get a core dump yet, will post it if we get one.

[PettitWesley]

I did get a core dump, but its a huge, half a GB, and when I try to read it, I don't get much:

Core was generated by `/fluent-bit/bin/fluent-bit -c /fluent-bit/etc/fluent-bit.conf'.
Program terminated with signal SIGABRT, Aborted.
#0  0x00007f1c81662ca0 in __gettextparse () from /lib64/libc.so.6
[Current thread is 1 (Thread 0x7f1c703ff700 (LWP 13))]
Missing separate debuginfos, use: debuginfo-install bzip2-libs-1.0.6-13.amzn2.0.3.x86_64 cyrus-sasl-lib-2.1.26-24.amzn2.x86_64 elfutils-libelf-0.176-2.amzn2.x86_64 elfutils-libs-0.176-2.amzn2.x86_64 glibc-2.26-58.amzn2.x86_64 keyutils-libs-1.5.8-3.amzn2.0.2.x86_64 krb5-libs-1.15.1-37.amzn2.2.4.x86_64 libcap-2.54-1.amzn2.0.1.x86_64 libcom_err-1.42.9-19.amzn2.x86_64 libcrypt-2.26-58.amzn2.x86_64 libgcc-7.3.1-15.amzn2.x86_64 libgcrypt-1.5.3-14.amzn2.0.3.x86_64 libgpg-error-1.12-3.amzn2.0.3.x86_64 libselinux-2.5-12.amzn2.0.2.x86_64 lz4-1.7.5-2.amzn2.0.1.x86_64 pcre-8.32-17.amzn2.0.2.x86_64 systemd-libs-219-78.amzn2.0.18.x86_64 xz-libs-5.2.2-1.amzn2.0.3.x86_64 zlib-1.2.7-19.amzn2.0.1.x86_64
(gdb) bt
#0  0x00007f1c81662ca0 in __gettextparse () from /lib64/libc.so.6
#1  0x0000000000000000 in ?? ()
(gdb) list
1203    /tmp/fluent-bit-1.9.4/src/fluent-bit.c: No such file or directory.

@ptsneves Do you happen to know if there's likely reason for this? It is just because I need to try installing the separate debug info for each of these packages?

[PettitWesley]

Now that I installed all of the debug infos, I get:

warning: Could not load shared library symbols for 2 libraries, e.g. /lib64/libssl.so.1.1.
Use the "info sharedlibrary" command to see the complete listing.
Do you need "set solib-search-path" or "set sysroot"?
[Thread debugging using libthread_db enabled]
Using host libthread_db library "/lib64/libthread_db.so.1".
Core was generated by `/fluent-bit/bin/fluent-bit -c /fluent-bit/etc/fluent-bit.conf'.
Program terminated with signal SIGABRT, Aborted.
#0  0x00007f1c81662ca0 in __gettextlex (arg=0x0, lval=<synthetic pointer>) at plural.y:373
373   arg->cp = exp;
[Current thread is 1 (Thread 0x7f1c703ff700 (LWP 13))]
(gdb) bt
#0  0x00007f1c81662ca0 in __gettextlex (arg=0x0, lval=<synthetic pointer>) at plural.y:373
#1  __gettextparse (arg=0x0) at plural.c:1424
#2  0x0000000000000000 in ?? ()

Which is still not something I know how what to do with...

[ptsneves]

@PettitWesley You must have not installed the fluent-bit debug symbols as the only symbols are gettextlex. I believe if you provide them we can get closer. It is a clear null pointer derreference.

[PettitWesley]

Alrighty so we installed all the debuginfos and then got it to pick up the raw fluent bit code and dumped the stack for all threads. Unfortunately I am still struggling to understand what's going on here:

<http://www.gnu.org/software/gdb/documentation/>.
For help, type "help".
Type "apropos word" to search for commands related to "word"...
Reading symbols from fluent-bit-bin...done.
 
warning: core file may not match specified executable file.
[New LWP 13]
[New LWP 12]
[New LWP 14]
[New LWP 9]
[New LWP 15]
[New LWP 11]
[New LWP 16]
[New LWP 10]
[Thread debugging using libthread_db enabled]
Using host libthread_db library "/lib64/libthread_db.so.1".
Core was generated by `/fluent-bit/bin/fluent-bit -c /fluent-bit/etc/fluent-bit.conf'.
Program terminated with signal SIGABRT, Aborted.
#0  __GI_raise (sig=sig@entry=6) at ../sysdeps/unix/sysv/linux/raise.c:51
51	}
[Current thread is 1 (Thread 0x7f1c703ff700 (LWP 13))]
(gdb) thread apply all bt full
 
Thread 8 (Thread 0x7f1c7e5ff700 (LWP 10)):
#0  0x00007f1c8171e88c in epoll_wait (epfd=8, events=0x7f1c7d601000, maxevents=256, timeout=-1) at ../sysdeps/unix/sysv/linux/epoll_wait.c:30
        resultvar = 18446744073709551612
        sc_cancel_oldtype = 0
#1  0x0000000000a47b72 in _mk_event_wait_2 (loop=0x7f1c7d60a000, timeout=-1) at /tmp/fluent-bit-1.9.4/lib/monkey/mk_core/mk_event_epoll.c:404
        ctx = 0x7f1c7d600000
        ret = 0
#2  0x0000000000a47f0e in mk_event_wait (loop=0x7f1c7d60a000) at /tmp/fluent-bit-1.9.4/lib/monkey/mk_core/mk_event.c:184
No locals.
#3  0x00000000004f6375 in flb_engine_start (config=0x7f1c7ea19f00) at /tmp/fluent-bit-1.9.4/src/flb_engine.c:760
        __flb_event_priority_live_foreach_iter = 1
        __flb_event_priority_live_foreach_n_events = 0
        ret = 0
        ts = 1658818390012293448
        tmp = "24.0K\000\000\000\000\000\000\000\000\000\000"
        t_flush = {tm = {tv_sec = 5, tv_nsec = 0}}
        event = 0x0
        evl = 0x7f1c7d60a000
        evl_bktq = 0x7f1c7d613000
        sched = 0x7f1c7d6553a0
        dns_ctx = {lookups = {prev = 0x7f1c7e5f4a70, next = 0x7f1c7e5f4a70}, lookups_drop = {prev = 0x7f1c7e5f4a80, next = 0x7f1c7e5f4a80}}
#4  0x00000000004d3a1e in flb_lib_worker (data=0x7f1c7ea18000) at /tmp/fluent-bit-1.9.4/src/flb_lib.c:626
        ret = 0
        ctx = 0x7f1c7ea18000
        config = 0x7f1c7ea19f00
#5  0x00007f1c8310544b in start_thread (arg=0x7f1c7e5ff700) at pthread_create.c:465
        pd = 0x7f1c7e5ff700
        now = <optimized out>
        unwind_buf = {cancel_jmp_buf = {{jmp_buf = {139760356030208, -1178532097430040805, 140720376096638, 140720376096639, 139760360325120,
                140720376096800, 1269452091937083163, 1268895163278511899}, mask_was_saved = 0}}, priv = {pad = {0x0, 0x0, 0x0, 0x0}, data = {
              prev = 0x0, cleanup = 0x0, canceltype = 0}}}
        not_first_call = <optimized out>
#6  0x00007f1c8171e56f in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:95
---Type <return> to continue, or q <return> to quit---
No locals.
 
Thread 7 (Thread 0x7f1c6d9fe700 (LWP 16)):
#0  0x00007f1c8171e88c in epoll_wait (epfd=193, events=0x7f1c6ca0c000, maxevents=256, timeout=-1) at ../sysdeps/unix/sysv/linux/epoll_wait.c:30
        resultvar = 18446744073709551612
        sc_cancel_oldtype = 0
#1  0x0000000000a47b72 in _mk_event_wait_2 (loop=0x7f1c6ca15000, timeout=-1) at /tmp/fluent-bit-1.9.4/lib/monkey/mk_core/mk_event_epoll.c:404
        ctx = 0x7f1c6ca00040
        ret = 0
#2  0x0000000000a47f0e in mk_event_wait (loop=0x7f1c6ca15000) at /tmp/fluent-bit-1.9.4/lib/monkey/mk_core/mk_event.c:184
No locals.
#3  0x0000000000a411e2 in mk_server_worker_loop (server=0x7f1c714b1200) at /tmp/fluent-bit-1.9.4/lib/monkey/mk_server/mk_server.c:506
        __i = 1
        __ctx = 0x7f1c6ca00040
        ret = 0
        timeout_fd = 199
        val = 4008636142
        event = 0x0
        evl = 0x7f1c6ca15000
        list = 0x7f1c6ca00050
        head = 0x7f1c6ca00050
        conn = 0x7f1c6d9f3ac0
        sched = 0x7f1c6e401000
        listener = 0x7f1c6ca0b080
        server_timeout = 0x7f1c6ca1e050
        __i = 0
        __ctx = 0x7f1c6ca00040
#4  0x0000000000a37ae9 in mk_sched_launch_worker_loop (data=0x7f1c6e400008) at /tmp/fluent-bit-1.9.4/lib/monkey/mk_server/mk_scheduler.c:417
        ret = 0
        wid = 0
        len = 13
        thread_name = 0x7f1c6ca23000 "\001"
        head = 0x7f1c714b13e8
        wcb = 0x7f1c714c8e00
---Type <return> to continue, or q <return> to quit---
        sched = 0x7f1c6e401000
        notif = 0x7f1c6ca1e000
        thinfo = 0x7f1c6e400008
        ctx = 0x7f1c6e400000
        server = 0x7f1c714b1200
#5  0x00007f1c8310544b in start_thread (arg=0x7f1c6d9fe700) at pthread_create.c:465
        pd = 0x7f1c6d9fe700
        now = <optimized out>
        unwind_buf = {cancel_jmp_buf = {{jmp_buf = {139760075007744, -1178532097430040805, 139760102230478, 139760102230479, 139760085499912,
                139760102230624, 1269408659617173275, 1268895163278511899}, mask_was_saved = 0}}, priv = {pad = {0x0, 0x0, 0x0, 0x0}, data = {
              prev = 0x0, cleanup = 0x0, canceltype = 0}}}
        not_first_call = <optimized out>
#6  0x00007f1c8171e56f in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:95
No locals.
 
Thread 6 (Thread 0x7f1c7d5ff700 (LWP 11)):
#0  0x00007f1c8171e88c in epoll_wait (epfd=11, events=0x7f1c7d61c000, maxevents=32, timeout=-1) at ../sysdeps/unix/sysv/linux/epoll_wait.c:30
        resultvar = 18446744073709551612
        sc_cancel_oldtype = 0
#1  0x0000000000a47b72 in _mk_event_wait_2 (loop=0x7f1c7d60a018, timeout=-1) at /tmp/fluent-bit-1.9.4/lib/monkey/mk_core/mk_event_epoll.c:404
        ctx = 0x7f1c7d600010
        ret = 0
#2  0x0000000000a47f0e in mk_event_wait (loop=0x7f1c7d60a018) at /tmp/fluent-bit-1.9.4/lib/monkey/mk_core/mk_event.c:184
No locals.
#3  0x00000000004d41cb in log_worker_collector (data=0x7f1c7d615000) at /tmp/fluent-bit-1.9.4/src/flb_log.c:131
        __i = 1
        __ctx = 0x7f1c7d600010
        run = 1
        event = 0x0
        log = 0x7f1c7d615000
#4  0x00000000004ff6e0 in step_callback (data=0x7f1c7d6290c0) at /tmp/fluent-bit-1.9.4/src/flb_worker.c:43
        worker = 0x7f1c7d6290c0
#5  0x00007f1c8310544b in start_thread (arg=0x7f1c7d5ff700) at pthread_create.c:465
        pd = 0x7f1c7d5ff700
---Type <return> to continue, or q <return> to quit---
        now = <optimized out>
        unwind_buf = {cancel_jmp_buf = {{jmp_buf = {139760339252992, -1178532097430040805, 139760355985566, 139760355985567, 139760339423424,
                139760355985728, 1269445494867316507, 1268895163278511899}, mask_was_saved = 0}}, priv = {pad = {0x0, 0x0, 0x0, 0x0}, data = {
              prev = 0x0, cleanup = 0x0, canceltype = 0}}}
        not_first_call = <optimized out>
#6  0x00007f1c8171e56f in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:95
No locals.
 
Thread 5 (Thread 0x7f1c6e3ff700 (LWP 15)):
#0  0x00007f1c816eec2b in __GI___nanosleep (requested_time=requested_time@entry=0x7f1c6e3f4b10, remaining=remaining@entry=0x7f1c6e3f4b10)
    at ../sysdeps/unix/sysv/linux/nanosleep.c:27
        resultvar = 18446744073709551100
        sc_cancel_oldtype = 2
#1  0x00007f1c816eeb4a in __sleep (seconds=0) at ../sysdeps/posix/sleep.c:55
        save_errno = 0
        ts = {tv_sec = 0, tv_nsec = 298820888}
#2  0x0000000000a3fe42 in mk_clock_worker_init (data=0x7f1c714b1200) at /tmp/fluent-bit-1.9.4/lib/monkey/mk_server/mk_clock.c:133
        cur_time = 1658818389
        server = 0x7f1c714b1200
#3  0x00007f1c8310544b in start_thread (arg=0x7f1c6e3ff700) at pthread_create.c:465
        pd = 0x7f1c6e3ff700
        now = <optimized out>
        unwind_buf = {cancel_jmp_buf = {{jmp_buf = {139760085497600, -1178532097430040805, 139760102230526, 139760102230527, 139760136557056,
                139760102230688, 1269416082931273499, 1268895163278511899}, mask_was_saved = 0}}, priv = {pad = {0x0, 0x0, 0x0, 0x0}, data = {
              prev = 0x0, cleanup = 0x0, canceltype = 0}}}
        not_first_call = <optimized out>
#4  0x00007f1c8171e56f in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:95
No locals.
 
Thread 4 (Thread 0x7f1c8352aac0 (LWP 9)):
#0  0x00007f1c816eec2b in __GI___nanosleep (requested_time=requested_time@entry=0x7ffc0407a890, remaining=remaining@entry=0x7ffc0407a890)
    at ../sysdeps/unix/sysv/linux/nanosleep.c:27
        resultvar = 18446744073709551100
        sc_cancel_oldtype = 0
---Type <return> to continue, or q <return> to quit---
#1  0x00007f1c816eeb4a in __sleep (seconds=0) at ../sysdeps/posix/sleep.c:55
        save_errno = 25
        ts = {tv_sec = 0, tv_nsec = 298961207}
#2  0x00000000004566e4 in flb_main (argc=3, argv=0x7ffc0407aa38) at /tmp/fluent-bit-1.9.4/src/fluent-bit.c:1198
        opt = -1
        ret = 0
        json = 0x1 <error: Cannot access memory at address 0x1>
        last_plugin = -1
        cfg_file = 0x7f1c7ea18300 "\005"
        cf = 0x7f1c7ea1f000
        tmp = 0x7f1c7ea1f000
        service = 0x7f1c7ea22000
        s = 0x7f1c80944060
        long_opts = {{name = 0xa4d908 "storage_path", has_arg = 1, flag = 0x0, val = 98}, {name = 0xa4d915 "config", has_arg = 1, flag = 0x0,
            val = 99}, {name = 0xa4d79f "daemon", has_arg = 0, flag = 0x0, val = 100}, {name = 0xa4d91c "dry-run", has_arg = 0, flag = 0x0,
            val = 68}, {name = 0xa4d7a6 "flush", has_arg = 1, flag = 0x0, val = 102}, {name = 0xa4d924 "http", has_arg = 0, flag = 0x0, val = 72},
          {name = 0xa4d7b2 "log_file", has_arg = 1, flag = 0x0, val = 108}, {name = 0xa4d929 "port", has_arg = 1, flag = 0x0, val = 80}, {
            name = 0xa4d5b6 "custom", has_arg = 1, flag = 0x0, val = 67}, {name = 0xa4d5bd "input", has_arg = 1, flag = 0x0, val = 105}, {
            name = 0xa4d7ac "match", has_arg = 1, flag = 0x0, val = 109}, {name = 0xa4d5ca "output", has_arg = 1, flag = 0x0, val = 111}, {
            name = 0xa4d5c3 "filter", has_arg = 1, flag = 0x0, val = 70}, {name = 0xa4d6e4 "parser", has_arg = 1, flag = 0x0, val = 82}, {
            name = 0xa4d92e "prop", has_arg = 1, flag = 0x0, val = 112}, {name = 0xa4d933 "plugin", has_arg = 1, flag = 0x0, val = 101}, {
            name = 0xa4d7bb "tag", has_arg = 1, flag = 0x0, val = 116}, {name = 0xa4d93a "sp-task", has_arg = 1, flag = 0x0, val = 84}, {
            name = 0xa4d942 "version", has_arg = 0, flag = 0x0, val = 86}, {name = 0xa4d94a "verbose", has_arg = 0, flag = 0x0, val = 118}, {
            name = 0xa4d952 "workdir", has_arg = 1, flag = 0x0, val = 119}, {name = 0xa4d95a "quiet", has_arg = 0, flag = 0x0, val = 113}, {
            name = 0xa4d960 "help", has_arg = 0, flag = 0x0, val = 104}, {name = 0xa4d965 "help-json", has_arg = 0, flag = 0x0, val = 74}, {
            name = 0xa4d96f "coro_stack_size", has_arg = 1, flag = 0x0, val = 115}, {name = 0xa4d97f "sosreport", has_arg = 0, flag = 0x0,
            val = 83}, {name = 0xa4d7bf "http_server", has_arg = 0, flag = 0x0, val = 72}, {name = 0xa4d989 "http_listen", has_arg = 1,
            flag = 0x0, val = 76}, {name = 0xa4d995 "http_port", has_arg = 1, flag = 0x0, val = 80}, {name = 0x0, has_arg = 0, flag = 0x0,
            val = 0}}
#3  0x000000000045676b in main (argc=3, argv=0x7ffc0407aa38) at /tmp/fluent-bit-1.9.4/src/fluent-bit.c:1217
No locals.
 
Thread 3 (Thread 0x7f1c6f3ff700 (LWP 14)):
#0  0x00007f1c8171e88c in epoll_wait (epfd=190, events=0x7f1c714b6060, maxevents=8, timeout=-1) at ../sysdeps/unix/sysv/linux/epoll_wait.c:30
---Type <return> to continue, or q <return> to quit---
        resultvar = 18446744073709551612
        sc_cancel_oldtype = 0
#1  0x0000000000a47b72 in _mk_event_wait_2 (loop=0x7f1c714d6988, timeout=-1) at /tmp/fluent-bit-1.9.4/lib/monkey/mk_core/mk_event_epoll.c:404
        ctx = 0x7f1c714eb6e0
        ret = 0
#2  0x0000000000a47f0e in mk_event_wait (loop=0x7f1c714d6988) at /tmp/fluent-bit-1.9.4/lib/monkey/mk_core/mk_event.c:184
No locals.
#3  0x0000000000a2f683 in mk_lib_worker (data=0x7f1c714d69a0) at /tmp/fluent-bit-1.9.4/lib/monkey/mk_server/mk_lib.c:154
        fd = 0
        bytes = 0
        val = 0
        server = 0x7f1c714b1200
        event = 0x0
        ctx = 0x7f1c714d69a0
        __i = 0
        __ctx = 0x0
#4  0x00007f1c8310544b in start_thread (arg=0x7f1c6f3ff700) at pthread_create.c:465
        pd = 0x7f1c6f3ff700
        now = <optimized out>
        unwind_buf = {cancel_jmp_buf = {{jmp_buf = {139760102274816, -1178532097430040805, 139760355985614, 139760355985615, 139760136710560,
                139760355985776, 1269413883908017947, 1268895163278511899}, mask_was_saved = 0}}, priv = {pad = {0x0, 0x0, 0x0, 0x0}, data = {
              prev = 0x0, cleanup = 0x0, canceltype = 0}}}
        not_first_call = <optimized out>
#5  0x00007f1c8171e56f in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:95
No locals.
 
Thread 2 (Thread 0x7f1c712ba700 (LWP 12)):
#0  mbedtls_internal_sha256_process (ctx=0x7f1c746bc730,
    data=0x7f1c72937840 "2FnZVRlbXBsYXRlXCI6XCJFeGVjdXRpbmcgYW4gaW1wbGljaXQgaGFuZGxlciBtZXRob2QgLSBNb2RlbFN0YXRlIGlzIHtWYWxpZGF0aW9uU3RhdGV9XCIsXCJQcm9wZXJ0aWVzXCI6e1wiVmFsaWRhdGlvblN0YXRlXCI6XCJWYWxpZFwiLFwiRXZlbnRJZFwiOntcI"...) at /tmp/fluent-bit-1.9.4/lib/mbedtls-2.28.0/library/sha256.c:209
        local = {temp1 = 0, temp2 = 0, W = {843476570, 1448242274, 1480749913, 1481796696, 1128871512, 1128941157, 1196845668, 1481797730,
            1835231065, 1463052129, 1462859618, 1198287457, 1481729889, 1195799898, 1199074403, 0 <repeats 49 times>}, A = {66496397, 4232882364,
            3734377519, 3847249253, 2457485059, 322807692, 3663508526, 3133988016}}
        i = 15
---Type <return> to continue, or q <return> to quit---
#1  0x00000000007ef104 in mbedtls_sha256_update_ret (ctx=0x7f1c746bc730,
    input=0x7f1c72937840 "2FnZVRlbXBsYXRlXCI6XCJFeGVjdXRpbmcgYW4gaW1wbGljaXQgaGFuZGxlciBtZXRob2QgLSBNb2RlbFN0YXRlIGlzIHtWYWxpZGF0aW9uU3RhdGV9XCIsXCJQcm9wZXJ0aWVzXCI6e1wiVmFsaWRhdGlvblN0YXRlXCI6XCJWYWxpZFwiLFwiRXZlbnRJZFwiOntcI"..., ilen=475618)
    at /tmp/fluent-bit-1.9.4/lib/mbedtls-2.28.0/library/sha256.c:310
        ret = 0
        fill = 64
        left = 0
#2  0x00000000007ef17e in mbedtls_sha256_update (ctx=0x7f1c746bc730,
    input=0x7f1c728aa4c0 "{\"StreamName\":\"words\",\"Records\":[{\"Data\":\"eyJsb2dfcHJvY2Vzc2VkIjp7IlByb3BlcnRpZXMiOnsiRXZlbnRJZCI6eyJOYW1lIjoiRXhlY3V0ZWRJbXBsaWNpdEhhbmRsZXJNZXRob2QiLCJJZCI6MTA0fSwiQWN0aW9uUmVzdWx0IjoiTWljcm9zb2Z0Lk"..., ilen=1054050)
    at /tmp/fluent-bit-1.9.4/lib/mbedtls-2.28.0/library/sha256.c:328
No locals.
#3  0x000000000063bc0f in flb_signv4_canonical_request (c=0x7f1c705a1ea0, normalize_uri=1, amz_date_header=1,
    amzdate=0x7f1c746bc8f0 "20220726T065310Z",
    security_token=0x7f1c76840610 "IQoJb3JpZ2luX2VjEG4aCXVzLWVhc3QtMiJGMEQCIA55RUHhRpIS7YMALrg41V4x42+qbdyEOANoNuT98ugIAiBg6ygq4qZmEPfYVgb1ONz2NEt1DuAg5CmLanTRj9byKyrcBAi3//////////8BEAMaDDE0NDcxODcxMTQ3MCIM8nP0i3jZCTz86qVnKrAEO0Qoi5h+"..., s3_mode=0, signed_headers=0x7f1c746bc8c8)
    at /tmp/fluent-bit-1.9.4/src/flb_signv4.c:752
        i = 32540
        len = 1
        items = 0
        post_params = 0
        size = 577
        val = 0x7f1c767994e0 "application/x-amz-json-1.1"
        arr = 0x7f1c746bc8a8
        cr = 0x7f1c705c0b10 "POST\n/\n\n"
        uri = 0x7f1c7510e208 "/"
        tmp = 0x7f1c705c0b10 "POST\n/\n\n"
        params = 0x0
        payload_hash = 0x0
        kv = 0x4db7ed <flb_malloc+38>
        list_tmp = {prev = 0x3000000030, next = 0x7f1c712f1e10}
        head = 0x7f1c746bc940
        sha256_buf = '\000' <repeats 63 times>
        sha256_ctx = {total = {1054050, 0}, state = {66496397, 4232882364, 3734377519, 3847249253, 2457485059, 322807692, 3663508526, 3133988016},
---Type <return> to continue, or q <return> to quit---
          buffer = '\000' <repeats 63 times>, is224 = 0}
#4  0x000000000063cc44 in flb_signv4_do (c=0x7f1c705a1ea0, normalize_uri=1, amz_date_header=1, t_now=1658818390,
    region=0x7f1c7ea7d810 "us-east-1", service=0xa7d94c "kinesis", s3_mode=0, provider=0x7f1c7135b040)
    at /tmp/fluent-bit-1.9.4/src/flb_signv4.c:1148
        amzdate = "20220726T065310Z\000\246\245\000\000\000\000\000\033\000\000\000\000\000\000"
        datestamp = "20220726\000\036Zp\034\177\000\000\000\311kt\001\000\000\000\340\320wv\034\177\000"
        gmt = 0x7f1c70416df8
        cr = 0x7f1c705a1ea0 "\300\320Yp\034\177"
        string_to_sign = 0xa7df80 "X-Amz-Target"
        signature = 0xc <error: Cannot access memory at address 0xc>
        signed_headers = 0x7f1c768575d0 ""
        auth_header = 0xa7df8d "Kinesis_20131202.PutRecords"
        creds = 0x7f1c7510ef78
#5  0x0000000000830486 in request_do (aws_client=0x7f1c7d6a6ac0, method=1, uri=0xa7e9ff "/",
    body=0x7f1c728aa4c0 "{\"StreamName\":\"words\",\"Records\":[{\"Data\":\"eyJsb2dfcHJvY2Vzc2VkIjp7IlByb3BlcnRpZXMiOnsiRXZlbnRJZCI6eyJOYW1lIjoiRXhlY3V0ZWRJbXBsaWNpdEhhbmRsZXJNZXRob2QiLCJJZCI6MTA0fSwiQWN0aW9uUmVzdWx0IjoiTWljcm9zb2Z0Lk"..., body_len=1054050,
    dynamic_headers=0xe74740 <put_records_target_header>, dynamic_headers_len=1) at /tmp/fluent-bit-1.9.4/src/aws/flb_aws_util.c:397
        b_sent = 10820789
        ret = 0
        u_conn = 0x7f1c7059d0c0
        signature = 0x0
        i = 1
        normalize_uri = 1
        header = {key = 0xa7df80 "X-Amz-Target", key_len = 12, val = 0xa7df8d "Kinesis_20131202.PutRecords", val_len = 27}
        c = 0x7f1c705a1ea0
        tmp = 0x7f1c00000000 <error: Cannot access memory at address 0x7f1c00000000>
        user_agent_prefix = 0x7f1c71386c40 "\242"
#6  0x000000000082fa02 in flb_aws_client_request (aws_client=0x7f1c7d6a6ac0, method=1, uri=0xa7e9ff "/",
    body=0x7f1c728aa4c0 "{\"StreamName\":\"words\",\"Records\":[{\"Data\":\"eyJsb2dfcHJvY2Vzc2VkIjp7IlByb3BlcnRpZXMiOnsiRXZlbnRJZCI6eyJOYW1lIjoiRXhlY3V0ZWRJbXBsaWNpdEhhbmRsZXJNZXRob2QiLCJJZCI6MTA0fSwiQWN0aW9uUmVzdWx0IjoiTWljcm9zb2Z0Lk"..., body_len=1054050,
    dynamic_headers=0xe74740 <put_records_target_header>, dynamic_headers_len=1) at /tmp/fluent-bit-1.9.4/src/aws/flb_aws_util.c:160
        c = 0x0
#7  0x00000000005ef041 in put_records (ctx=0x7f1c7d670680, buf=0x7f1c7041c150, payload_size=1054050, num_records=500)
    at /tmp/fluent-bit-1.9.4/plugins/out_kinesis_streams/kinesis_api.c:893
---Type <return> to continue, or q <return> to quit---
        c = 0x0
        kinesis_client = 0x7f1c7d6a6ac0
        error = 0x7f1c746bcaf0 "@\313kt\034\177"
        failed_records = 0
#8  0x00000000005ed926 in send_log_events (ctx=0x7f1c7d670680, buf=0x7f1c7041c150)
    at /tmp/fluent-bit-1.9.4/plugins/out_kinesis_streams/kinesis_api.c:441
        ret = 0
        offset = 1054050
        i = 500
        event = 0x7f1c767a77e0
#9  0x00000000005edcbe in add_event (ctx=0x7f1c7d670680, buf=0x7f1c7041c150, obj=0x7f1c746bcc40, tms=0x7f1c746bcbd0)
    at /tmp/fluent-bit-1.9.4/plugins/out_kinesis_streams/kinesis_api.c:516
        ret = 0
        event = 0x7f1c767a77e0
        retry_add = 0
        event_bytes = 2400
#10 0x00000000005ee037 in process_and_send_to_kinesis (ctx=0x7f1c7d670680, buf=0x7f1c7041c150,
    data=0x7f1c7b62510a <error: Cannot access memory at address 0x7f1c7b62510a>, bytes=2059220)
    at /tmp/fluent-bit-1.9.4/plugins/out_kinesis_streams/kinesis_api.c:616
        off = 700420
        i = 499
        map_size = 6
        result = {zone = 0x7f1c7041a5f8, data = {type = MSGPACK_OBJECT_ARRAY, via = {boolean = 2, u64 = 139758235811842, i64 = 139758235811842,
              f64 = 6.9049743038010949e-310, array = {size = 2, ptr = 0x7f1c7042f808}, map = {size = 2, ptr = 0x7f1c7042f808}, str = {size = 2,
                ptr = 0x7f1c7042f808 "\t"}, bin = {size = 2, ptr = 0x7f1c7042f808 "\t"}, ext = {type = 2 '\002', size = 32540,
                ptr = 0x7f1c7042f808 "\t"}}}}
        obj = 0x7f1c7042f820
        map = {type = MSGPACK_OBJECT_MAP, via = {boolean = 6, u64 = 6, i64 = 6, f64 = 2.9643938750474793e-323, array = {size = 6,
              ptr = 0x7f1c7042f838}, map = {size = 6, ptr = 0x7f1c7042f838}, str = {size = 6, ptr = 0x7f1c7042f838 "\005"}, bin = {size = 6,
              ptr = 0x7f1c7042f838 "\005"}, ext = {type = 6 '\006', size = 0, ptr = 0x7f1c7042f838 "\005"}}}
        root = {type = MSGPACK_OBJECT_ARRAY, via = {boolean = 2, u64 = 139758235811842, i64 = 139758235811842, f64 = 6.9049743038010949e-310,
            array = {size = 2, ptr = 0x7f1c7042f808}, map = {size = 2, ptr = 0x7f1c7042f808}, str = {size = 2, ptr = 0x7f1c7042f808 "\t"}, bin = {
              size = 2, ptr = 0x7f1c7042f808 "\t"}, ext = {type = 2 '\002', size = 32540, ptr = 0x7f1c7042f808 "\t"}}}
        kv = 0x5eaca3 <flb_malloc+38>
---Type <return> to continue, or q <return> to quit---
        key = {type = 72, via = {boolean = 196, u64 = 4576196, i64 = 4576196, f64 = 2.2609412322361291e-317, array = {size = 4576196,
              ptr = 0x7f1c712b04a0}, map = {size = 4576196, ptr = 0x7f1c712b04a0}, str = {size = 4576196, ptr = 0x7f1c712b04a0 "\360Sl\025"},
            bin = {size = 4576196, ptr = 0x7f1c712b04a0 "\360Sl\025"}, ext = {type = -60 '\304', size = 0, ptr = 0x7f1c712b04a0 "\360Sl\025"}}}
        val = {type = 72, via = {boolean = false, u64 = 5242880, i64 = 5242880, f64 = 2.5903268932681547e-317, array = {size = 5242880,
              ptr = 0x500000 <msgpack_pack_ext_body+24>}, map = {size = 5242880, ptr = 0x500000 <msgpack_pack_ext_body+24>}, str = {
              size = 5242880,
              ptr = 0x500000 <msgpack_pack_ext_body+24> "H\213@\bH\213U\370H\213\nH\213U\350H\213u\360H\211\317\377\320\311\303UH\211\345H\203\354 \211}\354dH\213\004%"}, bin = {size = 5242880,
              ptr = 0x500000 <msgpack_pack_ext_body+24> "H\213@\bH\213U\370H\213\nH\213U\350H\213u\360H\211\317\377\320\311\303UH\211\345H\203\354 \211}\354dH\213\004%"}, ext = {type = 0 '\000', size = 0,
              ptr = 0x500000 <msgpack_pack_ext_body+24> "H\213@\bH\213U\370H\213\nH\213U\350H\213u\360H\211\317\377\320\311\303UH\211\345H\203\354 \211}\354dH\213\004%"}}}
        key_str = 0x0
        key_str_size = 0
        j = 32540
        ret = 0
        check = 0
        found = 0
        tms = {tm = {tv_sec = 1658817989, tv_nsec = 569724276}}
#11 0x00000000005ec0be in cb_kinesis_flush (event_chunk=0x7f1c714ca848, out_flush=0x7f1c705c6500, i_ins=0x7f1c7d643000,
    out_context=0x7f1c7d670680, config=0x7f1c7ea19f00) at /tmp/fluent-bit-1.9.4/plugins/out_kinesis_streams/kinesis.c:336
        ctx = 0x7f1c7d670680
        ret = -1
        buf = 0x7f1c7041c150
#12 0x00000000004e60f4 in output_pre_cb_flush () at /tmp/fluent-bit-1.9.4/include/fluent-bit/flb_output.h:517
        coro = 0x7f1c76795500
        out_p = 0x7f1c7ea52060
        params = 0x7f1c70415000
        persisted_params = {event_chunk = 0x7f1c714ca848, out_flush = 0x7f1c705c6500, i_ins = 0x7f1c7d643000, out_plugin = 0x7f1c7ea52060,
          out_context = 0x7f1c7d670680, config = 0x7f1c7ea19f00, coro = 0x7f1c76795500}
#13 0x0000000000a4a1a7 in co_init () at /tmp/fluent-bit-1.9.4/lib/monkey/deps/flb_libco/amd64.c:117
No locals.
#14 0x5a6a4d7463444d74 in ?? ()
No symbol table info available.
---Type <return> to continue, or q <return> to quit---
#15 0x4d544e3659444d55 in ?? ()
No symbol table info available.
#16 0x557a4e7551444d36 in ?? ()
No symbol table info available.

and here is the other core dump:

Reading symbols from fluent-bit-bin...done.
 
warning: core file may not match specified executable file.
[New LWP 12]
[New LWP 8]
[New LWP 11]
[New LWP 9]
[New LWP 13]
[New LWP 14]
[New LWP 15]
[New LWP 10]
[Thread debugging using libthread_db enabled]
Using host libthread_db library "/lib64/libthread_db.so.1".
Core was generated by `/fluent-bit/bin/fluent-bit -c /fluent-bit/etc/fluent-bit.conf'.
Program terminated with signal SIGABRT, Aborted.
#0  __GI_raise (sig=sig@entry=6) at ../sysdeps/unix/sysv/linux/raise.c:51
51	}
[Current thread is 1 (Thread 0x7fcb60fff700 (LWP 12))]
(gdb) bt
#0  __GI_raise (sig=sig@entry=6) at ../sysdeps/unix/sysv/linux/raise.c:51
#1  0x00007fcb7453e148 in __GI_abort () at abort.c:79
#2  0x0000000000455433 in flb_signal_handler (signal=11) at /tmp/fluent-bit-1.9.4/src/fluent-bit.c:581
#3  <signal handler called>
#4  0x0000000000a2ccc6 in template_execute (ctx=0x7fcb60012f60, data=0x7fcb7080610a "", len=2057756, off=0x7fcb60013780) at /tmp/fluent-bit-1.9.4/lib/msgpack-c/include/msgpack/unpack_template.h:172
#5  0x0000000000a2e7d8 in msgpack_unpack_next (result=0x7fcb60013860, data=0x7fcb7080610a "", len=2057756, off=0x7fcb60013880) at /tmp/fluent-bit-1.9.4/lib/msgpack-c/src/unpack.c:677
#6  0x00000000005ee066 in process_and_send_to_kinesis (ctx=0x7fcb704a1b60, buf=0x7fcb6001c000, data=0x7fcb7080610a "", bytes=2057756) at /tmp/fluent-bit-1.9.4/plugins/out_kinesis_streams/kinesis_api.c:556
#7  0x00000000005ec0be in cb_kinesis_flush (event_chunk=0x7fcb62041708, out_flush=0x7fcb6000b000, i_ins=0x7fcb7180a280, out_context=0x7fcb704a1b60, config=0x7fcb71819340) at /tmp/fluent-bit-1.9.4/plugins/out_kinesis_streams/kinesis.c:336
#8  0x00000000004e60f4 in output_pre_cb_flush () at /tmp/fluent-bit-1.9.4/include/fluent-bit/flb_output.h:517
#9  0x0000000000a4a1a7 in co_init () at /tmp/fluent-bit-1.9.4/lib/monkey/deps/flb_libco/amd64.c:117
#10 0x0000000000000000 in ?? ()
(gdb) list
46	  int ret = INLINE_SYSCALL (tgkill, 3, pid, tid, sig);
47
48	  __libc_signal_restore_set (&set);
49
50	  return ret;
51	}
52	libc_hidden_def (raise)
53	weak_alias (raise, gsignal)
(gdb) thread apply all bt full
 
Thread 8 (Thread 0x7fcb703ff700 (LWP 10)):
#0  0x00007fcb745f888c in epoll_wait (epfd=11, events=0x7fcb7041c000, maxevents=32, timeout=-1) at ../sysdeps/unix/sysv/linux/epoll_wait.c:30
        resultvar = 18446744073709551612
        sc_cancel_oldtype = 0
#1  0x0000000000a47b72 in _mk_event_wait_2 (loop=0x7fcb7040a018, timeout=-1) at /tmp/fluent-bit-1.9.4/lib/monkey/mk_core/mk_event_epoll.c:404
        ctx = 0x7fcb70400010
        ret = 0
#2  0x0000000000a47f0e in mk_event_wait (loop=0x7fcb7040a018) at /tmp/fluent-bit-1.9.4/lib/monkey/mk_core/mk_event.c:184
No locals.
#3  0x00000000004d41cb in log_worker_collector (data=0x7fcb70415000) at /tmp/fluent-bit-1.9.4/src/flb_log.c:131
        __i = 1
        __ctx = 0x7fcb70400010
        run = 1
        event = 0x0
        log = 0x7fcb70415000
#4  0x00000000004ff6e0 in step_callback (data=0x7fcb704290c0) at /tmp/fluent-bit-1.9.4/src/flb_worker.c:43
        worker = 0x7fcb704290c0
#5  0x00007fcb75fdf44b in start_thread (arg=0x7fcb703ff700) at pthread_create.c:465
        pd = 0x7fcb703ff700
        now = <optimized out>
        unwind_buf = {cancel_jmp_buf = {{jmp_buf = {140511738328832, -7268922882397343170, 140511755061406, 140511755061407, 140511738499264, 140511755061568,
                7244454894633972286, 7244447771109585470}, mask_was_saved = 0}}, priv = {pad = {0x0, 0x0, 0x0, 0x0}, data = {prev = 0x0, cleanup = 0x0,
              canceltype = 0}}}
        not_first_call = <optimized out>
#6  0x00007fcb745f856f in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:95
No locals.
 
Thread 7 (Thread 0x7fcb5e5fe700 (LWP 15)):
#0  0x00007fcb745f888c in epoll_wait (epfd=211, events=0x7fcb5d60c000, maxevents=256, timeout=-1) at ../sysdeps/unix/sysv/linux/epoll_wait.c:30
        resultvar = 18446744073709551612
        sc_cancel_oldtype = 0
#1  0x0000000000a47b72 in _mk_event_wait_2 (loop=0x7fcb5d615000, timeout=-1) at /tmp/fluent-bit-1.9.4/lib/monkey/mk_core/mk_event_epoll.c:404
        ctx = 0x7fcb5d600040
        ret = 0
#2  0x0000000000a47f0e in mk_event_wait (loop=0x7fcb5d615000) at /tmp/fluent-bit-1.9.4/lib/monkey/mk_core/mk_event.c:184
No locals.
---Type <return> to continue, or q <return> to quit---
#3  0x0000000000a411e2 in mk_server_worker_loop (server=0x7fcb6202b200) at /tmp/fluent-bit-1.9.4/lib/monkey/mk_server/mk_server.c:506
        __i = 1
        __ctx = 0x7fcb5d600040
        ret = 0
        timeout_fd = 217
        val = 4008636142
        event = 0x0
        evl = 0x7fcb5d615000
        list = 0x7fcb5d600050
        head = 0x7fcb5d600050
        conn = 0x7fcb5e5f3ac0
        sched = 0x7fcb5f001000
        listener = 0x7fcb5d60b080
        server_timeout = 0x7fcb5d61e050
        __i = 0
        __ctx = 0x7fcb5d600040
#4  0x0000000000a37ae9 in mk_sched_launch_worker_loop (data=0x7fcb5f000008) at /tmp/fluent-bit-1.9.4/lib/monkey/mk_server/mk_scheduler.c:417
        ret = 0
        wid = 0
        len = 13
        thread_name = 0x7fcb5d623000 "\001"
        head = 0x7fcb6202b3e8
        wcb = 0x7fcb62059100
        sched = 0x7fcb5f001000
        notif = 0x7fcb5d61e000
        thinfo = 0x7fcb5f000008
        ctx = 0x7fcb5f000000
        server = 0x7fcb6202b200
#5  0x00007fcb75fdf44b in start_thread (arg=0x7fcb5e5fe700) at pthread_create.c:465
        pd = 0x7fcb5e5fe700
        now = <optimized out>
        unwind_buf = {cancel_jmp_buf = {{jmp_buf = {140511438432000, -7268922882397343170, 140511465654734, 140511465654735, 140511448924168, 140511465654880,
                7244362814293242430, 7244447771109585470}, mask_was_saved = 0}}, priv = {pad = {0x0, 0x0, 0x0, 0x0}, data = {prev = 0x0, cleanup = 0x0,
              canceltype = 0}}}
        not_first_call = <optimized out>
#6  0x00007fcb745f856f in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:95
No locals.
---Type <return> to continue, or q <return> to quit---
 
Thread 6 (Thread 0x7fcb5efff700 (LWP 14)):
#0  0x00007fcb745c8c2b in __GI___nanosleep (requested_time=requested_time@entry=0x7fcb5eff4b10, remaining=remaining@entry=0x7fcb5eff4b10)
    at ../sysdeps/unix/sysv/linux/nanosleep.c:27
        resultvar = 18446744073709551100
        sc_cancel_oldtype = 2
#1  0x00007fcb745c8b4a in __sleep (seconds=0) at ../sysdeps/posix/sleep.c:55
        save_errno = 0
        ts = {tv_sec = 0, tv_nsec = 346195906}
#2  0x0000000000a3fe42 in mk_clock_worker_init (data=0x7fcb6202b200) at /tmp/fluent-bit-1.9.4/lib/monkey/mk_server/mk_clock.c:133
        cur_time = 1658818696
        server = 0x7fcb6202b200
#3  0x00007fcb75fdf44b in start_thread (arg=0x7fcb5efff700) at pthread_create.c:465
        pd = 0x7fcb5efff700
        now = <optimized out>
        unwind_buf = {cancel_jmp_buf = {{jmp_buf = {140511448921856, -7268922882397343170, 140511465654782, 140511465654783, 140511499432448, 140511465654944,
                7244361985901425214, 7244447771109585470}, mask_was_saved = 0}}, priv = {pad = {0x0, 0x0, 0x0, 0x0}, data = {prev = 0x0, cleanup = 0x0,
              canceltype = 0}}}
        not_first_call = <optimized out>
#4  0x00007fcb745f856f in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:95
No locals.
 
Thread 5 (Thread 0x7fcb5ffff700 (LWP 13)):
#0  0x00007fcb745f888c in epoll_wait (epfd=208, events=0x7fcb61fcc260, maxevents=8, timeout=-1) at ../sysdeps/unix/sysv/linux/epoll_wait.c:30
        resultvar = 18446744073709551612
        sc_cancel_oldtype = 0
#1  0x0000000000a47b72 in _mk_event_wait_2 (loop=0x7fcb6204a988, timeout=-1) at /tmp/fluent-bit-1.9.4/lib/monkey/mk_core/mk_event_epoll.c:404
        ctx = 0x7fcb620586e0
        ret = 0
#2  0x0000000000a47f0e in mk_event_wait (loop=0x7fcb6204a988) at /tmp/fluent-bit-1.9.4/lib/monkey/mk_core/mk_event.c:184
No locals.
#3  0x0000000000a2f683 in mk_lib_worker (data=0x7fcb6204a9a0) at /tmp/fluent-bit-1.9.4/lib/monkey/mk_server/mk_lib.c:154
        fd = 0
        bytes = 0
        val = 0
        server = 0x7fcb6202b200
        event = 0x0
---Type <return> to continue, or q <return> to quit---
        ctx = 0x7fcb6204a9a0
        __i = 0
        __ctx = 0x0
#4  0x00007fcb75fdf44b in start_thread (arg=0x7fcb5ffff700) at pthread_create.c:465
        pd = 0x7fcb5ffff700
        now = <optimized out>
        unwind_buf = {cancel_jmp_buf = {{jmp_buf = {140511465699072, -7268922882397343170, 140511755061454, 140511755061455, 140511499561376, 140511755061616,
                7244364184924680766, 7244447771109585470}, mask_was_saved = 0}}, priv = {pad = {0x0, 0x0, 0x0, 0x0}, data = {prev = 0x0, cleanup = 0x0,
              canceltype = 0}}}
        not_first_call = <optimized out>
#5  0x00007fcb745f856f in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:95
No locals.
 
Thread 4 (Thread 0x7fcb713ff700 (LWP 9)):
#0  0x00007fcb745f888c in epoll_wait (epfd=8, events=0x7fcb70401000, maxevents=256, timeout=-1) at ../sysdeps/unix/sysv/linux/epoll_wait.c:30
        resultvar = 18446744073709551612
        sc_cancel_oldtype = 0
#1  0x0000000000a47b72 in _mk_event_wait_2 (loop=0x7fcb7040a000, timeout=-1) at /tmp/fluent-bit-1.9.4/lib/monkey/mk_core/mk_event_epoll.c:404
        ctx = 0x7fcb70400000
        ret = 0
#2  0x0000000000a47f0e in mk_event_wait (loop=0x7fcb7040a000) at /tmp/fluent-bit-1.9.4/lib/monkey/mk_core/mk_event.c:184
No locals.
#3  0x00000000004f6375 in flb_engine_start (config=0x7fcb71819340) at /tmp/fluent-bit-1.9.4/src/flb_engine.c:760
        __flb_event_priority_live_foreach_iter = 1
        __flb_event_priority_live_foreach_n_events = 0
        ret = 0
        ts = 1658818697028235355
        tmp = "24.0K\000\000\000\000\000\000\000\000\000\000"
        t_flush = {tm = {tv_sec = 5, tv_nsec = 0}}
        event = 0x0
        evl = 0x7fcb7040a000
        evl_bktq = 0x7fcb70413000
        sched = 0x7fcb704565a0
        dns_ctx = {lookups = {prev = 0x7fcb713f4a70, next = 0x7fcb713f4a70}, lookups_drop = {prev = 0x7fcb713f4a80, next = 0x7fcb713f4a80}}
#4  0x00000000004d3a1e in flb_lib_worker (data=0x7fcb71818000) at /tmp/fluent-bit-1.9.4/src/flb_lib.c:626
        ret = 0
        ctx = 0x7fcb71818000
---Type <return> to continue, or q <return> to quit---
        config = 0x7fcb71819340
#5  0x00007fcb75fdf44b in start_thread (arg=0x7fcb713ff700) at pthread_create.c:465
        pd = 0x7fcb713ff700
        now = <optimized out>
        unwind_buf = {cancel_jmp_buf = {{jmp_buf = {140511755106048, -7268922882397343170, 140732979375166, 140732979375167, 140511759400960, 140732979375328,
                7244457093657227838, 7244447771109585470}, mask_was_saved = 0}}, priv = {pad = {0x0, 0x0, 0x0, 0x0}, data = {prev = 0x0, cleanup = 0x0,
              canceltype = 0}}}
        not_first_call = <optimized out>
#6  0x00007fcb745f856f in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:95
No locals.
 
Thread 3 (Thread 0x7fcb61e27700 (LWP 11)):
#0  0x00000000004f2871 in flb_utils_write_str (
    buf=0x7fcb63fe02cc "{\"log_processed\":{\"Properties\":{\"ConnectionId\":\"0HMJ594C2UQEH\",\"CorrelationId\":\"61a6927548c54a6293e19efc61bc8488\",\"ActionResult\":\"\",\"RequestId\":\"0HMJ594C2UQEH:00000002\",\"EventId\":{\"Id\":102,\"Name\":\"Exe"..., off=0x7fcb611a7428, size=4907123,
    str=0x7fcb6e48d1f4 <error: Cannot access memory at address 0x7fcb6e48d1f4>, str_len=64) at /tmp/fluent-bit-1.9.4/src/flb_utils.c:666
        i = 62
        b = 1677591244
        ret = 0
        written = 62
        required = 64
        len = 10
        hex_bytes = 0
        is_valid = 1629123624
        utf_sequence_number = 0
        utf_sequence_length = 4907123
        codepoint = 0
        state = 0
        tmp = '\000' <repeats 12 times>, "\n\000\000"
        available = 4905684
        c = 98
        p = 0x7fcb63fe08a9 "i9JbmRleCIsIlJlcXVlc3RJZCI6IjBITUo1OTRDMlVRQjI6MDAwMDAwMDIiLCJTb3VyY2VDb250ZXh0IjoiTWljcm9zb2Z0LkFzcE5ldENvcmUuUm91dGluZy5FbmRwb2ludE1pZGRsZXdhcmUiLCJDb3JyZWxhdGlvbklkIjoiODY3MDYwMjJhNjY4NDEzZTlkOWU2M"...
        s = 0x7fcb6e48ce4b <error: Cannot access memory at address 0x7fcb6e48ce4b>
#1  0x00000000004da283 in msgpack2json (
    buf=0x7fcb63fe02cc "{\"log_processed\":{\"Properties\":{\"ConnectionId\":\"0HMJ594C2UQEH\",\"CorrelationId\":\"61a6927548c54a6293e19efc61bc8488\",\"ActionResult\":\"\",\"RequestId\":\"0HMJ594C2UQEH:00000002\",\"EventId\":{\"Id\":102,\"Name\":\"Exe"..., off=0x7fcb611a7428, left=4907123, o=0x7fcb61039d00)
---Type <return> to continue, or q <return> to quit---
    at /tmp/fluent-bit-1.9.4/src/flb_pack.c:590
        i = 32715
        dup = 0
        ret = 0
        loop = 1629123624
        packed = 32715
        __FUNCTION__ = "msgpack2json"
#2  0x00000000004da7e7 in msgpack2json (
    buf=0x7fcb63fe02cc "{\"log_processed\":{\"Properties\":{\"ConnectionId\":\"0HMJ594C2UQEH\",\"CorrelationId\":\"61a6927548c54a6293e19efc61bc8488\",\"ActionResult\":\"\",\"RequestId\":\"0HMJ594C2UQEH:00000002\",\"EventId\":{\"Id\":102,\"Name\":\"Exe"..., off=0x7fcb611a7428, left=4907123, o=0x7fcb61039940)
    at /tmp/fluent-bit-1.9.4/src/flb_pack.c:680
        k = {type = MSGPACK_OBJECT_STR, via = {boolean = 9, u64 = 9, i64 = 9, f64 = 4.4465908125712189e-323, array = {size = 9, ptr = 0x7fcb6e48d1e9}, map = {
              size = 9, ptr = 0x7fcb6e48d1e9}, str = {size = 9, ptr = 0x7fcb6e48d1e9 <error: Cannot access memory at address 0x7fcb6e48d1e9>}, bin = {size = 9,
              ptr = 0x7fcb6e48d1e9 <error: Cannot access memory at address 0x7fcb6e48d1e9>}, ext = {type = 9 '\t', size = 0,
              ptr = 0x7fcb6e48d1e9 <error: Cannot access memory at address 0x7fcb6e48d1e9>}}}
        p = 0x7fcb61039c58
        i = 3
        dup = 0
        ret = 0
        loop = 4
        packed = 3
        __FUNCTION__ = "msgpack2json"
#3  0x00000000004da7e7 in msgpack2json (
    buf=0x7fcb63fe02cc "{\"log_processed\":{\"Properties\":{\"ConnectionId\":\"0HMJ594C2UQEH\",\"CorrelationId\":\"61a6927548c54a6293e19efc61bc8488\",\"ActionResult\":\"\",\"RequestId\":\"0HMJ594C2UQEH:00000002\",\"EventId\":{\"Id\":102,\"Name\":\"Exe"..., off=0x7fcb611a7428, left=4907123, o=0x7fcb611a7600)
    at /tmp/fluent-bit-1.9.4/src/flb_pack.c:680
        k = {type = MSGPACK_OBJECT_STR, via = {boolean = 10, u64 = 10, i64 = 10, f64 = 4.9406564584124654e-323, array = {size = 10, ptr = 0x7fcb6e48d16d},
            map = {size = 10, ptr = 0x7fcb6e48d16d}, str = {size = 10, ptr = 0x7fcb6e48d16d <error: Cannot access memory at address 0x7fcb6e48d16d>}, bin = {
              size = 10, ptr = 0x7fcb6e48d16d <error: Cannot access memory at address 0x7fcb6e48d16d>}, ext = {type = 10 '\n', size = 0,
              ptr = 0x7fcb6e48d16d <error: Cannot access memory at address 0x7fcb6e48d16d>}}}
        p = 0x7fcb61039838
        i = 5
        dup = 0
        ret = 0
        loop = 6
        packed = 5
        __FUNCTION__ = "msgpack2json"
---Type <return> to continue, or q <return> to quit---
#4  0x00000000004da8f2 in flb_msgpack_to_json (
    json_str=0x7fcb63fe02cc "{\"log_processed\":{\"Properties\":{\"ConnectionId\":\"0HMJ594C2UQEH\",\"CorrelationId\":\"61a6927548c54a6293e19efc61bc8488\",\"ActionResult\":\"\",\"RequestId\":\"0HMJ594C2UQEH:00000002\",\"EventId\":{\"Id\":102,\"Name\":\"Exe"..., json_size=4907124, obj=0x7fcb611a7600)
    at /tmp/fluent-bit-1.9.4/src/flb_pack.c:717
        ret = -1
        off = 1439
#5  0x00000000005eccb1 in process_event (ctx=0x7fcb70470680, buf=0x7fcb6101c310, obj=0x7fcb611a7600, tms=0x7fcb611a7590)
    at /tmp/fluent-bit-1.9.4/plugins/out_kinesis_streams/kinesis_api.c:230
        written = 0
        ret = 32715
        size = 2423
        b64_len = 2152
        event = 0x7fcb61050600
        tmp_buf_ptr = 0x7fcb63fe02cc "{\"log_processed\":{\"Properties\":{\"ConnectionId\":\"0HMJ594C2UQEH\",\"CorrelationId\":\"61a6927548c54a6293e19efc61bc8488\",\"ActionResult\":\"\",\"RequestId\":\"0HMJ594C2UQEH:00000002\",\"EventId\":{\"Id\":102,\"Name\":\"Exe"...
        time_key_ptr = 0x0
        time_stamp = {tm_sec = 1629123879, tm_min = 32715, tm_hour = 64, tm_mday = 0, tm_mon = 1629123983, tm_year = 32715, tm_wday = 4824680, tm_yday = 0,
          tm_isdst = 4096, tm_gmtoff = 64, tm_zone = 0x0}
        tmp = 0x1000
        len = 4612485
        tmp_size = 4907664
#6  0x00000000005eda2c in add_event (ctx=0x7fcb70470680, buf=0x7fcb6101c310, obj=0x7fcb611a7600, tms=0x7fcb611a7590)
    at /tmp/fluent-bit-1.9.4/plugins/out_kinesis_streams/kinesis_api.c:469
        ret = 32715
        event = 0x7fcb611a7620
        retry_add = 0
        event_bytes = 0
#7  0x00000000005ee037 in process_and_send_to_kinesis (ctx=0x7fcb70470680, buf=0x7fcb6101c310,
    data=0x7fcb6e45510a <error: Cannot access memory at address 0x7fcb6e45510a>, bytes=2059220)
    at /tmp/fluent-bit-1.9.4/plugins/out_kinesis_streams/kinesis_api.c:616
        off = 229674
        i = 163
        map_size = 6
        result = {zone = 0x7fcb6101b388, data = {type = MSGPACK_OBJECT_ARRAY, via = {boolean = 2, u64 = 140509855088642, i64 = 140509855088642,
              f64 = 6.9421092301429872e-310, array = {size = 2, ptr = 0x7fcb61039808}, map = {size = 2, ptr = 0x7fcb61039808}, str = {size = 2,
                ptr = 0x7fcb61039808 "\t"}, bin = {size = 2, ptr = 0x7fcb61039808 "\t"}, ext = {type = 2 '\002', size = 32715, ptr = 0x7fcb61039808 "\t"}}}}
        obj = 0x7fcb61039820
---Type <return> to continue, or q <return> to quit---
        map = {type = MSGPACK_OBJECT_MAP, via = {boolean = 6, u64 = 6, i64 = 6, f64 = 2.9643938750474793e-323, array = {size = 6, ptr = 0x7fcb61039838}, map = {
              size = 6, ptr = 0x7fcb61039838}, str = {size = 6, ptr = 0x7fcb61039838 "\005"}, bin = {size = 6, ptr = 0x7fcb61039838 "\005"}, ext = {
              type = 6 '\006', size = 0, ptr = 0x7fcb61039838 "\005"}}}
        root = {type = MSGPACK_OBJECT_ARRAY, via = {boolean = 2, u64 = 140509855088642, i64 = 140509855088642, f64 = 6.9421092301429872e-310, array = {size = 2,
              ptr = 0x7fcb61039808}, map = {size = 2, ptr = 0x7fcb61039808}, str = {size = 2, ptr = 0x7fcb61039808 "\t"}, bin = {size = 2,
              ptr = 0x7fcb61039808 "\t"}, ext = {type = 2 '\002', size = 32715, ptr = 0x7fcb61039808 "\t"}}}
        kv = 0x5eaca3 <flb_malloc+38>
        key = {type = 72, via = {boolean = 196, u64 = 4576196, i64 = 4576196, f64 = 2.2609412322361291e-317, array = {size = 4576196, ptr = 0x7fcb61e1d560},
            map = {size = 4576196, ptr = 0x7fcb61e1d560}, str = {size = 4576196, ptr = 0x7fcb61e1d560 "\001"}, bin = {size = 4576196,
              ptr = 0x7fcb61e1d560 "\001"}, ext = {type = -60 '\304', size = 0, ptr = 0x7fcb61e1d560 "\001"}}}
        val = {type = 72, via = {boolean = false, u64 = 5242880, i64 = 5242880, f64 = 2.5903268932681547e-317, array = {size = 5242880,
              ptr = 0x500000 <msgpack_pack_ext_body+24>}, map = {size = 5242880, ptr = 0x500000 <msgpack_pack_ext_body+24>}, str = {size = 5242880,
              ptr = 0x500000 <msgpack_pack_ext_body+24> "H\213@\bH\213U\370H\213\nH\213U\350H\213u\360H\211\317\377\320\311\303UH\211\345H\203\354 \211}\354dH\213\004%"}, bin = {size = 5242880,
              ptr = 0x500000 <msgpack_pack_ext_body+24> "H\213@\bH\213U\370H\213\nH\213U\350H\213u\360H\211\317\377\320\311\303UH\211\345H\203\354 \211}\354dH\213\004%"}, ext = {type = 0 '\000', size = 0,
              ptr = 0x500000 <msgpack_pack_ext_body+24> "H\213@\bH\213U\370H\213\nH\213U\350H\213u\360H\211\317\377\320\311\303UH\211\345H\203\354 \211}\354dH\213\004%"}}}
        key_str = 0x0
        key_str_size = 0
        j = 32715
        ret = 0
        check = 0
        found = 0
        tms = {tm = {tv_sec = 1658817989, tv_nsec = 503870779}}
#8  0x00000000005ec0be in cb_kinesis_flush (event_chunk=0x7fcb7042d040, out_flush=0x7fcb6100b200, i_ins=0x7fcb70436000, out_context=0x7fcb70470680,
    config=0x7fcb71819340) at /tmp/fluent-bit-1.9.4/plugins/out_kinesis_streams/kinesis.c:336
        ctx = 0x7fcb70470680
        ret = -1
        buf = 0x7fcb6101c310
#9  0x00000000004e60f4 in output_pre_cb_flush () at /tmp/fluent-bit-1.9.4/include/fluent-bit/flb_output.h:517
        coro = 0x7fcb6100c200
        out_p = 0x7fcb71852060
        params = 0x7fcb61015000
        persisted_params = {event_chunk = 0x7fcb7042d040, out_flush = 0x7fcb6100b200, i_ins = 0x7fcb70436000, out_plugin = 0x7fcb71852060,
          out_context = 0x7fcb70470680, config = 0x7fcb71819340, coro = 0x7fcb6100c200}
#10 0x0000000000a4a1a7 in co_init () at /tmp/fluent-bit-1.9.4/lib/monkey/deps/flb_libco/amd64.c:117
---Type <return> to continue, or q <return> to quit---
No locals.
#11 0x00007fcb611a7780 in ?? ()
No symbol table info available.
#12 0x00007fcb611a7780 in ?? ()
No symbol table info available.
#13 0x0000000000000000 in ?? ()
No symbol table info available.
 
Thread 2 (Thread 0x7fcb76404ac0 (LWP 8)):
#0  0x00007fcb745c8c2b in __GI___nanosleep (requested_time=requested_time@entry=0x7ffef33e6d50, remaining=remaining@entry=0x7ffef33e6d50)
    at ../sysdeps/unix/sysv/linux/nanosleep.c:27
        resultvar = 18446744073709551100
        sc_cancel_oldtype = 0
#1  0x00007fcb745c8b4a in __sleep (seconds=0) at ../sysdeps/posix/sleep.c:55
        save_errno = 25
        ts = {tv_sec = 0, tv_nsec = 346371293}
#2  0x00000000004566e4 in flb_main (argc=3, argv=0x7ffef33e6ef8) at /tmp/fluent-bit-1.9.4/src/fluent-bit.c:1198
        opt = -1
        ret = 0
        json = 0x1 <error: Cannot access memory at address 0x1>
        last_plugin = -1
        cfg_file = 0x7fcb71818300 "\005"
        cf = 0x7fcb7181f000
        tmp = 0x7fcb7181f000
        service = 0x7fcb71822000
        s = 0x7fcb7381e060
        long_opts = {{name = 0xa4d908 "storage_path", has_arg = 1, flag = 0x0, val = 98}, {name = 0xa4d915 "config", has_arg = 1, flag = 0x0, val = 99}, {
            name = 0xa4d79f "daemon", has_arg = 0, flag = 0x0, val = 100}, {name = 0xa4d91c "dry-run", has_arg = 0, flag = 0x0, val = 68}, {
            name = 0xa4d7a6 "flush", has_arg = 1, flag = 0x0, val = 102}, {name = 0xa4d924 "http", has_arg = 0, flag = 0x0, val = 72}, {
            name = 0xa4d7b2 "log_file", has_arg = 1, flag = 0x0, val = 108}, {name = 0xa4d929 "port", has_arg = 1, flag = 0x0, val = 80}, {
            name = 0xa4d5b6 "custom", has_arg = 1, flag = 0x0, val = 67}, {name = 0xa4d5bd "input", has_arg = 1, flag = 0x0, val = 105}, {
            name = 0xa4d7ac "match", has_arg = 1, flag = 0x0, val = 109}, {name = 0xa4d5ca "output", has_arg = 1, flag = 0x0, val = 111}, {
            name = 0xa4d5c3 "filter", has_arg = 1, flag = 0x0, val = 70}, {name = 0xa4d6e4 "parser", has_arg = 1, flag = 0x0, val = 82}, {
            name = 0xa4d92e "prop", has_arg = 1, flag = 0x0, val = 112}, {name = 0xa4d933 "plugin", has_arg = 1, flag = 0x0, val = 101}, {name = 0xa4d7bb "tag",
            has_arg = 1, flag = 0x0, val = 116}, {name = 0xa4d93a "sp-task", has_arg = 1, flag = 0x0, val = 84}, {name = 0xa4d942 "version", has_arg = 0,
            flag = 0x0, val = 86}, {name = 0xa4d94a "verbose", has_arg = 0, flag = 0x0, val = 118}, {name = 0xa4d952 "workdir", has_arg = 1, flag = 0x0,
            val = 119}, {name = 0xa4d95a "quiet", has_arg = 0, flag = 0x0, val = 113}, {name = 0xa4d960 "help", has_arg = 0, flag = 0x0, val = 104}, {
---Type <return> to continue, or q <return> to quit---
            name = 0xa4d965 "help-json", has_arg = 0, flag = 0x0, val = 74}, {name = 0xa4d96f "coro_stack_size", has_arg = 1, flag = 0x0, val = 115}, {
            name = 0xa4d97f "sosreport", has_arg = 0, flag = 0x0, val = 83}, {name = 0xa4d7bf "http_server", has_arg = 0, flag = 0x0, val = 72}, {
            name = 0xa4d989 "http_listen", has_arg = 1, flag = 0x0, val = 76}, {name = 0xa4d995 "http_port", has_arg = 1, flag = 0x0, val = 80}, {name = 0x0,
            has_arg = 0, flag = 0x0, val = 0}}
#3  0x000000000045676b in main (argc=3, argv=0x7ffef33e6ef8) at /tmp/fluent-bit-1.9.4/src/fluent-bit.c:1217
No locals.
 
Thread 1 (Thread 0x7fcb60fff700 (LWP 12)):
#0  __GI_raise (sig=sig@entry=6) at ../sysdeps/unix/sysv/linux/raise.c:51
        set = {__val = {1024, 0 <repeats 15 times>}}
        pid = <optimized out>
        tid = <optimized out>
#1  0x00007fcb7453e148 in __GI_abort () at abort.c:79
        save_stage = 1
        act = {__sigaction_handler = {sa_handler = 0x0, sa_sigaction = 0x0}, sa_mask = {__val = {0, 140511759474912, 140511839092736, 0, 0, 0, 0,
              140511465775280, 5013559, 0, 140511839092736, 4536502, 4536427, 15264144, 4294967296, 0}}, sa_flags = 4536795, sa_restorer = 0x0}
        sigs = {__val = {32, 0 <repeats 15 times>}}
#2  0x0000000000455433 in flb_signal_handler (signal=11) at /tmp/fluent-bit-1.9.4/src/fluent-bit.c:581
        len = 22
        ts = "[2022/07/26 06:58:16] \000\000\366#\001`\313\177\000"
        s = "[engine] caught signal ("
        now = 1658818696
        cur = 0x7fcb748b5600 <_tmbuf>
#3  <signal handler called>
No locals.
#4  0x0000000000a2ccc6 in template_execute (ctx=0x7fcb60012f60, data=0x7fcb7080610a "", len=2057756, off=0x7fcb60013780)
    at /tmp/fluent-bit-1.9.4/lib/msgpack-c/include/msgpack/unpack_template.h:172
        p = 0x7fcb708b0e86 ""
        pe = 0x7fcb709fc726 ""
        top = 0
        n = 0x0
        cs = 0
        user = 0x7fcb60012f60
        obj = {type = 1610690256, via = {boolean = 212, u64 = 10677460, i64 = 10677460, f64 = 5.2753661708440763e-317, array = {size = 10677460,
              ptr = 0x7fcb60012f10}, map = {size = 10677460, ptr = 0x7fcb60012f10}, str = {size = 10677460, ptr = 0x7fcb60012f10 "\240\067\001`\313\177"},
            bin = {size = 10677460, ptr = 0x7fcb60012f10 "\240\067\001`\313\177"}, ext = {type = -44 '\324', size = 0,
              ptr = 0x7fcb60012f10 "\240\067\001`\313\177"}}}
---Type <return> to continue, or q <return> to quit---
        ret = 1610690320
        trail = 0
        stack = 0x7fcb60012f80
        c = 0x0
        __PRETTY_FUNCTION__ = "template_execute"
#5  0x0000000000a2e7d8 in msgpack_unpack_next (result=0x7fcb60013860, data=0x7fcb7080610a "", len=2057756, off=0x7fcb60013880)
    at /tmp/fluent-bit-1.9.4/lib/msgpack-c/src/unpack.c:677
        e = 0
        ctx = {user = {z = 0x7fcb60013860, referenced = false}, cs = 0, trail = 0, top = 0, stack = {{obj = {type = MSGPACK_OBJECT_NIL, via = {boolean = false,
                  u64 = 140511465865216, i64 = 140511465865216, f64 = 6.9421888130798211e-310, array = {size = 1610776576, ptr = 0x7fcb60012ed0}, map = {
                    size = 1610776576, ptr = 0x7fcb60012ed0}, str = {size = 1610776576, ptr = 0x7fcb60012ed0 ""}, bin = {size = 1610776576,
                    ptr = 0x7fcb60012ed0 ""}, ext = {type = 0 '\000', size = 32715, ptr = 0x7fcb60012ed0 ""}}}, count = 0, ct = 0, map_key = {
                type = MSGPACK_OBJECT_NIL, via = {boolean = false, u64 = 0, i64 = 0, f64 = 0, array = {size = 0, ptr = 0x0}, map = {size = 0, ptr = 0x0}, str = {
                    size = 0, ptr = 0x0}, bin = {size = 0, ptr = 0x0}, ext = {type = 0 '\000', size = 0, ptr = 0x0}}}}, {obj = {type = MSGPACK_OBJECT_NIL,
                via = {boolean = false, u64 = 0, i64 = 0, f64 = 0, array = {size = 0, ptr = 0x0}, map = {size = 0, ptr = 0x0}, str = {size = 0, ptr = 0x0},
                  bin = {size = 0, ptr = 0x0}, ext = {type = 0 '\000', size = 0, ptr = 0x0}}}, count = 0, ct = 0, map_key = {type = MSGPACK_OBJECT_NIL, via = {
                  boolean = false, u64 = 0, i64 = 0, f64 = 0, array = {size = 0, ptr = 0x0}, map = {size = 0, ptr = 0x0}, str = {size = 0, ptr = 0x0}, bin = {
                    size = 0, ptr = 0x0}, ext = {type = 0 '\000', size = 0, ptr = 0x0}}}} <repeats 11 times>, {obj = {type = MSGPACK_OBJECT_NIL, via = {
                  boolean = false, u64 = 0, i64 = 0, f64 = 0, array = {size = 0, ptr = 0x1098410 <je_extent_mutex_pool+4656>}, map = {size = 0,
                    ptr = 0x1098410 <je_extent_mutex_pool+4656>}, str = {size = 0, ptr = 0x1098410 <je_extent_mutex_pool+4656> ""}, bin = {size = 0,
                    ptr = 0x1098410 <je_extent_mutex_pool+4656> ""}, ext = {type = 0 '\000', size = 0, ptr = 0x1098410 <je_extent_mutex_pool+4656> ""}}},
              count = 3574451219, ct = 1627346064, map_key = {type = 1612802304, via = {boolean = 8, u64 = 140511756069640, i64 = 140511756069640,
                  f64 = 6.9422031510834381e-310, array = {size = 1900981000, ptr = 0x492d1c <extent_lock_from_addr+428>}, map = {size = 1900981000,
                    ptr = 0x492d1c <extent_lock_from_addr+428>}, str = {size = 1900981000,
                    ptr = 0x492d1c <extent_lock_from_addr+428> "\205\300uKI\203G8\001M;g0t\tI\203G(\001M\211g0I\213\006H\301\340\020H\301\370\020H\203\340\376L9\350t\024A\306Gh"}, bin = {size = 1900981000,
                    ptr = 0x492d1c <extent_lock_from_addr+428> "\205\300uKI\203G8\001M;g0t\tI\203G(\001M\211g0I\213\006H\301\340\020H\301\370\020H\203\340\376L9\350t\024A\306Gh"}, ext = {type = 8 '\b', size = 32715,
                    ptr = 0x492d1c <extent_lock_from_addr+428> "\205\300uKI\203G8\001M;g0t\tI\203G(\001M\211g0I\213\006H\301\340\020H\301\370\020H\203\340\376L9\350t\024A\306Gh"}}}}, {obj = {type = MSGPACK_OBJECT_NIL, via = {boolean = false, u64 = 72057594037927936, i64 = 72057594037927936, f64 = 7.2911220195563975e-304,
                  array = {size = 0, ptr = 0x0}, map = {size = 0, ptr = 0x0}, str = {size = 0, ptr = 0x0}, bin = {size = 0, ptr = 0x0}, ext = {type = 0 '\000',
                    size = 16777216, ptr = 0x0}}}, count = 140511467811304, ct = 1627346064, map_key = {type = 1612802560, via = {boolean = false, u64 = 0,
                  i64 = 0, f64 = 0, array = {size = 0, ptr = 0xd50dcc13}, map = {size = 0, ptr = 0xd50dcc13}, str = {size = 0,
                    ptr = 0xd50dcc13 <error: Cannot access memory at address 0xd50dcc13>}, bin = {size = 0,
                    ptr = 0xd50dcc13 <error: Cannot access memory at address 0xd50dcc13>}, ext = {type = 0 '\000', size = 0,
                    ptr = 0xd50dcc13 <error: Cannot access memory at address 0xd50dcc13>}}}}, {obj = {type = MSGPACK_OBJECT_NIL, via = {boolean = 45,
                  u64 = 4817453, i64 = 4817453, f64 = 2.3801380277548507e-317, array = {size = 4817453, ptr = 0x0}, map = {size = 4817453, ptr = 0x0}, str = {
---Type <return> to continue, or q <return> to quit---
                    size = 4817453, ptr = 0x0}, bin = {size = 4817453, ptr = 0x0}, ext = {type = 45 '-', size = 0, ptr = 0x0}}}, count = 140511467800832,
              ct = 1627346112, map_key = {type = MSGPACK_OBJECT_BOOLEAN, via = {boolean = false, u64 = 0, i64 = 0, f64 = 0, array = {size = 0,
                    ptr = 0x7fcb600135b8}, map = {size = 0, ptr = 0x7fcb600135b8}, str = {size = 0, ptr = 0x7fcb600135b8 "`\207", <incomplete sequence \343>},
                  bin = {size = 0, ptr = 0x7fcb600135b8 "`\207", <incomplete sequence \343>}, ext = {type = 0 '\000', size = 0,
                    ptr = 0x7fcb600135b8 "`\207", <incomplete sequence \343>}}}}, {obj = {type = MSGPACK_OBJECT_NIL, via = {boolean = 144,
                  u64 = 140511482434704, i64 = 140511482434704, f64 = 6.9421896317213001e-310, array = {size = 1627346064, ptr = 0x0}, map = {size = 1627346064,
                    ptr = 0x0}, str = {size = 1627346064, ptr = 0x0}, bin = {size = 1627346064, ptr = 0x0}, ext = {type = -112 '\220', size = 32715,
                    ptr = 0x0}}}, count = 140511465780664, ct = 1627346112, map_key = {type = 1612712192, via = {boolean = 232, u64 = 140511467811304,
                  i64 = 140511467811304, f64 = 6.9421889092293436e-310, array = {size = 1612722664, ptr = 0x498ea9 <extent_record+217>}, map = {
                    size = 1612722664, ptr = 0x498ea9 <extent_record+217>}, str = {size = 1612722664,
                    ptr = 0x498ea9 <extent_record+217> "H\203\304 I\211\300\200|$\037"}, bin = {size = 1612722664,
                    ptr = 0x498ea9 <extent_record+217> "H\203\304 I\211\300\200|$\037"}, ext = {type = -24 '\350', size = 32715,
                    ptr = 0x498ea9 <extent_record+217> "H\203\304 I\211\300\200|$\037"}}}}, {obj = {type = 1610691519, via = {boolean = false, u64 = 0, i64 = 0,
                  f64 = 0, array = {size = 0, ptr = 0x1}, map = {size = 0, ptr = 0x1}, str = {size = 0, ptr = 0x1 <error: Cannot access memory at address 0x1>},
                  bin = {size = 0, ptr = 0x1 <error: Cannot access memory at address 0x1>}, ext = {type = 0 '\000', size = 0,
                    ptr = 0x1 <error: Cannot access memory at address 0x1>}}}, count = 4820742, ct = 1610691519, map_key = {type = 1612722728, via = {
                  boolean = false, u64 = 0, i64 = 0, f64 = 0, array = {size = 0, ptr = 0x0}, map = {size = 0, ptr = 0x0}, str = {size = 0, ptr = 0x0}, bin = {
                    size = 0, ptr = 0x0}, ext = {type = 0 '\000', size = 0, ptr = 0x0}}}}, {obj = {type = MSGPACK_OBJECT_NIL, via = {boolean = false, u64 = 0,
                  i64 = 0, f64 = 0, array = {size = 0, ptr = 0x0}, map = {size = 0, ptr = 0x0}, str = {size = 0, ptr = 0x0}, bin = {size = 0, ptr = 0x0}, ext = {
                    type = 0 '\000', size = 0, ptr = 0x0}}}, count = 0, ct = 0, map_key = {type = MSGPACK_OBJECT_NIL, via = {boolean = false, u64 = 0, i64 = 0,
                  f64 = 0, array = {size = 0, ptr = 0x0}, map = {size = 0, ptr = 0x0}, str = {size = 0, ptr = 0x0}, bin = {size = 0, ptr = 0x0}, ext = {
                    type = 0 '\000', size = 0, ptr = 0x0}}}}, {obj = {type = MSGPACK_OBJECT_NIL, via = {boolean = false, u64 = 0, i64 = 0, f64 = 0, array = {
                    size = 0, ptr = 0x0}, map = {size = 0, ptr = 0x0}, str = {size = 0, ptr = 0x0}, bin = {size = 0, ptr = 0x0}, ext = {type = 0 '\000',
                    size = 0, ptr = 0x0}}}, count = 0, ct = 0, map_key = {type = MSGPACK_OBJECT_NIL, via = {boolean = false, u64 = 0, i64 = 0, f64 = 0, array = {
                    size = 0, ptr = 0x0}, map = {size = 0, ptr = 0x0}, str = {size = 0, ptr = 0x0}, bin = {size = 0, ptr = 0x0}, ext = {type = 0 '\000',
                    size = 0, ptr = 0x0}}}}, {obj = {type = MSGPACK_OBJECT_NIL, via = {boolean = false, u64 = 0, i64 = 0, f64 = 0, array = {size = 0,
                    ptr = 0x0}, map = {size = 0, ptr = 0x0}, str = {size = 0, ptr = 0x0}, bin = {size = 0, ptr = 0x0}, ext = {type = 0 '\000', size = 0,
                    ptr = 0x0}}}, count = 0, ct = 0, map_key = {type = MSGPACK_OBJECT_NIL, via = {boolean = false, u64 = 0, i64 = 0, f64 = 0, array = {size = 0,
                    ptr = 0x0}, map = {size = 0, ptr = 0x0}, str = {size = 0, ptr = 0x0}, bin = {size = 0, ptr = 0x0}, ext = {type = 0 '\000', size = 0,
                    ptr = 0x0}}}}, {obj = {type = MSGPACK_OBJECT_NIL, via = {boolean = false, u64 = 0, i64 = 0, f64 = 0, array = {size = 0, ptr = 0x0}, map = {
                    size = 0, ptr = 0x0}, str = {size = 0, ptr = 0x0}, bin = {size = 0, ptr = 0x0}, ext = {type = 0 '\000', size = 0, ptr = 0x0}}}, count = 0,
              ct = 0, map_key = {type = MSGPACK_OBJECT_NIL, via = {boolean = false, u64 = 0, i64 = 0, f64 = 0, array = {size = 0, ptr = 0x0}, map = {size = 0,
                    ptr = 0x0}, str = {size = 0, ptr = 0x0}, bin = {size = 0, ptr = 0x0}, ext = {type = 0 '\000', size = 0, ptr = 0x0}}}}, {obj = {
                type = MSGPACK_OBJECT_NIL, via = {boolean = false, u64 = 0, i64 = 0, f64 = 0, array = {size = 0, ptr = 0x0}, map = {size = 0, ptr = 0x0}, str = {
                    size = 0, ptr = 0x0}, bin = {size = 0, ptr = 0x0}, ext = {type = 0 '\000', size = 0, ptr = 0x0}}}, count = 0, ct = 0, map_key = {
                type = MSGPACK_OBJECT_NIL, via = {boolean = false, u64 = 0, i64 = 0, f64 = 0, array = {size = 0, ptr = 0x0}, map = {size = 0, ptr = 0x0}, str = {
                    size = 0, ptr = 0x0}, bin = {size = 0, ptr = 0x0}, ext = {type = 0 '\000', size = 0, ptr = 0x0}}}}, {obj = {type = MSGPACK_OBJECT_NIL,
---Type <return> to continue, or q <return> to quit---
                via = {boolean = false, u64 = 0, i64 = 0, f64 = 0, array = {size = 0, ptr = 0x0}, map = {size = 0, ptr = 0x0}, str = {size = 0, ptr = 0x0},
                  bin = {size = 0, ptr = 0x0}, ext = {type = 0 '\000', size = 0, ptr = 0x0}}}, count = 0, ct = 0, map_key = {type = MSGPACK_OBJECT_NIL, via = {
                  boolean = false, u64 = 0, i64 = 0, f64 = 0, array = {size = 0, ptr = 0x0}, map = {size = 0, ptr = 0x0}, str = {size = 0, ptr = 0x0}, bin = {
                    size = 0, ptr = 0x0}, ext = {type = 0 '\000', size = 0, ptr = 0x0}}}}, {obj = {type = MSGPACK_OBJECT_NIL, via = {boolean = false,
                  u64 = 140511467800832, i64 = 140511467800832, f64 = 6.942188908711958e-310, array = {size = 1612712192, ptr = 0x7fcb60ff5490}, map = {
                    size = 1612712192, ptr = 0x7fcb60ff5490}, str = {size = 1612712192, ptr = 0x7fcb60ff5490 ""}, bin = {size = 1612712192,
                    ptr = 0x7fcb60ff5490 ""}, ext = {type = 0 '\000', size = 32715, ptr = 0x7fcb60ff5490 ""}}}, count = 140511467811304, ct = 1073741824,
              map_key = {type = 1073741824, via = {boolean = 192, u64 = 140511482434752, i64 = 140511482434752, f64 = 6.9421896317236716e-310, array = {
                    size = 1627346112, ptr = 0x465ff0 <je_arena_extents_dirty_dalloc+32>}, map = {size = 1627346112,
                    ptr = 0x465ff0 <je_arena_extents_dirty_dalloc+32>}, str = {size = 1627346112,
                    ptr = 0x465ff0 <je_arena_extents_dirty_dalloc+32> "H\213\203\230c"}, bin = {size = 1627346112,
                    ptr = 0x465ff0 <je_arena_extents_dirty_dalloc+32> "H\213\203\230c"}, ext = {type = -64 '\300', size = 32715,
                    ptr = 0x465ff0 <je_arena_extents_dirty_dalloc+32> "H\213\203\230c"}}}}, {obj = {type = 1612712192, via = {boolean = 144,
                  u64 = 140511482434704, i64 = 140511482434704, f64 = 6.9421896317213001e-310, array = {size = 1627346064, ptr = 0x7fcb60216a00}, map = {
                    size = 1627346064, ptr = 0x7fcb60216a00}, str = {size = 1627346064, ptr = 0x7fcb60216a00 "\003`\255\003"}, bin = {size = 1627346064,
                    ptr = 0x7fcb60216a00 "\003`\255\003"}, ext = {type = -112 '\220', size = 32715, ptr = 0x7fcb60216a00 "\003`\255\003"}}},
              count = 140510928830464, ct = 1073741824, map_key = {type = 4841224, via = {boolean = 181, u64 = 10820789, i64 = 10820789,
                  f64 = 5.3461801057968564e-317, array = {size = 10820789, ptr = 0xe38760 <je_extent_hooks_default>}, map = {size = 10820789,
                    ptr = 0xe38760 <je_extent_hooks_default>}, str = {size = 10820789, ptr = 0xe38760 <je_extent_hooks_default> " 'I"}, bin = {size = 10820789,
                    ptr = 0xe38760 <je_extent_hooks_default> " 'I"}, ext = {type = -75 '\265', size = 0, ptr = 0xe38760 <je_extent_hooks_default> " 'I"}}}}, {
              obj = {type = 1627346064, via = {boolean = 16, u64 = 140511756069648, i64 = 140511756069648, f64 = 6.9422031510838333e-310, array = {
                    size = 1900981008, ptr = 0x7fcb60ff5560}, map = {size = 1900981008, ptr = 0x7fcb60ff5560}, str = {size = 1900981008,
                    ptr = 0x7fcb60ff5560 "\001"}, bin = {size = 1900981008, ptr = 0x7fcb60ff5560 "\001"}, ext = {type = 16 '\020', size = 32715,
                    ptr = 0x7fcb60ff5560 "\001"}}}, count = 140510928830464, ct = 1073741824, map_key = {type = 4583947, via = {boolean = 16,
                  u64 = 140511465780752, i64 = 140511465780752, f64 = 6.942188808906745e-310, array = {size = 1610692112,
                    ptr = 0x51ca7f <flb_kv_item_destroy+96>}, map = {size = 1610692112, ptr = 0x51ca7f <flb_kv_item_destroy+96>}, str = {size = 1610692112,
                    ptr = 0x51ca7f <flb_kv_item_destroy+96> "\220\311\303UH\211\345H\203\354\060H\211}\330H\213E\330H\213@\bH\211E\360H\213E\360H\213@\bH\211E\370\353\064H\213E\360H\211E\350H\213E\350H\203\350\020H\211E\340H\213E\340H\211\307\350W\377\377\377H\213E\370H\211E\360H\213E\360H\213@\bH\211E\370H\213E\360H;E\330u\302\220\311\303UH\211\345H\203\354\060H\211}\330H\211u\320H\203", <incomplete sequence \330>}, bin = {size = 1610692112,
                    ptr = 0x51ca7f <flb_kv_item_destroy+96> "\220\311\303UH\211\345H\203\354\060H\211}\330H\213E\330H\213@\bH\211E\360H\213E\360H\213@\bH\211E\370\353\064H\213E\360H\211E\350H\213E\350H\203\350\020H\211E\340H\213E\340H\211\307\350W\377\377\377H\213E\370H\211E\360H\213E\360H\213@\bH\211E\370H\213E\360H;E\330u\302\220\311\303UH\211\345H\203\354\060H\211}\330H\211u\320H\203", <incomplete sequence \330>}, ext = {type = 16 '\020', size = 32715,
                    ptr = 0x51ca7f <flb_kv_item_destroy+96> "\220\311\303UH\211\345H\203\354\060H\211}\330H\213E\330H\213@\bH\211E\360H\213E\360H\213@\bH\211E\370\353\064H\213E\360H\211E\350H\213E\350H\203\350\020H\211E\340H\213E\340H\211\307\350W\377\377\377H\213E\370H\211E\360H\213E\360H\213@\bH\211E\370H\213E\360H;E\330u\302\220\311\303UH\211\345H\203\354\060H\211}\330H\211u\320H\203", <incomplete sequence \330>}}}}, {obj = {type = 1904294296, via = {boolean = 224,
                  u64 = 140511626673888, i64 = 140511626673888, f64 = 6.94219675808386e-310, array = {size = 1771585248, ptr = 0x7fcb60013650}, map = {
                    size = 1771585248, ptr = 0x7fcb60013650}, str = {size = 1771585248, ptr = 0x7fcb60013650 ""}, bin = {size = 1771585248,
---Type <return> to continue, or q <return> to quit---
                    ptr = 0x7fcb60013650 ""}, ext = {type = -32 '\340', size = 32715, ptr = 0x7fcb60013650 ""}}}, count = 5360328, ct = 1610692224, map_key = {
                type = 1904294296, via = {boolean = 112, u64 = 140511465780848, i64 = 140511465780848, f64 = 6.9421888089114881e-310, array = {
                    size = 1610692208, ptr = 0x0}, map = {size = 1610692208, ptr = 0x0}, str = {size = 1610692208, ptr = 0x0}, bin = {size = 1610692208,
                    ptr = 0x0}, ext = {type = 112 'p', size = 32715, ptr = 0x0}}}}, {obj = {type = MSGPACK_OBJECT_NIL, via = {boolean = false, u64 = 0, i64 = 0,
                  f64 = 0, array = {size = 0, ptr = 0x0}, map = {size = 0, ptr = 0x0}, str = {size = 0, ptr = 0x0}, bin = {size = 0, ptr = 0x0}, ext = {
                    type = 0 '\000', size = 0, ptr = 0x0}}}, count = 5383678, ct = 1610692240, map_key = {type = 1612296800, via = {boolean = 144,
                  u64 = 140511465780880, i64 = 140511465780880, f64 = 6.9421888089130691e-310, array = {size = 1610692240,
                    ptr = 0x52516c <flb_http_client_destroy+87>}, map = {size = 1610692240, ptr = 0x52516c <flb_http_client_destroy+87>}, str = {
                    size = 1610692240,
                    ptr = 0x52516c <flb_http_client_destroy+87> "\220\311\303UH\211\345H\203\354 H\211}\370H\211u\360H\211U\350H\213U\350H\213u\360H\213E\370\271\001"}, bin = {size = 1610692240,
                    ptr = 0x52516c <flb_http_client_destroy+87> "\220\311\303UH\211\345H\203\354 H\211}\370H\211u\360H\211U\350H\213U\350H\213u\360H\213E\370\271\001"}, ext = {type = -112 '\220', size = 32715,
                    ptr = 0x52516c <flb_http_client_destroy+87> "\220\311\303UH\211\345H\203\354 H\211}\370H\211u\360H\211U\350H\213U\350H\213u\360H\213E\370\271\001"}}}}, {obj = {type = 1610692336, via = {boolean = 96, u64 = 140511467385440, i64 = 140511467385440, f64 = 6.9421888881888664e-310, array = {
                    size = 1612296800, ptr = 0x7fcb600136f0}, map = {size = 1612296800, ptr = 0x7fcb600136f0}, str = {size = 1612296800,
                    ptr = 0x7fcb600136f0 "@7\001`\313\177"}, bin = {size = 1612296800, ptr = 0x7fcb600136f0 "@7\001`\313\177"}, ext = {type = 96 '`',
                    size = 32715, ptr = 0x7fcb600136f0 "@7\001`\313\177"}}}, count = 6222733, ct = 11001938, map_key = {type = 1053164, via = {boolean = false,
                  u64 = 140511465816064, i64 = 140511465816064, f64 = 6.9421888106513897e-310, array = {size = 1610727424, ptr = 0x7fcb704a1b60}, map = {
                    size = 1610727424, ptr = 0x7fcb704a1b60}, str = {size = 1610727424, ptr = 0x7fcb704a1b60 " \335\366a\313\177"}, bin = {size = 1610727424,
                    ptr = 0x7fcb704a1b60 " \335\366a\313\177"}, ext = {type = 0 '\000', size = 32715, ptr = 0x7fcb704a1b60 " \335\366a\313\177"}}}}, {obj = {
                type = 1610692336, via = {boolean = 160, u64 = 140511499480992, i64 = 140511499480992, f64 = 6.9421904739198291e-310, array = {
                    size = 1644392352, ptr = 0xa7e147}, map = {size = 1644392352, ptr = 0xa7e147}, str = {size = 1644392352, ptr = 0xa7e147 ","}, bin = {
                    size = 1644392352, ptr = 0xa7e147 ","}, ext = {type = -96 '\240', size = 32715, ptr = 0xa7e147 ","}}}, count = 140511467385440,
              ct = 1610692416, map_key = {type = 1053164, via = {boolean = 64, u64 = 140511465781056, i64 = 140511465781056, f64 = 6.9421888089217646e-310,
                  array = {size = 1610692416, ptr = 0x5ed926 <send_log_events+1101>}, map = {size = 1610692416, ptr = 0x5ed926 <send_log_events+1101>}, str = {
                    size = 1610692416, ptr = 0x5ed926 <send_log_events+1101> "\211E\350\203", <incomplete sequence \350>}, bin = {size = 1610692416,
                    ptr = 0x5ed926 <send_log_events+1101> "\211E\350\203", <incomplete sequence \350>}, ext = {type = 64 '@', size = 32715,
                    ptr = 0x5ed926 <send_log_events+1101> "\211E\350\203", <incomplete sequence \350>}}}}, {obj = {type = 1610727424, via = {boolean = 96,
                  u64 = 140511738993504, i64 = 140511738993504, f64 = 6.9422023074102219e-310, array = {size = 1883904864, ptr = 0x40}, map = {
                    size = 1883904864, ptr = 0x40}, str = {size = 1883904864, ptr = 0x40 <error: Cannot access memory at address 0x40>}, bin = {
                    size = 1883904864, ptr = 0x40 <error: Cannot access memory at address 0x40>}, ext = {type = 96 '`', size = 32715,
                    ptr = 0x40 <error: Cannot access memory at address 0x40>}}}, count = 6214853, ct = 1610727424, map_key = {type = 1883904864, via = {
                  boolean = false, u64 = 0, i64 = 0, f64 = 0, array = {size = 0, ptr = 0x0}, map = {size = 0, ptr = 0x0}, str = {size = 0, ptr = 0x0}, bin = {
                    size = 0, ptr = 0x0}, ext = {type = 0 '\000', size = 0, ptr = 0x0}}}}, {obj = {type = 1610692512, via = {boolean = 212, u64 = 6216916,
                  i64 = 6216916, f64 = 3.0715646186807791e-317, array = {size = 6216916, ptr = 0x7fcb600137d0}, map = {size = 6216916, ptr = 0x7fcb600137d0},
                  str = {size = 6216916, ptr = 0x7fcb600137d0 "\203\220\337b"}, bin = {size = 6216916, ptr = 0x7fcb600137d0 "\203\220\337b"}, ext = {
---Type <return> to continue, or q <return> to quit---
                    type = -44 '\324', size = 0, ptr = 0x7fcb600137d0 "\203\220\337b"}}}, count = 140511465781312, ct = 1610727424, map_key = {
                type = 1883904864, via = {boolean = 160, u64 = 140511465861280, i64 = 140511465861280, f64 = 6.9421888128853569e-310, array = {
                    size = 1610772640, ptr = 0x600137d0}, map = {size = 1610772640, ptr = 0x600137d0}, str = {size = 1610772640,
                    ptr = 0x600137d0 <error: Cannot access memory at address 0x600137d0>}, bin = {size = 1610772640,
                    ptr = 0x600137d0 <error: Cannot access memory at address 0x600137d0>}, ext = {type = -96 '\240', size = 32715,
                    ptr = 0x600137d0 <error: Cannot access memory at address 0x600137d0>}}}}}}
        noff = 699772
#6  0x00000000005ee066 in process_and_send_to_kinesis (ctx=0x7fcb704a1b60, buf=0x7fcb6001c000, data=0x7fcb7080610a "", bytes=2057756)
    at /tmp/fluent-bit-1.9.4/plugins/out_kinesis_streams/kinesis_api.c:556
        off = 699772
        i = 500
        map_size = 6
        result = {zone = 0x0, data = {type = MSGPACK_OBJECT_NIL, via = {boolean = false, u64 = 0, i64 = 0, f64 = 0, array = {size = 0, ptr = 0x0}, map = {
                size = 0, ptr = 0x0}, str = {size = 0, ptr = 0x0}, bin = {size = 0, ptr = 0x0}, ext = {type = 0 '\000', size = 0, ptr = 0x0}}}}
        obj = 0x7fcb60028020
        map = {type = MSGPACK_OBJECT_MAP, via = {boolean = 6, u64 = 6, i64 = 6, f64 = 2.9643938750474793e-323, array = {size = 6, ptr = 0x7fcb60028038}, map = {
              size = 6, ptr = 0x7fcb60028038}, str = {size = 6, ptr = 0x7fcb60028038 "\005"}, bin = {size = 6, ptr = 0x7fcb60028038 "\005"}, ext = {
              type = 6 '\006', size = 0, ptr = 0x7fcb60028038 "\005"}}}
        root = {type = MSGPACK_OBJECT_ARRAY, via = {boolean = 2, u64 = 140509855088642, i64 = 140509855088642, f64 = 6.9421092301429872e-310, array = {size = 2,
              ptr = 0x7fcb60028008}, map = {size = 2, ptr = 0x7fcb60028008}, str = {size = 2, ptr = 0x7fcb60028008 "\t"}, bin = {size = 2,
              ptr = 0x7fcb60028008 "\t"}, ext = {type = 2 '\002', size = 32715, ptr = 0x7fcb60028008 "\t"}}}
        kv = 0x5eaca3 <flb_malloc+38>
        key = {type = 4294965248, via = {boolean = 71, u64 = 4576839, i64 = 4576839, f64 = 2.261258916446405e-317, array = {size = 4576839, ptr = 0x0}, map = {
              size = 4576839, ptr = 0x0}, str = {size = 4576839, ptr = 0x0}, bin = {size = 4576839, ptr = 0x0}, ext = {type = 71 'G', size = 0, ptr = 0x0}}}
        val = {type = 1612712192, via = {boolean = 128, u64 = 16000, i64 = 16000, f64 = 7.9050503334599447e-320, array = {size = 16000, ptr = 0x4000}, map = {
              size = 16000, ptr = 0x4000}, str = {size = 16000, ptr = 0x4000 <error: Cannot access memory at address 0x4000>}, bin = {size = 16000,
              ptr = 0x4000 <error: Cannot access memory at address 0x4000>}, ext = {type = -128 '\200', size = 0,
              ptr = 0x4000 <error: Cannot access memory at address 0x4000>}}}
        key_str = 0x0
        key_str_size = 0
        j = 32715
        ret = 0
        check = 0
        found = 0
        tms = {tm = {tv_sec = 1658818691, tv_nsec = 486443254}}
#7  0x00000000005ec0be in cb_kinesis_flush (event_chunk=0x7fcb62041708, out_flush=0x7fcb6000b000, i_ins=0x7fcb7180a280, out_context=0x7fcb704a1b60,
    config=0x7fcb71819340) at /tmp/fluent-bit-1.9.4/plugins/out_kinesis_streams/kinesis.c:336
---Type <return> to continue, or q <return> to quit---
        ctx = 0x7fcb704a1b60
        ret = 0
        buf = 0x7fcb6001c000
#8  0x00000000004e60f4 in output_pre_cb_flush () at /tmp/fluent-bit-1.9.4/include/fluent-bit/flb_output.h:517
        coro = 0x7fcb6000c000
        out_p = 0x7fcb71852060
        params = 0x7fcb60015000
        persisted_params = {event_chunk = 0x7fcb62041708, out_flush = 0x7fcb6000b000, i_ins = 0x7fcb7180a280, out_plugin = 0x7fcb71852060,
          out_context = 0x7fcb704a1b60, config = 0x7fcb71819340, coro = 0x7fcb6000c000}
#9  0x0000000000a4a1a7 in co_init () at /tmp/fluent-bit-1.9.4/lib/monkey/deps/flb_libco/amd64.c:117
No locals.
#10 0x0000000000000000 in ?? ()
No symbol table info available.

[JoeShi]

@ptsneves Do you have any idea about this issue? We run into the same issue.

[ptsneves]

@JoeShi no, did not have time yet. Sorry

[tarruda]

@PettitWesley is the SEGFAULT reproducible if you set workers 1 in the [OUTPUT] config?

[PettitWesley]

@tarruda in 1.9.4 S3 already had default 1 workers enabled, as does Kinesis. So this was reproduced with 1 worker.

[tarruda]

@PettitWesley do you think you could create a simple fluent-bit config that reproduces the segfault? It would help me a lot in debugging this

[PettitWesley]

@tarruda the only config I have repro'd it with is basically what is shown in the issue. Since I don't know what causes it I didn't try removing anything from the config.