openCTI platform - fatal error about "JavaScript heap out of memory"

[Deventual]

Description

the openCTI platform service within docker compose keep crashing every 10 minutes or so.

Environment

1. OS (where OpenCTI server runs): { Ubuntu server 18.04.6 LTS }
2. OpenCTI version: { latest openCTI 6.5.1 }
3. OpenCTI client: { frontend }
4. Other environment details:

Additional information

<--- Last few GCs --->

[7:0x7fbf8c931660] 728893 ms: Scavenge 7887.2 (8128.0) -> 7883.1 (8128.0) MB, 13.08 / 0.02 ms (average mu = 0.168, current mu = 0.026) allocation failure;
[7:0x7fbf8c931660] 728916 ms: Scavenge 7887.3 (8128.0) -> 7883.9 (8139.5) MB, 18.25 / 0.00 ms (average mu = 0.168, current mu = 0.026) allocation failure;
[7:0x7fbf8c931660] 735012 ms: Mark-Compact 7896.2 (8139.5) -> 7888.3 (8149.5) MB, 6018.49 / 0.02 ms (average mu = 0.100, current mu = 0.035) allocation failure; scavenge might not succeed

<--- JS stacktrace --->

FATAL ERROR: Reached heap limit Allocation failed - JavaScript heap out of memory
----- Native stack trace -----

{"level":"info","message":"[MIGRATION] Read 3 migrations from the database","timestamp":"2025-02-10T12:42:41.637Z"}
{"level":"info","message":"[MIGRATION] Platform already up to date, nothing to migrate","timestamp":"2025-02-10T12:42:41.640Z"}
{"level":"info","message":"[MIGRATION] Migration process completed","timestamp":"2025-02-10T12:42:41.641Z"}
{"category":"APP","cause":{"attributes":{"genre":"BUSINESS","http_status":500,"participantIds":["external-reference--5b3eee59-cc9e-5266-9fd7-dd75402a8aa1"]},"code":"LOCK_ERROR","message":"Execution timeout, too many concurrent call on the same entities","name":"LOCK_ERROR","stack":"GraphQLError: Execution timeout, too many concurrent call on the same entities\n at error (/opt/opencti/build/back.js:1690:2273)\n at LockTimeoutError (/opt/opencti/build/back.js:1690:5707)\n at createEntityRaw (/opt/opencti/build/back.js:2451:131876)\n at processTicksAndRejections (node:internal/process/task_queues:95:5)\n at createEntity (/opt/opencti/build/back.js:2451:132052)\n at addExternalReference (/opt/opencti/build/back.js:3077:97581)"},"level":"error","message":"Execution timeout, too many concurrent call on the same entities","operation":"Unspecified","size":262,"source":"backend","time":2603,"timestamp":"2025-02-10T12:42:51.176Z","type":"WRITE_ERROR","user":{"applicant_id":"88ec0c6a-13ce-5e39-b486-354fe4a7084f","call_retry_number":"322","group_ids":[],"ip":"::ffff:172.19.0.23","organization_ids":[],"socket":"query","user_id":"88ec0c6a-13ce-5e39-b486-354fe4a7084f","user_metadata":{}},"version":"6.5.1"}
{"category":"APP","cause":{"attributes":{"genre":"BUSINESS","http_status":500,"participantIds":["external-reference--5b3eee59-cc9e-5266-9fd7-dd75402a8aa1"]},"code":"LOCK_ERROR","message":"Execution timeout, too many concurrent call on the same entities","name":"LOCK_ERROR","stack":"GraphQLError: Execution timeout, too many concurrent call on the same entities\n at error (/opt/opencti/build/back.js:1690:2273)\n at LockTimeoutError (/opt/opencti/build/back.js:1690:5707)\n at createEntityRaw (/opt/opencti/build/back.js:2451:131876)\n at processTicksAndRejections (node:internal/process/task_queues:95:5)\n at createEntity (/opt/opencti/build/back.js:2451:132052)\n at addExternalReference (/opt/opencti/build/back.js:3077:97581)"},"level":"error","message":"Execution timeout, too many concurrent call on the same entities","operation":"Unspecified","size":262,"source":"backend","time":3185,"timestamp":"2025-02-10T12:42:53.968Z","type":"WRITE_ERROR","user":{"applicant_id":"88ec0c6a-13ce-5e39-b486-354fe4a7084f","call_retry_number":"335","group_ids":[],"ip":"::ffff:172.19.0.23","organization_ids":[],"socket":"query","user_id":"88ec0c6a-13ce-5e39-b486-354fe4a7084f","user_metadata":{}},"version":"6.5.1"}
{"category":"APP","cause":{"attributes":{"genre":"BUSINESS","http_status":500,"participantIds":["rulemanager--4ec066c4-c662-50c0-ad53-d46e0d831ecf","rule_engine_settings"]},"code":"LOCK_ERROR","message":"Execution timeout, too many concurrent call on the same entities","name":"LOCK_ERROR","stack":"GraphQLError: Execution timeout, too many concurrent call on the same entities\n at error (/opt/opencti/build/back.js:1690:2273)\n at LockTimeoutError (/opt/opencti/build/back.js:1690:5707)\n at createEntityRaw (/opt/opencti/build/back.js:2451:131876)\n at processTicksAndRejections (node:internal/process/task_queues:95:5)\n at createEntity (/opt/opencti/build/back.js:2451:132052)\n at getInitRuleManager (/opt/opencti/build/back.js:3079:2742406)\n at ruleHandler (/opt/opencti/build/back.js:3079:2742772)\n at /opt/opencti/build/back.js:3079:2743628\n at _SetIntervalAsyncTimer.runHandlerAndScheduleTimeout_fn (/opt/opencti/build/back.js:908:470121)\n at Timeout._onTimeout (/opt/opencti/build/back.js:908:469908)"},"level":"error","manager":"RULE_ENGINE","message":"[OPENCTI-MODULE] Rule engine handler error","source":"backend","timestamp":"2025-02-10T12:42:55.572Z","version":"6.5.1"}
{"category":"APP","cause":{"attributes":{"genre":"BUSINESS","http_status":500,"participantIds":["external-reference--5b3eee59-cc9e-5266-9fd7-dd75402a8aa1"]},"code":"LOCK_ERROR","message":"Execution timeout, too many concurrent call on the same entities","name":"LOCK_ERROR","stack":"GraphQLError: Execution timeout, too many concurrent call on the same entities\n at error (/opt/opencti/build/back.js:1690:2273)\n at LockTimeoutError (/opt/opencti/build/back.js:1690:5707)\n at createEntityRaw (/opt/opencti/build/back.js:2451:131876)\n at processTicksAndRejections (node:internal/process/task_queues:95:5)\n at createEntity (/opt/opencti/build/back.js:2451:132052)\n at addExternalReference (/opt/opencti/build/back.js:3077:97581)"},"level":"error","message":"Execution timeout, too many concurrent call on the same entities","operation":"Unspecified","size":262,"source":"backend","time":3206,"timestamp":"2025-02-10T12:43:04.001Z","type":"WRITE_ERROR","user":{"applicant_id":"88ec0c6a-13ce-5e39-b486-354fe4a7084f","call_retry_number":"330","group_ids":[],"ip":"::ffff:172.19.0.24","organization_ids":[],"socket":"query","user_id":"88ec0c6a-13ce-5e39-b486-354fe4a7084f","user_metadata":{}},"version":"6.5.1"}
{"category":"APP","cause":{"attributes":{"genre":"BUSINESS","http_status":500,"participantIds":["external-reference--5b3eee59-cc9e-5266-9fd7-dd75402a8aa1"]},"code":"LOCK_ERROR","message":"Execution timeout, too many concurrent call on the same entities","name":"LOCK_ERROR","stack":"GraphQLError: Execution timeout, too many concurrent call on the same entities\n at error (/opt/opencti/build/back.js:1690:2273)\n at LockTimeoutError (/opt/opencti/build/back.js:1690:5707)\n at createEntityRaw (/opt/opencti/build/back.js:2451:131876)\n at processTicksAndRejections (node:internal/process/task_queues:95:5)\n at createEntity (/opt/opencti/build/back.js:2451:132052)\n at addExternalReference (/opt/opencti/build/back.js:3077:97581)"},"level":"error","message":"Execution timeout, too many concurrent call on the same entities","operation":"Unspecified","size":262,"source":"backend","time":3045,"timestamp":"2025-02-10T12:43:04.896Z","type":"WRITE_ERROR","user":{"applicant_id":"88ec0c6a-13ce-5e39-b486-354fe4a7084f","call_retry_number":"341","group_ids":[],"ip":"::ffff:172.19.0.17","organization_ids":[],"socket":"query","user_id":"88ec0c6a-13ce-5e39-b486-354fe4a7084f","user_metadata":{}},"version":"6.5.1"}
{"category":"APP","cause":{"attributes":{"genre":"BUSINESS","http_status":500,"participantIds":["rulemanager--bba10628-05ee-5ab0-aec7-2a28157cafce","rule_engine_settings"]},"code":"LOCK_ERROR","message":"Execution timeout, too many concurrent call on the same entities","name":"LOCK_ERROR","stack":"GraphQLError: Execution timeout, too many concurrent call on the same entities\n at error (/opt/opencti/build/back.js:1690:2273)\n at LockTimeoutError (/opt/opencti/build/back.js:1690:5707)\n at createEntityRaw (/opt/opencti/build/back.js:2451:131876)\n at processTicksAndRejections (node:internal/process/task_queues:95:5)\n at createEntity (/opt/opencti/build/back.js:2451:132052)\n at getInitRuleManager (/opt/opencti/build/back.js:3079:2742406)\n at ruleHandler (/opt/opencti/build/back.js:3079:2742772)\n at /opt/opencti/build/back.js:3079:2743628\n at _SetIntervalAsyncTimer.runHandlerAndScheduleTimeout_fn (/opt/opencti/build/back.js:908:470121)\n at Timeout._onTimeout (/opt/opencti/build/back.js:908:469908)"},"level":"error","manager":"RULE_ENGINE","message":"[OPENCTI-MODULE] Rule engine handler error","source":"backend","timestamp":"2025-02-10T12:43:08.972Z","version":"6.5.1"}
{"category":"APP","cause":{"attributes":{"genre":"BUSINESS","http_status":500,"participantIds":["rulemanager--dc0b392b-9864-5901-8873-659aa0ae786d","rule_engine_settings"]},"code":"LOCK_ERROR","message":"Execution timeout, too many concurrent call on the same entities","name":"LOCK_ERROR","stack":"GraphQLError: Execution timeout, too many concurrent call on the same entities\n at error (/opt/opencti/build/back.js:1690:2273)\n at LockTimeoutError (/opt/opencti/build/back.js:1690:5707)\n at createEntityRaw (/opt/opencti/build/back.js:2451:131876)\n at processTicksAndRejections (node:internal/process/task_queues:95:5)\n at createEntity (/opt/opencti/build/back.js:2451:132052)\n at getInitRuleManager (/opt/opencti/build/back.js:3079:2742406)\n at ruleHandler (/opt/opencti/build/back.js:3079:2742772)\n at /opt/opencti/build/back.js:3079:2743628\n at _SetIntervalAsyncTimer.runHandlerAndScheduleTimeout_fn (/opt/opencti/build/back.js:908:470121)\n at Timeout._onTimeout (/opt/opencti/build/back.js:908:469908)"},"level":"error","manager":"RULE_ENGINE","message":"[OPENCTI-MODULE] Rule engine handler error","source":"backend","timestamp":"2025-02-10T12:43:22.182Z","version":"6.5.1"}
{"category":"APP","level":"error","message":"[OPENCTI] Engine unhandled rejection","reason":"Error: Cannot set headers after they are sent to the client\n at ServerResponse.setHeader (node:_http_outgoing:655:11)\n at ServerResponse.res.set.res.header (/opt/opencti/build/back.js:1328:32243)\n at ServerResponse.res.send (/opt/opencti/build/back.js:1328:26398)\n at ServerResponse.res.json (/opt/opencti/build/back.js:1328:27800)\n at ServerResponse.res.send (/opt/opencti/build/back.js:1328:26279)\n at /opt/opencti/build/back.js:3498:11541\n at processTicksAndRejections (node:internal/process/task_queues:95:5)","source":"backend","timestamp":"2025-02-10T12:43:38.358Z","version":"6.5.1"}
{"category":"APP","cause":{"attributes":{"genre":"BUSINESS","http_status":500},"code":"UNSUPPORTED_ERROR","message":"Enterprise edition is not enabled","name":"UNSUPPORTED_ERROR","stack":"GraphQLError: Enterprise edition is not enabled\n at error (/opt/opencti/build/back.js:1690:2273)\n at UnsupportedError (/opt/opencti/build/back.js:1690:4480)\n at checkEnterpriseEdition (/opt/opencti/build/back.js:2734:535)\n at processTicksAndRejections (node:internal/process/task_queues:95:5)\n at findAll80 (/opt/opencti/build/back.js:3394:279)"},"level":"error","message":"Enterprise edition is not enabled","operation":"Unspecified","size":183,"source":"backend","time":3,"timestamp":"2025-02-10T12:50:13.426Z","type":"READ_ERROR","user":{"group_ids":[],"ip":"::ffff:109.67.115.76","organization_ids":[],"referer":"http://x.x.x.x:8080/dashboard/settings/accesses/dissemination_list","socket":"query","user_id":"88ec0c6a-13ce-5e39-b486-354fe4a7084f","user_metadata":{}},"version":"6.5.1"}
{"category":"APP","cause":{"code":"UNKNOWN_ERROR","message":"stream is not readable","name":"InternalServerError","stack":"InternalServerError: stream is not readable\n at readStream2 (/opt/opencti/build/back.js:1292:432328)\n at getRawBody (/opt/opencti/build/back.js:1292:431531)\n at read3 (/opt/opencti/build/back.js:1292:438194)\n at /opt/opencti/build/back.js:1292:446452\n at Layer.handle_request (/opt/opencti/build/back.js:1313:4957)\n at trim_prefix (/opt/opencti/build/back.js:1313:12526)\n at /opt/opencti/build/back.js:1313:11811\n at Function.proto.process_params (/opt/opencti/build/back.js:1313:12689)\n at next (/opt/opencti/build/back.js:1313:11686)\n at /opt/opencti/build/back.js:921:61590\n at Layer.handle_request (/opt/opencti/build/back.js:1313:4957)\n at trim_prefix (/opt/opencti/build/back.js:1313:12526)\n at /opt/opencti/build/back.js:1313:11811\n at Function.proto.process_params (/opt/opencti/build/back.js:1313:12689)\n at next (/opt/opencti/build/back.js:1313:11686)\n at /opt/opencti/build/back.js:921:5975"},"level":"error","message":"Http call interceptor fail","referer":"http://x.x.x.x:8080/dashboard/data/ingestion/connectors/37a4891a-0d1f-413c-bd71-aae2e446b91c","source":"backend","timestamp":"2025-02-10T12:52:01.970Z","version":"6.5.1"}
{"category":"APP","cause":{"code":"UNKNOWN_ERROR","message":"socket hang up","name":"Error","stack":"Error: socket hang up\n at Function.AxiosError.from (/opt/opencti/build/back.js:1750:220145)\n at RedirectableRequest. (/opt/opencti/build/back.js:1753:16871)\n at RedirectableRequest.emit (node:events:518:28)\n at ClientRequest.eventHandlers. (/opt/opencti/build/back.js:306:10004)\n at ClientRequest.emit (node:events:518:28)\n at emitErrorEvent (node:_http_client:101:11)\n at Socket.socketOnEnd (node:_http_client:528:5)\n at Socket.emit (node:events:530:35)\n at endReadableNT (node:internal/streams/readable:1698:12)\n at processTicksAndRejections (node:internal/process/task_queues:82:21)\n at Axios.request (/opt/opencti/build/back.js:1755:3316)\n at processTicksAndRejections (node:internal/process/task_queues:95:5)\n at getConnectorQueueDetails (/opt/opencti/build/back.js:1896:145727)"},"connectorId":"37a4891a-0d1f-413c-bd71-aae2e446b91c","level":"error","message":"Get connector queue details fail","source":"backend","timestamp":"2025-02-10T12:54:20.374Z","version":"6.5.1"}

<--- Last few GCs --->

[7:0x7f0b93d90660] 750379 ms: Scavenge (reduce) 7917.0 (8142.8) -> 7916.9 (8143.8) MB, 3359.77 / 0.00 ms (average mu = 0.211, current mu = 0.063) allocation failure;
[7:0x7f0b93d90660] 757021 ms: Mark-Compact (reduce) 7917.9 (8143.8) -> 7917.8 (8144.8) MB, 6640.00 / 0.00 ms (average mu = 0.304, current mu = 0.347) allocation failure; scavenge might not succeed

<--- JS stacktrace --->

FATAL ERROR: Reached heap limit Allocation failed - JavaScript heap out of memory
----- Native stack trace -----

{"level":"info","message":"[MIGRATION] Read 3 migrations from the database","timestamp":"2025-02-10T12:55:21.829Z"}
{"level":"info","message":"[MIGRATION] Platform already up to date, nothing to migrate","timestamp":"2025-02-10T12:55:21.833Z"}
{"level":"info","message":"[MIGRATION] Migration process completed","timestamp":"2025-02-10T12:55:21.834Z"}
{"category":"APP","cause":{"attributes":{"genre":"BUSINESS","http_status":500,"participantIds":["external-reference--f33542f6-df61-569e-b090-992243589a85"]},"code":"LOCK_ERROR","message":"Execution timeout, too many concurrent call on the same entities","name":"LOCK_ERROR","stack":"GraphQLError: Execution timeout, too many concurrent call on the same entities\n at error (/opt/opencti/build/back.js:1690:2273)\n at LockTimeoutError (/opt/opencti/build/back.js:1690:5707)\n at createEntityRaw (/opt/opencti/build/back.js:2451:131876)\n at processTicksAndRejections (node:internal/process/task_queues:95:5)\n at createEntity (/opt/opencti/build/back.js:2451:132052)\n at addExternalReference (/opt/opencti/build/back.js:3077:97581)"},"level":"error","message":"Execution timeout, too many concurrent call on the same entities","operation":"Unspecified","size":365,"source":"backend","time":3226,"timestamp":"2025-02-10T12:55:37.881Z","type":"WRITE_ERROR","user":{"applicant_id":"88ec0c6a-13ce-5e39-b486-354fe4a7084f","call_retry_number":"323","group_ids":[],"ip":"::ffff:172.19.0.23","organization_ids":[],"socket":"query","user_id":"88ec0c6a-13ce-5e39-b486-354fe4a7084f","user_metadata":{}},"version":"6.5.1"}
{"category":"APP","cause":{"attributes":{"genre":"BUSINESS","http_status":500,"participantIds":["external-reference--f33542f6-df61-569e-b090-992243589a85"]},"code":"LOCK_ERROR","message":"Execution timeout, too many concurrent call on the same entities","name":"LOCK_ERROR","stack":"GraphQLError: Execution timeout, too many concurrent call on the same entities\n at error (/opt/opencti/build/back.js:1690:2273)\n at LockTimeoutError (/opt/opencti/build/back.js:1690:5707)\n at createEntityRaw (/opt/opencti/build/back.js:2451:131876)\n at processTicksAndRejections (node:internal/process/task_queues:95:5)\n at createEntity (/opt/opencti/build/back.js:2451:132052)\n at addExternalReference (/opt/opencti/build/back.js:3077:97581)"},"level":"error","message":"Execution timeout, too many concurrent call on the same entities","operation":"Unspecified","size":365,"source":"backend","time":2673,"timestamp":"2025-02-10T12:55:38.627Z","type":"WRITE_ERROR","user":{"applicant_id":"88ec0c6a-13ce-5e39-b486-354fe4a7084f","call_retry_number":"332","group_ids":[],"ip":"::ffff:172.19.0.24","organization_ids":[],"socket":"query","user_id":"88ec0c6a-13ce-5e39-b486-354fe4a7084f","user_metadata":{}},"version":"6.5.1"}
{"category":"APP","cause":{"attributes":{"genre":"BUSINESS","http_status":500,"participantIds":["external-reference--f33542f6-df61-569e-b090-992243589a85"]},"code":"LOCK_ERROR","message":"Execution timeout, too many concurrent call on the same entities","name":"LOCK_ERROR","stack":"GraphQLError: Execution timeout, too many concurrent call on the same entities\n at error (/opt/opencti/build/back.js:1690:2273)\n at LockTimeoutError (/opt/opencti/build/back.js:1690:5707)\n at createEntityRaw (/opt/opencti/build/back.js:2451:131876)\n at processTicksAndRejections (node:internal/process/task_queues:95:5)\n at createEntity (/opt/opencti/build/back.js:2451:132052)\n at addExternalReference (/opt/opencti/build/back.js:3077:97581)"},"level":"error","message":"Execution timeout, too many concurrent call on the same entities","operation":"Unspecified","size":365,"source":"backend","time":2633,"timestamp":"2025-02-10T12:55:40.199Z","type":"WRITE_ERROR","user":{"applicant_id":"88ec0c6a-13ce-5e39-b486-354fe4a7084f","call_retry_number":"337","group_ids":[],"ip":"::ffff:172.19.0.23","organization_ids":[],"socket":"query","user_id":"88ec0c6a-13ce-5e39-b486-354fe4a7084f","user_metadata":{}},"version":"6.5.1"}
{"category":"APP","cause":{"attributes":{"genre":"BUSINESS","http_status":500,"participantIds":["external-reference--f3a01fa8-8368-5e41-b999-5de89fe8bfef"]},"code":"LOCK_ERROR","message":"Execution timeout, too many concurrent call on the same entities","name":"LOCK_ERROR","stack":"GraphQLError: Execution timeout, too many concurrent call on the same entities\n at error (/opt/opencti/build/back.js:1690:2273)\n at LockTimeoutError (/opt/opencti/build/back.js:1690:5707)\n at createEntityRaw (/opt/opencti/build/back.js:2451:131876)\n at processTicksAndRejections (node:internal/process/task_queues:95:5)\n at createEntity (/opt/opencti/build/back.js:2451:132052)\n at addExternalReference (/opt/opencti/build/back.js:3077:97581)"},"level":"error","message":"Execution timeout, too many concurrent call on the same entities","operation":"Unspecified","size":308,"source":"backend","time":2831,"timestamp":"2025-02-10T12:55:40.864Z","type":"WRITE_ERROR","user":{"applicant_id":"88ec0c6a-13ce-5e39-b486-354fe4a7084f","call_retry_number":"323","group_ids":[],"ip":"::ffff:172.19.0.23","organization_ids":[],"socket":"query","user_id":"88ec0c6a-13ce-5e39-b486-354fe4a7084f","user_metadata":{}},"version":"6.5.1"}
{"category":"APP","cause":{"attributes":{"genre":"BUSINESS","http_status":500,"participantIds":["external-reference--f3a01fa8-8368-5e41-b999-5de89fe8bfef"]},"code":"LOCK_ERROR","message":"Execution timeout, too many concurrent call on the same entities","name":"LOCK_ERROR","stack":"GraphQLError: Execution timeout, too many concurrent call on the same entities\n at error (/opt/opencti/build/back.js:1690:2273)\n at LockTimeoutError (/opt/opencti/build/back.js:1690:5707)\n at createEntityRaw (/opt/opencti/build/back.js:2451:131876)\n at processTicksAndRejections (node:internal/process/task_queues:95:5)\n at createEntity (/opt/opencti/build/back.js:2451:132052)\n at addExternalReference (/opt/opencti/build/back.js:3077:97581)"},"level":"error","message":"Execution timeout, too many concurrent call on the same entities","operation":"Unspecified","size":308,"source":"backend","time":3114,"timestamp":"2025-02-10T12:55:41.748Z","type":"WRITE_ERROR","user":{"applicant_id":"88ec0c6a-13ce-5e39-b486-354fe4a7084f","call_retry_number":"332","group_ids":[],"ip":"::ffff:172.19.0.24","organization_ids":[],"socket":"query","user_id":"88ec0c6a-13ce-5e39-b486-354fe4a7084f","user_metadata":{}},"version":"6.5.1"}
{"category":"APP","cause":{"attributes":{"genre":"BUSINESS","http_status":500,"participantIds":["external-reference--f3a01fa8-8368-5e41-b999-5de89fe8bfef"]},"code":"LOCK_ERROR","message":"Execution timeout, too many concurrent call on the same entities","name":"LOCK_ERROR","stack":"GraphQLError: Execution timeout, too many concurrent call on the same entities\n at error (/opt/opencti/build/back.js:1690:2273)\n at LockTimeoutError (/opt/opencti/build/back.js:1690:5707)\n at createEntityRaw (/opt/opencti/build/back.js:2451:131876)\n at processTicksAndRejections (node:internal/process/task_queues:95:5)\n at createEntity (/opt/opencti/build/back.js:2451:132052)\n at addExternalReference (/opt/opencti/build/back.js:3077:97581)"},"level":"error","message":"Execution timeout, too many concurrent call on the same entities","operation":"Unspecified","size":308,"source":"backend","time":3616,"timestamp":"2025-02-10T12:55:43.822Z","type":"WRITE_ERROR","user":{"applicant_id":"88ec0c6a-13ce-5e39-b486-354fe4a7084f","call_retry_number":"337","group_ids":[],"ip":"::ffff:172.19.0.23","organization_ids":[],"socket":"query","user_id":"88ec0c6a-13ce-5e39-b486-354fe4a7084f","user_metadata":{}},"version":"6.5.1"}
{"category":"APP","cause":{"attributes":{"genre":"BUSINESS","http_status":500,"participantIds":["external-reference--f33542f6-df61-569e-b090-992243589a85"]},"code":"LOCK_ERROR","message":"Execution timeout, too many concurrent call on the same entities","name":"LOCK_ERROR","stack":"GraphQLError: Execution timeout, too many concurrent call on the same entities\n at error (/opt/opencti/build/back.js:1690:2273)\n at LockTimeoutError (/opt/opencti/build/back.js:1690:5707)\n at createEntityRaw (/opt/opencti/build/back.js:2451:131876)\n at processTicksAndRejections (node:internal/process/task_queues:95:5)\n at createEntity (/opt/opencti/build/back.js:2451:132052)\n at addExternalReference (/opt/opencti/build/back.js:3077:97581)"},"level":"error","message":"Execution timeout, too many concurrent call on the same entities","operation":"Unspecified","size":365,"source":"backend","time":3313,"timestamp":"2025-02-10T12:55:45.819Z","type":"WRITE_ERROR","user":{"applicant_id":"88ec0c6a-13ce-5e39-b486-354fe4a7084f","call_retry_number":"344","group_ids":[],"ip":"::ffff:172.19.0.17","organization_ids":[],"socket":"query","user_id":"88ec0c6a-13ce-5e39-b486-354fe4a7084f","user_metadata":{}},"version":"6.5.1"}
{"category":"APP","cause":{"attributes":{"genre":"BUSINESS","http_status":500,"participantIds":["external-reference--38cfe4c7-e867-56ee-b776-c0c3b99766cd"]},"code":"LOCK_ERROR","message":"Execution timeout, too many concurrent call on the same entities","name":"LOCK_ERROR","stack":"GraphQLError: Execution timeout, too many concurrent call on the same entities\n at error (/opt/opencti/build/back.js:1690:2273)\n at LockTimeoutError (/opt/opencti/build/back.js:1690:5707)\n at createEntityRaw (/opt/opencti/build/back.js:2451:131876)\n at processTicksAndRejections (node:internal/process/task_queues:95:5)\n at createEntity (/opt/opencti/build/back.js:2451:132052)\n at addExternalReference (/opt/opencti/build/back.js:3077:97581)"},"level":"error","message":"Execution timeout, too many concurrent call on the same entities","operation":"Unspecified","size":317,"source":"backend","time":2613,"timestamp":"2025-02-10T12:55:46.834Z","type":"WRITE_ERROR","user":{"applicant_id":"88ec0c6a-13ce-5e39-b486-354fe4a7084f","call_retry_number":"338","group_ids":[],"ip":"::ffff:172.19.0.24","organization_ids":[],"socket":"query","user_id":"88ec0c6a-13ce-5e39-b486-354fe4a7084f","user_metadata":{}},"version":"6.5.1"}
{"category":"APP","cause":{"attributes":{"genre":"BUSINESS","http_status":500,"participantIds":["external-reference--f3a01fa8-8368-5e41-b999-5de89fe8bfef"]},"code":"LOCK_ERROR","message":"Execution timeout, too many concurrent call on the same entities","name":"LOCK_ERROR","stack":"GraphQLError: Execution timeout, too many concurrent call on the same entities\n at error (/opt/opencti/build/back.js:1690:2273)\n at LockTimeoutError (/opt/opencti/build/back.js:1690:5707)\n at createEntityRaw (/opt/opencti/build/back.js:2451:131876)\n at processTicksAndRejections (node:internal/process/task_queues:95:5)\n at createEntity (/opt/opencti/build/back.js:2451:132052)\n at addExternalReference (/opt/opencti/build/back.js:3077:97581)"},"level":"error","message":"Execution timeout, too many concurrent call on the same entities","operation":"Unspecified","size":308,"source":"backend","time":3075,"timestamp":"2025-02-10T12:55:48.902Z","type":"WRITE_ERROR","user":{"applicant_id":"88ec0c6a-13ce-5e39-b486-354fe4a7084f","call_retry_number":"344","group_ids":[],"ip":"::ffff:172.19.0.17","organization_ids":[],"socket":"query","user_id":"88ec0c6a-13ce-5e39-b486-354fe4a7084f","user_metadata":{}},"version":"6.5.1"}

Screenshots (optional)

[marieflorescontact]

There is potentially a lock issue with external-reference--5b3eee59-cc9e-5266-9fd7-dd75402a8aa1. Could you check the element history if there is no conflict between rule and something else?

[Deventual]

Yes, but it doesn’t seem that this is the cause of the web app crash, am I right?

[richard-julien]

Hi @Deventual , its really difficult to diagnose OOM with an error logs.
I know this is a lot to ask but to get more information we will need you to install pyroscope https://pyroscope.io/ and configure OpenCTI like this

"app": {
    "telemetry": {
      "pyroscope": {
        "enabled": false,
        "identifier": "OpenCTI",
        "exporter": "YOUR_PYROSCOPE"
      },
    },
}

Then you will be able to see a flamegraph of your instance with cpu and memory usage.
After that if you agree to have a live debug session with your pyroscope interface in screen sharing I think we will be able to understand what could be the cause of the OOM.

Thanks