[GATEWAY V2]: Bifurcate connect / connection-acquire timeout between Gateway V1 and Gateway V2 endpoints.#48174
Build #20260304.3 had test failures
Details
- Failed: 14 (0.02%)
- Passed: 56,396 (91.27%)
- Other: 5,381 (8.71%)
- Total: 61,791
Annotations
azure-pipelines / java - cosmos - tests
c.a.c.CosmosItemSerializerTest.CosmosItemSerializerTest::batchAndChangeFeedWithObjectNode[Gateway with Session consistency](WriteRetriesWithTrackingId|RequestOptions_EnvelopWrappingItemSerializer|Client_DEFAULT)[com.azure.cosmos.CosmosItemSerializerTest$EnvelopWrappingItemSerializer@3438309c](4)
{"innerErrorMessage":null,"cosmosDiagnostics":{"userAgent":"azsdk-java-cosmos/4.79.0-beta.1 Linux/6.11.0-1018-azure JRE/25.0.1","activityId":"8125cfad-65d3-4b3f-8d10-9fd01e95031f","requestLatencyInMs":60039,"requestStartTimeUTC":"2026-03-04T23:20:04.235923275Z","requestEndTimeUTC":"2026-03-04T23:21:04.275589298Z","responseStatisticsList":[],"supplementalResponseStatisticsList":[],"addressResolutionStatistics":{},"regionsContacted":["west central us"],"retryContext":{"statusAndSubStatusCodes":[[408,10002]],"retryCount":1,"retryLatency":0},"metadataDiagnosticsContext":{"metadataDiagnosticList":null,"empty":true},"serializationDiagnosticsContext":{"serializationDiagnosticsList":null},"gatewayStatisticsList":[{"sessionToken":null,"operationType":"Batch","resourceType":"Document","statusCode":408,"subStatusCode":10002,"requestCharge":0.0,"requestTimeline":[{"eventName":"connectionAcquired","startTimeUTC":"2026-03-04T23:20:04.236125594Z","durationInMilliSecs":0.083073},{"eventName":"connectionConfigured","startTimeUTC":"2026-03-04T23:20:04.236208667Z","durationInMilliSecs":0.020288},{"eventName":"requestSent","startTimeUTC":"2026-03-04T23:20:04.236228955Z","durationInMilliSecs":0.406382},{"eventName":"transitTime","startTimeUTC":"2026-03-04T23:20:04.236635337Z","durationInMilliSecs":60038.887838},{"eventName":"received","startTimeUTC":null,"durationInMilliSecs":0.0}],"partitionKeyRangeId":null,"responsePayloadSizeInBytes":0,"httpNetworkResponseTimeout":"PT1M","exceptionResponseHeaders":"{x-ms-substatus=10002}","endpoint":"https://tbf3cce10641e4fd1-westcentralus.documents.azure.com:443/dbs/RxJava.SDKTest.SharedDatabase_20260304T230139_YlY/colls/9904c1e8-a267-447a-82aa-39cd6b8a689e/docs","isHubRegionProcessingOnly":"false","perPartitionCircuitBreakerInfoHolder":{"perPartitionCircuitBreakerInfoHolder":null},"perPartitionAutomaticFailoverInfoHolder":null,"channelId":"48ae4454","parentChannelId":"48ae4454"}],"samplingRateSnapshot":1.0,"bloomFilterInsertionCountSnapshot":0,"systemInformation":{"usedMemory":"77096 KB","availableMemory":"4117208 KB","systemCpuLoad":"(2026-03-04T23:20:37.814639954Z 0.8%), (2026-03-04T23:20:42.814638395Z 1.0%), (2026-03-04T23:20:47.814637505Z 0.7%), (2026-03-04T23:20:52.814641670Z 0.4%), (2026-03-04T23:20:57.814638064Z 0.5%), (2026-03-04T23:21:02.814640024Z 2.0%)","availableProcessors":4},"clientCfgs":{"id":267,"machineId":"uuid:d96ab7e0-f23a-4234-ae16-3366143bee96","connectionMode":"GATEWAY","numberOfClients":1,"isPpafEnabled":"false","isFalseProgSessionTokenMergeEnabled":"true","excrgns":"[]","clientEndpoints":{"https://tbf3cce10641e4fd1.documents.azure.com:443/":267},"connCfg":{"rntbd":null,"gw":"(cps:1000, nrto:PT1M, icto:PT1M, cto:PT45S, gwV2Cto:n/a, p:false, http2:(enabled:false, maxc:1000, minc:8, maxs:30))","other":"(ed: true, cs: false, rv: true)"},"consistencyCfg":"(consistency: Session, readConsistencyStrategy: null, mm: false, prgns: [])","proactiveInitCfg":"","e2ePolicyCfg":"","sessionRetryCfg":""}}}
Raw output
{"ClassName":"CosmosException","userAgent":"azsdk-java-cosmos/4.79.0-beta.1 Linux/6.11.0-1018-azure JRE/25.0.1","statusCode":408,"resourceAddress":"https://tbf3cce10641e4fd1-westcentralus.documents.azure.com:443/dbs/RxJava.SDKTest.SharedDatabase_20260304T230139_YlY/colls/9904c1e8-a267-447a-82aa-39cd6b8a689e/docs","innerErrorMessage":null,"causeInfo":"[class: class io.netty.handler.timeout.ReadTimeoutException, message: null]","responseHeaders":"{x-ms-substatus=10002}","requestHeaders":"[x-ms-cosmos-batch-atomic=true, x-ms-cosmos-sdk-supportedcapabilities=1, x-ms-cosmos-intended-collection-rid=X49aAKikFQw=, Accept=application/json, x-ms-session-token=0:-1#1214, x-ms-cosmos-is-batch-request=true, x-ms-date=Wed, 04 Mar 2026 23:20:04 GMT, x-ms-cosmos-batch-continue-on-error=false, x-ms-documentdb-partitionkey=[\"54c40724-0761-43f4-9dee-0be54bfb005e\"], x-ms-consistency-level=Session, Content-Type=application/json]","cosmosDiagnostics":{"userAgent":"azsdk-java-cosmos/4.79.0-beta.1 Linux/6.11.0-1018-azure JRE/25.0.1","activityId":"8125cfad-65d3-4b3f-8d10-9fd01e95031f","requestLatencyInMs":60039,"requestStartTimeUTC":"2026-03-04T23:20:04.235923275Z","requestEndTimeUTC":"2026-03-04T23:21:04.275589298Z","responseStatisticsList":[],"supplementalResponseStatisticsList":[],"addressResolutionStatistics":{},"regionsContacted":["west central us"],"retryContext":{"statusAndSubStatusCodes":[[408,10002]],"retryCount":1,"retryLatency":0},"metadataDiagnosticsContext":{"metadataDiagnosticList":null,"empty":true},"serializationDiagnosticsContext":{"serializationDiagnosticsList":null},"gatewayStatisticsList":[{"sessionToken":null,"operationType":"Batch","resourceType":"Document","statusCode":408,"subStatusCode":10002,"requestCharge":0.0,"requestTimeline":[{"eventName":"connectionAcquired","startTimeUTC":"2026-03-04T23:20:04.236125594Z","durationInMilliSecs":0.083073},{"eventName":"connectionConfigured","startTimeUTC":"2026-03-04T23:20:04.236208667Z","durationInMilliSecs":0.020288},{"eventName":"requestSent","startTimeUTC":"2026-03-04T23:20:04.236228955Z","durationInMilliSecs":0.406382},{"eventName":"transitTime","startTimeUTC":"2026-03-04T23:20:04.236635337Z","durationInMilliSecs":60038.887838},{"eventName":"received","startTimeUTC":null,"durationInMilliSecs":0.0}],"partitionKeyRangeId":null,"responsePayloadSizeInBytes":0,"httpNetworkResponseTimeout":"PT1M","exceptionResponseHeaders":"{x-ms-substatus=10002}","endpoint":"https://tbf3cce10641e4fd1-westcentralus.documents.azure.com:443/dbs/RxJava.SDKTest.SharedDatabase_20260304T230139_YlY/colls/9904c1e8-a267-447a-82aa-39cd6b8a689e/docs","isHubRegionProcessingOnly":"false","perPartitionCircuitBreakerInfoHolder":{"perPartitionCircuitBreakerInfoHolder":null},"perPartitionAutomaticFailoverInfoHolder":null,"channelId":"48ae4454","parentChannelId":"48ae4454"}],"samplingRateSnapshot":1.0,"bloomFilterInsertionCountSnapshot":0,"systemInformation":{"usedMemory":"77273 KB","availableMemory":"4117031 KB","systemCpuLoad":"(2026-03-04T23:20:37.814639954Z 0.8%), (2026-03-04T23:20:42.814638395Z 1.0%), (2026-03-04T23:20:47.814637505Z 0.7%), (2026-03-04T23:20:52.814641670Z 0.4%), (2026-03-04T23:20:57.814638064Z 0.5%), (2026-03-04T23:21:02.814640024Z 2.0%)","availableProcessors":4},"clientCfgs":{"id":267,"machineId":"uuid:d96ab7e0-f23a-4234-ae16-3366143bee96","connectionMode":"GATEWAY","numberOfClients":1,"isPpafEnabled":"false","isFalseProgSessionTokenMergeEnabled":"true","excrgns":"[]","clientEndpoints":{"https://tbf3cce10641e4fd1.documents.azure.com:443/":267},"connCfg":{"rntbd":null,"gw":"(cps:1000, nrto:PT1M, icto:PT1M, cto:PT45S, gwV2Cto:n/a, p:false, http2:(enabled:false, maxc:1000, minc:8, maxs:30))","other":"(ed: true, cs: false, rv: true)"},"consistencyCfg":"(consistency: Session, readConsistencyStrategy: null, mm: false, prgns: [])","proactiveInitCfg":"","e2ePolicyCfg":"","sessionRetryCfg":""}}}
at com.azure.cosmos.BridgeInternal.createCosmosException(BridgeInternal.java:416)
at com.azure.cosmos.implementat
azure-pipelines / java - cosmos - tests
c.a.c.CosmosItemTest.CosmosItemTest::readManyWithTwoSecondariesNotReachable[Direct Tcp with BoundedStaleness consistency]
{"innerErrorMessage":"Service is currently unavailable, please retry after a while. More info: https://aka.ms/cosmosdb-tsg-service-unavailable-java: Message: {\"innerErrorMessage\":\"READ Quorum size of 2 is not met for the request.\"}","cosmosDiagnostics":{"userAgent":"azsdk-java-cosmos/4.79.0-beta.1 Linux/6.11.0-1018-azure JRE/25.0.1","activityId":"ed07b6fa-f2e0-439c-895c-9d45f5e2be6b","requestLatencyInMs":60259,"requestStartTimeUTC":"2026-03-04T23:45:27.188206111Z","requestEndTimeUTC":"2026-03-04T23:46:27.447883489Z","responseStatisticsList":[{"storeResult":{"storePhysicalAddress":"rntbd://cdb-ms-prod-westcentralus1-be10.documents.azure.com:14003/apps/8974a354-e4f8-4edb-8e8d-d46f0a9d3da4/services/c8bbddbe-878c-433f-bcdb-008eb61081cf/partitions/6c730c88-26f7-4e5e-a9ad-1c2793e63beb/replicas/134171167846900235s/","lsn":-1,"quorumAckedLSN":-1,"currentReplicaSetSize":-1,"globalCommittedLsn":-1,"globalNRegionCommittedGlsn":-1,"partitionKeyRangeId":"0","isValid":false,"statusCode":410,"subStatusCode":21005,"isGone":true,"isNotFound":false,"isInvalidPartition":false,"isThroughputControlRequestRateTooLarge":false,"requestCharge":0.0,"itemLSN":-1,"sessionToken":null,"backendLatencyInMs":null,"retryAfterInMs":null,"exceptionMessage":"Fault injection server error [The requested resource is no longer available at the server.]","exceptionResponseHeaders":"{x-ms-substatus=21005}","faultInjectionRuleId":"extremelyLongConnectDelay","replicaStatusList":{"Ignoring":["14061:P:Connected"],"Attempting":["14003:S:Connected","14313:S:Connected","14037:S:Connected"]},"transportRequestTimeline":[{"eventName":"created","startTimeUTC":"2026-03-04T23:45:27.188379436Z","durationInMilliSecs":0.003006},{"eventName":"queued","startTimeUTC":"2026-03-04T23:45:27.188382442Z","durationInMilliSecs":1.4E-4},{"eventName":"channelAcquisitionStarted","startTimeUTC":"2026-03-04T23:45:27.188382582Z","durationInMilliSecs":0.112633},{"eventName":"pipelined","startTimeUTC":"2026-03-04T23:45:27.188495215Z","durationInMilliSecs":0.089536},{"eventName":"transitTime","startTimeUTC":null,"durationInMilliSecs":0.0},{"eventName":"decodeTime","startTimeUTC":null,"durationInMilliSecs":0.0},{"eventName":"received","startTimeUTC":null,"durationInMilliSecs":0.0},{"eventName":"completed","startTimeUTC":"2026-03-04T23:45:27.188584751Z","durationInMilliSecs":0.002174}],"rntbdRequestLengthInBytes":-1,"rntbdResponseLengthInBytes":-1,"requestPayloadLengthInBytes":4629,"responsePayloadLengthInBytes":-1,"channelStatistics":{"channelId":"ea7b2167","channelTaskQueueSize":1,"pendingRequestsCount":0,"lastReadTime":"2026-03-04T23:45:25.109004883Z","waitForConnectionInit":false},"serviceEndpointStatistics":{"availableChannels":1,"acquiredChannels":0,"executorTaskQueueSize":0,"inflightRequests":1,"lastSuccessfulRequestTime":"2026-03-04T23:35:01.384Z","lastRequestTime":"2026-03-04T23:45:25.108Z","createdTime":"2026-03-04T23:34:45.087089238Z","isClosed":false,"cerMetrics":{}}},"requestResponseTimeUTC":"2026-03-04T23:45:27.188618824Z","requestStartTimeUTC":"2026-03-04T23:45:27.188379436Z","requestResourceType":"Document","requestOperationType":"Query","requestSessionToken":null,"e2ePolicyCfg":"{e2eto=PT1M10S, as=}","excludedRegions":null,"sessionTokenEvaluationResults":[],"perPartitionCircuitBreakerInfoHolder":null,"perPartitionAutomaticFailoverInfoHolder":null,"isHubRegionProcessingOnly":"false","perPartitionFailoverInfoHolder":null},{"storeResult":{"storePhysicalAddress":"rntbd://cdb-ms-prod-westcentralus1-be10.documents.azure.com:14037/apps/8974a354-e4f8-4edb-8e8d-d46f0a9d3da4/services/c8bbddbe-878c-433f-bcdb-008eb61081cf/partitions/6c730c88-26f7-4e5e-a9ad-1c2793e63beb/replicas/134171167846900237s/","lsn":-1,"quorumAckedLSN":-1,"currentReplicaSetSize":-1,"globalCommittedLsn":-1,"globalNRegionCommittedGlsn":-1,"partitionKeyRangeId":"0","isValid":false,"statusCode":410,"subStatusCode":21005,"isGone":true,"isNotFound":false,"isInvalidPartition":false,"isThroughputControlRequestRateTooLarge":false,"r
Raw output
{"ClassName":"ServiceUnavailableException","userAgent":"azsdk-java-cosmos/4.79.0-beta.1 Linux/6.11.0-1018-azure JRE/25.0.1","statusCode":503,"resourceAddress":null,"innerErrorMessage":"Service is currently unavailable, please retry after a while. More info: https://aka.ms/cosmosdb-tsg-service-unavailable-java: Message: {\"innerErrorMessage\":\"READ Quorum size of 2 is not met for the request.\"}","causeInfo":"[class: class com.azure.cosmos.implementation.GoneException, message: {\"innerErrorMessage\":\"READ Quorum size of 2 is not met for the request.\"}]","responseHeaders":"{x-ms-substatus=21007}","cosmosDiagnostics":{"userAgent":"azsdk-java-cosmos/4.79.0-beta.1 Linux/6.11.0-1018-azure JRE/25.0.1","activityId":"ed07b6fa-f2e0-439c-895c-9d45f5e2be6b","requestLatencyInMs":60259,"requestStartTimeUTC":"2026-03-04T23:45:27.188206111Z","requestEndTimeUTC":"2026-03-04T23:46:27.447883489Z","responseStatisticsList":[{"storeResult":{"storePhysicalAddress":"rntbd://cdb-ms-prod-westcentralus1-be10.documents.azure.com:14003/apps/8974a354-e4f8-4edb-8e8d-d46f0a9d3da4/services/c8bbddbe-878c-433f-bcdb-008eb61081cf/partitions/6c730c88-26f7-4e5e-a9ad-1c2793e63beb/replicas/134171167846900235s/","lsn":-1,"quorumAckedLSN":-1,"currentReplicaSetSize":-1,"globalCommittedLsn":-1,"globalNRegionCommittedGlsn":-1,"partitionKeyRangeId":"0","isValid":false,"statusCode":410,"subStatusCode":21005,"isGone":true,"isNotFound":false,"isInvalidPartition":false,"isThroughputControlRequestRateTooLarge":false,"requestCharge":0.0,"itemLSN":-1,"sessionToken":null,"backendLatencyInMs":null,"retryAfterInMs":null,"exceptionMessage":"Fault injection server error [The requested resource is no longer available at the server.]","exceptionResponseHeaders":"{x-ms-substatus=21005}","faultInjectionRuleId":"extremelyLongConnectDelay","replicaStatusList":{"Ignoring":["14061:P:Connected"],"Attempting":["14003:S:Connected","14313:S:Connected","14037:S:Connected"]},"transportRequestTimeline":[{"eventName":"created","startTimeUTC":"2026-03-04T23:45:27.188379436Z","durationInMilliSecs":0.003006},{"eventName":"queued","startTimeUTC":"2026-03-04T23:45:27.188382442Z","durationInMilliSecs":1.4E-4},{"eventName":"channelAcquisitionStarted","startTimeUTC":"2026-03-04T23:45:27.188382582Z","durationInMilliSecs":0.112633},{"eventName":"pipelined","startTimeUTC":"2026-03-04T23:45:27.188495215Z","durationInMilliSecs":0.089536},{"eventName":"transitTime","startTimeUTC":null,"durationInMilliSecs":0.0},{"eventName":"decodeTime","startTimeUTC":null,"durationInMilliSecs":0.0},{"eventName":"received","startTimeUTC":null,"durationInMilliSecs":0.0},{"eventName":"completed","startTimeUTC":"2026-03-04T23:45:27.188584751Z","durationInMilliSecs":0.002174}],"rntbdRequestLengthInBytes":-1,"rntbdResponseLengthInBytes":-1,"requestPayloadLengthInBytes":4629,"responsePayloadLengthInBytes":-1,"channelStatistics":{"channelId":"ea7b2167","channelTaskQueueSize":1,"pendingRequestsCount":0,"lastReadTime":"2026-03-04T23:45:25.109004883Z","waitForConnectionInit":false},"serviceEndpointStatistics":{"availableChannels":1,"acquiredChannels":0,"executorTaskQueueSize":0,"inflightRequests":1,"lastSuccessfulRequestTime":"2026-03-04T23:35:01.384Z","lastRequestTime":"2026-03-04T23:45:25.108Z","createdTime":"2026-03-04T23:34:45.087089238Z","isClosed":false,"cerMetrics":{}}},"requestResponseTimeUTC":"2026-03-04T23:45:27.188618824Z","requestStartTimeUTC":"2026-03-04T23:45:27.188379436Z","requestResourceType":"Document","requestOperationType":"Query","requestSessionToken":null,"e2ePolicyCfg":"{e2eto=PT1M10S, as=}","excludedRegions":null,"sessionTokenEvaluationResults":[],"perPartitionCircuitBreakerInfoHolder":null,"perPartitionAutomaticFailoverInfoHolder":null,"isHubRegionProcessingOnly":"false","perPartitionFailoverInfoHolder":null},{"storeResult":{"storePhysicalAddress":"rntbd://cdb-ms-prod-westcentralus1-be10.documents.azure.com:14037/apps/8974a354-e4f8-4edb-8e8d-d46f0a9d3da4/services/c8bbddbe-878c-433f-bcdb-008eb61081cf/partitions/6
Check failure on line 1 in c.a.c.CosmosItemSerializerTest.batchAndChangeFeedWithObjectNode
azure-pipelines / java - cosmos - tests
c.a.c.CosmosItemSerializerTest.batchAndChangeFeedWithObjectNode
{"innerErrorMessage":null,"cosmosDiagnostics":{"userAgent":"azsdk-java-cosmos/4.79.0-beta.1 Linux/6.11.0-1018-azure JRE/25.0.1","activityId":"8125cfad-65d3-4b3f-8d10-9fd01e95031f","requestLatencyInMs":60039,"requestStartTimeUTC":"2026-03-04T23:20:04.235923275Z","requestEndTimeUTC":"2026-03-04T23:21:04.275589298Z","responseStatisticsList":[],"supplementalResponseStatisticsList":[],"addressResolutionStatistics":{},"regionsContacted":["west central us"],"retryContext":{"statusAndSubStatusCodes":[[408,10002]],"retryCount":1,"retryLatency":0},"metadataDiagnosticsContext":{"metadataDiagnosticList":null,"empty":true},"serializationDiagnosticsContext":{"serializationDiagnosticsList":null},"gatewayStatisticsList":[{"sessionToken":null,"operationType":"Batch","resourceType":"Document","statusCode":408,"subStatusCode":10002,"requestCharge":0.0,"requestTimeline":[{"eventName":"connectionAcquired","startTimeUTC":"2026-03-04T23:20:04.236125594Z","durationInMilliSecs":0.083073},{"eventName":"connectionConfigured","startTimeUTC":"2026-03-04T23:20:04.236208667Z","durationInMilliSecs":0.020288},{"eventName":"requestSent","startTimeUTC":"2026-03-04T23:20:04.236228955Z","durationInMilliSecs":0.406382},{"eventName":"transitTime","startTimeUTC":"2026-03-04T23:20:04.236635337Z","durationInMilliSecs":60038.887838},{"eventName":"received","startTimeUTC":null,"durationInMilliSecs":0.0}],"partitionKeyRangeId":null,"responsePayloadSizeInBytes":0,"httpNetworkResponseTimeout":"PT1M","exceptionResponseHeaders":"{x-ms-substatus=10002}","endpoint":"https://tbf3cce10641e4fd1-westcentralus.documents.azure.com:443/dbs/RxJava.SDKTest.SharedDatabase_20260304T230139_YlY/colls/9904c1e8-a267-447a-82aa-39cd6b8a689e/docs","isHubRegionProcessingOnly":"false","perPartitionCircuitBreakerInfoHolder":{"perPartitionCircuitBreakerInfoHolder":null},"perPartitionAutomaticFailoverInfoHolder":null,"channelId":"48ae4454","parentChannelId":"48ae4454"}],"samplingRateSnapshot":1.0,"bloomFilterInsertionCountSnapshot":0,"systemInformation":{"usedMemory":"153562 KB","availableMemory":"4040742 KB","systemCpuLoad":"empty","availableProcessors":4},"clientCfgs":{"id":267,"machineId":"uuid:d96ab7e0-f23a-4234-ae16-3366143bee96","connectionMode":"GATEWAY","numberOfClients":0,"isPpafEnabled":"false","isFalseProgSessionTokenMergeEnabled":"true","excrgns":"[]","clientEndpoints":{"https://tbf3cce10641e4fd1.documents.azure.com:443/":534,"https://13.71.194.15:443/":6},"connCfg":{"rntbd":null,"gw":"(cps:1000, nrto:PT1M, icto:PT1M, cto:PT45S, gwV2Cto:n/a, p:false, http2:(enabled:false, maxc:1000, minc:8, maxs:30))","other":"(ed: true, cs: false, rv: true)"},"consistencyCfg":"(consistency: Session, readConsistencyStrategy: null, mm: false, prgns: [])","proactiveInitCfg":"","e2ePolicyCfg":"","sessionRetryCfg":""}}}
Raw output
{"ClassName":"CosmosException","userAgent":"azsdk-java-cosmos/4.79.0-beta.1 Linux/6.11.0-1018-azure JRE/25.0.1","statusCode":408,"resourceAddress":"https://tbf3cce10641e4fd1-westcentralus.documents.azure.com:443/dbs/RxJava.SDKTest.SharedDatabase_20260304T230139_YlY/colls/9904c1e8-a267-447a-82aa-39cd6b8a689e/docs","innerErrorMessage":null,"causeInfo":"[class: class io.netty.handler.timeout.ReadTimeoutException, message: null]","responseHeaders":"{x-ms-substatus=10002}","requestHeaders":"[x-ms-cosmos-batch-atomic=true, x-ms-cosmos-sdk-supportedcapabilities=1, x-ms-cosmos-intended-collection-rid=X49aAKikFQw=, Accept=application/json, x-ms-session-token=0:-1#1214, x-ms-cosmos-is-batch-request=true, x-ms-date=Wed, 04 Mar 2026 23:20:04 GMT, x-ms-cosmos-batch-continue-on-error=false, x-ms-documentdb-partitionkey=[\"54c40724-0761-43f4-9dee-0be54bfb005e\"], x-ms-consistency-level=Session, Content-Type=application/json]","cosmosDiagnostics":{"userAgent":"azsdk-java-cosmos/4.79.0-beta.1 Linux/6.11.0-1018-azure JRE/25.0.1","activityId":"8125cfad-65d3-4b3f-8d10-9fd01e95031f","requestLatencyInMs":60039,"requestStartTimeUTC":"2026-03-04T23:20:04.235923275Z","requestEndTimeUTC":"2026-03-04T23:21:04.275589298Z","responseStatisticsList":[],"supplementalResponseStatisticsList":[],"addressResolutionStatistics":{},"regionsContacted":["west central us"],"retryContext":{"statusAndSubStatusCodes":[[408,10002]],"retryCount":1,"retryLatency":0},"metadataDiagnosticsContext":{"metadataDiagnosticList":null,"empty":true},"serializationDiagnosticsContext":{"serializationDiagnosticsList":null},"gatewayStatisticsList":[{"sessionToken":null,"operationType":"Batch","resourceType":"Document","statusCode":408,"subStatusCode":10002,"requestCharge":0.0,"requestTimeline":[{"eventName":"connectionAcquired","startTimeUTC":"2026-03-04T23:20:04.236125594Z","durationInMilliSecs":0.083073},{"eventName":"connectionConfigured","startTimeUTC":"2026-03-04T23:20:04.236208667Z","durationInMilliSecs":0.020288},{"eventName":"requestSent","startTimeUTC":"2026-03-04T23:20:04.236228955Z","durationInMilliSecs":0.406382},{"eventName":"transitTime","startTimeUTC":"2026-03-04T23:20:04.236635337Z","durationInMilliSecs":60038.887838},{"eventName":"received","startTimeUTC":null,"durationInMilliSecs":0.0}],"partitionKeyRangeId":null,"responsePayloadSizeInBytes":0,"httpNetworkResponseTimeout":"PT1M","exceptionResponseHeaders":"{x-ms-substatus=10002}","endpoint":"https://tbf3cce10641e4fd1-westcentralus.documents.azure.com:443/dbs/RxJava.SDKTest.SharedDatabase_20260304T230139_YlY/colls/9904c1e8-a267-447a-82aa-39cd6b8a689e/docs","isHubRegionProcessingOnly":"false","perPartitionCircuitBreakerInfoHolder":{"perPartitionCircuitBreakerInfoHolder":null},"perPartitionAutomaticFailoverInfoHolder":null,"channelId":"48ae4454","parentChannelId":"48ae4454"}],"samplingRateSnapshot":1.0,"bloomFilterInsertionCountSnapshot":0,"systemInformation":{"usedMemory":"153562 KB","availableMemory":"4040742 KB","systemCpuLoad":"empty","availableProcessors":4},"clientCfgs":{"id":267,"machineId":"uuid:d96ab7e0-f23a-4234-ae16-3366143bee96","connectionMode":"GATEWAY","numberOfClients":0,"isPpafEnabled":"false","isFalseProgSessionTokenMergeEnabled":"true","excrgns":"[]","clientEndpoints":{"https://tbf3cce10641e4fd1.documents.azure.com:443/":534,"https://13.71.194.15:443/":6},"connCfg":{"rntbd":null,"gw":"(cps:1000, nrto:PT1M, icto:PT1M, cto:PT45S, gwV2Cto:n/a, p:false, http2:(enabled:false, maxc:1000, minc:8, maxs:30))","other":"(ed: true, cs: false, rv: true)"},"consistencyCfg":"(consistency: Session, readConsistencyStrategy: null, mm: false, prgns: [])","proactiveInitCfg":"","e2ePolicyCfg":"","sessionRetryCfg":""}}}
at com.azure.cosmos.BridgeInternal.createCosmosException(BridgeInternal.java:416)
at com.azure.cosmos.implementation.RxGatewayStoreModel.lambda$toDocumentServiceResponse$7(RxGatewayStoreModel.java:600)
at reactor.core.publisher.FluxOnErrorResume$ResumeSubscriber.onError(FluxOnErrorResume.java:94)
at reactor.co
Check failure on line 1 in c.a.c.CosmosItemTest.readManyWithTwoSecondariesNotReachable
azure-pipelines / java - cosmos - tests
c.a.c.CosmosItemTest.readManyWithTwoSecondariesNotReachable
{"innerErrorMessage":"Service is currently unavailable, please retry after a while. More info: https://aka.ms/cosmosdb-tsg-service-unavailable-java: Message: {\"innerErrorMessage\":\"READ Quorum size of 2 is not met for the request.\"}","cosmosDiagnostics":{"userAgent":"azsdk-java-cosmos/4.79.0-beta.1 Linux/6.11.0-1018-azure JRE/25.0.1","activityId":"ed07b6fa-f2e0-439c-895c-9d45f5e2be6b","requestLatencyInMs":60259,"requestStartTimeUTC":"2026-03-04T23:45:27.188206111Z","requestEndTimeUTC":"2026-03-04T23:46:27.447883489Z","responseStatisticsList":[{"storeResult":{"storePhysicalAddress":"rntbd://cdb-ms-prod-westcentralus1-be10.documents.azure.com:14003/apps/8974a354-e4f8-4edb-8e8d-d46f0a9d3da4/services/c8bbddbe-878c-433f-bcdb-008eb61081cf/partitions/6c730c88-26f7-4e5e-a9ad-1c2793e63beb/replicas/134171167846900235s/","lsn":-1,"quorumAckedLSN":-1,"currentReplicaSetSize":-1,"globalCommittedLsn":-1,"globalNRegionCommittedGlsn":-1,"partitionKeyRangeId":"0","isValid":false,"statusCode":410,"subStatusCode":21005,"isGone":true,"isNotFound":false,"isInvalidPartition":false,"isThroughputControlRequestRateTooLarge":false,"requestCharge":0.0,"itemLSN":-1,"sessionToken":null,"backendLatencyInMs":null,"retryAfterInMs":null,"exceptionMessage":"Fault injection server error [The requested resource is no longer available at the server.]","exceptionResponseHeaders":"{x-ms-substatus=21005}","faultInjectionRuleId":"extremelyLongConnectDelay","replicaStatusList":{"Ignoring":["14061:P:Connected"],"Attempting":["14003:S:Connected","14313:S:Connected","14037:S:Connected"]},"transportRequestTimeline":[{"eventName":"created","startTimeUTC":"2026-03-04T23:45:27.188379436Z","durationInMilliSecs":0.003006},{"eventName":"queued","startTimeUTC":"2026-03-04T23:45:27.188382442Z","durationInMilliSecs":1.4E-4},{"eventName":"channelAcquisitionStarted","startTimeUTC":"2026-03-04T23:45:27.188382582Z","durationInMilliSecs":0.112633},{"eventName":"pipelined","startTimeUTC":"2026-03-04T23:45:27.188495215Z","durationInMilliSecs":0.089536},{"eventName":"transitTime","startTimeUTC":null,"durationInMilliSecs":0.0},{"eventName":"decodeTime","startTimeUTC":null,"durationInMilliSecs":0.0},{"eventName":"received","startTimeUTC":null,"durationInMilliSecs":0.0},{"eventName":"completed","startTimeUTC":"2026-03-04T23:45:27.188584751Z","durationInMilliSecs":0.002174}],"rntbdRequestLengthInBytes":-1,"rntbdResponseLengthInBytes":-1,"requestPayloadLengthInBytes":4629,"responsePayloadLengthInBytes":-1,"channelStatistics":{"channelId":"ea7b2167","channelTaskQueueSize":1,"pendingRequestsCount":0,"lastReadTime":"2026-03-04T23:45:25.109004883Z","waitForConnectionInit":false},"serviceEndpointStatistics":{"availableChannels":1,"acquiredChannels":0,"executorTaskQueueSize":0,"inflightRequests":1,"lastSuccessfulRequestTime":"2026-03-04T23:35:01.384Z","lastRequestTime":"2026-03-04T23:45:25.108Z","createdTime":"2026-03-04T23:34:45.087089238Z","isClosed":false,"cerMetrics":{}}},"requestResponseTimeUTC":"2026-03-04T23:45:27.188618824Z","requestStartTimeUTC":"2026-03-04T23:45:27.188379436Z","requestResourceType":"Document","requestOperationType":"Query","requestSessionToken":null,"e2ePolicyCfg":"{e2eto=PT1M10S, as=}","excludedRegions":null,"sessionTokenEvaluationResults":[],"perPartitionCircuitBreakerInfoHolder":null,"perPartitionAutomaticFailoverInfoHolder":null,"isHubRegionProcessingOnly":"false","perPartitionFailoverInfoHolder":null},{"storeResult":{"storePhysicalAddress":"rntbd://cdb-ms-prod-westcentralus1-be10.documents.azure.com:14037/apps/8974a354-e4f8-4edb-8e8d-d46f0a9d3da4/services/c8bbddbe-878c-433f-bcdb-008eb61081cf/partitions/6c730c88-26f7-4e5e-a9ad-1c2793e63beb/replicas/134171167846900237s/","lsn":-1,"quorumAckedLSN":-1,"currentReplicaSetSize":-1,"globalCommittedLsn":-1,"globalNRegionCommittedGlsn":-1,"partitionKeyRangeId":"0","isValid":false,"statusCode":410,"subStatusCode":21005,"isGone":true,"isNotFound":false,"isInvalidPartition":false,"isThroughputControlRequestRateTooLarge":false,"r
Raw output
{"ClassName":"ServiceUnavailableException","userAgent":"azsdk-java-cosmos/4.79.0-beta.1 Linux/6.11.0-1018-azure JRE/25.0.1","statusCode":503,"resourceAddress":null,"innerErrorMessage":"Service is currently unavailable, please retry after a while. More info: https://aka.ms/cosmosdb-tsg-service-unavailable-java: Message: {\"innerErrorMessage\":\"READ Quorum size of 2 is not met for the request.\"}","causeInfo":"[class: class com.azure.cosmos.implementation.GoneException, message: {\"innerErrorMessage\":\"READ Quorum size of 2 is not met for the request.\"}]","responseHeaders":"{x-ms-substatus=21007}","cosmosDiagnostics":{"userAgent":"azsdk-java-cosmos/4.79.0-beta.1 Linux/6.11.0-1018-azure JRE/25.0.1","activityId":"ed07b6fa-f2e0-439c-895c-9d45f5e2be6b","requestLatencyInMs":60259,"requestStartTimeUTC":"2026-03-04T23:45:27.188206111Z","requestEndTimeUTC":"2026-03-04T23:46:27.447883489Z","responseStatisticsList":[{"storeResult":{"storePhysicalAddress":"rntbd://cdb-ms-prod-westcentralus1-be10.documents.azure.com:14003/apps/8974a354-e4f8-4edb-8e8d-d46f0a9d3da4/services/c8bbddbe-878c-433f-bcdb-008eb61081cf/partitions/6c730c88-26f7-4e5e-a9ad-1c2793e63beb/replicas/134171167846900235s/","lsn":-1,"quorumAckedLSN":-1,"currentReplicaSetSize":-1,"globalCommittedLsn":-1,"globalNRegionCommittedGlsn":-1,"partitionKeyRangeId":"0","isValid":false,"statusCode":410,"subStatusCode":21005,"isGone":true,"isNotFound":false,"isInvalidPartition":false,"isThroughputControlRequestRateTooLarge":false,"requestCharge":0.0,"itemLSN":-1,"sessionToken":null,"backendLatencyInMs":null,"retryAfterInMs":null,"exceptionMessage":"Fault injection server error [The requested resource is no longer available at the server.]","exceptionResponseHeaders":"{x-ms-substatus=21005}","faultInjectionRuleId":"extremelyLongConnectDelay","replicaStatusList":{"Ignoring":["14061:P:Connected"],"Attempting":["14003:S:Connected","14313:S:Connected","14037:S:Connected"]},"transportRequestTimeline":[{"eventName":"created","startTimeUTC":"2026-03-04T23:45:27.188379436Z","durationInMilliSecs":0.003006},{"eventName":"queued","startTimeUTC":"2026-03-04T23:45:27.188382442Z","durationInMilliSecs":1.4E-4},{"eventName":"channelAcquisitionStarted","startTimeUTC":"2026-03-04T23:45:27.188382582Z","durationInMilliSecs":0.112633},{"eventName":"pipelined","startTimeUTC":"2026-03-04T23:45:27.188495215Z","durationInMilliSecs":0.089536},{"eventName":"transitTime","startTimeUTC":null,"durationInMilliSecs":0.0},{"eventName":"decodeTime","startTimeUTC":null,"durationInMilliSecs":0.0},{"eventName":"received","startTimeUTC":null,"durationInMilliSecs":0.0},{"eventName":"completed","startTimeUTC":"2026-03-04T23:45:27.188584751Z","durationInMilliSecs":0.002174}],"rntbdRequestLengthInBytes":-1,"rntbdResponseLengthInBytes":-1,"requestPayloadLengthInBytes":4629,"responsePayloadLengthInBytes":-1,"channelStatistics":{"channelId":"ea7b2167","channelTaskQueueSize":1,"pendingRequestsCount":0,"lastReadTime":"2026-03-04T23:45:25.109004883Z","waitForConnectionInit":false},"serviceEndpointStatistics":{"availableChannels":1,"acquiredChannels":0,"executorTaskQueueSize":0,"inflightRequests":1,"lastSuccessfulRequestTime":"2026-03-04T23:35:01.384Z","lastRequestTime":"2026-03-04T23:45:25.108Z","createdTime":"2026-03-04T23:34:45.087089238Z","isClosed":false,"cerMetrics":{}}},"requestResponseTimeUTC":"2026-03-04T23:45:27.188618824Z","requestStartTimeUTC":"2026-03-04T23:45:27.188379436Z","requestResourceType":"Document","requestOperationType":"Query","requestSessionToken":null,"e2ePolicyCfg":"{e2eto=PT1M10S, as=}","excludedRegions":null,"sessionTokenEvaluationResults":[],"perPartitionCircuitBreakerInfoHolder":null,"perPartitionAutomaticFailoverInfoHolder":null,"isHubRegionProcessingOnly":"false","perPartitionFailoverInfoHolder":null},{"storeResult":{"storePhysicalAddress":"rntbd://cdb-ms-prod-westcentralus1-be10.documents.azure.com:14037/apps/8974a354-e4f8-4edb-8e8d-d46f0a9d3da4/services/c8bbddbe-878c-433f-bcdb-008eb61081cf/partitions/6