Power Platform Community Forum Thread Details

Topic all things not working properly.

Error code 3000. But when i create new bot it works just fine.

Here's full error code when i try to open Topics.

Is there anything i can do besides creating new bot from scratch and connect all the connections again?

Any answer is appreciated

An unexpected server error occurred. Please try again.

Diagnostic Trace Id: Unexpected error: {"diagnosticInformation":"Trace id not available. Headers: {\"content-type\":\"application/json\",\"x-ms-activity-vector\":\"00.00.00.00\",\"x-ms-islandgateway\":\"GA0000001\",\"x-ms-service-request-id\":\"53835de2-5caa-4544-9312-98040e24196a\"}","errorMessage":"The operation was canceled.\r\nCancellation Token has expired: True. Learn more at: https://aka.ms/cosmosdb-tsg-request-timeout\r\nCosmosDiagnostics: {\"Summary\":{\"DirectCalls\":{\"(410, 20001)\":8},\"GatewayCalls\":{\"(0, 0)\":2}},\"name\":\"ReadItemAsync\",\"id\":\"0e3c051a-f253-46f0-8b4d-5c1d3e79da78\",\"start time\":\"03:30:03:849\",\"duration in milliseconds\":40912.7016,\"data\":{\"Client Configuration\":{\"Client Created Time Utc\":\"2023-02-03T19:28:22.2728871Z\",\"MachineId\":\"hashedVmId:12599399-3fcf-ede9-8982-9375a6ccd989\",\"VM Region\":\"southeastasia\",\"NumberOfClientsCreated\":8,\"NumberOfActiveClients\":8,\"ConnectionMode\":\"Direct\",\"User Agent\":\"cosmos-netstandard-sdk/3.31.0|2|X64|Microsoft Windows 10.0.17763 |.NET Framework 4.7.3946.0|N|\",\"ConnectionConfig\":{\"gw\":\"(cps:50, urto:10, p:False, httpf: False)\",\"rntbd\":\"(cto: 5, icto: -1, mrpc: 30, mcpe: 65535, erd: True, pr: ReuseUnicastPort)\",\"other\":\"(ed:False, be:False)\"},\"ConsistencyConfig\":\"(consistency: NotSet, prgns:[], apprgn: )\",\"ProcessorCount\":16}},\"children\":[{\"name\":\"Microsoft.Azure.Cosmos.Handlers.RequestInvokerHandler\",\"id\":\"29e5411e-e41c-43f1-a2fe-9f307bca8da7\",\"start time\":\"03:30:03:849\",\"duration in milliseconds\":40912.2361,\"children\":[{\"name\":\"Microsoft.CCI.Store.Core.ObjectStorage.TelemetryHandler\",\"id\":\"24444222-4b7e-4d62-8c88-0216aca608f6\",\"start time\":\"03:30:03:849\",\"duration in milliseconds\":40912.0588,\"children\":[{\"name\":\"Microsoft.Azure.Cosmos.Handlers.DiagnosticsHandler\",\"id\":\"40ffc0f8-8a83-4e6d-b08c-4f91d2410692\",\"start time\":\"03:30:03:849\",\"duration in milliseconds\":10906.6713,\"children\":[{\"name\":\"Microsoft.Azure.Cosmos.Handlers.RetryHandler\",\"id\":\"c7cd3917-c76c-4a0d-a27d-102a4b66adca\",\"start time\":\"03:30:03:849\",\"duration in milliseconds\":10906.6046,\"children\":[{\"name\":\"Microsoft.Azure.Cosmos.Handlers.RouterHandler\",\"id\":\"f1f3459f-9f88-41c8-9289-69c6488ab6f7\",\"start time\":\"03:30:03:849\",\"duration in milliseconds\":10906.4786,\"children\":[{\"name\":\"Microsoft.Azure.Cosmos.Handlers.TransportHandler\",\"id\":\"ceb9415b-47b9-4a43-a1b3-647952b052df\",\"start time\":\"03:30:03:849\",\"duration in milliseconds\":10906.4372,\"children\":[{\"name\":\"Microsoft.Azure.Documents.ServerStoreModel Transport Request\",\"id\":\"34052c06-e1e3-49c5-9750-c2c2663719b2\",\"start time\":\"03:30:03:849\",\"duration in milliseconds\":10906.2711,\"data\":{\"Client Side Request Stats\":{\"Id\":\"AggregatedClientSideRequestStatistics\",\"ContactedReplicas\":[],\"RegionsContacted\":[],\"FailedReplicas\":[\"rntbd://cdb-ms-prod-southeastasia1-be11.documents.azure.com:14341/apps/ee6e8986-453c-4d8f-b354-d4ceeaf2334b/services/77502dc5-1713-4f86-8e2a-4052d8bc58fe/partitions/e77d4ed9-c358-43a9-8a56-ab65a4c9e274/replicas/133089522008538228p/\",\"rntbd://cdb-ms-prod-southeastasia1-be11.documents.azure.com:14310/apps/ee6e8986-453c-4d8f-b354-d4ceeaf2334b/services/77502dc5-1713-4f86-8e2a-4052d8bc58fe/partitions/e77d4ed9-c358-43a9-8a56-ab65a4c9e274/replicas/133089522100574305s/\"],\"AddressResolutionStatistics\":[{\"StartTimeUTC\":\"2023-02-08T03:30:09.7559367Z\",\"EndTimeUTC\":\"9999-12-31T23:59:59.9999999\",\"TargetEndpoint\":\"https://mscciprodapc01scu0cd01-southeastasia.documents.azure.com//addresses/?$resolveFor=dbs%2f12wmAA%3d%3d%2fcolls%2f12wmAMZ8Z%2bM%3d%2fdocs&$filter=protocol eq rntbd&$partitionKeyRangeIds=12\"}],\"StoreResponseStatistics\":[{\"ResponseTimeUTC\":\"2023-02-08T03:30:09.7559367Z\",\"ResourceType\":\"Document\",\"OperationType\":\"Read\",\"RequestSessionToken\":\"12:3#3484111#25=-1\",\"LocationEndpoint\":\"https://mscciprodapc01scu0cd01-southeastasia.documents.azure.com/\",\"StoreResult\":{\"ActivityId\":\"ad0e790e-73a9-47ab-a3bd-2624d5595cce\",\"StatusCode\":\"Gone\",\"SubStatusCode\":\"TransportGenerated410\",\"LSN\":-1,\"PartitionKeyRangeId\":null,\"GlobalCommittedLSN\":-1,\"ItemLSN\":-1,\"UsingLocalLSN\":true,\"QuorumAckedLSN\":-1,\"SessionToken\":null,\"CurrentWriteQuorum\":-1,\"CurrentReplicaSetSize\":-1,\"NumberOfReadRegions\":-1,\"IsValid\":false,\"StorePhysicalAddress\":\"rntbd://cdb-ms-prod-southeastasia1-be11.documents.azure.com:14341/apps/ee6e8986-453c-4d8f-b354-d4ceeaf2334b/services/77502dc5-1713-4f86-8e2a-4052d8bc58fe/partitions/e77d4ed9-c358-43a9-8a56-ab65a4c9e274/replicas/133089522008538228p/\",\"RequestCharge\":0,\"RetryAfterInMs\":null,\"BELatencyInMs\":null,\"transportRequestTimeline\":{\"requestTimeline\":[{\"event\": \"Created\", \"startTimeUtc\": \"2023-02-08T03:30:03.8495844Z\", \"durationInMs\": 0.0057},{\"event\": \"ChannelAcquisitionStarted\", \"startTimeUtc\": \"2023-02-08T03:30:03.8495901Z\", \"durationInMs\": 5905.9216},{\"event\": \"Failed\", \"startTimeUtc\": \"2023-02-08T03:30:09.7555117Z\", \"durationInMs\": 0}],\"serviceEndpointStats\":{\"inflightRequests\":1,\"openConnections\":1},\"connectionStats\":{\"waitforConnectionInit\":\"True\"}},\"TransportException\":\"A client transport error occurred: The connection attempt timed out. (Time: 2023-02-08T03:30:09.7559367Z, activity ID: ad0e790e-73a9-47ab-a3bd-2624d5595cce, error code: ConnectTimeout [0x0006], base error: HRESULT 0x80131500, URI: rntbd://cdb-ms-prod-southeastasia1-be11.documents.azure.com:14341/, connection: <not connected> -> rntbd://cdb-ms-prod-southeastasia1-be11.documents.azure.com:14341/, payload sent: False)\"}},{\"ResponseTimeUTC\":\"2023-02-08T03:30:14.7560305Z\",\"ResourceType\":\"Document\",\"OperationType\":\"Read\",\"RequestSessionToken\":\"12:3#3484111#25=-1\",\"LocationEndpoint\":\"https://mscciprodapc01scu0cd01-southeastasia.documents.azure.com/\",\"StoreResult\":{\"ActivityId\":\"ad0e790e-73a9-47ab-a3bd-2624d5595cce\",\"StatusCode\":\"Gone\",\"SubStatusCode\":\"TransportGenerated410\",\"LSN\":-1,\"PartitionKeyRangeId\":null,\"GlobalCommittedLSN\":-1,\"ItemLSN\":-1,\"UsingLocalLSN\":true,\"QuorumAckedLSN\":-1,\"SessionToken\":null,\"CurrentWriteQuorum\":-1,\"CurrentReplicaSetSize\":-1,\"NumberOfReadRegions\":-1,\"IsValid\":false,\"StorePhysicalAddress\":\"rntbd://cdb-ms-prod-southeastasia1-be11.documents.azure.com:14310/apps/ee6e8986-453c-4d8f-b354-d4ceeaf2334b/services/77502dc5-1713-4f86-8e2a-4052d8bc58fe/partitions/e77d4ed9-c358-43a9-8a56-ab65a4c9e274/replicas/133089522100574305s/\",\"RequestCharge\":0,\"RetryAfterInMs\":null,\"BELatencyInMs\":null,\"transportRequestTimeline\":{\"requestTimeline\":[{\"event\": \"Created\", \"startTimeUtc\": \"2023-02-08T03:30:09.7559367Z\", \"durationInMs\": 0.0042},{\"event\": \"ChannelAcquisitionStarted\", \"startTimeUtc\": \"2023-02-08T03:30:09.7559409Z\", \"durationInMs\": 4998.2316},{\"event\": \"Failed\", \"startTimeUtc\": \"2023-02-08T03:30:14.7541725Z\", \"durationInMs\": 0}],\"serviceEndpointStats\":{\"inflightRequests\":1,\"openConnections\":1},\"connectionStats\":{\"waitforConnectionInit\":\"True\"}},\"TransportException\":\"A client transport error occurred: The connection attempt timed out. (Time: 2023-02-08T03:30:14.7560305Z, activity ID: ad0e790e-73a9-47ab-a3bd-2624d5595cce, error code: ConnectTimeout [0x0006], base error: HRESULT 0x80131500, URI: rntbd://cdb-ms-prod-southeastasia1-be11.documents.azure.com:14310/, connection: <not connected> -> rntbd://cdb-ms-prod-southeastasia1-be11.documents.azure.com:14310/, payload sent: False)\"}}],\"HttpResponseStats\":[{\"StartTimeUTC\":\"2023-02-08T03:30:09.7559367Z\",\"DurationInMs\":515.6393,\"RequestUri\":\"https://mscciprodapc01scu0cd01-southeastasia.documents.azure.com//addresses/?$resolveFor=dbs%2f12wmAA%3d%3d%2fcolls%2f12wmAMZ8Z%2bM%3d%2fdocs&$filter=protocol eq rntbd&$partitionKeyRangeIds=12\",\"ResourceType\":\"Document\",\"HttpMethod\":\"GET\",\"ActivityId\":\"ad0e790e-73a9-47ab-a3bd-2624d5595cce\",\"ExceptionType\":\"System.Threading.Tasks.TaskCanceledException\",\"ExceptionMessage\":\"A task was canceled.\"},{\"StartTimeUTC\":\"2023-02-08T03:30:10.2715760Z\",\"DurationInMs\":5000.1209,\"RequestUri\":\"https://mscciprodapc01scu0cd01-southeastasia.documents.azure.com//addresses/?$resolveFor=dbs%2f12wmAA%3d%3d%2fcolls%2f12wmAMZ8Z%2bM%3d%2fdocs&$filter=protocol eq rntbd&$partitionKeyRangeIds=12\",\"ResourceType\":\"Document\",\"HttpMethod\":\"GET\",\"ActivityId\":\"ad0e790e-73a9-47ab-a3bd-2624d5595cce\",\"ExceptionType\":\"System.Threading.Tasks.TaskCanceledException\",\"ExceptionMessage\":\"A task was canceled.\"}]}}}]}]}]}]},{\"name\":\"Microsoft.Azure.Cosmos.Handlers.DiagnosticsHandler\",\"id\":\"1cd0bed4-c4ac-4e92-afd0-28667c4b7763\",\"start time\":\"03:30:14:756\",\"duration in milliseconds\":10001.6789,\"children\":[{\"name\":\"Microsoft.Azure.Cosmos.Handlers.RetryHandler\",\"id\":\"38ce2550-b125-4648-a712-b249e4d6bddd\",\"start time\":\"03:30:14:756\",\"duration in milliseconds\":10001.617900000001,\"children\":[{\"name\":\"Microsoft.Azure.Cosmos.Handlers.RouterHandler\",\"id\":\"b88d770d-eeec-4022-8506-69e10731c878\",\"start time\":\"03:30:14:756\",\"duration in milliseconds\":10001.5148,\"children\":[{\"name\":\"Microsoft.Azure.Cosmos.Handlers.TransportHandler\",\"id\":\"85060ab4-9a35-438a-bb18-8ddcf5f4cf99\",\"start time\":\"03:30:14:756\",\"duration in milliseconds\":10001.480300000001,\"children\":[{\"name\":\"Microsoft.Azure.Documents.ServerStoreModel Transport Request\",\"id\":\"dde6d51f-9e4c-4884-b8a0-e18ad66b8026\",\"start time\":\"03:30:14:756\",\"duration in milliseconds\":10001.305900000001,\"data\":{\"Client Side Request Stats\":{\"Id\":\"AggregatedClientSideRequestStatistics\",\"ContactedReplicas\":[],\"RegionsContacted\":[],\"FailedReplicas\":[\"rntbd://cdb-ms-prod-southeastasia1-be11.documents.azure.com:14310/apps/ee6e8986-453c-4d8f-b354-d4ceeaf2334b/services/77502dc5-1713-4f86-8e2a-4052d8bc58fe/partitions/e77d4ed9-c358-43a9-8a56-ab65a4c9e274/replicas/133089522100574305s/\",\"rntbd://cdb-ms-prod-southeastasia1-be11.documents.azure.com:14341/apps/ee6e8986-453c-4d8f-b354-d4ceeaf2334b/services/77502dc5-1713-4f86-8e2a-4052d8bc58fe/partitions/e77d4ed9-c358-43a9-8a56-ab65a4c9e274/replicas/133089522008538228p/\"],\"AddressResolutionStatistics\":[],\"StoreResponseStatistics\":[{\"ResponseTimeUTC\":\"2023-02-08T03:30:19.7561053Z\",\"ResourceType\":\"Document\",\"OperationType\":\"Read\",\"RequestSessionToken\":\"12:3#3484111#25=-1\",\"LocationEndpoint\":\"https://mscciprodapc01scu0cd01-southeastasia.documents.azure.com/\",\"StoreResult\":{\"ActivityId\":\"ad0e790e-73a9-47ab-a3bd-2624d5595cce\",\"StatusCode\":\"Gone\",\"SubStatusCode\":\"TransportGenerated410\",\"LSN\":-1,\"PartitionKeyRangeId\":null,\"GlobalCommittedLSN\":-1,\"ItemLSN\":-1,\"UsingLocalLSN\":true,\"QuorumAckedLSN\":-1,\"SessionToken\":null,\"CurrentWriteQuorum\":-1,\"CurrentReplicaSetSize\":-1,\"NumberOfReadRegions\":-1,\"IsValid\":false,\"StorePhysicalAddress\":\"rntbd://cdb-ms-prod-southeastasia1-be11.documents.azure.com:14310/apps/ee6e8986-453c-4d8f-b354-d4ceeaf2334b/services/77502dc5-1713-4f86-8e2a-4052d8bc58fe/partitions/e77d4ed9-c358-43a9-8a56-ab65a4c9e274/replicas/133089522100574305s/\",\"RequestCharge\":0,\"RetryAfterInMs\":null,\"BELatencyInMs\":null,\"transportRequestTimeline\":{\"requestTimeline\":[{\"event\": \"Created\", \"startTimeUtc\": \"2023-02-08T03:30:14.7560305Z\", \"durationInMs\": 0.0031},{\"event\": \"ChannelAcquisitionStarted\", \"startTimeUtc\": \"2023-02-08T03:30:14.7560336Z\", \"durationInMs\": 4998.8925},{\"event\": \"Failed\", \"startTimeUtc\": \"2023-02-08T03:30:19.7549261Z\", \"durationInMs\": 0}],\"serviceEndpointStats\":{\"inflightRequests\":1,\"openConnections\":1},\"connectionStats\":{\"waitforConnectionInit\":\"True\"}},\"TransportException\":\"A client transport error occurred: The connection attempt timed out. (Time: 2023-02-08T03:30:19.7561053Z, activity ID: ad0e790e-73a9-47ab-a3bd-2624d5595cce, error code: ConnectTimeout [0x0006], base error: HRESULT 0x80131500, URI: rntbd://cdb-ms-prod-southeastasia1-be11.documents.azure.com:14310/, connection: <not connected> -> rntbd://cdb-ms-prod-southeastasia1-be11.documents.azure.com:14310/, payload sent: False)\"}},{\"ResponseTimeUTC\":\"2023-02-08T03:30:24.7561882Z\",\"ResourceType\":\"Document\",\"OperationType\":\"Read\",\"RequestSessionToken\":\"12:3#3484111#25=-1\",\"LocationEndpoint\":\"https://mscciprodapc01scu0cd01-southeastasia.documents.azure.com/\",\"StoreResult\":{\"ActivityId\":\"ad0e790e-73a9-47ab-a3bd-2624d5595cce\",\"StatusCode\":\"Gone\",\"SubStatusCode\":\"TransportGenerated410\",\"LSN\":-1,\"PartitionKeyRangeId\":null,\"GlobalCommittedLSN\":-1,\"ItemLSN\":-1,\"UsingLocalLSN\":true,\"QuorumAckedLSN\":-1,\"SessionToken\":null,\"CurrentWriteQuorum\":-1,\"CurrentReplicaSetSize\":-1,\"NumberOfReadRegions\":-1,\"IsValid\":false,\"StorePhysicalAddress\":\"rntbd://cdb-ms-prod-southeastasia1-be11.documents.azure.com:14341/apps/ee6e8986-453c-4d8f-b354-d4ceeaf2334b/services/77502dc5-1713-4f86-8e2a-4052d8bc58fe/partitions/e77d4ed9-c358-43a9-8a56-ab65a4c9e274/replicas/133089522008538228p/\",\"RequestCharge\":0,\"RetryAfterInMs\":null,\"BELatencyInMs\":null,\"transportRequestTimeline\":{\"requestTimeline\":[{\"event\": \"Created\", \"startTimeUtc\": \"2023-02-08T03:30:19.7561053Z\", \"durationInMs\": 0.0031},{\"event\": \"ChannelAcquisitionStarted\", \"startTimeUtc\": \"2023-02-08T03:30:19.7561084Z\", \"durationInMs\": 4999.3903},{\"event\": \"Failed\", \"startTimeUtc\": \"2023-02-08T03:30:24.7554987Z\", \"durationInMs\": 0}],\"serviceEndpointStats\":{\"inflightRequests\":1,\"openConnections\":1},\"connectionStats\":{\"waitforConnectionInit\":\"True\"}},\"TransportException\":\"A client transport error occurred: The connection attempt timed out. (Time: 2023-02-08T03:30:24.7561882Z, activity ID: ad0e790e-73a9-47ab-a3bd-2624d5595cce, error code: ConnectTimeout [0x0006], base error: HRESULT 0x80131500, URI: rntbd://cdb-ms-prod-southeastasia1-be11.documents.azure.com:14341/, connection: <not connected> -> rntbd://cdb-ms-prod-southeastasia1-be11.documents.azure.com:14341/, payload sent: False)\"}}]}}}]}]}]}]},{\"name\":\"Microsoft.Azure.Cosmos.Handlers.DiagnosticsHandler\",\"id\":\"d145f76d-79b5-4da9-afd3-36f1641d080f\",\"start time\":\"03:30:24:756\",\"duration in milliseconds\":9998.862000000001,\"children\":[{\"name\":\"Microsoft.Azure.Cosmos.Handlers.RetryHandler\",\"id\":\"509b32b6-7db1-4ad3-8e5b-4d70fd99df63\",\"start time\":\"03:30:24:756\",\"duration in milliseconds\":9998.7642,\"children\":[{\"name\":\"Microsoft.Azure.Cosmos.Handlers.RouterHandler\",\"id\":\"b2f94740-5ef9-4860-9dfd-e458cb048a8c\",\"start time\":\"03:30:24:756\",\"duration in milliseconds\":9998.5913,\"children\":[{\"name\":\"Microsoft.Azure.Cosmos.Handlers.TransportHandler\",\"id\":\"14da9d05-56bb-43f4-8573-5abf9d553c0b\",\"start time\":\"03:30:24:756\",\"duration in milliseconds\":9998.5278,\"children\":[{\"name\":\"Microsoft.Azure.Documents.ServerStoreModel Transport Request\",\"id\":\"5acd99e1-e0ba-426b-a6c7-b239d8d93d22\",\"start time\":\"03:30:24:756\",\"duration in milliseconds\":9998.3299,\"data\":{\"Client Side Request Stats\":{\"Id\":\"AggregatedClientSideRequestStatistics\",\"ContactedReplicas\":[],\"RegionsContacted\":[],\"FailedReplicas\":[\"rntbd://cdb-ms-prod-southeastasia1-be11.documents.azure.com:14341/apps/ee6e8986-453c-4d8f-b354-d4ceeaf2334b/services/77502dc5-1713-4f86-8e2a-4052d8bc58fe/partitions/e77d4ed9-c358-43a9-8a56-ab65a4c9e274/replicas/133089522008538228p/\",\"rntbd://cdb-ms-prod-southeastasia1-be11.documents.azure.com:14310/apps/ee6e8986-453c-4d8f-b354-d4ceeaf2334b/services/77502dc5-1713-4f86-8e2a-4052d8bc58fe/partitions/e77d4ed9-c358-43a9-8a56-ab65a4c9e274/replicas/133089522100574305s/\"],\"AddressResolutionStatistics\":[],\"StoreResponseStatistics\":[{\"ResponseTimeUTC\":\"2023-02-08T03:30:29.7562645Z\",\"ResourceType\":\"Document\",\"OperationType\":\"Read\",\"RequestSessionToken\":\"12:3#3484111#25=-1\",\"LocationEndpoint\":\"https://mscciprodapc01scu0cd01-southeastasia.documents.azure.com/\",\"StoreResult\":{\"ActivityId\":\"ad0e790e-73a9-47ab-a3bd-2624d5595cce\",\"StatusCode\":\"Gone\",\"SubStatusCode\":\"TransportGenerated410\",\"LSN\":-1,\"PartitionKeyRangeId\":null,\"GlobalCommittedLSN\":-1,\"ItemLSN\":-1,\"UsingLocalLSN\":true,\"QuorumAckedLSN\":-1,\"SessionToken\":null,\"CurrentWriteQuorum\":-1,\"CurrentReplicaSetSize\":-1,\"NumberOfReadRegions\":-1,\"IsValid\":false,\"StorePhysicalAddress\":\"rntbd://cdb-ms-prod-southeastasia1-be11.documents.azure.com:14341/apps/ee6e8986-453c-4d8f-b354-d4ceeaf2334b/services/77502dc5-1713-4f86-8e2a-4052d8bc58fe/partitions/e77d4ed9-c358-43a9-8a56-ab65a4c9e274/replicas/133089522008538228p/\",\"RequestCharge\":0,\"RetryAfterInMs\":null,\"BELatencyInMs\":null,\"transportRequestTimeline\":{\"requestTimeline\":[{\"event\": \"Created\", \"startTimeUtc\": \"2023-02-08T03:30:24.7561882Z\", \"durationInMs\": 0.002},{\"event\": \"ChannelAcquisitionStarted\", \"startTimeUtc\": \"2023-02-08T03:30:24.7561902Z\", \"durationInMs\": 4997.3798},{\"event\": \"Failed\", \"startTimeUtc\": \"2023-02-08T03:30:29.7535700Z\", \"durationInMs\": 0}],\"serviceEndpointStats\":{\"inflightRequests\":1,\"openConnections\":1},\"connectionStats\":{\"waitforConnectionInit\":\"True\"}},\"TransportException\":\"A client transport error occurred: The connection attempt timed out. (Time: 2023-02-08T03:30:29.7562645Z, activity ID: ad0e790e-73a9-47ab-a3bd-2624d5595cce, error code: ConnectTimeout [0x0006], base error: HRESULT 0x80131500, URI: rntbd://cdb-ms-prod-southeastasia1-be11.documents.azure.com:14341/, connection: <not connected> -> rntbd://cdb-ms-prod-southeastasia1-be11.documents.azure.com:14341/, payload sent: False)\"}},{\"ResponseTimeUTC\":\"2023-02-08T03:30:34.7563337Z\",\"ResourceType\":\"Document\",\"OperationType\":\"Read\",\"RequestSessionToken\":\"12:3#3484111#25=-1\",\"LocationEndpoint\":\"https://mscciprodapc01scu0cd01-southeastasia.documents.azure.com/\",\"StoreResult\":{\"ActivityId\":\"ad0e790e-73a9-47ab-a3bd-2624d5595cce\",\"StatusCode\":\"Gone\",\"SubStatusCode\":\"TransportGenerated410\",\"LSN\":-1,\"PartitionKeyRangeId\":null,\"GlobalCommittedLSN\":-1,\"ItemLSN\":-1,\"UsingLocalLSN\":true,\"QuorumAckedLSN\":-1,\"SessionToken\":null,\"CurrentWriteQuorum\":-1,\"CurrentReplicaSetSize\":-1,\"NumberOfReadRegions\":-1,\"IsValid\":false,\"StorePhysicalAddress\":\"rntbd://cdb-ms-prod-southeastasia1-be11.documents.azure.com:14310/apps/ee6e8986-453c-4d8f-b354-d4ceeaf2334b/services/77502dc5-1713-4f86-8e2a-4052d8bc58fe/partitions/e77d4ed9-c358-43a9-8a56-ab65a4c9e274/replicas/133089522100574305s/\",\"RequestCharge\":0,\"RetryAfterInMs\":null,\"BELatencyInMs\":null,\"transportRequestTimeline\":{\"requestTimeline\":[{\"event\": \"Created\", \"startTimeUtc\": \"2023-02-08T03:30:29.7562645Z\", \"durationInMs\": 0.0019},{\"event\": \"ChannelAcquisitionStarted\", \"startTimeUtc\": \"2023-02-08T03:30:29.7562664Z\", \"durationInMs\": 4999.3667},{\"event\": \"Failed\", \"startTimeUtc\": \"2023-02-08T03:30:34.7556331Z\", \"durationInMs\": 0}],\"serviceEndpointStats\":{\"inflightRequests\":1,\"openConnections\":1},\"connectionStats\":{\"waitforConnectionInit\":\"True\"}},\"TransportException\":\"A client transport error occurred: The connection attempt timed out. (Time: 2023-02-08T03:30:34.7563337Z, activity ID: ad0e790e-73a9-47ab-a3bd-2624d5595cce, error code: ConnectTimeout [0x0006], base error: HRESULT 0x80131500, URI: rntbd://cdb-ms-prod-southeastasia1-be11.documents.azure.com:14310/, connection: <not connected> -> rntbd://cdb-ms-prod-southeastasia1-be11.documents.azure.com:14310/, payload sent: False)\"}}]}}}]}]}]}]},{\"name\":\"Microsoft.Azure.Cosmos.Handlers.DiagnosticsHandler\",\"id\":\"35214b2d-82f2-4874-a446-24392800bca4\",\"start time\":\"03:30:34:756\",\"duration in milliseconds\":10001.9558,\"children\":[{\"name\":\"Microsoft.Azure.Cosmos.Handlers.RetryHandler\",\"id\":\"5494a08d-2c85-4d42-97db-537ce95fb7a7\",\"start time\":\"03:30:34:756\",\"duration in milliseconds\":10001.7364,\"children\":[{\"name\":\"Microsoft.Azure.Cosmos.Handlers.RouterHandler\",\"id\":\"f074750f-a7c0-42da-806c-b153cf211a44\",\"start time\":\"03:30:34:756\",\"duration in milliseconds\":10001.4017,\"children\":[{\"name\":\"Microsoft.Azure.Cosmos.Handlers.TransportHandler\",\"id\":\"efa7c1c7-40c8-42f2-92d2-0d130682bed5\",\"start time\":\"03:30:34:756\",\"duration in milliseconds\":10001.3428,\"children\":[{\"name\":\"Microsoft.Azure.Documents.ServerStoreModel Transport Request\",\"id\":\"4c78a2cd-d844-4ec5-9da5-af8ed1a744cb\",\"start time\":\"03:30:34:756\",\"duration in milliseconds\":10001.149300000001,\"data\":{\"Client Side Request Stats\":{\"Id\":\"AggregatedClientSideRequestStatistics\",\"ContactedReplicas\":[],\"RegionsContacted\":[],\"FailedReplicas\":[\"rntbd://cdb-ms-prod-southeastasia1-be11.documents.azure.com:14310/apps/ee6e8986-453c-4d8f-b354-d4ceeaf2334b/services/77502dc5-1713-4f86-8e2a-4052d8bc58fe/partitions/e77d4ed9-c358-43a9-8a56-ab65a4c9e274/replicas/133089522100574305s/\",\"rntbd://cdb-ms-prod-southeastasia1-be11.documents.azure.com:14341/apps/ee6e8986-453c-4d8f-b354-d4ceeaf2334b/services/77502dc5-1713-4f86-8e2a-4052d8bc58fe/partitions/e77d4ed9-c358-43a9-8a56-ab65a4c9e274/replicas/133089522008538228p/\"],\"AddressResolutionStatistics\":[],\"StoreResponseStatistics\":[{\"ResponseTimeUTC\":\"2023-02-08T03:30:39.7564051Z\",\"ResourceType\":\"Document\",\"OperationType\":\"Read\",\"RequestSessionToken\":\"12:3#3484111#25=-1\",\"LocationEndpoint\":\"https://mscciprodapc01scu0cd01-southeastasia.documents.azure.com/\",\"StoreResult\":{\"ActivityId\":\"ad0e790e-73a9-47ab-a3bd-2624d5595cce\",\"StatusCode\":\"Gone\",\"SubStatusCode\":\"TransportGenerated410\",\"LSN\":-1,\"PartitionKeyRangeId\":null,\"GlobalCommittedLSN\":-1,\"ItemLSN\":-1,\"UsingLocalLSN\":true,\"QuorumAckedLSN\":-1,\"SessionToken\":null,\"CurrentWriteQuorum\":-1,\"CurrentReplicaSetSize\":-1,\"NumberOfReadRegions\":-1,\"IsValid\":false,\"StorePhysicalAddress\":\"rntbd://cdb-ms-prod-southeastasia1-be11.documents.azure.com:14310/apps/ee6e8986-453c-4d8f-b354-d4ceeaf2334b/services/77502dc5-1713-4f86-8e2a-4052d8bc58fe/partitions/e77d4ed9-c358-43a9-8a56-ab65a4c9e274/replicas/133089522100574305s/\",\"RequestCharge\":0,\"RetryAfterInMs\":null,\"BELatencyInMs\":null,\"transportRequestTimeline\":{\"requestTimeline\":[{\"event\": \"Created\", \"startTimeUtc\": \"2023-02-08T03:30:34.7563337Z\", \"durationInMs\": 0.0031},{\"event\": \"ChannelAcquisitionStarted\", \"startTimeUtc\": \"2023-02-08T03:30:34.7563368Z\", \"durationInMs\": 4997.3457},{\"event\": \"Failed\", \"startTimeUtc\": \"2023-02-08T03:30:39.7536825Z\", \"durationInMs\": 0}],\"serviceEndpointStats\":{\"inflightRequests\":1,\"openConnections\":1},\"connectionStats\":{\"waitforConnectionInit\":\"True\"}},\"TransportException\":\"A client transport error occurred: The connection attempt timed out. (Time: 2023-02-08T03:30:39.7564051Z, activity ID: ad0e790e-73a9-47ab-a3bd-2624d5595cce, error code: ConnectTimeout [0x0006], base error: HRESULT 0x80131500, URI: rntbd://cdb-ms-prod-southeastasia1-be11.documents.azure.com:14310/, connection: <not connected> -> rntbd://cdb-ms-prod-southeastasia1-be11.documents.azure.com:14310/, payload sent: False)\"}},{\"ResponseTimeUTC\":\"2023-02-08T03:30:44.7565059Z\",\"ResourceType\":\"Document\",\"OperationType\":\"Read\",\"RequestSessionToken\":\"12:3#3484111#25=-1\",\"LocationEndpoint\":\"https://mscciprodapc01scu0cd01-southeastasia.documents.azure.com/\",\"StoreResult\":{\"ActivityId\":\"ad0e790e-73a9-47ab-a3bd-2624d5595cce\",\"StatusCode\":\"Gone\",\"SubStatusCode\":\"TransportGenerated410\",\"LSN\":-1,\"PartitionKeyRangeId\":null,\"GlobalCommittedLSN\":-1,\"ItemLSN\":-1,\"UsingLocalLSN\":true,\"QuorumAckedLSN\":-1,\"SessionToken\":null,\"CurrentWriteQuorum\":-1,\"CurrentReplicaSetSize\":-1,\"NumberOfReadRegions\":-1,\"IsValid\":false,\"StorePhysicalAddress\":\"rntbd://cdb-ms-prod-southeastasia1-be11.documents.azure.com:14341/apps/ee6e8986-453c-4d8f-b354-d4ceeaf2334b/services/77502dc5-1713-4f86-8e2a-4052d8bc58fe/partitions/e77d4ed9-c358-43a9-8a56-ab65a4c9e274/replicas/133089522008538228p/\",\"RequestCharge\":0,\"RetryAfterInMs\":null,\"BELatencyInMs\":null,\"transportRequestTimeline\":{\"requestTimeline\":[{\"event\": \"Created\", \"startTimeUtc\": \"2023-02-08T03:30:39.7564051Z\", \"durationInMs\": 0.0032},{\"event\": \"ChannelAcquisitionStarted\", \"startTimeUtc\": \"2023-02-08T03:30:39.7564083Z\", \"durationInMs\": 5001.3229},{\"event\": \"Failed\", \"startTimeUtc\": \"2023-02-08T03:30:44.7577312Z\", \"durationInMs\": 0}],\"serviceEndpointStats\":{\"inflightRequests\":1,\"openConnections\":1},\"connectionStats\":{\"waitforConnectionInit\":\"True\"}},\"TransportException\":\"A client transport error occurred: The connection attempt timed out. (Time: 2023-02-08T03:30:44.7565059Z, activity ID: ad0e790e-73a9-47ab-a3bd-2624d5595cce, error code: ConnectTimeout [0x0006], base error: HRESULT 0x80131500, URI: rntbd://cdb-ms-prod-southeastasia1-be11.documents.azure.com:14341/, connection: <not connected> -> rntbd://cdb-ms-prod-southeastasia1-be11.documents.azure.com:14341/, payload sent: False)\"}}]}}}]}]}]}]}]}]},{\"name\":\"CosmosOperationCanceledException\",\"id\":\"0f020a7c-b657-49b2-a217-01e8682fde07\",\"start time\":\"03:30:44:756\",\"duration in milliseconds\":0.0046,\"data\":{\"Operation Cancelled Exception\":\"System.OperationCanceledException: The operation was canceled.\\r\\n at System.Threading.CancellationToken.ThrowOperationCanceledException()\\r\\n at Microsoft.Azure.Documents.RequestRetryUtility.<ProcessRequestAsync>d__2`2.MoveNext()\\r\\n--- End of stack trace from previous location where exception was thrown ---\\r\\n at System.Runtime.ExceptionServices.ExceptionDispatchInfo.Throw()\\r\\n at System.Runtime.CompilerServices.TaskAwaiter.ThrowForNonSuccess(Task task)\\r\\n at System.Runtime.CompilerServices.TaskAwaiter.HandleNonSuccessAndDebuggerNotification(Task task)\\r\\n at Microsoft.Azure.Documents.StoreClient.<ProcessMessageAsync>d__19.MoveNext()\\r\\n--- End of stack trace from previous location where exception was thrown ---\\r\\n at System.Runtime.ExceptionServices.ExceptionDispatchInfo.Throw()\\r\\n at System.Runtime.CompilerServices.TaskAwaiter.ThrowForNonSuccess(Task task)\\r\\n at System.Runtime.CompilerServices.TaskAwaiter.HandleNonSuccessAndDebuggerNotification(Task task)\\r\\n at Microsoft.Azure.Cosmos.Handlers.TransportHandler.<ProcessMessageAsync>d__3.MoveNext()\\r\\n--- End of stack trace from previous location where exception was thrown ---\\r\\n at System.Runtime.ExceptionServices.ExceptionDispatchInfo.Throw()\\r\\n at System.Runtime.CompilerServices.TaskAwaiter.ThrowForNonSuccess(Task task)\\r\\n at System.Runtime.CompilerServices.TaskAwaiter.HandleNonSuccessAndDebuggerNotification(Task task)\\r\\n at Microsoft.Azure.Cosmos.Handlers.TransportHandler.<SendAsync>d__2.MoveNext()\\r\\n--- End of stack trace from previous location where exception was thrown ---\\r\\n at System.Runtime.ExceptionServices.ExceptionDispatchInfo.Throw()\\r\\n at System.Runtime.CompilerServices.TaskAwaiter.ThrowForNonSuccess(Task task)\\r\\n at System.Runtime.CompilerServices.TaskAwaiter.HandleNonSuccessAndDebuggerNotification(Task task)\\r\\n at Microsoft.Azure.Cosmos.Handlers.RouterHandler.<SendAsync>d__3.MoveNext()\\r\\n--- End of stack trace from previous location where exception was thrown ---\\r\\n at System.Runtime.ExceptionServices.ExceptionDispatchInfo.Throw()\\r\\n at System.Runtime.CompilerServices.TaskAwaiter.ThrowForNonSuccess(Task task)\\r\\n at System.Runtime.CompilerServices.TaskAwaiter.HandleNonSuccessAndDebuggerNotification(Task task)\\r\\n at Microsoft.Azure.Cosmos.RequestHandler.<SendAsync>d__6.MoveNext()\\r\\n--- End of stack trace from previous location where exception was thrown ---\\r\\n at System.Runtime.ExceptionServices.ExceptionDispatchInfo.Throw()\\r\\n at System.Runtime.CompilerServices.TaskAwaiter.ThrowForNonSuccess(Task task)\\r\\n at System.Runtime.CompilerServices.TaskAwaiter.HandleNonSuccessAndDebuggerNotification(Task task)\\r\\n at Microsoft.Azure.Cosmos.Handlers.AbstractRetryHandler.<ExecuteHttpRequestAsync>d__2.MoveNext()\\r\\n--- End of stack trace from previous location where exception was thrown ---\\r\\n at System.Runtime.ExceptionServices.ExceptionDispatchInfo.Throw()\\r\\n at System.Runtime.CompilerServices.TaskAwaiter.ThrowForNonSuccess(Task task)\\r\\n at System.Runtime.CompilerServices.TaskAwaiter.HandleNonSuccessAndDebuggerNotification(Task task)\\r\\n at Microsoft.Azure.Cosmos.Handlers.AbstractRetryHandler.<SendAsync>d__1.MoveNext()\\r\\n--- End of stack trace from previous location where exception was thrown ---\\r\\n at System.Runtime.ExceptionServices.ExceptionDispatchInfo.Throw()\\r\\n at System.Runtime.CompilerServices.TaskAwaiter.ThrowForNonSuccess(Task task)\\r\\n at System.Runtime.CompilerServices.TaskAwaiter.HandleNonSuccessAndDebuggerNotification(Task task)\\r\\n at Microsoft.Azure.Cosmos.RequestHandler.<SendAsync>d__6.MoveNext()\\r\\n--- End of stack trace from previous location where exception was thrown ---\\r\\n at System.Runtime.ExceptionServices.ExceptionDispatchInfo.Throw()\\r\\n at System.Runtime.CompilerServices.TaskAwaiter.ThrowForNonSuccess(Task task)\\r\\n at System.Runtime.CompilerServices.TaskAwaiter.HandleNonSuccessAndDebuggerNotification(Task task)\\r\\n at Microsoft.Azure.Cosmos.Handlers.DiagnosticsHandler.<SendAsync>d__0.MoveNext()\\r\\n--- End of stack trace from previous location where exception was thrown ---\\r\\n at System.Runtime.ExceptionServices.ExceptionDispatchInfo.Throw()\\r\\n at System.Runtime.CompilerServices.TaskAwaiter.ThrowForNonSuccess(Task task)\\r\\n at System.Runtime.CompilerServices.TaskAwaiter.HandleNonSuccessAndDebuggerNotification(Task task)\\r\\n at Microsoft.Azure.Cosmos.RequestHandler.<SendAsync>d__6.MoveNext()\\r\\n--- End of stack trace from previous location where exception was thrown ---\\r\\n at System.Runtime.ExceptionServices.ExceptionDispatchInfo.Throw()\\r\\n at System.Runtime.CompilerServices.TaskAwaiter.ThrowForNonSuccess(Task task)\\r\\n at System.Runtime.CompilerServices.TaskAwaiter.HandleNonSuccessAndDebuggerNotification(Task task)\\r\\n at Microsoft.CCI.Store.Core.ObjectStorage.TelemetryHandler.<>c__DisplayClass7_0.<<SendRequestAsync>b__0>d.MoveNext() in C:\\\\__w\\\\1\\\\s\\\\src\\\\Infrastructure\\\\Common\\\\Microsoft.CCI.Store.Core\\\\ObjectStorage\\\\Telemetry\\\\TelemetryHandler.cs:line 124\\r\\n--- End of stack trace from previous location where exception was thrown ---\\r\\n at System.Runtime.ExceptionServices.ExceptionDispatchInfo.Throw()\\r\\n at System.Runtime.CompilerServices.TaskAwaiter.ThrowForNonSuccess(Task task)\\r\\n at System.Runtime.CompilerServices.TaskAwaiter.HandleNonSuccessAndDebuggerNotification(Task task)\\r\\n at Microsoft.CCI.Common.ExecuteWithRetry.<RunAsync>d__31`1.MoveNext() in C:\\\\__w\\\\1\\\\s\\\\src\\\\Infrastructure\\\\Common\\\\Microsoft.CCI.Common\\\\ExecuteWithRetry.cs:line 620\\r\\n--- End of stack trace from previous location where exception was thrown ---\\r\\n at System.Runtime.ExceptionServices.ExceptionDispatchInfo.Throw()\\r\\n at System.Runtime.CompilerServices.TaskAwaiter.ThrowForNonSuccess(Task task)\\r\\n at System.Runtime.CompilerServices.TaskAwaiter.HandleNonSuccessAndDebuggerNotification(Task task)\\r\\n at Microsoft.CCI.Store.Core.ObjectStorage.TelemetryHandler.<SendRequestAsync>d__7.MoveNext() in C:\\\\__w\\\\1\\\\s\\\\src\\\\Infrastructure\\\\Common\\\\Microsoft.CCI.Store.Core\\\\ObjectStorage\\\\Telemetry\\\\TelemetryHandler.cs:line 128\\r\\n--- End of stack trace from previous location where exception was thrown ---\\r\\n at System.Runtime.ExceptionServices.ExceptionDispatchInfo.Throw()\\r\\n at System.Runtime.CompilerServices.TaskAwaiter.ThrowForNonSuccess(Task task)\\r\\n at System.Runtime.CompilerServices.TaskAwaiter.HandleNonSuccessAndDebuggerNotification(Task task)\\r\\n at Microsoft.CCI.Store.Core.ObjectStorage.TelemetryHandler.<>c__DisplayClass6_0.<<SendAsync>b__0>d.MoveNext() in C:\\\\__w\\\\1\\\\s\\\\src\\\\Infrastructure\\\\Common\\\\Microsoft.CCI.Store.Core\\\\ObjectStorage\\\\Telemetry\\\\TelemetryHandler.cs:line 62\\r\\n--- End of stack trace from previous location where exception was thrown ---\\r\\n at System.Runtime.ExceptionServices.ExceptionDispatchInfo.Throw()\\r\\n at System.Runtime.CompilerServices.TaskAwaiter.ThrowForNonSuccess(Task task)\\r\\n at System.Runtime.CompilerServices.TaskAwaiter.HandleNonSuccessAndDebuggerNotification(Task task)\\r\\n at Microsoft.PowerApps.CoreFramework.ActivityLoggerExtensions.<ExecuteAsync>d__11`1.MoveNext()\\r\\n--- End of stack trace from previous location where exception was thrown ---\\r\\n at System.Runtime.ExceptionServices.ExceptionDispatchInfo.Throw()\\r\\n at System.Runtime.CompilerServices.TaskAwaiter.ThrowForNonSuccess(Task task)\\r\\n at System.Runtime.CompilerServices.TaskAwaiter.HandleNonSuccessAndDebuggerNotification(Task task)\\r\\n at Microsoft.PowerApps.CoreFramework.ActivityLoggerExtensions.<ExecuteAsync>d__3`1.MoveNext()\\r\\n--- End of stack trace from previous location where exception was thrown ---\\r\\n at System.Runtime.ExceptionServices.ExceptionDispatchInfo.Throw()\\r\\n at System.Runtime.CompilerServices.TaskAwaiter.ThrowForNonSuccess(Task task)\\r\\n at System.Runtime.CompilerServices.TaskAwaiter.HandleNonSuccessAndDebuggerNotification(Task task)\\r\\n at Microsoft.Azure.Cosmos.RequestHandler.<SendAsync>d__6.MoveNext()\\r\\n--- End of stack trace from previous location where exception was thrown ---\\r\\n at System.Runtime.ExceptionServices.ExceptionDispatchInfo.Throw()\\r\\n at System.Runtime.CompilerServices.TaskAwaiter.ThrowForNonSuccess(Task task)\\r\\n at System.Runtime.CompilerServices.TaskAwaiter.HandleNonSuccessAndDebuggerNotification(Task task)\\r\\n at Microsoft.Azure.Cosmos.Handlers.RequestInvokerHandler.<SendAsync>d__7.MoveNext()\\r\\n--- End of stack trace from previous location where exception was thrown ---\\r\\n at System.Runtime.ExceptionServices.ExceptionDispatchInfo.Throw()\\r\\n at System.Runtime.CompilerServices.TaskAwaiter.ThrowForNonSuccess(Task task)\\r\\n at System.Runtime.CompilerServices.TaskAwaiter.HandleNonSuccessAndDebuggerNotification(Task task)\\r\\n at Microsoft.Azure.Cosmos.Handlers.RequestInvokerHandler.<SendAsync>d__9.MoveNext()\\r\\n--- End of stack trace from previous location where exception was thrown ---\\r\\n at System.Runtime.ExceptionServices.ExceptionDispatchInfo.Throw()\\r\\n at System.Runtime.CompilerServices.TaskAwaiter.ThrowForNonSuccess(Task task)\\r\\n at System.Runtime.CompilerServices.TaskAwaiter.HandleNonSuccessAndDebuggerNotification(Task task)\\r\\n at Microsoft.Azure.Cosmos.ContainerCore.<ProcessItemStreamAsync>d__89.MoveNext()\\r\\n--- End of stack trace from previous location where exception was thrown ---\\r\\n at System.Runtime.ExceptionServices.ExceptionDispatchInfo.Throw()\\r\\n at System.Runtime.CompilerServices.TaskAwaiter.ThrowForNonSuccess(Task task)\\r\\n at System.Runtime.CompilerServices.TaskAwaiter.HandleNonSuccessAndDebuggerNotification(Task task)\\r\\n at Microsoft.Azure.Cosmos.ContainerCore.<ReadItemAsync>d__58`1.MoveNext()\\r\\n--- End of stack trace from previous location where exception was thrown ---\\r\\n at System.Runtime.ExceptionServices.ExceptionDispatchInfo.Throw()\\r\\n at System.Runtime.CompilerServices.TaskAwaiter.ThrowForNonSuccess(Task task)\\r\\n at System.Runtime.CompilerServices.TaskAwaiter.HandleNonSuccessAndDebuggerNotification(Task task)\\r\\n at Microsoft.Azure.Cosmos.ClientContextCore.<RunWithDiagnosticsHelperAsync>d__39`1.MoveNext()\"}}]}","errorCode":500,"errorStatus":500,"errorInfo":""}