The first snapshot stuck in inconsistent state after 6.8 -> 7.12 upgrade #75339
Description
Elasticsearch version (bin/elasticsearch --version): 6.8.16 upgraded to 7.12 (or 7.13, it looks like minor version doesn't matter here)
Plugins installed: [repository-s3]
Steps to reproduce:
Steps to reproduce for Elastic Cloud:
- Create minimalistic 6.8.16 Cloud deployment - single node 1GB ES, 1GB Kibana
- Upgrade to 7.12 / 7.13 (it's going to be a a rolling upgrade which eventually succeeds)
- Apply a noop plan (which will trigger snapshot capture)
- Watch Elasticsearch struggle to take a snapshot, Cloud change is eventually going to timeout claiming that Elasticsearch failed to capture a snapshot.
If we examine _snapshot API, we're going to find an oddity:
GET _snapshot/found-snapshots/cloud-snapshot-2021.06.30-tal3bny0rfq17eumwbhnmg
reports "state": "IN_PROGRESS", whereas:
_snapshot/found-snapshots/_status
reports "state": "SUCCESS"
Apparently, that snapshot is going to remain in this "inconsistent" state forever. FWIW, this doesn't prevent the snapshot mechanism from being operational - if we delete aforementioned snapshot and capture new one - it's going to work without any issues.
Full API responses:
//_snapshot/found-snapshots/*
//_snapshot/found-snapshots/cloud-snapshot-2021.06.30-tal3bny0rfq17eumwbhnmg
{
"snapshots": [
{
"snapshot": "cloud-snapshot-2021.06.30-tal3bny0rfq17eumwbhnmg",
"uuid": "XqY6xh7ZTbGAC7KOABdYnw",
"version_id": 7130299,
"version": "7.13.2",
"indices": [
".kibana_7.13.2_001",
".apm-custom-link",
".kibana-event-log-7.13.2-000001",
".apm-agent-configuration",
".kibana_task_manager_pre7.4.0_001",
".kibana_security_session_1",
".ds-ilm-history-5-2021.06.30-000001",
".kibana_task_manager_7.13.2_001",
".kibana_1",
".security-6",
".tasks"
],
"data_streams": [
"ilm-history-5"
],
"include_global_state": true,
"metadata": {
"policy": "cloud-snapshot-policy"
},
"state": "IN_PROGRESS",
"start_time": "2021-06-30T08:11:20.691Z",
"start_time_in_millis": 1625040680691,
"end_time": "1970-01-01T00:00:00.000Z",
"end_time_in_millis": 0,
"duration_in_millis": 0,
"failures": [],
"shards": {
"total": 0,
"failed": 0,
"successful": 0
},
"feature_states": [
{
"feature_name": "security",
"indices": [
".security-6"
]
},
{
"feature_name": "kibana",
"indices": [
".kibana_task_manager_pre7.4.0_001",
".kibana_task_manager_7.13.2_001",
".kibana_security_session_1",
".kibana_7.13.2_001",
".kibana_1",
".apm-agent-configuration",
".apm-custom-link"
]
},
{
"feature_name": "tasks",
"indices": [
".tasks"
]
}
]
}
]
}
//_snapshot/found-snapshots/_status
{
"snapshots": [
{
"snapshot": "cloud-snapshot-2021.06.30-tal3bny0rfq17eumwbhnmg",
"repository": "found-snapshots",
"uuid": "XqY6xh7ZTbGAC7KOABdYnw",
"state": "SUCCESS",
"include_global_state": true,
"shards_stats": {
"initializing": 0,
"started": 0,
"finalizing": 0,
"done": 11,
"failed": 0,
"total": 11
},
"stats": {
"incremental": {
"file_count": 176,
"size_in_bytes": 2624566
},
"processed": {
"file_count": 137,
"size_in_bytes": 2608294
},
"total": {
"file_count": 186,
"size_in_bytes": 2638788
},
"start_time_in_millis": 1625040680691,
"time_in_millis": 1573599
},
"indices": {
".kibana_7.13.2_001": {
"shards_stats": {
"initializing": 0,
"started": 0,
"finalizing": 0,
"done": 1,
"failed": 0,
"total": 1
},
"stats": {
"incremental": {
"file_count": 45,
"size_in_bytes": 2255550
},
"processed": {
"file_count": 35,
"size_in_bytes": 2251118
},
"total": {
"file_count": 45,
"size_in_bytes": 2255550
},
"start_time_in_millis": 1625040683194,
"time_in_millis": 27894
},
"shards": {
"0": {
"stage": "DONE",
"stats": {
"incremental": {
"file_count": 45,
"size_in_bytes": 2255550
},
"processed": {
"file_count": 35,
"size_in_bytes": 2251118
},
"total": {
"file_count": 45,
"size_in_bytes": 2255550
},
"start_time_in_millis": 1625040683194,
"time_in_millis": 27894
}
}
}
},
".apm-custom-link": {
"shards_stats": {
"initializing": 0,
"started": 0,
"finalizing": 0,
"done": 1,
"failed": 0,
"total": 1
},
"stats": {
"incremental": {
"file_count": 1,
"size_in_bytes": 208
},
"processed": {
"file_count": 0,
"size_in_bytes": 0
},
"total": {
"file_count": 1,
"size_in_bytes": 208
},
"start_time_in_millis": 1625040680895,
"time_in_millis": 409
},
"shards": {
"0": {
"stage": "DONE",
"stats": {
"incremental": {
"file_count": 1,
"size_in_bytes": 208
},
"processed": {
"file_count": 0,
"size_in_bytes": 0
},
"total": {
"file_count": 1,
"size_in_bytes": 208
},
"start_time_in_millis": 1625040680895,
"time_in_millis": 409
}
}
}
},
".kibana-event-log-7.13.2-000001": {
"shards_stats": {
"initializing": 0,
"started": 0,
"finalizing": 0,
"done": 1,
"failed": 0,
"total": 1
},
"stats": {
"incremental": {
"file_count": 4,
"size_in_bytes": 5748
},
"processed": {
"file_count": 2,
"size_in_bytes": 5080
},
"total": {
"file_count": 4,
"size_in_bytes": 5748
},
"start_time_in_millis": 1625040682188,
"time_in_millis": 6100
},
"shards": {
"0": {
"stage": "DONE",
"stats": {
"incremental": {
"file_count": 4,
"size_in_bytes": 5748
},
"processed": {
"file_count": 2,
"size_in_bytes": 5080
},
"total": {
"file_count": 4,
"size_in_bytes": 5748
},
"start_time_in_millis": 1625040682188,
"time_in_millis": 6100
}
}
}
},
".kibana_task_manager_pre7.4.0_001": {
"shards_stats": {
"initializing": 0,
"started": 0,
"finalizing": 0,
"done": 1,
"failed": 0,
"total": 1
},
"stats": {
"incremental": {
"file_count": 4,
"size_in_bytes": 6620
},
"processed": {
"file_count": 2,
"size_in_bytes": 5952
},
"total": {
"file_count": 4,
"size_in_bytes": 6620
},
"start_time_in_millis": 1625040681988,
"time_in_millis": 5411
},
"shards": {
"0": {
"stage": "DONE",
"stats": {
"incremental": {
"file_count": 4,
"size_in_bytes": 6620
},
"processed": {
"file_count": 2,
"size_in_bytes": 5952
},
"total": {
"file_count": 4,
"size_in_bytes": 6620
},
"start_time_in_millis": 1625040681988,
"time_in_millis": 5411
}
}
}
},
".apm-agent-configuration": {
"shards_stats": {
"initializing": 0,
"started": 0,
"finalizing": 0,
"done": 1,
"failed": 0,
"total": 1
},
"stats": {
"incremental": {
"file_count": 1,
"size_in_bytes": 208
},
"processed": {
"file_count": 0,
"size_in_bytes": 0
},
"total": {
"file_count": 1,
"size_in_bytes": 208
},
"start_time_in_millis": 1625040681304,
"time_in_millis": 283
},
"shards": {
"0": {
"stage": "DONE",
"stats": {
"incremental": {
"file_count": 1,
"size_in_bytes": 208
},
"processed": {
"file_count": 0,
"size_in_bytes": 0
},
"total": {
"file_count": 1,
"size_in_bytes": 208
},
"start_time_in_millis": 1625040681304,
"time_in_millis": 283
}
}
}
},
".ds-ilm-history-5-2021.06.30-000001": {
"shards_stats": {
"initializing": 0,
"started": 0,
"finalizing": 0,
"done": 1,
"failed": 0,
"total": 1
},
"stats": {
"incremental": {
"file_count": 10,
"size_in_bytes": 26486
},
"processed": {
"file_count": 6,
"size_in_bytes": 24944
},
"total": {
"file_count": 10,
"size_in_bytes": 26486
},
"start_time_in_millis": 1625040682994,
"time_in_millis": 10394
},
"shards": {
"0": {
"stage": "DONE",
"stats": {
"incremental": {
"file_count": 10,
"size_in_bytes": 26486
},
"processed": {
"file_count": 6,
"size_in_bytes": 24944
},
"total": {
"file_count": 10,
"size_in_bytes": 26486
},
"start_time_in_millis": 1625040682994,
"time_in_millis": 10394
}
}
}
},
".kibana_security_session_1": {
"shards_stats": {
"initializing": 0,
"started": 0,
"finalizing": 0,
"done": 1,
"failed": 0,
"total": 1
},
"stats": {
"incremental": {
"file_count": 1,
"size_in_bytes": 208
},
"processed": {
"file_count": 0,
"size_in_bytes": 0
},
"total": {
"file_count": 1,
"size_in_bytes": 208
},
"start_time_in_millis": 1625040681587,
"time_in_millis": 200
},
"shards": {
"0": {
"stage": "DONE",
"stats": {
"incremental": {
"file_count": 1,
"size_in_bytes": 208
},
"processed": {
"file_count": 0,
"size_in_bytes": 0
},
"total": {
"file_count": 1,
"size_in_bytes": 208
},
"start_time_in_millis": 1625040681587,
"time_in_millis": 200
}
}
}
},
".kibana_task_manager_7.13.2_001": {
"shards_stats": {
"initializing": 0,
"started": 0,
"finalizing": 0,
"done": 1,
"failed": 0,
"total": 1
},
"stats": {
"incremental": {
"file_count": 53,
"size_in_bytes": 77870
},
"processed": {
"file_count": 48,
"size_in_bytes": 75189
},
"total": {
"file_count": 53,
"size_in_bytes": 77870
},
"start_time_in_millis": 1625040683887,
"time_in_millis": 29801
},
"shards": {
"0": {
"stage": "DONE",
"stats": {
"incremental": {
"file_count": 53,
"size_in_bytes": 77870
},
"processed": {
"file_count": 48,
"size_in_bytes": 75189
},
"total": {
"file_count": 53,
"size_in_bytes": 77870
},
"start_time_in_millis": 1625040683887,
"time_in_millis": 29801
}
}
}
},
".kibana_1": {
"shards_stats": {
"initializing": 0,
"started": 0,
"finalizing": 0,
"done": 1,
"failed": 0,
"total": 1
},
"stats": {
"incremental": {
"file_count": 0,
"size_in_bytes": 0
},
"total": {
"file_count": 10,
"size_in_bytes": 14222
},
"start_time_in_millis": 1625040684089,
"time_in_millis": 603
},
"shards": {
"0": {
"stage": "DONE",
"stats": {
"incremental": {
"file_count": 0,
"size_in_bytes": 0
},
"total": {
"file_count": 10,
"size_in_bytes": 14222
},
"start_time_in_millis": 1625040684089,
"time_in_millis": 603
}
}
}
},
".security-6": {
"shards_stats": {
"initializing": 0,
"started": 0,
"finalizing": 0,
"done": 1,
"failed": 0,
"total": 1
},
"stats": {
"incremental": {
"file_count": 41,
"size_in_bytes": 216412
},
"processed": {
"file_count": 34,
"size_in_bytes": 213171
},
"total": {
"file_count": 41,
"size_in_bytes": 216412
},
"start_time_in_millis": 1625040682388,
"time_in_millis": 27012
},
"shards": {
"0": {
"stage": "DONE",
"stats": {
"incremental": {
"file_count": 41,
"size_in_bytes": 216412
},
"processed": {
"file_count": 34,
"size_in_bytes": 213171
},
"total": {
"file_count": 41,
"size_in_bytes": 216412
},
"start_time_in_millis": 1625040682388,
"time_in_millis": 27012
}
}
}
},
".tasks": {
"shards_stats": {
"initializing": 0,
"started": 0,
"finalizing": 0,
"done": 1,
"failed": 0,
"total": 1
},
"stats": {
"incremental": {
"file_count": 16,
"size_in_bytes": 35256
},
"processed": {
"file_count": 10,
"size_in_bytes": 32840
},
"total": {
"file_count": 16,
"size_in_bytes": 35256
},
"start_time_in_millis": 1625040680691,
"time_in_millis": 15600
},
"shards": {
"0": {
"stage": "DONE",
"stats": {
"incremental": {
"file_count": 16,
"size_in_bytes": 35256
},
"processed": {
"file_count": 10,
"size_in_bytes": 32840
},
"total": {
"file_count": 16,
"size_in_bytes": 35256
},
"start_time_in_millis": 1625040680691,
"time_in_millis": 15600
}
}
}
}
}
}
]
}