How to fix ClusterBlockException: blocked by: [SERVICE_UNAVAILABLE/1/state not recovered / initialized] in Elasticsearch

# Check cluster health
curl -X GET "localhost:9200/_cluster/health?pretty" -u "username:password"

# List pending cluster tasks (may timeout during recovery)
curl -X GET "localhost:9200/_cluster/pending_tasks?pretty" -u "username:password"

# Check cluster state version
curl -X GET "localhost:9200/_cluster/state/metadata?pretty" -u "username:password"

# Get node discovery status
curl -X GET "localhost:9200/_nodes?pretty" -u "username:password"

# Check node count
curl -X GET "localhost:9200/_cat/nodes?v" -u "username:password"

# Check disk usage on all nodes
curl -X GET "localhost:9200/_nodes/stats/fs?pretty" -u "username:password"

# More detailed allocation information
curl -X GET "localhost:9200/_cat/allocation?v&pretty" -u "username:password"

# On each node, check filesystem directly
df -h /path/to/elasticsearch/data

# Delete old indices to free space
curl -X DELETE "localhost:9200/old-index-name" -u "username:password"

# Add new disk volume and restart node
# Or delete expired snapshots if present
curl -X DELETE "localhost:9200/_snapshot/repo-name/snapshot-name" -u "username:password"

# Check node connectivity from primary
curl -X GET "localhost:9200/_nodes?pretty" -u "username:password"

# On each node, check the logs for connection errors
tail -f /var/log/elasticsearch/elasticsearch.log | grep -i "connection\|error\|discovery"

# Verify network connectivity between nodes manually
ping <other-node-ip>
telnet <other-node-ip> 9300

# Check discovery settings in elasticsearch.yml:
# discovery.seed_hosts: ["node1:9300", "node2:9300", "node3:9300"]
# cluster.initial_master_nodes: ["node1", "node2", "node3"]

# Monitor recovery progress
watch -n 5 'curl -s "localhost:9200/_cluster/health?pretty" -u "username:password" | head -20'

# Watch cluster events in real-time
curl -X GET "localhost:9200/_cluster/health?wait_for_status=yellow&timeout=30s&pretty" -u "username:password"

# For large clusters, recovery can take several minutes to hours
# Increase the timeout and monitor logs

# Check current gateway settings
curl -X GET "localhost:9200/_cluster/settings?include_defaults=true&filter_path=**.gateway.*&pretty" -u "username:password"

# For a 3-node cluster, check these settings in elasticsearch.yml:
# gateway.recover_after_nodes: 2
# gateway.recover_after_time: 5m
# gateway.expected_nodes: 3

# Verify minimum master nodes setting
curl -X GET "localhost:9200/_cluster/settings?include_defaults=true&filter_path=**.discovery.zen.*&pretty" -u "username:password"

# For Elasticsearch 7.0+, use this instead:
curl -X GET "localhost:9200/_cluster/settings?include_defaults=true&filter_path=**.cluster.initial_master_nodes&pretty" -u "username:password"

curl -X PUT "localhost:9200/_cluster/settings?pretty" -u "username:password" -H 'Content-Type: application/json' -d'
{
  "persistent": {
    "gateway.recover_after_nodes": 2,
    "gateway.expected_nodes": 3,
    "gateway.recover_after_time": "5m"
  }
}
'

# CAUTION: Only use these steps if recovery has not completed after 30+ minutes
# and all nodes are present and reachable

# Option 1: Restart the master node
# 1. Identify current master
curl -X GET "localhost:9200/_nodes?filter_path=nodes.*.name,nodes.*.master_node&pretty" -u "username:password"

# 2. Stop the master node gracefully
# On the master node server:
pkill -TERM -f "org.elasticsearch.bootstrap.Elasticsearch"

# 3. Wait 30 seconds, then restart it
systemctl restart elasticsearch
# or
./bin/elasticsearch

# Option 2: Temporarily disable minimum master nodes requirement
curl -X PUT "localhost:9200/_cluster/settings?pretty" -u "username:password" -H 'Content-Type: application/json' -d'
{
  "transient": {
    "discovery.zen.minimum_master_nodes": 1
  }
}
'

# Option 3: Reset cluster state (destructive - last resort only!)
# Stop all nodes completely, delete data/nodes directory, restart fresh
# This will lose all cluster state but allows starting over:
systemctl stop elasticsearch
rm -rf /var/lib/elasticsearch/nodes/0
systemctl start elasticsearch