How to fix SearchPhaseExecutionException: all shards failed in Elasticsearch

# Check cluster health (look for RED status)
curl -X GET "localhost:9200/_cluster/health?pretty" -u "username:password"

# Get detailed shard allocation information
curl -X GET "localhost:9200/_cat/shards?v&h=index,shard,prirep,state,docs,store,ip,node,unassigned.reason&s=state" -u "username:password"

# Check for unassigned shards specifically
curl -X GET "localhost:9200/_cat/shards?v&h=index,shard,prirep,state,unassigned.reason&s=state&state=UNASSIGNED" -u "username:password"

# View cluster allocation explanation
curl -X GET "localhost:9200/_cluster/allocation/explain?pretty" -u "username:password"

# Check node stats and resource usage
curl -X GET "localhost:9200/_nodes/stats?pretty" -u "username:password"

# Check node disk usage (look for high usage)
curl -X GET "localhost:9200/_cat/nodes?v&h=name,heap.percent,ram.percent,cpu,load_1m,disk.used_percent,disk.avail" -u "username:password"

# Check thread pool statistics for search queues
curl -X GET "localhost:9200/_cat/thread_pool/search?v&h=node_name,name,active,queue,rejected,completed" -u "username:password"

# Check JVM heap and garbage collection
curl -X GET "localhost:9200/_nodes/hot_threads?pretty" -u "username:password"

# Check disk watermarks
curl -X GET "localhost:9200/_cluster/settings?include_defaults=true&filter_path=*.disk.*" -u "username:password"

# Free up disk space or add more storage
# Temporarily adjust watermarks (if absolutely necessary)
curl -X PUT "localhost:9200/_cluster/settings" -u "username:password" -H 'Content-Type: application/json' -d'
{
  "transient": {
    "cluster.routing.allocation.disk.watermark.low": "90%",
    "cluster.routing.allocation.disk.watermark.high": "95%",
    "cluster.routing.allocation.disk.watermark.flood_stage": "98%"
  }
}
'

# Retry shard allocation
curl -X POST "localhost:9200/_cluster/reroute?retry_failed=true" -u "username:password"

# Manually allocate specific shard (use with caution)
curl -X POST "localhost:9200/_cluster/reroute" -u "username:password" -H 'Content-Type: application/json' -d'
{
  "commands": [
    {
      "allocate_stale_primary": {
        "index": "my-index",
        "shard": 0,
        "node": "target-node-name",
        "accept_data_loss": true
      }
    }
  ]
}
'

# List available snapshots
curl -X GET "localhost:9200/_snapshot/my-repository/_all?pretty" -u "username:password"

# Restore specific indices
curl -X POST "localhost:9200/_snapshot/my-repository/my-snapshot/_restore" -u "username:password" -H 'Content-Type: application/json' -d'
{
  "indices": ["my-index-*"],
  "ignore_unavailable": true,
  "include_global_state": false
}
'

# Create new index with proper settings
curl -X PUT "localhost:9200/my-index-new" -u "username:password" -H 'Content-Type: application/json' -d'
{
  "settings": {
    "number_of_shards": 3,
    "number_of_replicas": 1
  }
}
'

# Reindex from source (if you have another data source)
curl -X POST "localhost:9200/_reindex" -u "username:password" -H 'Content-Type: application/json' -d'
{
  "source": {
    "index": "my-index-backup"
  },
  "dest": {
    "index": "my-index-new"
  }
}
'

# Only if you can afford to lose the data
curl -X DELETE "localhost:9200/my-corrupted-index" -u "username:password"

# Example: Elasticsearch configuration (elasticsearch.yml)
cluster.routing.allocation.disk.threshold_enabled: true
cluster.routing.allocation.disk.watermark.low: 85%
cluster.routing.allocation.disk.watermark.high: 90%
cluster.routing.allocation.disk.watermark.flood_stage: 95%

# Enable shard allocation awareness
cluster.routing.allocation.awareness.attributes: rack,zone

# Set appropriate shard limits
cluster.max_shards_per_node: 1000
cluster.max_shards_per_node.frozen: 3000

# Configure thread pools
thread_pool.search.queue_size: 1000
thread_pool.search.size: 50