How to fix EsRejectedExecutionException: rejected execution of search in Elasticsearch

# Get detailed search thread pool statistics
curl -X GET "localhost:9200/_cat/thread_pool/search?v&h=id,name,active,queue,rejected,completed" -u "username:password"

# Example output:
# id    name    active  queue  rejected  completed
# node1 search  7       1000   245       7265472
# node2 search  2       50     3         7123456

# Get node stats with detailed thread pool info
curl -X GET "localhost:9200/_nodes/stats/thread_pool/search" -u "username:password" | jq '.nodes'

# Check for HIGH rejection rates
# If 'rejected' count is increasing rapidly, you have an active problem

# Check CPU utilization per node
curl -X GET "localhost:9200/_cat/nodes?v&h=name,ip,cpu,heap.percent,ram.percent" -u "username:password"

# Check load on each node
curl -X GET "localhost:9200/_nodes/stats/os?filter_path=**.load_average" -u "username:password"

# Check JVM garbage collection frequency (GC pauses cause rejections)
curl -X GET "localhost:9200/_nodes/stats/jvm?filter_path=**.gc" -u "username:password"

# Detailed node info
curl -X GET "localhost:9200/_nodes?pretty" -u "username:password"

# Enable adaptive replica selection (ES 6.1+)
curl -X PUT "localhost:9200/_cluster/settings" -u "username:password" -H 'Content-Type: application/json' -d'
{
  "transient": {
    "cluster.routing.use_adaptive_replica_selection": true
  }
}
'

# Verify it's enabled
curl -X GET "localhost:9200/_cluster/settings" -u "username:password"

// BAD: Large result sets with aggregations
{
  "size": 10000,
  "query": { "match_all": {} },
  "aggs": {
    "all_users": {
      "terms": {
        "field": "user_id.keyword",
        "size": 50000
      }
    }
  }
}

// GOOD: Small result sets with date filtering
{
  "size": 100,
  "query": {
    "bool": {
      "must": [
        { "range": { "timestamp": { "gte": "now-1h" } } }
      ]
    }
  },
  "aggs": {
    "top_users": {
      "terms": {
        "field": "user_id.keyword",
        "size": 100,
        "execution_hint": "map"
      }
    }
  }
}

// For pagination, use search_after instead of large 'from'
{
  "size": 100,
  "sort": [{ "timestamp": "desc" }, { "_id": "asc" }],
  "search_after": [1640995200000, "doc-id-123"],
  "query": { "range": { "timestamp": { "gte": "now-30d" } } }
}

# In elasticsearch.yml
# The search thread pool size formula is: int((# of available_processors * 3) / 2) + 1

# For a 4-core node, default size = 7 threads
# For an 8-core node, default size = 13 threads

# Configure search thread pool
thread_pool:
  search:
    type: fixed  # Don't change type, it's always fixed for search
    size: 13     # For 8 cores: (8 * 3 / 2) + 1 = 13
    queue_size: 1000  # Default, rarely needs adjustment

# If you want to accept more queued requests (not recommended):
# thread_pool:
#   search:
#     queue_size: 2000

# You can also add search_throttled pool for background searches
  search_throttled:
    type: fixed
    size: 3
    queue_size: 100

# Reload configuration without restart (for supported settings)
curl -X PUT "localhost:9200/_cluster/settings" -u "username:password" -H 'Content-Type: application/json' -d'
{
  "persistent": {
    "thread_pool.search.queue_size": 1000
  }
}
'

// JavaScript/Node.js example with exponential backoff
async function searchWithRetry(query, maxRetries = 5) {
  let retries = 0;
  let backoff = 100; // Start with 100ms

  while (retries < maxRetries) {
    try {
      const response = await client.search({
        index: 'my-index',
        body: query
      });
      return response;
    } catch (error) {
      if (error.statusCode === 429 && retries < maxRetries) {
        retries++;
        const delay = backoff * Math.pow(2, retries - 1); // Exponential backoff
        console.log(`Rejected, retrying in ${delay}ms (attempt ${retries}/${maxRetries})`);
        await new Promise(resolve => setTimeout(resolve, delay));
      } else {
        throw error;
      }
    }
  }
  throw new Error('Max retries exceeded');
}

// Usage
try {
  const results = await searchWithRetry({
    query: { match_all: {} },
    size: 100
  });
  console.log(results);
} catch (error) {
  console.error('Search failed:', error.message);
}

# Python example with retry logic
import time
from elasticsearch import Elasticsearch
from elasticsearch.exceptions import ConnectionError

client = Elasticsearch(['http://localhost:9200'])

def search_with_retry(query, max_retries=5):
    retries = 0
    backoff = 0.1  # Start with 100ms

    while retries < max_retries:
        try:
            return client.search(index='my-index', body=query)
        except ConnectionError as e:
            if '429' in str(e) and retries < max_retries:
                retries += 1
                delay = backoff * (2 ** (retries - 1))
                print(f'Rejected, retrying in {delay:.1f}s (attempt {retries}/{max_retries})')
                time.sleep(delay)
            else:
                raise

# Usage
try:
    results = search_with_retry({'query': {'match_all': {}}, 'size': 100})
except Exception as e:
    print(f'Search failed: {e}')

# Vertical scaling: Increase heap size (elasticsearch.yml)
# Recommended: 50% of available RAM, not exceeding 32GB
-Xms16g
-Xmx16g

# You must restart the node after changing heap size

# Check current heap
curl -X GET "localhost:9200/_nodes/stats/jvm?filter_path=**.mem" -u "username:password"

# Horizontal scaling: Add more nodes to the cluster
# Each new node adds:
# - More CPU cores for search threads
# - More heap for caching
# - More processing capacity overall

# After adding nodes, check cluster status
curl -X GET "localhost:9200/_cluster/health?pretty" -u "username:password"

# Check new node has joined
curl -X GET "localhost:9200/_cat/nodes?v" -u "username:password"

# Rebalance shards across new nodes
curl -X POST "localhost:9200/_cluster/reroute?retry_failed" -u "username:password"

# Create a monitoring query to track rejection rate
curl -X POST "localhost:9200/_search" -u "username:password" -H 'Content-Type: application/json' -d'
{
  "index": ".monitoring-es-*",
  "query": {
    "bool": {
      "must": [
        { "term": { "type": "node_stats" } },
        { "exists": { "field": "node_stats.thread_pool.search.rejected" } }
      ]
    }
  },
  "size": 100,
  "sort": [{ "timestamp": { "order": "desc" } }]
}
'

# Check rejection trend
curl -X GET "localhost:9200/_nodes/stats/thread_pool/search?filter_path=**.rejected" -u "username:password"

# With Elasticsearch alerting or external monitoring tool
# Alert when search rejections > 10 in 5 minutes
- alert: ElasticsearchSearchRejections
  expr: rate(elasticsearch_thread_pool_search_rejected[5m]) > 10
  for: 5m
  annotations:
    summary: "Elasticsearch is rejecting search requests"
    description: "{{ $labels.instance }} is rejecting search requests"