How to fix ElasticsearchTimeoutException: java.util.concurrent.TimeoutException: Timeout waiting for task in Elasticsearch

# Increase search timeout at request level
curl -X GET "localhost:9200/my-index/_search" -u "username:password" -H 'Content-Type: application/json' -d'
{
  "query": {
    "match_all": {}
  },
  "timeout": "60s"  # Increase from default 30s
}
'

# Set timeout in application code (Java example)
SearchRequest searchRequest = new SearchRequest("my-index");
searchRequest.source(new SearchSourceBuilder()
    .query(QueryBuilders.matchAllQuery())
    .timeout(TimeValue.timeValueSeconds(60)));

# For bulk operations
BulkRequest bulkRequest = new BulkRequest();
bulkRequest.timeout(TimeValue.timeValueMinutes(2));

# Check current cluster settings
curl -X GET "localhost:9200/_cluster/settings?include_defaults=true&filter_path=**.timeout" -u "username:password"

# Update cluster-level timeout settings
curl -X PUT "localhost:9200/_cluster/settings" -u "username:password" -H 'Content-Type: application/json' -d'
{
  "persistent": {
    "search.default_search_timeout": "60s",
    "indices.memory.index_buffer_size": "10%"
  }
}
'

# Enable slow query logging
curl -X PUT "localhost:9200/my-index/_settings" -u "username:password" -H 'Content-Type: application/json' -d'
{
  "index.search.slowlog.threshold.query.warn": "10s",
  "index.search.slowlog.threshold.query.info": "5s",
  "index.search.slowlog.threshold.query.debug": "2s",
  "index.search.slowlog.threshold.query.trace": "500ms",
  "index.search.slowlog.level": "info"
}
'

# Check slow query logs
curl -X GET "localhost:9200/_nodes/hot_threads" -u "username:password"
curl -X GET "localhost:9200/_cat/thread_pool?v" -u "username:password"

# Profile a slow query to see where time is spent
curl -X GET "localhost:9200/my-index/_search" -u "username:password" -H 'Content-Type: application/json' -d'
{
  "query": {
    "match": {
      "content": "search term"
    }
  },
  "profile": true
}
'

# Common query optimizations:
# 1. Use filters instead of queries for yes/no conditions
# 2. Limit aggregation size with "size": 0 for count-only aggregations
# 3. Use "terminate_after" to limit total hits scanned
# 4. Avoid expensive joins (nested/ parent-child) when possible
# 5. Use keyword fields for exact matches instead of text analysis

# Example optimized query
curl -X GET "localhost:9200/my-index/_search" -u "username:password" -H 'Content-Type: application/json' -d'
{
  "query": {
    "bool": {
      "filter": [
        {"term": {"status": "active"}},
        {"range": {"timestamp": {"gte": "now-7d/d"}}}
      ],
      "must": [
        {"match": {"title": "important"}}
      ]
    }
  },
  "aggs": {
    "category_count": {
      "terms": {
        "field": "category.keyword",
        "size": 10  # Limit aggregation buckets
      }
    }
  },
  "terminate_after": 10000,  # Stop after 10k documents
  "timeout": "30s"
}
'

# Check cluster health and resource usage
curl -X GET "localhost:9200/_cluster/health?pretty" -u "username:password"
curl -X GET "localhost:9200/_cat/nodes?v&h=name,heap.percent,ram.percent,cpu,load_1m" -u "username:password"
curl -X GET "localhost:9200/_cat/indices?v&h=index,docs.count,store.size,pri.store.size" -u "username:password"

# Adjust thread pool settings for better concurrency
curl -X PUT "localhost:9200/_cluster/settings" -u "username:password" -H 'Content-Type: application/json' -d'
{
  "persistent": {
    "thread_pool.search.size": 20,
    "thread_pool.search.queue_size": 1000,
    "thread_pool.write.size": 10,
    "thread_pool.write.queue_size": 500,
    "thread_pool.get.size": 10,
    "thread_pool.get.queue_size": 1000
  }
}
'

# Increase circuit breaker limits if hitting memory limits
curl -X PUT "localhost:9200/_cluster/settings" -u "username:password" -H 'Content-Type: application/json' -d'
{
  "persistent": {
    "indices.breaker.total.limit": "70%",
    "indices.breaker.fielddata.limit": "40%",
    "indices.breaker.request.limit": "60%"
  }
}
'

# Optimize index settings for better performance
curl -X PUT "localhost:9200/my-index/_settings" -u "username:password" -H 'Content-Type: application/json' -d'
{
  "index": {
    "refresh_interval": "30s",  # Reduce refresh frequency for indexing-heavy workloads
    "number_of_replicas": 1,    # Adjust based on read/write ratio
    "translog.durability": "async",  # For better indexing performance
    "translog.sync_interval": "5s"
  }
}
'

# Consider adding more nodes or upgrading hardware if consistently resource-bound
# - Add data nodes for storage/IO capacity
# - Add coordinating nodes for query processing
# - Increase heap size (but not beyond 50% of total RAM)
# - Use SSDs for better disk performance

// JavaScript/Node.js example with exponential backoff
const { Client } = require('@elastic/elasticsearch');
const client = new Client({ node: 'http://localhost:9200' });

async function searchWithRetry(index, query, maxRetries = 3) {
  let lastError;

  for (let attempt = 1; attempt <= maxRetries; attempt++) {
    try {
      const response = await client.search({
        index: index,
        body: query,
        requestTimeout: 60000, // 60 seconds
      });

      return response;
    } catch (error) {
      lastError = error;

      // Check if it's a timeout error
      if (error.meta?.body?.error?.type === 'elasticsearch_timeout_exception' ||
          error.meta?.body?.error?.type === 'timeout_exception') {

        console.warn(`Search timeout (attempt ${attempt}/${maxRetries}): ${error.message}`);

        if (attempt < maxRetries) {
          // Exponential backoff: 1s, 2s, 4s, etc.
          const delay = Math.pow(2, attempt - 1) * 1000;
          await new Promise(resolve => setTimeout(resolve, delay));
          continue;
        }
      } else {
        // Non-timeout error, rethrow immediately
        throw error;
      }
    }
  }

  throw new Error(`Search failed after ${maxRetries} attempts: ${lastError.message}`);
}

// Java example with retry template
import org.elasticsearch.action.search.SearchRequest;
import org.elasticsearch.action.search.SearchResponse;
import org.elasticsearch.client.RequestOptions;
import org.elasticsearch.client.RestHighLevelClient;
import org.springframework.retry.annotation.Backoff;
import org.springframework.retry.annotation.Retryable;

@Service
public class SearchService {

  @Retryable(
    value = {ElasticsearchTimeoutException.class},
    maxAttempts = 3,
    backoff = @Backoff(delay = 1000, multiplier = 2)
  )
  public SearchResponse searchWithRetry(SearchRequest request) throws IOException {
    return client.search(request, RequestOptions.DEFAULT);
  }
}

// Python example with tenacity library
from elasticsearch import Elasticsearch
from tenacity import retry, stop_after_attempt, wait_exponential

client = Elasticsearch(["localhost:9200"])

@retry(
    stop=stop_after_attempt(3),
    wait=wait_exponential(multiplier=1, min=1, max=10)
)
def search_with_retry(index, query):
    try:
        return client.search(index=index, body=query, request_timeout=60)
    except Exception as e:
        if "timeout" in str(e).lower() or "TimeoutException" in str(e):
            raise
        else:
            # Non-timeout errors shouldn't trigger retry
            raise Exception("Non-retryable error") from e

# Monitor timeout metrics
curl -X GET "localhost:9200/_nodes/stats/indices,thread_pool?pretty" -u "username:password"

# Check for rejected threads (indicates queue saturation)
curl -X GET "localhost:9200/_cat/thread_pool?v&h=name,active,queue,rejected,completed" -u "username:password"

# Monitor search latency percentiles
curl -X GET "localhost:9200/_nodes/stats/indices/search?pretty" -u "username:password"

# Set up alerting rules (example using Elasticsearch Watcher)
curl -X PUT "localhost:9200/_watcher/watch/search_timeout_alert" -u "username:password" -H 'Content-Type: application/json' -d'
{
  "trigger": {
    "schedule": {
      "interval": "1m"
    }
  },
  "input": {
    "search": {
      "request": {
        "indices": [".monitoring-es-*"],
        "body": {
          "query": {
            "bool": {
              "must": [
                {
                  "range": {
                    "timestamp": {
                      "gte": "now-5m"
                    }
                  }
                },
                {
                  "term": {
                    "type": "search_slowlog"
                  }
                }
              ]
            }
          },
          "aggs": {
            "timeout_count": {
              "value_count": {
                "field": "search_slowlog.total_time"
              }
            }
          }
        }
      }
    }
  },
  "condition": {
    "compare": {
      "ctx.payload.aggregations.timeout_count.value": {
        "gt": 10
      }
    }
  },
  "actions": {
    "send_email": {
      "email": {
        "to": ["[email protected]"],
        "subject": "High search timeout rate detected",
        "body": "{{ctx.payload.aggregations.timeout_count.value}} search timeouts detected in the last 5 minutes."
      }
    }
  }
}
'

# Use APM or application performance monitoring
# - Track 95th/99th percentile response times
# - Set alerts when latency exceeds thresholds
# - Monitor error rates for timeout exceptions
# - Track resource utilization (CPU, memory, disk I/O)

# Common monitoring thresholds:
# - Search latency > 5s (warning), > 10s (critical)
# - Indexing latency > 2s (warning), > 5s (critical)
# - Thread pool queue > 50% capacity
# - JVM GC pauses > 1s
# - Disk utilization > 80%