How to fix NodeDisconnectedException: [node] disconnected in Elasticsearch

# Check overall cluster health
curl -X GET "localhost:9200/_cluster/health?pretty" -u "username:password"

# Check cluster state for detailed node information
curl -X GET "localhost:9200/_cluster/state?pretty&filter_path=metadata,nodes" -u "username:password"

# List all nodes in the cluster
curl -X GET "localhost:9200/_cat/nodes?v&h=name,ip,heap.percent,ram.percent,cpu,load_1m,node.role,master" -u "username:password"

# Check for unassigned shards (often a symptom of node disconnection)
curl -X GET "localhost:9200/_cat/shards?v&h=index,shard,prirep,state,unassigned.reason,node" -u "username:password" | grep UNASSIGNED

# Check master node status
curl -X GET "localhost:9200/_cat/master?v" -u "username:password"

# From each node, test connectivity to other nodes (adjust ports as needed)
# Test transport port (default 9300)
nc -zv other-node-ip 9300

# Test HTTP port (default 9200)
nc -zv other-node-ip 9200

# Check DNS resolution
nslookup other-node-hostname

# Check firewall rules
sudo iptables -L -n | grep 9300
sudo iptables -L -n | grep 9200

# For cloud environments, check security groups/network ACLs
# AWS: Check security group inbound rules for ports 9200 and 9300
# GCP: Check firewall rules
# Azure: Check network security groups

# Check network interface status and configuration
ip addr show
netstat -tulpn | grep java

# Test with telnet (if available)
telnet other-node-ip 9300

# Check Elasticsearch logs (location varies by installation)
# Systemd installations:
sudo journalctl -u elasticsearch --since "1 hour ago" | tail -100

# Tarball installations:
tail -100 /path/to/elasticsearch/logs/elasticsearch.log

# Look for specific error patterns:
grep -i "disconnect" /var/log/elasticsearch/elasticsearch.log
grep -i "timeout" /var/log/elasticsearch/elasticsearch.log
grep -i "heartbeat" /var/log/elasticsearch/elasticsearch.log
grep -i "master_not_discovered" /var/log/elasticsearch/elasticsearch.log

# Check for out of memory errors
grep -i "outofmemory" /var/log/elasticsearch/elasticsearch.log
grep -i "gc" /var/log/elasticsearch/elasticsearch.log | tail -20

# Check for long garbage collection pauses
# Enable GC logging in jvm.options if not already:
# -Xlog:gc*,gc+age=trace,safepoint:file=logs/gc.log:utctime,pid,tags:filecount=32,filesize=64m

# Check system logs for hardware/OS issues
dmesg | tail -50
sudo tail -50 /var/log/syslog

# Gracefully restart the Elasticsearch service
sudo systemctl restart elasticsearch

# Or for tarball installations
# First, find the PID
ps aux | grep elasticsearch

# Send SIGTERM for graceful shutdown
kill -TERM <pid>

# Wait for shutdown, then restart
/path/to/elasticsearch/bin/elasticsearch -d -p pid

# Monitor startup logs
tail -f /var/log/elasticsearch/elasticsearch.log

# After restart, check if node rejoins cluster
curl -X GET "localhost:9200/_cat/nodes?v" -u "username:password"

# Check cluster health recovery
watch -n 5 'curl -s "localhost:9200/_cluster/health?pretty" -u "username:password"'

# If shards are recovering, monitor progress
curl -X GET "localhost:9200/_cat/recovery?v&active_only" -u "username:password"

# Check current discovery settings
curl -X GET "localhost:9200/_cluster/settings?include_defaults=true&filter_path=**.discovery.*" -u "username:password" | jq .

# Increase ping timeouts (adjust values based on your environment)
curl -X PUT "localhost:9200/_cluster/settings" -u "username:password" -H 'Content-Type: application/json' -d'
{
  "persistent": {
    "discovery.zen.ping_timeout": "30s",
    "discovery.zen.fd.ping_timeout": "2m",
    "discovery.zen.fd.ping_retries": 10,
    "discovery.zen.no_master_block": "write"
  }
}
'

# For Elasticsearch 7.x and above with cluster formation settings:
curl -X PUT "localhost:9200/_cluster/settings" -u "username:password" -H 'Content-Type: application/json' -d'
{
  "persistent": {
    "cluster.fault_detection.leader_check.interval": "2s",
    "cluster.fault_detection.leader_check.timeout": "10s",
    "cluster.fault_detection.follower_check.interval": "1s",
    "cluster.fault_detection.follower_check.timeout": "10s"
  }
}
'

# Adjust thread pool settings if nodes are overloaded
curl -X PUT "localhost:9200/_cluster/settings" -u "username:password" -H 'Content-Type: application/json' -d'
{
  "persistent": {
    "thread_pool.search.queue_size": 2000,
    "thread_pool.write.queue_size": 500
  }
}
'

# After changing settings, restart nodes if needed
# Settings marked "persistent" survive restarts
# Settings marked "transient" are reset on full cluster restart

# Configure Elasticsearch monitoring (X-Pack or open source alternatives)
# Enable monitoring in elasticsearch.yml:
xpack.monitoring.enabled: true
xpack.monitoring.elasticsearch.collection.enabled: true

# Set up alerts for node disconnections
# Example using Elasticsearch Watcher or external monitoring:

# Create a watch for node disconnections
curl -X PUT "localhost:9200/_watcher/watch/node_disconnect_alert" -u "username:password" -H 'Content-Type: application/json' -d'
{
  "trigger": {
    "schedule": {
      "interval": "30s"
    }
  },
  "input": {
    "search": {
      "request": {
        "indices": [".monitoring-es-*"],
        "body": {
          "query": {
            "bool": {
              "must": [
                {
                  "range": {
                    "timestamp": {
                      "gte": "now-1m"
                    }
                  }
                },
                {
                  "term": {
                    "type": "node_stats"
                  }
                }
              ]
            }
          },
          "aggs": {
            "nodes": {
              "terms": {
                "field": "node.name",
                "size": 10
              }
            }
          },
          "size": 0
        }
      }
    }
  },
  "condition": {
    "compare": {
      "ctx.payload.aggregations.nodes.buckets.length": {
        "lt": "{{expected_node_count}}"
      }
    }
  },
  "actions": {
    "send_email": {
      "email": {
        "to": ["[email protected]"],
        "subject": "Elasticsearch Node Disconnected",
        "body": "Node count dropped from {{expected_node_count}} to {{ctx.payload.aggregations.nodes.buckets.length}}"
      }
    }
  }
}
'

# Monitor key metrics:
# - Node count over time
# - Network errors in transport layer
# - GC duration and frequency
# - Heap usage trends
# - Thread pool queue sizes
# - Disk I/O latency

# Set up external monitoring (Prometheus + Grafana example):
# 1. Install and configure Elasticsearch exporter
# 2. Set up Prometheus to scrape metrics
# 3. Create Grafana dashboards for:
#    - Node status and count
#    - Network connectivity
#    - Resource utilization
#    - Cluster health status