How to fix AuthenticationException: unable to authenticate user in Elasticsearch

# Check if security is enabled in elasticsearch.yml
grep "xpack.security.enabled" /etc/elasticsearch/elasticsearch.yml

# Expected output:
# xpack.security.enabled: true

# Test basic connectivity to Elasticsearch
curl -X GET "http://localhost:9200/" -H 'Content-Type: application/json'

# If security is enabled, you should see:
# {"error":{"root_cause":[{"type":"security_exception","reason":"missing authentication credentials for REST request [/]"}]}}

# This confirms security is active and requiring authentication

# Edit /etc/elasticsearch/elasticsearch.yml
xpack.security.enabled: true
xpack.security.transport.ssl.enabled: true
xpack.security.http.ssl.enabled: true  # Optional but recommended

sudo systemctl restart elasticsearch

# Wait for cluster to be ready
curl -k -u elastic:password "https://localhost:9200/_cluster/health?wait_for_status=yellow&timeout=30s"

# Test with elastic superuser
curl -X GET "http://localhost:9200/_security/_authenticate" \
  -u elastic:your_password \
  -H 'Content-Type: application/json'

# Successful response shows your user details:
# {
#   "username" : "elastic",
#   "roles" : [ "superuser" ],
#   "full_name" : null,
#   "email" : null,
#   "metadata" : { },
#   "enabled" : true,
#   "authentication_realm" : {
#     "name" : "reserved",
#     "type" : "reserved"
#   },
#   "lookup_realm" : {
#     "name" : "reserved",
#     "type" : "reserved"
#   }
# }

# If you get 401 error, password is incorrect

# Reset elastic user password (Elasticsearch 8.x+)
cd /usr/share/elasticsearch
sudo bin/elasticsearch-reset-password -u elastic

# Output will show the new password:
# Password for the [elastic] user successfully reset.
# New value: xYz9-AbCd3fGhIjKl

# For Elasticsearch 7.x, use setup-passwords:
sudo bin/elasticsearch-setup-passwords auto

# Or set it interactively:
sudo bin/elasticsearch-setup-passwords interactive

// Node.js client
const { Client } = require('@elastic/elasticsearch');
const client = new Client({
  node: 'https://localhost:9200',
  auth: {
    username: 'elastic',
    password: 'xYz9-AbCd3fGhIjKl'  // Updated password
  },
  tls: {
    rejectUnauthorized: false  // Only for dev/testing
  }
});

# Python client
from elasticsearch import Elasticsearch

es = Elasticsearch(
    ['https://localhost:9200'],
    basic_auth=('elastic', 'xYz9-AbCd3fGhIjKl'),  # Updated password
    verify_certs=False  # Only for dev/testing
)

# Password with special characters: P@ssw0rd!123
# Needs URL encoding in connection strings

# INCORRECT - special characters not encoded:
curl -u elastic:P@ssw0rd!123 "http://localhost:9200/_cat/indices"

# CORRECT - special characters URL encoded:
# @ -> %40
# ! -> %21
curl -u elastic:P%40ssw0rd%21123 "http://localhost:9200/_cat/indices"

# Connection string format:
# https://username:password@hostname:port

# Example with encoded password:
export ELASTICSEARCH_URL="https://elastic:P%40ssw0rd%21123@localhost:9200"

# Or use the elasticsearch-keystore for secure credential storage:
sudo /usr/share/elasticsearch/bin/elasticsearch-keystore add bootstrap.password
# Enter password when prompted (special characters handled automatically)

// JavaScript - no manual encoding needed
const client = new Client({
  node: 'https://localhost:9200',
  auth: {
    username: 'elastic',
    password: 'P@ssw0rd!123'  // Client handles encoding
  }
});

# Python - no manual encoding needed
es = Elasticsearch(
    ['https://localhost:9200'],
    basic_auth=('elastic', 'P@ssw0rd!123')  # Client handles encoding
)

# Check certificate validity and expiration
openssl s_client -connect localhost:9200 -showcerts

# Look for:
# - Certificate chain completeness
# - Expiration date (notAfter)
# - Subject Alternative Name (SAN) matching hostname

# Verify certificate dates
echo | openssl s_client -connect localhost:9200 2>/dev/null | openssl x509 -noout -dates

# Output:
# notBefore=Jan 15 10:23:45 2025 GMT
# notAfter=Jan 15 10:23:45 2026 GMT   # Check this hasn't passed

# If certificate is expired, regenerate it:
cd /usr/share/elasticsearch
sudo bin/elasticsearch-certutil cert --ca elastic-stack-ca.p12 \
  --name elasticsearch-node1 \
  --dns localhost \
  --dns elasticsearch.example.com \
  --ip 192.168.1.100 \
  --out config/certs/elasticsearch-node1.p12

# Update elasticsearch.yml with new certificate:
xpack.security.transport.ssl.keystore.path: certs/elasticsearch-node1.p12
xpack.security.transport.ssl.truststore.path: certs/elasticsearch-node1.p12

# Set the keystore password:
sudo bin/elasticsearch-keystore add xpack.security.transport.ssl.keystore.secure_password
sudo bin/elasticsearch-keystore add xpack.security.transport.ssl.truststore.secure_password

# Restart Elasticsearch
sudo systemctl restart elasticsearch

// Node.js - disable cert verification (dev only)
const client = new Client({
  node: 'https://localhost:9200',
  auth: { username: 'elastic', password: 'password' },
  tls: {
    rejectUnauthorized: false  // WARNING: Only for dev/testing
  }
});

# Python - disable cert verification (dev only)
import urllib3
urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)

es = Elasticsearch(
    ['https://localhost:9200'],
    basic_auth=('elastic', 'password'),
    verify_certs=False  # WARNING: Only for dev/testing
)

# Export the CA certificate
openssl pkcs12 -in config/certs/elastic-stack-ca.p12 -out ca.crt -nokeys

# Configure client to trust the CA
export ELASTICSEARCH_CA_PATH="/path/to/ca.crt"

// Node.js - use CA certificate
const fs = require('fs');
const client = new Client({
  node: 'https://localhost:9200',
  auth: { username: 'elastic', password: 'password' },
  tls: {
    ca: fs.readFileSync('/path/to/ca.crt'),
    rejectUnauthorized: true
  }
});

# View current authentication realms
curl -X GET "http://localhost:9200/_xpack/security/realm/_all" \
  -u elastic:password \
  -H 'Content-Type: application/json'

# Check Elasticsearch logs for authentication failures
sudo tail -f /var/log/elasticsearch/elasticsearch.log | grep -i "authentication"

# Common log patterns indicating issues:
# - "failed to authenticate user" - wrong credentials
# - "realm [ldap1] failed to authenticate" - LDAP connection issue
# - "SSL handshake failed" - certificate problem
# - "password authentication failed" - incorrect password

# Native realm (default)
xpack.security.authc.realms.native.native1:
  order: 0

# File realm example
xpack.security.authc.realms.file.file1:
  order: 1

# LDAP realm example
xpack.security.authc.realms.ldap.ldap1:
  order: 2
  url: "ldaps://ldap.example.com:636"
  bind_dn: "cn=admin,dc=example,dc=com"
  bind_password: "ldap_password"
  user_search.base_dn: "ou=users,dc=example,dc=com"
  user_search.filter: "(cn={0})"
  group_search.base_dn: "ou=groups,dc=example,dc=com"

# Test LDAP connection
ldapsearch -x -H ldaps://ldap.example.com:636 \
  -D "cn=admin,dc=example,dc=com" \
  -W \
  -b "ou=users,dc=example,dc=com" \
  "(cn=testuser)"

# If LDAP is unreachable, check:
# - Network connectivity to LDAP server
# - LDAP server is running and accepting connections
# - bind_dn and bind_password are correct
# - SSL certificate for LDAPS is valid

# Check file realm users (Elasticsearch 8.x)
cat /etc/elasticsearch/users

# Verify file permissions
ls -la /etc/elasticsearch/users
# Should be: -rw-r----- elasticsearch elasticsearch

# Add user to file realm
sudo /usr/share/elasticsearch/bin/elasticsearch-users useradd myuser -p password -r admin

# Verify user was added
sudo /usr/share/elasticsearch/bin/elasticsearch-users list

# Enable debug logging for security
curl -X PUT "http://localhost:9200/_cluster/settings" \
  -u elastic:password \
  -H 'Content-Type: application/json' -d'
{
  "transient": {
    "logger.org.elasticsearch.xpack.security": "DEBUG",
    "logger.org.elasticsearch.xpack.security.authc": "TRACE"
  }
}
'

# Monitor logs for detailed authentication flow
sudo tail -f /var/log/elasticsearch/elasticsearch.log

# Create an API key for programmatic access
curl -X POST "http://localhost:9200/_security/api_key" \
  -u elastic:password \
  -H 'Content-Type: application/json' -d'
{
  "name": "my-application-key",
  "role_descriptors": {
    "my_role": {
      "cluster": ["monitor"],
      "index": [
        {
          "names": ["my-index-*"],
          "privileges": ["read", "write"]
        }
      ]
    }
  },
  "expiration": "30d"
}
'

# Response contains API key:
# {
#   "id": "VuaCfGcBCdbkQm-e5aOx",
#   "name": "my-application-key",
#   "expiration": 1740000000000,
#   "api_key": "ui2lp2axTNmsyakw9tvNnw",
#   "encoded": "VnVhQ2ZHY0JDZGJrUW0tZTVhT3g6dWkybHAyYXhUTm1zeWFrdzl0dk5udw=="
# }

# Use API key for authentication (base64 encoded)
curl -X GET "http://localhost:9200/_cat/indices" \
  -H "Authorization: ApiKey VnVhQ2ZHY0JDZGJrUW0tZTVhT3g6dWkybHAyYXhUTm1zeWFrdzl0dk5udw=="

# Or use id:api_key format
curl -X GET "http://localhost:9200/_cat/indices" \
  -H "Authorization: ApiKey $(echo -n 'VuaCfGcBCdbkQm-e5aOx:ui2lp2axTNmsyakw9tvNnw' | base64)"

// Node.js with API key
const { Client } = require('@elastic/elasticsearch');
const client = new Client({
  node: 'https://localhost:9200',
  auth: {
    apiKey: 'VnVhQ2ZHY0JDZGJrUW0tZTVhT3g6dWkybHAyYXhUTm1zeWFrdzl0dk5udw=='
  }
});

// Or use id and api_key separately
const client = new Client({
  node: 'https://localhost:9200',
  auth: {
    apiKey: {
      id: 'VuaCfGcBCdbkQm-e5aOx',
      api_key: 'ui2lp2axTNmsyakw9tvNnw'
    }
  }
});

# Python with API key
from elasticsearch import Elasticsearch

es = Elasticsearch(
    ['https://localhost:9200'],
    api_key=('VuaCfGcBCdbkQm-e5aOx', 'ui2lp2axTNmsyakw9tvNnw'),
    verify_certs=True
)

# List all API keys
curl -X GET "http://localhost:9200/_security/api_key" \
  -u elastic:password

# Get specific API key info
curl -X GET "http://localhost:9200/_security/api_key?id=VuaCfGcBCdbkQm-e5aOx" \
  -u elastic:password

# Invalidate (delete) API key
curl -X DELETE "http://localhost:9200/_security/api_key" \
  -u elastic:password \
  -H 'Content-Type: application/json' -d'
{
  "id": "VuaCfGcBCdbkQm-e5aOx"
}
'

# Invalidate all API keys for a user
curl -X DELETE "http://localhost:9200/_security/api_key" \
  -u elastic:password \
  -H 'Content-Type: application/json' -d'
{
  "username": "myuser"
}
'