How to fix IndexClosedException: closed in Elasticsearch

# List all indices with their status
curl -X GET "localhost:9200/_cat/indices?v&h=index,status,health,pri,rep,docs.count,store.size" -u "username:password"

# Check specific index status
curl -X GET "localhost:9200/my-index" -u "username:password"

# Get detailed index information including why it might be closed
curl -X GET "localhost:9200/my-index/_settings?include_defaults=true" -u "username:password"

# Check ILM policy status for the index
curl -X GET "localhost:9200/my-index/_ilm/explain" -u "username:password"

# List only closed indices
curl -X GET "localhost:9200/_cat/indices?v&h=index,status&status=close" -u "username:password"

# Open a single index
curl -X POST "localhost:9200/my-index/_open" -u "username:password"

# Open multiple indices
curl -X POST "localhost:9200/index1,index2,index3/_open" -u "username:password"

# Open indices using wildcard pattern
curl -X POST "localhost:9200/logs-*/_open" -u "username:password"

# Open all closed indices (use with caution)
curl -X POST "localhost:9200/_all/_open" -u "username:password"

# Open with timeout settings (useful for large indices)
curl -X POST "localhost:9200/my-index/_open?timeout=5m" -u "username:password" -H 'Content-Type: application/json' -d'
{
  "settings": {
    "index.number_of_replicas": 1
  }
}
'

# Check recovery status
curl -X GET "localhost:9200/_cat/recovery?v&h=index,stage,time,bytes_percent" -u "username:password"

# Verify index is now open and searchable
curl -X GET "localhost:9200/my-index" -u "username:password"

# Test search on the reopened index
curl -X GET "localhost:9200/my-index/_search" -u "username:password" -H 'Content-Type: application/json' -d'
{
  "query": {
    "match_all": {}
  },
  "size": 1
}
'

# Check shard allocation status
curl -X GET "localhost:9200/_cat/shards/my-index?v&h=index,shard,prirep,state,unassigned.reason" -u "username:password"

# Monitor cluster health
curl -X GET "localhost:9200/_cluster/health?wait_for_status=yellow&timeout=50s" -u "username:password"

# Check current ILM policy
curl -X GET "localhost:9200/_ilm/policy/my-policy" -u "username:password"

# Update ILM policy to prevent unwanted closures
curl -X PUT "localhost:9200/_ilm/policy/my-policy" -u "username:password" -H 'Content-Type: application/json' -d'
{
  "policy": {
    "phases": {
      "hot": {
        "min_age": "0ms",
        "actions": {
          "rollover": {
            "max_size": "50gb",
            "max_age": "30d"
          },
          "set_priority": {
            "priority": 100
          }
        }
      },
      "warm": {
        "min_age": "30d",
        "actions": {
          "set_priority": {
            "priority": 50
          },
          "shrink": {
            "number_of_shards": 1
          }
        }
      },
      "cold": {
        "min_age": "60d",
        "actions": {
          "set_priority": {
            "priority": 0
          }
        }
      },
      "delete": {
        "min_age": "90d",
        "actions": {
          "delete": {}
        }
      }
    }
  }
}
'

# Remove close action from policy if not needed
# Or adjust min_age to keep indices open longer

# Check which indices are affected by ILM
curl -X GET "localhost:9200/_ilm/explain" -u "username:password"

# Manually move index to different phase
curl -X POST "localhost:9200/_ilm/move/my-index" -u "username:password" -H 'Content-Type: application/json' -d'
{
  "current_phase": "hot",
  "next_phase": "warm"
}
'

// Example: Check index status before operations
const { Client } = require('@elastic/elasticsearch');
const client = new Client({ node: 'http://localhost:9200' });

async function safeSearch(indexName, query) {
  try {
    const response = await client.search({
      index: indexName,
      body: query
    });
    return response;
  } catch (error) {
    if (error.meta?.body?.error?.type === 'index_closed_exception') {
      console.log('Index ' + indexName + ' is closed. Attempting to open...');

      try {
        // Try to open the index
        await client.indices.open({ index: indexName });
        console.log('Index ' + indexName + ' opened successfully.');

        // Wait a moment for index to become ready
        await new Promise(resolve => setTimeout(resolve, 1000));

        // Retry the search
        const retryResponse = await client.search({
          index: indexName,
          body: query
        });
        return retryResponse;
      } catch (openError) {
        console.error('Failed to open index ' + indexName + ':', openError.message);
        throw new Error('Index is closed and could not be opened: ' + openError.message);
      }
    }
    throw error;
  }
}

// Example: Bulk operations with closed index handling
async function safeBulkWithClosedIndexHandling(operations) {
  const response = await client.bulk({
    body: operations,
    refresh: true
  });

  if (response.errors) {
    const closedIndices = new Set();

    // Check for closed index errors
    response.items.forEach(item => {
      if (item.index?.error?.type === 'index_closed_exception') {
        const indexName = item.index.error.index;
        closedIndices.add(indexName);
      }
    });

    // Open closed indices and retry
    for (const indexName of closedIndices) {
      console.log('Opening closed index: ' + indexName);
      try {
        await client.indices.open({ index: indexName });
        console.log('Successfully opened index: ' + indexName);
      } catch (error) {
        console.error('Failed to open index ' + indexName + ':', error.message);
      }
    }

    // Retry failed operations if indices were opened
    if (closedIndices.size > 0) {
      await new Promise(resolve => setTimeout(resolve, 2000)); // Wait for indices to be ready
      return await client.bulk({
        body: operations,
        refresh: true
      });
    }
  }

  return response;
}

# Elasticsearch monitoring queries
# Check for closed indices (run periodically)
curl -X GET "localhost:9200/_cat/indices?h=index,status&status=close" -u "username:password"

# Set up alert using Elasticsearch alerting or external monitoring
# Example: Check every 5 minutes, alert if critical indices are closed

# Kibana alert example (if using Kibana):
# 1. Create index pattern: "metricbeat-*" or your monitoring indices
# 2. Create threshold alert:
#    - Condition: count() > 0
#    - Filter: event.dataset: "elasticsearch.index" AND index.status: "close"
#    - For indices: "production-*"

# Alternative: Use Elasticsearch Watcher
curl -X PUT "localhost:9200/_watcher/watch/closed_index_alert" -u "username:password" -H 'Content-Type: application/json' -d'
{
  "trigger": {
    "schedule": {
      "interval": "5m"
    }
  },
  "input": {
    "search": {
      "request": {
        "indices": [".monitoring-es-*"],
        "body": {
          "query": {
            "bool": {
              "must": [
                {
                  "term": {
                    "type": "indices"
                  }
                },
                {
                  "term": {
                    "index_stats.primaries.status": "close"
                  }
                },
                {
                  "wildcard": {
                    "index_stats.index": "production-*"
                  }
                }
              ]
            }
          }
        }
      }
    }
  },
  "condition": {
    "compare": {
      "ctx.payload.hits.total": {
        "gt": 0
      }
    }
  },
  "actions": {
    "send_email": {
      "email": {
        "to": ["[email protected]"],
        "subject": "Alert: Production indices closed",
        "body": "{{ctx.payload.hits.total}} production indices are closed: {{#ctx.payload.hits.hits}}{{_source.index_stats.index}} {{/ctx.payload.hits.hits}}"
      }
    }
  }
}
'