How to fix ClusterBlockException: blocked by: [FORBIDDEN/12/index read-only / allow delete (api)] in Elasticsearch

# Check cluster health with disk awareness
curl -X GET "localhost:9200/_cluster/health?pretty" -u "username:password"

# Check disk usage per node
curl -X GET "localhost:9200/_cat/allocation?v&pretty" -u "username:password"

# Detailed node stats including disk
curl -X GET "localhost:9200/_nodes/stats/fs?pretty" -u "username:password"

# Check which indices are blocked
curl -X GET "localhost:9200/_cluster/state?filter_path=metadata.indices.*.settings.index.blocks.*&pretty" -u "username:password"

# Get specific index settings to see blocks
curl -X GET "localhost:9200/my-index/_settings?pretty&flat_settings=true" -u "username:password"

# Delete old indices (be careful!)
curl -X DELETE "localhost:9200/old-index-*" -u "username:password"

# Use ILM (Index Lifecycle Management) to manage indices
curl -X GET "localhost:9200/_ilm/policy?pretty" -u "username:password"

# Force merge to reduce segment count and free space
curl -X POST "localhost:9200/my-index/_forcemerge?max_num_segments=1&pretty" -u "username:password"

# Clear cache (temporary relief)
curl -X POST "localhost:9200/my-index/_cache/clear?pretty" -u "username:password"

# Delete by query to remove old documents
curl -X POST "localhost:9200/my-index/_delete_by_query?pretty" -u "username:password" -H 'Content-Type: application/json' -d'
{
  "query": {
    "range": {
      "@timestamp": {
        "lt": "now-30d/d"
      }
    }
  }
}
'

# Check current disk watermark settings
curl -X GET "localhost:9200/_cluster/settings?include_defaults=true&filter_path=**.disk.*&pretty" -u "username:password"

# Temporarily raise flood stage threshold (use with caution!)
curl -X PUT "localhost:9200/_cluster/settings?pretty" -u "username:password" -H 'Content-Type: application/json' -d'
{
  "transient": {
    "cluster.routing.allocation.disk.watermark.flood_stage": "97%",
    "cluster.routing.allocation.disk.watermark.high": "92%",
    "cluster.routing.allocation.disk.watermark.low": "88%"
  }
}
'

# Disable disk-based allocation temporarily (emergency only!)
curl -X PUT "localhost:9200/_cluster/settings?pretty" -u "username:password" -H 'Content-Type: application/json' -d'
{
  "transient": {
    "cluster.routing.allocation.disk.threshold_enabled": false
  }
}
'

# Remove read-only blocks from indices
curl -X PUT "localhost:9200/_all/_settings?pretty" -u "username:password" -H 'Content-Type: application/json' -d'
{
  "index.blocks.read_only_allow_delete": null
}
'

# Add a new data node with more disk space
# Update elasticsearch.yml on new node:
# node.roles: [data]
# Then start the node

# Check shard distribution
curl -X GET "localhost:9200/_cat/shards?v&h=index,shard,prirep,state,node,store&s=store:desc&pretty" -u "username:password"

# Rebalance shards manually
curl -X PUT "localhost:9200/_cluster/settings?pretty" -u "username:password" -H 'Content-Type: application/json' -d'
{
  "transient": {
    "cluster.routing.allocation.enable": "all"
  }
}
'

# Exclude full nodes from allocation
curl -X PUT "localhost:9200/_cluster/settings?pretty" -u "username:password" -H 'Content-Type: application/json' -d'
{
  "transient": {
    "cluster.routing.allocation.exclude._name": "full-node-name"
  }
}
'

# Move specific indices to nodes with more space
curl -X POST "localhost:9200/_cluster/reroute?pretty" -u "username:password" -H 'Content-Type: application/json' -d'
{
  "commands": [
    {
      "move": {
        "index": "large-index",
        "shard": 0,
        "from_node": "full-node",
        "to_node": "empty-node"
      }
    }
  ]
}
'

# Configure index lifecycle management (ILM)
curl -X PUT "localhost:9200/_ilm/policy/disk-space-policy?pretty" -u "username:password" -H 'Content-Type: application/json' -d'
{
  "policy": {
    "phases": {
      "hot": {
        "min_age": "0ms",
        "actions": {
          "rollover": {
            "max_size": "50gb",
            "max_age": "7d"
          },
          "set_priority": {
            "priority": 100
          }
        }
      },
      "delete": {
        "min_age": "30d",
        "actions": {
          "delete": {}
        }
      }
    }
  }
}
'

# Set up index templates with ILM
curl -X PUT "localhost:9200/_index_template/logs-template?pretty" -u "username:password" -H 'Content-Type: application/json' -d'
{
  "index_patterns": ["logs-*"],
  "template": {
    "settings": {
      "number_of_shards": 1,
      "number_of_replicas": 1,
      "index.lifecycle.name": "disk-space-policy",
      "index.lifecycle.rollover_alias": "logs"
    }
  }
}
'

# Configure snapshot/restore for backup before deletion
curl -X PUT "localhost:9200/_snapshot/my-backup-repo?pretty" -u "username:password" -H 'Content-Type: application/json' -d'
{
  "type": "fs",
  "settings": {
    "location": "/mnt/elasticsearch-backups"
  }
}
'