How to fix RepositoryException: [repository] could not read repository data from index blob in Elasticsearch

# 1. Identify all Elasticsearch instances accessing this repository
# Check configuration on each node:
grep -r "repository_name" /etc/elasticsearch/elasticsearch.yml

# 2. Temporarily disable repository access on all but one cluster
# Edit elasticsearch.yml on other clusters and comment out repository settings:
# repositories:
#   repository_name:
#     type: s3
#     settings:
#       bucket: my-bucket

# 3. Verify which cluster has repository access
curl -X GET "localhost:9200/_snapshot/repository_name" -u "username:password"

# Test repository connectivity
curl -X GET "localhost:9200/_snapshot/my_repository" -u "username:password"

# Get detailed repository information
curl -X GET "localhost:9200/_snapshot/my_repository?verbose=true" -u "username:password"

# Attempt repository verification
curl -X POST "localhost:9200/_snapshot/my_repository/_verify" -u "username:password"

# For filesystem repositories, check permissions and blob files
ls -la /path/to/repository/
ls -la /path/to/repository/indices/
ls -la /path/to/repository/metadata/

# Verify blob file exists and is readable
test -r /path/to/repository/indices/index-123 && echo "Readable" || echo "Not readable"

# For S3/GCS/Azure repositories, test credentials
# AWS S3 example:
aws s3 ls s3://my-bucket/my-repository/indices/ --region us-east-1

# 1. Identify the current repository state
curl -X GET "localhost:9200/_snapshot/my_repository?verbose=true" -u "username:password"
# Look at the "type" and "settings" fields

# 2. Delete and recreate the repository registration (NOT the physical files)
# This forces Elasticsearch to re-read the repository state from disk
curl -X DELETE "localhost:9200/_snapshot/my_repository" -u "username:password"

# 3. Re-register the repository with the same settings
curl -X PUT "localhost:9200/_snapshot/my_repository" -u "username:password" -H 'Content-Type: application/json' -d'
{
  "type": "fs",
  "settings": {
    "location": "/path/to/repository",
    "compress": true
  }
}
'

# For S3:
curl -X PUT "localhost:9200/_snapshot/my_repository" -u "username:password" -H 'Content-Type: application/json' -d'
{
  "type": "s3",
  "settings": {
    "bucket": "my-bucket",
    "region": "us-east-1"
  }
}
'

# 4. Verify the repository is accessible again
curl -X POST "localhost:9200/_snapshot/my_repository/_verify" -u "username:password"

# 5. List snapshots to confirm repository is readable
curl -X GET "localhost:9200/_snapshot/my_repository/_all" -u "username:password"

# For filesystem repositories:
# 1. Check blob file integrity
file /path/to/repository/indices/index-*
hexdump -C /path/to/repository/indices/index-* | head -20

# 2. Check filesystem health
fsck -n /path/to/repository  # Dry run, non-destructive
df -h /path/to/repository    # Check disk space
df -i /path/to/repository    # Check inode usage

# 3. Check for file corruption patterns
stat /path/to/repository/indices/index-*

# For S3 repositories:
# 1. Check blob object properties
aws s3api head-object --bucket my-bucket --key 'my-repository/indices/index-123'

# 2. Calculate object hash and verify integrity
aws s3api get-object --bucket my-bucket --key 'my-repository/indices/index-123' /tmp/index-blob
md5sum /tmp/index-blob

# 3. List all objects to identify missing or partial blobs
aws s3api list-objects-v2 --bucket my-bucket --prefix 'my-repository/indices/'

# If individual blobs are corrupted but others are intact:
# You can attempt selective deletion and repository reset for just those indices

# 1. Identify which snapshots can be accessed
curl -X GET "localhost:9200/_snapshot/my_repository/_all" -u "username:password"

# 2. Get detailed status of each snapshot
curl -X GET "localhost:9200/_snapshot/my_repository/snapshot_name?verbose=true" -u "username:password"

# 3. Delete corrupted snapshots one at a time
curl -X DELETE "localhost:9200/_snapshot/my_repository/corrupted_snapshot" -u "username:password"

# 4. Monitor repository repair progress
curl -X GET "localhost:9200/_snapshot/my_repository/_status" -u "username:password"

# 5. Once corrupted snapshots are removed, re-verify the repository
curl -X POST "localhost:9200/_snapshot/my_repository/_verify" -u "username:password"

# 6. Create new snapshots to rebuild healthy repository state
curl -X PUT "localhost:9200/_snapshot/my_repository/fresh_snapshot?wait_for_completion=true" -u "username:password" -H 'Content-Type: application/json' -d'
{
  "indices": "index1,index2",
  "ignore_unavailable": true,
  "include_global_state": false
}
'

# 1. Identify alternative snapshot sources
curl -X GET "localhost:9200/_snapshot" -u "username:password"

# 2. If you have a backup repository with the same snapshots:
curl -X POST "localhost:9200/_snapshot/backup_repository/backup_snapshot/_restore?wait_for_completion=true" -u "username:password" -H 'Content-Type: application/json' -d'
{
  "indices": "index1,index2",
  "ignore_unavailable": true,
  "include_global_state": false,
  "rename_pattern": "(.+)",
  "rename_replacement": "restored_$1"
}
'

# 3. Check restored indices
curl -X GET "localhost:9200/_cat/indices" -u "username:password"

# 4. Validate data integrity in restored indices
curl -X GET "localhost:9200/restored_index1/_stats" -u "username:password"

# 5. If no snapshots are available, reindex from source data
# Requires having the original data available elsewhere (Kafka, files, etc.)
curl -X POST "localhost:9200/_reindex" -u "username:password" -H 'Content-Type: application/json' -d'
{
  "source": { "remote": { "host": "http://old-cluster:9200" }, "index": "index1" },
  "dest": { "index": "index1" }
}
'

# 1. Implement Snapshot Lifecycle Management (SLM) with centralized policy
curl -X PUT "localhost:9200/_slm/policy/daily-central-snapshots" -u "username:password" -H 'Content-Type: application/json' -d'
{
  "schedule": "0 30 1 * * ?",
  "name": "<daily-snap-{now/d}>",
  "repository": "my_repository",
  "config": {
    "indices": ["*"],
    "ignore_unavailable": false,
    "include_global_state": false
  },
  "retention": {
    "expire_after": "30d",
    "min_count": 5,
    "max_count": 50
  }
}
'

# 2. Enable repository monitoring
curl -X GET "localhost:9200/_snapshot/_status" -u "username:password"
curl -X GET "localhost:9200/_slm/stats" -u "username:password"

# 3. Configure Elasticsearch to prevent concurrent access
# Add to elasticsearch.yml:
# - Ensure only one cluster has repository configured
# - Use distinct repository locations for different clusters
# - Document which cluster "owns" each repository

# 4. Regular verification schedule
# Add to crontab or scheduled job:
0 2 * * * curl -X POST "localhost:9200/_snapshot/my_repository/_verify" -u "username:password"

# 5. Monitor cluster logs for repository errors
# Set up log aggregation to alert on:
# - "RepositoryException"
# - "repository does not match its expected state"
# - "failed to read repository data"