How to fix ConcurrentSnapshotExecutionException: cannot snapshot while a snapshot/restore is in progress in Elasticsearch

# Check all currently running snapshots
curl -X GET "localhost:9200/_snapshot/_status" -u "username:password"

# Check snapshots for a specific repository
curl -X GET "localhost:9200/_snapshot/my_repository/_status" -u "username:password"

# List all snapshots in the repository
curl -X GET "localhost:9200/_snapshot/my_repository/_all" -u "username:password"

# Check for ongoing restore operations
curl -X GET "localhost:9200/_cat/recovery?v&active_only=true" -u "username:password"

# Get detailed snapshot information
curl -X GET "localhost:9200/_snapshot/my_repository/_current" -u "username:password"

# Option 1: Wait for the current snapshot to complete
# Monitor progress:
curl -X GET "localhost:9200/_snapshot/my_repository/current_snapshot" -u "username:password"

# Check snapshot statistics
curl -X GET "localhost:9200/_snapshot/my_repository/_current?verbose=true" -u "username:password"

# Option 2: Cancel a stuck or unwanted snapshot
# This stops the in-progress snapshot
curl -X DELETE "localhost:9200/_snapshot/my_repository/snapshot_name" -u "username:password"

# Option 3: Cancel a stuck restore operation
# Stop all ongoing restore operations
curl -X DELETE "localhost:9200/_snapshot/my_repository/snapshot_name/_restore" -u "username:password"

# Verify the operation was cancelled
curl -X GET "localhost:9200/_snapshot/_status" -u "username:password"

# Check existing SLM policies
curl -X GET "localhost:9200/_slm/policy" -u "username:password"

# Update SLM policy with longer interval
curl -X PUT "localhost:9200/_slm/policy/daily-snapshots" -u "username:password" -H 'Content-Type: application/json' -d'
{
  "schedule": "0 0 2 * * ?",
  "name": "<daily-snap-{now/d}>",
  "repository": "my_repository",
  "config": {
    "indices": ["*"],
    "ignore_unavailable": false,
    "include_global_state": true
  },
  "retention": {
    "expire_after": "30d",
    "min_count": 5,
    "max_count": 50
  }
}
'

# For manual cron-based snapshots, adjust timing
# Example: Change from every 30 minutes to every 2 hours
# Before: */30 * * * * (every 30 min)
# After:  0 */2 * * * (every 2 hours)

# Check current snapshot settings
curl -X GET "localhost:9200/_cluster/settings?include_defaults=true&filter_path=*.snapshot.*" -u "username:password"

# Set maximum concurrent snapshot operations (default is 1000)
# Note: This setting applies cluster-wide, not per-repository
curl -X PUT "localhost:9200/_cluster/settings" -u "username:password" -H 'Content-Type: application/json' -d'
{
  "persistent": {
    "snapshot.max_concurrent_operations": 5
  }
}
'

# For production clusters with multiple repositories:
# You can run snapshots to different repositories concurrently
curl -X PUT "localhost:9200/_snapshot/repo1/snapshot1?wait_for_completion=false" -u "username:password"
curl -X PUT "localhost:9200/_snapshot/repo2/snapshot2?wait_for_completion=false" -u "username:password"
# These will run concurrently since they use different repositories

# Create additional repositories for parallel snapshots
curl -X PUT "localhost:9200/_snapshot/repository_hourly" -u "username:password" -H 'Content-Type: application/json' -d'
{
  "type": "fs",
  "settings": {
    "location": "/mnt/snapshots/hourly",
    "compress": true
  }
}
'

curl -X PUT "localhost:9200/_snapshot/repository_daily" -u "username:password" -H 'Content-Type: application/json' -d'
{
  "type": "s3",
  "settings": {
    "bucket": "my-snapshots-daily",
    "region": "us-east-1",
    "compress": true
  }
}
'

# Create SLM policies for each repository
# Hourly snapshots to fast local storage
curl -X PUT "localhost:9200/_slm/policy/hourly-local" -u "username:password" -H 'Content-Type: application/json' -d'
{
  "schedule": "0 0 * * * ?",
  "repository": "repository_hourly",
  "config": {
    "indices": ["critical-*"],
    "include_global_state": false
  },
  "retention": {
    "expire_after": "24h",
    "min_count": 3,
    "max_count": 24
  }
}
'

# Daily snapshots to S3 for long-term retention
curl -X PUT "localhost:9200/_slm/policy/daily-s3" -u "username:password" -H 'Content-Type: application/json' -d'
{
  "schedule": "0 0 2 * * ?",
  "repository": "repository_daily",
  "config": {
    "indices": ["*"],
    "include_global_state": true
  },
  "retention": {
    "expire_after": "30d",
    "min_count": 7,
    "max_count": 30
  }
}
'

# These will run concurrently without conflicts

# Create a monitoring script to detect long-running snapshots
# Monitor snapshot duration
curl -X GET "localhost:9200/_snapshot/_status" -u "username:password" | jq '.snapshots[] | {repository: .repository, snapshot: .snapshot, state: .state, duration_seconds: (.stats.time_in_millis / 1000)}'

# Check SLM execution history
curl -X GET "localhost:9200/_slm/policy/daily-snapshots?human=true" -u "username:password"

# View SLM statistics
curl -X GET "localhost:9200/_slm/stats" -u "username:password"

# If snapshot is genuinely stuck (hours without progress):
# 1. Check cluster health
curl -X GET "localhost:9200/_cluster/health" -u "username:password"

# 2. Check master node logs for errors
# Look for repository connection issues, disk space, permissions

# 3. If deletion fails, restart the master node
# This clears stuck operations from cluster state
# Find master node:
curl -X GET "localhost:9200/_cat/master?v" -u "username:password"

# After identifying master, perform rolling restart:
# systemctl restart elasticsearch  (on master node only)

# 4. Verify repository after restart
curl -X POST "localhost:9200/_snapshot/my_repository/_verify" -u "username:password"