How to fix VersionConflictEngineException: [doc][1]: version conflict, current version [2] is different than the one provided [1] in Elasticsearch

# Get document with its current version
curl -X GET "localhost:9200/my-index/_doc/1" -u "username:password"

# Response includes version:
{
  "_index": "my-index",
  "_id": "1",
  "_version": 2,  # <-- Current version
  "_seq_no": 5,
  "_primary_term": 1,
  "found": true,
  "_source": {
    "title": "My Document",
    "content": "Some content"
  }
}

# Use this version in your update
curl -X POST "localhost:9200/my-index/_update/1" -u "username:password" -H 'Content-Type: application/json' -d'
{
  "doc": {
    "content": "Updated content"
  },
  "version": 2  # <-- Use the version from GET response
}
'

const { Client } = require('@elastic/elasticsearch');
const client = new Client({ node: 'http://localhost:9200' });

async function safeUpdate(documentId, updates) {
  try {
    // 1. Get current document with version
    const getResponse = await client.get({
      index: 'my-index',
      id: documentId
    });

    const currentVersion = getResponse._version;

    // 2. Update with current version
    const updateResponse = await client.update({
      index: 'my-index',
      id: documentId,
      body: {
        doc: updates,
        version: currentVersion
      }
    });

    return updateResponse;
  } catch (error) {
    if (error.meta?.body?.error?.type === 'version_conflict_engine_exception') {
      console.error('Version conflict for document', documentId);
      // Handle conflict (retry, notify user, etc.)
    }
    throw error;
  }
}

# Get document with seq_no and primary_term
curl -X GET "localhost:9200/my-index/_doc/1" -u "username:password"

# Response includes:
{
  "_index": "my-index",
  "_id": "1",
  "_version": 2,
  "_seq_no": 5,          # <-- Sequence number
  "_primary_term": 1,    # <-- Primary term
  "found": true,
  "_source": { ... }
}

# Update using seq_no and primary_term
curl -X POST "localhost:9200/my-index/_update/1" -u "username:password" -H 'Content-Type: application/json' -d'
{
  "doc": {
    "content": "Updated content"
  }
}'
-H 'If-Seq-No: 5' -H 'If-Primary-Term: 1'

# Or in the request body for update API
curl -X POST "localhost:9200/my-index/_update/1" -u "username:password" -H 'Content-Type: application/json' -d'
{
  "if_seq_no": 5,
  "if_primary_term": 1,
  "doc": {
    "content": "Updated content"
  }
}
'

async function safeUpdateWithSequence(documentId, updates) {
  try {
    // 1. Get current document with seq_no and primary_term
    const getResponse = await client.get({
      index: 'my-index',
      id: documentId
    });

    const seqNo = getResponse._seq_no;
    const primaryTerm = getResponse._primary_term;

    // 2. Update with sequence control
    const updateResponse = await client.update({
      index: 'my-index',
      id: documentId,
      if_seq_no: seqNo,
      if_primary_term: primaryTerm,
      body: {
        doc: updates
      }
    });

    return updateResponse;
  } catch (error) {
    if (error.meta?.body?.error?.type === 'version_conflict_engine_exception') {
      console.error('Sequence conflict for document', documentId);
      // The document was modified by another process
    }
    throw error;
  }
}

async function updateWithRetry(documentId, updates, maxRetries = 3) {
  let lastError;

  for (let attempt = 1; attempt <= maxRetries; attempt++) {
    try {
      // Get fresh document data
      const getResponse = await client.get({
        index: 'my-index',
        id: documentId
      });

      // Update with latest seq_no and primary_term
      const updateResponse = await client.update({
        index: 'my-index',
        id: documentId,
        if_seq_no: getResponse._seq_no,
        if_primary_term: getResponse._primary_term,
        body: {
          doc: updates
        },
        refresh: 'wait_for'  // Wait for refresh to ensure consistency
      });

      return updateResponse;

    } catch (error) {
      lastError = error;

      if (error.meta?.body?.error?.type === 'version_conflict_engine_exception') {
        console.log('Version conflict on attempt ' + attempt + '/' + maxRetries + ' for document ' + documentId);

        if (attempt < maxRetries) {
          // Exponential backoff: wait longer each retry
          const delay = Math.min(1000 * Math.pow(2, attempt - 1), 10000);
          await new Promise(resolve => setTimeout(resolve, delay));
          continue; // Try again
        }
      }

      // Not a version conflict or out of retries
      throw error;
    }
  }

  throw lastError; // All retries failed
}

// For bulk operations with conflict handling
async function bulkUpdateWithConflictHandling(operations) {
  const response = await client.bulk({
    body: operations,
    refresh: true
  });

  if (response.errors) {
    const retryOperations = [];

    response.items.forEach((item, index) => {
      if (item.update?.error?.type === 'version_conflict_engine_exception') {
        // Extract the failed operation for retry
        const originalOp = operations[index * 2]; // bulk format: {update: {...}}, then {doc: {...}}
        retryOperations.push(originalOp);
      }
    });

    if (retryOperations.length > 0) {
      console.log('Retrying ' + retryOperations.length + ' failed operations');
      // Implement retry logic for failed operations
    }
  }

  return response;
}

# Update with automatic retry on conflict (up to 3 times)
curl -X POST "localhost:9200/my-index/_update/1" -u "username:password" -H 'Content-Type: application/json' -d'
{
  "doc": {
    "content": "Updated content"
  },
  "retry_on_conflict": 3
}
'

# For scripted updates with retry
curl -X POST "localhost:9200/my-index/_update/1" -u "username:password" -H 'Content-Type: application/json' -d'
{
  "script": {
    "source": "ctx._source.views += params.increment",
    "params": {
      "increment": 1
    }
  },
  "retry_on_conflict": 5,
  "upsert": {
    "views": 1
  }
}
'

// Simple update with retry
await client.update({
  index: 'my-index',
  id: '1',
  body: {
    doc: { content: 'Updated' },
    retry_on_conflict: 3
  }
});

// For counter-like operations, use scripts
await client.update({
  index: 'my-index',
  id: '1',
  body: {
    script: {
      source: 'ctx._source.counter += params.increment',
      params: { increment: 1 }
    },
    retry_on_conflict: 5,
    upsert: { counter: 1 }
  }
});

// Bulk updates with retry
await client.bulk({
  body: [
    { update: { _index: 'my-index', _id: '1', retry_on_conflict: 3 } },
    { doc: { status: 'active' } },
    { update: { _index: 'my-index', _id: '2', retry_on_conflict: 3 } },
    { doc: { status: 'inactive' } }
  ]
});

// 1. Use append-only patterns where possible
// Instead of updating documents, append events
const event = {
  timestamp: new Date().toISOString(),
  type: 'user_action',
  data: { action: 'click', element: 'button' }
};

await client.index({
  index: 'user-events-2024.01.01',
  body: event
});

// 2. Use separate documents for frequently updated counters
await client.index({
  index: 'counters',
  id: 'user-1-clicks',
  body: {
    user_id: 'user-1',
    metric: 'clicks',
    count: 1,
    last_updated: new Date().toISOString()
  }
});

// Update counter with script (atomic, less conflict-prone)
await client.update({
  index: 'counters',
  id: 'user-1-clicks',
  body: {
    script: {
      source: 'ctx._source.count += params.increment; ctx._source.last_updated = params.timestamp',
      params: {
        increment: 1,
        timestamp: new Date().toISOString()
      }
    },
    retry_on_conflict: 5,
    upsert: {
      user_id: 'user-1',
      metric: 'clicks',
      count: 1,
      last_updated: new Date().toISOString()
    }
  }
});

// 3. Partition data to reduce contention
// Instead of one document for all users, use one per user
const userId = 'user-123';
await client.update({
  index: 'user-profiles',
  id: userId,  // Each user has their own document
  body: {
    doc: { last_login: new Date().toISOString() },
    retry_on_conflict: 3
  }
});

// 4. Use external versioning for external data sources
await client.index({
  index: 'external-data',
  id: 'record-1',
  body: record,
  version: externalVersion,  // From external system
  version_type: 'external'   // External versioning
});

// 5. Consider using Elasticsearch's built-in concurrency features
// For real-time analytics, use aggregations instead of updating counters
const searchResponse = await client.search({
  index: 'user-events-*',
  body: {
    query: { match: { user_id: 'user-1' } },
    aggs: {
      total_clicks: { value_count: { field: 'event_id' } }
    }
  }
});