How to fix ItemCollectionSizeLimitExceededException: Collection size exceeded in DynamoDB

# Use CloudWatch metrics to identify hot partitions
aws cloudwatch get-metric-statistics   --namespace AWS/DynamoDB   --metric-name ConsumedWriteCapacityUnits   --dimensions Name=TableName,Value=YourTableName   --start-time $(date -u -d "1 hour ago" +%Y-%m-%dT%H:%M:%SZ)   --end-time $(date -u +%Y-%m-%dT%H:%M:%SZ)   --period 300   --statistics Sum

# Check DynamoDB metrics for UserErrors
aws cloudwatch get-metric-statistics   --namespace AWS/DynamoDB   --metric-name UserErrors   --dimensions Name=TableName,Value=YourTableName Name=Operation,Value=PutItem   --start-time $(date -u -d "24 hours ago" +%Y-%m-%dT%H:%M:%SZ)   --end-time $(date -u +%Y-%m-%dT%H:%M:%SZ)   --period 3600   --statistics Sum

# Review application logs for failed write operations
# Look for patterns in partition key values that appear in error messages

// BEFORE: Single partition key for all metrics
const partitionKey = 'system-metrics';

// AFTER: Sharded partition keys
function getShardedPartitionKey(metricName) {
  const shardId = Math.floor(Math.random() * 100); // 0-99
  return `metric-${metricName}-shard-${shardId}`;
}

// For time-series data, include timestamp granularity
function getTimeBasedPartitionKey(entity, timestamp) {
  const date = new Date(timestamp);
  const month = date.toISOString().substring(0, 7); // YYYY-MM
  return `${entity}-${month}`;
}

// For user data, use composite keys
function getUserPartitionKey(userId, dataType) {
  return `user-${userId}-${dataType}`;
}

// Example: Delete items older than retention period
async function deleteOldItems(tableName, partitionKey, retentionDays) {
  const cutoffDate = new Date();
  cutoffDate.setDate(cutoffDate.getDate() - retentionDays);

  // Query for old items (assuming sort key is timestamp)
  const params = {
    TableName: tableName,
    KeyConditionExpression: 'pk = :pk AND sk < :cutoff',
    ExpressionAttributeValues: {
      ':pk': { S: partitionKey },
      ':cutoff': { N: cutoffDate.getTime().toString() }
    }
  };

  const items = [];
  let lastEvaluatedKey;

  do {
    if (lastEvaluatedKey) {
      params.ExclusiveStartKey = lastEvaluatedKey;
    }

    const result = await dynamodb.query(params).promise();
    items.push(...result.Items);
    lastEvaluatedKey = result.LastEvaluatedKey;
  } while (lastEvaluatedKey);

  // Batch delete in chunks of 25
  for (let i = 0; i < items.length; i += 25) {
    const chunk = items.slice(i, i + 25);
    const deleteRequests = chunk.map(item => ({
      DeleteRequest: {
        Key: { pk: item.pk, sk: item.sk }
      }
    }));

    await dynamodb.batchWriteItem({
      RequestItems: {
        [tableName]: deleteRequests
      }
    }).promise();
  }
}

// Enable TTL (Time to Live) for automatic expiration

// BEFORE: LSI on frequently updated attribute
// This creates large collections if many items share partition key

// AFTER: Use global secondary index (GSI) instead
// GSIs have their own partition key and don't contribute to item collection size

// Or: Denormalize data into separate tables
async function writeToDenormalizedTable(mainTable, summaryTable, item) {
  // Write detailed data to main table with fine-grained partition key
  await dynamodb.putItem({
    TableName: mainTable,
    Item: {
      pk: { S: `detail-${item.id}-${Date.now()}` },
      // ... other attributes
    }
  }).promise();

  // Write summary to separate table
  await dynamodb.updateItem({
    TableName: summaryTable,
    Key: { pk: { S: `summary-${item.category}` } },
    UpdateExpression: 'ADD #count :inc',
    ExpressionAttributeNames: { '#count': 'itemCount' },
    ExpressionAttributeValues: { ':inc': { N: '1' } }
  }).promise();
}

// Use sparse indexes for conditional data
// Only include items in GSI when they have specific attributes

// Monitor partition sizes
class PartitionMonitor {
  constructor(tableName) {
    this.tableName = tableName;
    this.partitionCounts = new Map();
    this.warningThreshold = 8 * 1024 * 1024 * 1024; // 8 GB warning
  }

  async trackWrite(partitionKey) {
    const count = (this.partitionCounts.get(partitionKey) || 0) + 1;
    this.partitionCounts.set(partitionKey, count);

    // Estimate size (adjust based on your average item size)
    const estimatedSize = count * 1024; // 1KB per item

    if (estimatedSize > this.warningThreshold) {
      console.warn(`Partition ${partitionKey} approaching 10 GB limit`);
      // Trigger mitigation: switch to sharded key, throttle writes, etc.
      return this.getShardedKey(partitionKey);
    }

    return partitionKey;
  }

  getShardedKey(originalKey) {
    const suffix = Math.floor(Math.random() * 10);
    return `${originalKey}-shard-${suffix}`;
  }
}