How to fix ItemCollectionSizeLimitExceededException: Item collection size limit of 10 GB exceeded in DynamoDB

# Use CloudWatch metrics to identify hot partitions
aws cloudwatch get-metric-statistics   --namespace AWS/DynamoDB   --metric-name ConsumedWriteCapacityUnits   --dimensions Name=TableName,Value=YourTableName   --start-time $(date -u -d "1 hour ago" +%Y-%m-%dT%H:%M:%SZ)   --end-time $(date -u +%Y-%m-%dT%H:%M:%SZ)   --period 300   --statistics Sum

# Scan for large item collections (careful - can be expensive)
aws dynamodb scan   --table-name YourTableName   --select COUNT   --filter-expression "begins_with(partitionKey, :prefix)"   --expression-attribute-values '{":prefix":{"S":"problematic-value"}}'

# Check for LSIs with large collections
aws dynamodb describe-table   --table-name YourTableName   --query 'Table.LocalSecondaryIndexes[*].IndexName'

// BEFORE: Single partition key for all metrics
const partitionKey = 'system-metrics';

// AFTER: Sharded partition keys
function getShardedPartitionKey(metricName) {
  const shardId = Math.floor(Math.random() * 100); // 0-99
  return `metric-${metricName}-shard-${shardId}`;
}

// For time-series data, include timestamp granularity
function getTimeBasedPartitionKey(entity, timestamp) {
  const date = new Date(timestamp);
  const month = date.toISOString().substring(0, 7); // YYYY-MM
  return `${entity}-${month}`;
}

// For user data, use composite keys
function getUserPartitionKey(userId, dataType) {
  return `user-${userId}-${dataType}`;
}

// Example: Delete items older than retention period
async function deleteOldItems(tableName, partitionKey, retentionDays) {
  const cutoffDate = new Date();
  cutoffDate.setDate(cutoffDate.getDate() - retentionDays);

  // Query for old items (assuming sort key is timestamp)
  const params = {
    TableName: tableName,
    KeyConditionExpression: 'pk = :pk AND sk < :cutoff',
    ExpressionAttributeValues: {
      ':pk': { S: partitionKey },
      ':cutoff': { N: cutoffDate.getTime().toString() }
    }
  };

  const items = [];
  let lastEvaluatedKey;

  do {
    if (lastEvaluatedKey) {
      params.ExclusiveStartKey = lastEvaluatedKey;
    }

    const result = await dynamodb.query(params).promise();
    items.push(...result.Items);
    lastEvaluatedKey = result.LastEvaluatedKey;
  } while (lastEvaluatedKey);

  // Batch delete in chunks of 25
  for (let i = 0; i < items.length; i += 25) {
    const chunk = items.slice(i, i + 25);
    const deleteRequests = chunk.map(item => ({
      DeleteRequest: {
        Key: { pk: item.pk, sk: item.sk }
      }
    }));

    await dynamodb.batchWriteItem({
      RequestItems: {
        [tableName]: deleteRequests
      }
    }).promise();
  }
}

// Alternative: Move to archival table
async function archiveOldItems(sourceTable, archiveTable, partitionKey) {
  // Similar pattern but with PutRequest to archive table
  // and DeleteRequest from source table in same batch
}

// BEFORE: LSI on frequently updated attribute
// This creates large collections if many items share partition key

// AFTER: Use global secondary index (GSI) instead
// GSIs have their own partition key and don't contribute to item collection size

// Or: Denormalize data into separate tables
async function writeToDenormalizedTable(mainTable, summaryTable, item) {
  // Write detailed data to main table with fine-grained partition key
  await dynamodb.putItem({
    TableName: mainTable,
    Item: {
      pk: { S: `detail-${item.id}-${Date.now()}` },
      // ... other attributes
    }
  }).promise();

  // Write summary to separate table
  await dynamodb.updateItem({
    TableName: summaryTable,
    Key: { pk: { S: `summary-${item.category}` } },
    UpdateExpression: 'ADD #count :inc',
    ExpressionAttributeNames: { '#count': 'itemCount' },
    ExpressionAttributeValues: { ':inc': { N: '1' } }
  }).promise();
}

// Use sparse indexes for conditional data
// Only include items in GSI when they have specific attributes

// Monitor partition sizes
class PartitionMonitor {
  constructor(tableName) {
    this.tableName = tableName;
    this.partitionCounts = new Map();
    this.warningThreshold = 8 * 1024 * 1024 * 1024; // 8 GB warning
  }

  async trackWrite(partitionKey) {
    const count = (this.partitionCounts.get(partitionKey) || 0) + 1;
    this.partitionCounts.set(partitionKey, count);

    // Estimate size (adjust based on your average item size)
    const estimatedSize = count * 1024; // 1KB per item

    if (estimatedSize > this.warningThreshold) {
      console.warn(`Partition ${partitionKey} approaching 10 GB limit`);
      // Trigger mitigation: switch to sharded key, throttle writes, etc.
      return this.getShardedKey(partitionKey);
    }

    return partitionKey;
  }

  getShardedKey(originalKey) {
    const suffix = Math.floor(Math.random() * 10);
    return `${originalKey}-shard-${suffix}`;
  }
}

// CloudWatch alarm for partition growth

// Problematic design: Single partition for all events
// Partition key: "events"
// Sort key: timestamp

// Better design: Time-based partitioning
// Partition key: "events-2024-01", "events-2024-02", etc.
// Sort key: timestamp

// Even better: Sharded time-based partitioning
// Partition key: "events-2024-01-shard-0" through "events-2024-01-shard-9"
// Sort key: timestamp

// For user data: Composite partition keys
// Partition key: "user-12345-profile", "user-12345-orders", "user-12345-prefs"
// Instead of: "user-12345" with different item types in sort key

// Use separate tables for different data access patterns
const tables = {
  userProfiles: 'UserProfilesTable',
  userOrders: 'UserOrdersTable',
  systemMetrics: 'SystemMetricsTable',
  auditLogs: 'AuditLogsTable'
};