How to fix BSONObjectTooLarge: object to insert too large in MongoDB

// Node.js example with MongoDB driver
const { MongoClient, BSON } = require('mongodb');

async function checkDocumentSize(document) {
  // Calculate BSON size
  const size = BSON.calculateObjectSize(document);
  console.log(`Document size: ${size} bytes`);

  if (size > 16 * 1024 * 1024) { // 16MB in bytes
    console.error('Document exceeds 16MB limit!');
    return false;
  }
  return true;
}

// Or using the driver's built-in check
const client = new MongoClient(uri);
const collection = client.db('mydb').collection('mycollection');

try {
  await collection.insertOne(largeDocument);
} catch (error) {
  if (error.message.includes('BSONObjectTooLarge')) {
    console.log('Document too large - implement size check');
  }
}

const { MongoClient } = require('mongodb');
const fs = require('fs');

async function storeLargeFile() {
  const client = new MongoClient(uri);
  const db = client.db('mydb');

  // Create GridFS bucket
  const bucket = new GridFSBucket(db, { bucketName: 'files' });

  // Upload large file
  const readStream = fs.createReadStream('large-video.mp4');
  const uploadStream = bucket.openUploadStream('large-video.mp4');

  readStream.pipe(uploadStream);

  uploadStream.on('error', (error) => {
    console.error('Upload failed:', error);
  });

  uploadStream.on('finish', () => {
    console.log('File uploaded successfully with ID:', uploadStream.id);
    // Store only the file reference in your document
    const document = {
      title: 'Video metadata',
      fileId: uploadStream.id,
      uploadedAt: new Date()
    };

    // This document will be small and within limits
    await db.collection('videos').insertOne(document);
  });
}

// Instead of one huge document:
// BAD: Single document with unlimited array
const badDocument = {
  userId: '123',
  activities: [/* thousands of activity objects */],
  // ... other fields
};

// GOOD: Separate collections with references
const userDocument = {
  userId: '123',
  profile: { /* user profile data */ },
  // Keep only recent activities or summary
  recentActivities: [/* last 100 activities */],
  totalActivities: 1500
};

// Activities in separate collection
const activityDocument = {
  userId: '123',
  activityType: 'login',
  timestamp: new Date(),
  details: { /* activity details */ }
};

// Query with joins when needed
const activities = await db.collection('activities')
  .find({ userId: '123' })
  .sort({ timestamp: -1 })
  .limit(100)
  .toArray();

// Strategy 1: Paginate array data
const paginatedDocument = {
  userId: '123',
  // Store array in chunks
  messagesPage1: [/* messages 1-100 */],
  messagesPage2: [/* messages 101-200 */],
  currentPage: 2,
  totalMessages: 250
};

// Strategy 2: Archive old data
async function archiveOldMessages(userId) {
  const user = await db.collection('users').findOne({ userId });

  if (user.messages.length > 1000) {
    // Move old messages to archive collection
    const oldMessages = user.messages.slice(0, 500);
    await db.collection('messageArchive').insertOne({
      userId,
      messages: oldMessages,
      archivedAt: new Date()
    });

    // Keep only recent messages
    await db.collection('users').updateOne(
      { userId },
      { $set: { messages: user.messages.slice(500) } }
    );
  }
}

// Strategy 3: Use capped arrays
const schema = {
  userId: String,
  // Keep only last 1000 messages
  recentMessages: {
    type: [MessageSchema],
    maxCount: 1000
  }
};

// Problematic: Single stage creating huge document
const badPipeline = [
  {
    $group: {
      _id: '$userId',
      allData: { $push: '$$ROOT' } // Pushes entire documents
    }
  }
];

// Better: Process in stages with limits
const optimizedPipeline = [
  // Stage 1: Filter and project only needed fields
  {
    $match: { timestamp: { $gte: startDate } }
  },
  {
    $project: {
      userId: 1,
      importantField: 1,
      // Don't include large fields
      largeBinaryField: 0
    }
  },

  // Stage 2: Group with aggregation, not array accumulation
  {
    $group: {
      _id: '$userId',
      count: { $sum: 1 },
      total: { $sum: '$amount' },
      // Use $addToSet for unique values instead of $push for all
      uniqueItems: { $addToSet: '$itemId' }
    }
  },

  // Stage 3: Add pagination with $skip and $limit
  { $skip: 0 },
  { $limit: 100 }
];

// For very large aggregations, consider:
// 1. Using $facet for parallel processing
// 2. Implementing map-reduce for distributed processing
// 3. Using $out to write intermediate results

// Regular document size checks
async function monitorDocumentSizes() {
  const collections = await db.listCollections().toArray();

  for (const collInfo of collections) {
    const collection = db.collection(collInfo.name);

    // Sample documents to check sizes
    const sample = await collection.find().limit(10).toArray();

    sample.forEach((doc, index) => {
      const size = JSON.stringify(doc).length;
      if (size > 10 * 1024 * 1024) { // Warn at 10MB
        console.warn(`Large document in ${collInfo.name}: ${size} bytes`);
      }
    });

    // Check for documents approaching limit
    const largeDocs = await collection.aggregate([
      {
        $addFields: {
          docSize: { $bsonSize: '$$ROOT' }
        }
      },
      {
        $match: {
          docSize: { $gt: 15 * 1024 * 1024 } // > 15MB
        }
      }
    ]).toArray();

    if (largeDocs.length > 0) {
      console.error(`Found ${largeDocs.length} documents approaching 16MB limit in ${collInfo.name}`);
    }
  }
}

// Set up alerts
const alertThreshold = 15 * 1024 * 1024; // 15MB
if (documentSize > alertThreshold) {
  sendAlert('Document approaching BSON size limit', {
    collection: collectionName,
    documentId: doc._id,
    size: documentSize
  });
}