How to fix Error: RSA key generation failed (crypto operation failed) in Node.js

# Check Node.js version
node --version

# Check OpenSSL version used by Node.js
node -e "console.log(process.versions.openssl)"

# On Linux, verify OpenSSL library is installed
openssl version

# On macOS with Homebrew
brew install openssl

# On Windows, ensure Node.js was installed with OpenSSL bundled
# or install OpenSSL from https://slproweb.com/products/Win32OpenSSL.html

import { generateKeyPair } from 'crypto';
import { promisify } from 'util';

const generateKeyPairAsync = promisify(generateKeyPair);

async function generateRSAKeys() {
  try {
    // Valid parameters for RSA key generation
    const { publicKey, privateKey } = await generateKeyPairAsync('rsa', {
      modulusLength: 2048,        // 2048 is standard; 4096 is secure but slower
      publicKeyEncoding: {
        type: 'spki',             // SubjectPublicKeyInfo format
        format: 'pem'             // PEM text format
      },
      privateKeyEncoding: {
        type: 'pkcs8',            // PKCS#8 format
        format: 'pem',            // PEM text format
        cipher: 'aes-256-cbc',    // Optional: encrypt private key
        passphrase: 'your-passphrase'
      }
    });

    return { publicKey, privateKey };
  } catch (err) {
    console.error('Key generation failed:', err.message);
    throw err;
  }
}

// Generate keys
generateRSAKeys().catch(err => {
  console.error('Fatal error:', err);
  process.exit(1);
});

# Check available memory on Linux
free -h

# Check memory on macOS
vm_stat

# Monitor process memory usage while generating keys
# In a separate terminal:
watch -n 1 'ps aux | grep node'

// Generate smaller keys first if memory is limited
import { generateKeyPair } from 'crypto';

// Use 2048-bit instead of 4096-bit for memory-constrained environments
const { publicKey, privateKey } = await generateKeyPairAsync('rsa', {
  modulusLength: 2048,  // Use 2048 instead of 4096 in memory-constrained systems
  publicKeyEncoding: { type: 'spki', format: 'pem' },
  privateKeyEncoding: { type: 'pkcs8', format: 'pem' }
});

# Run Docker container with increased memory
docker run --memory 2g myapp

# For Kubernetes, set resource requests/limits
kubectl set resources deployment myapp --limits=memory=2Gi

// WRONG: Multiple concurrent generations (high resource usage)
const keysArray = await Promise.all([
  generateKeyPairAsync('rsa', { modulusLength: 4096, ... }),
  generateKeyPairAsync('rsa', { modulusLength: 4096, ... }),
  generateKeyPairAsync('rsa', { modulusLength: 4096, ... })
]);

// CORRECT: Sequential generation (uses resources efficiently)
const keysArray = [];
for (let i = 0; i < 3; i++) {
  const keys = await generateKeyPairAsync('rsa', {
    modulusLength: 2048,
    publicKeyEncoding: { type: 'spki', format: 'pem' },
    privateKeyEncoding: { type: 'pkcs8', format: 'pem' }
  });
  keysArray.push(keys);
}

// Or use a queue pattern for controlled concurrency
import PQueue from 'p-queue';
const queue = new PQueue({ concurrency: 1 });

const keysArray = await Promise.all([
  queue.add(() => generateKeyPairAsync('rsa', { modulusLength: 2048, ... })),
  queue.add(() => generateKeyPairAsync('rsa', { modulusLength: 2048, ... })),
  queue.add(() => generateKeyPairAsync('rsa', { modulusLength: 2048, ... }))
]);

# Check entropy pool (Linux only)
cat /proc/sys/kernel/random/entropy_avail
# Should be > 100. If < 50, entropy is low.

# Install haveged or rng-tools to increase entropy
sudo apt-get install haveged

# On macOS, entropy is handled differently and usually not an issue

# Check open file limits
ulimit -n

# Check max processes
ulimit -u

# Increase limits if needed
ulimit -n 4096  # Increase max open files
ulimit -u 32768 # Increase max processes

# Dockerfile example
FROM node:18

# Ensure entropy source is available
RUN apt-get update && apt-get install -y haveged
RUN systemctl enable haveged

# Your application
COPY . /app
WORKDIR /app
CMD ["node", "app.js"]

# Update Node.js using nvm (recommended)
nvm install --lts
nvm use --lts

# Or update using your system package manager
# Ubuntu/Debian
sudo apt-get update && sudo apt-get upgrade nodejs

# macOS with Homebrew
brew upgrade node

# Verify the update
node --version
node -e "console.log(process.versions.openssl)"

# Rebuild native modules if using older node-gyp modules
npm rebuild

import { generateKeyPair } from 'crypto';

generateKeyPair('rsa', {
  modulusLength: 2048,
  publicKeyEncoding: { type: 'spki', format: 'pem' },
  privateKeyEncoding: { type: 'pkcs8', format: 'pem' }
}, (err, publicKey, privateKey) => {
  if (err) {
    console.error('Error after update:', err);
  } else {
    console.log('✓ Key generation successful after Node.js update');
  }
});