JSON in Node.js: Streams and Buffers for Large Data

Node.js Streams and Buffers are essential for handling JSON data efficiently, especially for large files, network responses, and real-time data processing. Instead of loading entire JSON payloads into memory, streams process data in chunks, enabling backpressure-aware pipelines that handle gigabytes of data with minimal memory. This guide covers streaming JSON parsing, transform streams for JSON, buffer management, and real-world patterns. Use our JSON Formatter for small payloads and JSON Validator for streaming validation.

When to Use Streams for JSON

Scenario	Memory (Standard)	Memory (Streaming)	Why Streams Win
1GB JSON file	2-4 GB RAM	50-100 MB	40x memory reduction
API response > 100MB	OOM risk	Constant memory	Processable regardless of size
Real-time log processing	Must buffer all data	Per-line processing	No buffering delay
Download + parse pipeline	Download, then parse	Parse while downloading	Lower total latency

Streaming JSON Parse (NDJSON)

const fs = require('fs');
const { Transform } = require('stream');
const readline = require('readline');

// Stream NDJSON file line by line
function createNDJSONReader(filePath) {
  const rl = readline.createInterface({
    input: fs.createReadStream(filePath),
    crlfDelay: Infinity
  });

  return new Transform({
    readableObjectMode: true,
    writableObjectMode: false,
    async transform(chunk, encoding, callback) {
      // Not used directly - readline emits 'line' events
    }
  });
}

// Example: Process large NDJSON file
async function processLargeJSON(filePath) {
  const fileStream = fs.createReadStream(filePath);
  const rl = readline.createInterface({ input: fileStream });

  const pipeline = rl.pipe(new Transform({
    readableObjectMode: true,
    writableObjectMode: false,
    transform(line, encoding, callback) {
      if (line.trim()) {
        try {
          const record = JSON.parse(line);
          callback(null, record);
        } catch (err) {
          console.error('Invalid JSON line:', err.message);
          callback(); // Skip bad lines
        }
      } else {
        callback();
      }
    }
  }));

  for await (const record of pipeline) {
    await processRecord(record);
  }
}

Streaming JSON Parse (Array)

// For standard JSON arrays, use clarinet or jsonstream2
import parser from 'clarinet';

function streamJSONArray(readableStream) {
  const jsonParser = parser.createStream();

  return new Transform({
    readableObjectMode: true,
    transform(chunk, encoding, callback) {
      jsonParser.write(chunk.toString());
      callback();
    }
  });
}

// Alternative: jsonstream2 for array streams
const JSONStream = require('jsonstream2');
const stream = fs.createReadStream('large-array.json')
  .pipe(JSONStream.parse('*'))  // Parse each array element
  .on('data', (item) => {
    // Process one array element at a time
    processItem(item);
  })
  .on('end', () => console.log('Done'));

Transform Stream for JSON Processing

// Transform stream that modifies JSON records
class JSONTransformStream extends Transform {
  constructor(transformFn, options = {}) {
    super({
      readableObjectMode: true,
      writableObjectMode: true,
      ...options
    });
    this.transformFn = transformFn;
  }

  _transform(record, encoding, callback) {
    try {
      const transformed = this.transformFn(record);
      callback(null, transformed);
    } catch (err) {
      callback(err);
    }
  }
}

// Usage: Add timestamp to each record
const addTimestamp = new JSONTransformStream((record) => ({
  ...record,
  processedAt: new Date().toISOString()
}));

// Pipeline: read NDJSON -> transform -> write NDJSON
const { pipeline } = require('stream');
pipeline(
  fs.createReadStream('input.ndjson'),
  new NDJSONLineReader(),
  addTimestamp,
  new NDJSONLineWriter(),
  fs.createWriteStream('output.ndjson'),
  (err) => console.log('Pipeline complete:', err)
);

Buffer Management for JSON

// Handling incomplete JSON in stream buffers
class JSONBuffer extends Transform {
  constructor(options = {}) {
    super(options);
    this.buffer = '';
    this.objectMode = false;
  }

  _transform(chunk, encoding, callback) {
    this.buffer += chunk.toString();

    // Try to find complete JSON objects
    let depth = 0;
    let start = -1;
    let complete = false;

    for (let i = 0; i < this.buffer.length; i++) {
      const char = this.buffer[i];

      if (char === '{' || char === '[') {
        if (depth === 0) start = i;
        depth++;
      } else if (char === '}' || char === ']') {
        depth--;
        if (depth === 0 && start >= 0) {
          const jsonStr = this.buffer.substring(start, i + 1);
          this.push(jsonStr);
          this.buffer = this.buffer.substring(i + 1);
          i = -1; // Reset to process remaining buffer
          start = -1;
        }
      }
    }
    callback();
  }

  _flush(callback) {
    if (this.buffer.trim()) {
      this.push(this.buffer);
    }
    callback();
  }
}

// Usage
const buffer = new JSONBuffer({ readableObjectMode: true });
buffer.on('data', (jsonStr) => {
  const data = JSON.parse(jsonStr);
  console.log('Complete JSON:', data);
});

Memory-Efficient Patterns

• Use Buffer.alloc() instead of new Buffer() for explicit memory allocation
• Set highWaterMark on streams to control buffer sizes (default 16KB, increase for throughput)
• Use pipeline() instead of .pipe() for automatic cleanup and error handling
• For large JSON objects, consider MongoDB or PostgreSQL JSONB instead of in-memory processing
• Use JSON Minifier to reduce file size before streaming
• Validate JSON structure with JSON Validator before expensive streaming operations

Buffer Size Benchmark

Buffer Size	Throughput (1GB file)	Memory	Use Case
16 KB (default)	45s	50 MB	Low-memory environments
256 KB	28s	80 MB	Balanced
1 MB	22s	150 MB	High-throughput, plenty of RAM

Next Steps

Format small JSON with JSON Formatter. Validate with JSON Validator. Minify large files with JSON Minifier before streaming.