Node.js Streams: Processing Large Data Efficiently | DevPrep

Node.js Streams: Processing Large Data Efficiently | DevPrep

// Bad: loads entire file into memory
const data = fs.readFileSync("huge-file.csv"); // 2GB in memory!

// Good: processes chunk by chunk
const stream = fs.createReadStream("huge-file.csv");
stream.on("data", (chunk) => processChunk(chunk)); // ~64KB at a time

// Compress a file
fs.createReadStream("input.txt")
  .pipe(zlib.createGzip())
  .pipe(fs.createWriteStream("input.txt.gz"));

// HTTP response streaming
app.get("/large-file", (req, res) => {
  const stream = fs.createReadStream("huge.csv");
  res.setHeader("Content-Type", "text/csv");
  stream.pipe(res);
});

const { Transform } = require("stream");

const upperCase = new Transform({
  transform(chunk, encoding, callback) {
    this.push(chunk.toString().toUpperCase());
    callback();
  }
});

process.stdin.pipe(upperCase).pipe(process.stdout);

async function processCSV(filePath) {
  const stream = fs.createReadStream(filePath, { encoding: "utf8" });
  let lineBuffer = "";
  
  for await (const chunk of stream) {
    lineBuffer += chunk;
    const lines = lineBuffer.split("\\n");
    lineBuffer = lines.pop(); // Keep incomplete line
    
    for (const line of lines) {
      await processLine(line);
    }
  }
}