"文件大小错误" (File Size Error) in Node.js

Question

当从大文件下载小的25MB数据块时，实际上下载的数据块远大于25MB。能否有人帮忙解决这个问题？

const fs = require('fs');
const files = fs.readdirSync('../in/');
files.map(fileName => {
  const readable = fs.createReadStream('../in/' + fileName);
  let size = 0;

  let temp_chunk = Buffer.alloc(0); // 初始化为Buffer
  let count = 0;

  readable.on("data", (chunk) => {
    temp_chunk = Buffer.concat([temp_chunk, chunk]); // 使用Buffer.concat合并
    size += chunk.length; // 直接使用chunk的长度，更准确高效

    let amount = size / 1000000;
    amount = amount.toFixed(2) + 'mb'; // 使用toFixed限制小数点后两位

    console.log(amount);
    if (amount > 24.5) {
      console.log(fileName + ' ' + count + ' downloaded');
      fs.writeFileSync('../out/' + fileName + count, temp_chunk);
      temp_chunk = Buffer.alloc(0); // 重置为新Buffer
      size = 0;
      count++;
    }
  });
});

我尝试通过读取temp_chunk的大小来解决问题，但这使下载速度大幅降低。

NathanOliver · Answer

使用Buffer来拼接数据块。直接使用+=操作符在字符串（如temp_chunk += chunk）上拼接二进制数据是不合适的，对于二进制数据可能导致意外结果。

解决方案

const fs = require('fs');
const files = fs.readdirSync('../in/');
files.forEach(fileName => {
  const readable = fs.createReadStream('../in/' + fileName);
  let size = 0;
  let temp_chunk = Buffer.alloc(0); // 初始化为空缓冲区
  let count = 0;

  readable.on("data", (chunk) => {
    temp_chunk = Buffer.concat([temp_chunk, chunk]);
    size += chunk.length; // 使用chunk.length获取字节大小

    let amount = size / 1000000; // 转换为兆字节

    console.log(amount + 'mb');
    if (amount > 24.5) {
      console.log(fileName + ' ' + count + ' 已下载');
      fs.writeFileSync('../out/' + fileName + count, temp_chunk);
      temp_chunk = Buffer.alloc(0); // 重置为空缓冲区
      size = 0;
      count++;
    }
  });

  readable.on("end", () => {
    if (temp_chunk.length > 0) {
      // 如果有剩余数据块，则写入
      console.log(fileName + ' ' + count + ' 已下载（最后的数据块）');
      fs.writeFileSync('../out/' + fileName + count, temp_chunk);
    }
  });
});

strager · Answer

你可以利用WHATWG标准流（自Node.js v17版本起成为其一部分），在发出前读取并缓存字节。采用这种模式还能增强代码的可移植性，使其能更容易地应用于其他环境（比如浏览器、Deno、Bun等）。

一个TransformStream（Node.js、 MDN）能够根据指定的字节长度来缓存数据，直到达到该长度后再发出，示例如下：

buffered_byte_stream.mjs

import { TransformStream } from "node:stream/web";

export class BufferedByteStream extends TransformStream {
  #buf = [];
  constructor(byteLength) {
    super({
      flush: (controller) => {
        if (this.#buf.length >= byteLength) {
          controller.enqueue(Uint8Array.from(this.#buf.splice(0, byteLength)));
        }
        if (this.#buf.length > 0) {
          controller.enqueue(Uint8Array.from(this.#buf.splice(0, this.#buf.length)));
        }
      },
      transform: (chunk, controller) => {
        this.#buf.push(...chunk);
        if (this.#buf.length >= byteLength) {
          controller.enqueue(Uint8Array.from(this.#buf.splice(0, byteLength)));
        }
      },
    });
  }
}

然后在你的主应用代码中使用它，大概会是这样的：

main.mjs

import * as fs from "node:fs/promises";
import * as path from "node:path";
import { WritableStream } from "node:stream/web";

import { BufferedByteStream } from "./buffered_byte_stream.mjs";

function createWritableDestination(filePath) {
  let index = 0;
  return new WritableStream({
    async write(u8Arr) {
      const { base, dir } = path.parse(filePath);
      const fileName = `${base}.part${(index++).toString().padStart(3, "0")}`;
      console.log(`下载完成: ${fileName}`);
      await fs.writeFile(path.join(dir, fileName), u8Arr);
    },
  });
}

const inputDir = "../in";
const outputDir = "../out";
const byteLengthLimit = 25e6; // 25MB

for (const entry of await fs.readdir(inputDir, { withFileTypes: true })) {
  if (!entry.isFile()) continue; // 忽略非文件
  const fileHandle = await fs.open(path.join(inputDir, entry.name));
  try {
    await fileHandle
      .readableWebStream({ type: "bytes" }) // 可读流<Uint8Array>
      .pipeThrough(new BufferedByteStream(byteLengthLimit))
      .pipeTo(createWritableDestination(path.join(outputDir, entry.name)));
  } finally {
    await fileHandle.close();
  }
}

Code in TypeScript Playground