feature: use bun archive instead of tar package (#678)

Co-authored-by: cubic-dev-ai[bot] <191113872+cubic-dev-ai[bot]@users.noreply.github.com>
Co-authored-by: autofix-ci[bot] <114827586+autofix-ci[bot]@users.noreply.github.com>

bun.lock

@@ -11,7 +11,6 @@
         "@kitajs/html": "^4.2.13",
         "elysia": "1.4.30",
         "sanitize-filename": "^1.6.4",
-        "tar": "^7.5.22",
       },
       "devDependencies": {
         "@eslint/js": "^10.0.1",
@@ -89,8 +88,6 @@
 
     "@humanwhocodes/retry": ["@humanwhocodes/retry@0.4.3", "", {}, "sha512-bV0Tgo9K4hfPCek+aMAn81RppFKv2ySDQeMoSZuvTASywNTnVJCArCZE2FWqpvIatKu7VMRLWlR1EazvVhDyhQ=="],
 
-    "@isaacs/fs-minipass": ["@isaacs/fs-minipass@4.0.1", "", { "dependencies": { "minipass": "^7.0.4" } }, "sha512-wgm9Ehl2jpeqP3zw/7mo3kRHFp5MEDhqAdwy1fTGkHAwnkGOVsgpvQhL8B5n1qlb01jV3n/bI0ZfZp5lWA1k4w=="],
-
     "@jridgewell/gen-mapping": ["@jridgewell/gen-mapping@0.3.13", "", { "dependencies": { "@jridgewell/sourcemap-codec": "^1.5.0", "@jridgewell/trace-mapping": "^0.3.24" } }, "sha512-2kkt/7niJ6MgEPxF0bYdQ6etZaA+fQvDcLKckhy1yIQOzaoKjBBjSj63/aLVjYE3qhRt5dvM+uUyfCg6UKCBbA=="],
 
     "@jridgewell/remapping": ["@jridgewell/remapping@2.3.5", "", { "dependencies": { "@jridgewell/gen-mapping": "^0.3.5", "@jridgewell/trace-mapping": "^0.3.24" } }, "sha512-LI9u/+laYG4Ds1TDKSJW2YPrIlcVYOwi2fUC6xB43lueCjgxV4lffOCZCtYFiH6TNOX+tQKXx97T4IKHbhyHEQ=="],
@@ -315,8 +312,6 @@
 
     "chalk": ["chalk@5.6.2", "", {}, "sha512-7NzBL0rN6fMUW+f7A6Io4h40qQlG+xGmtMxfbnH/K7TAtt8JQWVQK+6g0UXKMeVJoyV5EkkNsErQ8pVD3bLHbA=="],
 
-    "chownr": ["chownr@3.0.0", "", {}, "sha512-+IxzY9BZOQd/XuYPRmrvEVjF/nqj5kgT4kEq7VofrDoM1MxoRjEWkrCC3EtLi59TVawxTAn+orJwFQcrqEN1+g=="],
-
     "cliui": ["cliui@9.0.1", "", { "dependencies": { "string-width": "^7.2.0", "strip-ansi": "^7.1.0", "wrap-ansi": "^9.0.0" } }, "sha512-k7ndgKhwoQveBL+/1tqGJYNz097I7WOvwbmmU2AR5+magtbjPWQTS1C5vzGkBC8Ym8UWRzfKUzUUqFLypY4Q+w=="],
 
     "clsx": ["clsx@2.1.1", "", {}, "sha512-eYm0QWBtUrBWZWG0d386OGAw16Z995PiOVo2B7bjWSbHedGl5e0ZWaq65kOGgUSNesEIDkB9ISbTg/JK9dhCZA=="],
@@ -483,10 +478,6 @@
 
     "minimist": ["minimist@1.2.8", "", {}, "sha512-2yyAR8qBkN3YuheJanUpWC5U3bb5osDywNB8RzDVlDwDHbocAJveqqj1u8+SVD7jkWT4yvsHCpWqqWqAxb0zCA=="],
 
-    "minipass": ["minipass@7.1.3", "", {}, "sha512-tEBHqDnIoM/1rXME1zgka9g6Q2lcoCkxHLuc7ODJ5BxbP5d4c2Z5cGgtXAku59200Cx7diuHTOYfSBD8n6mm8A=="],
-
-    "minizlib": ["minizlib@3.1.0", "", { "dependencies": { "minipass": "^7.1.2" } }, "sha512-KZxYo1BUkWD2TVFLr0MQoM8vUUigWD3LlD83a/75BqC+4qE0Hb1Vo5v1FgcfaNXvfXzr+5EhQ6ing/CaBijTlw=="],
-
     "mri": ["mri@1.2.0", "", {}, "sha512-tzzskb3bG8LvYGFF/mDTpq3jpI6Q9wc3LEmBaghu+DdCssd1FakN7Bc0hVNmEyGq1bq3RgfkCb3cmQLpNPOroA=="],
 
     "ms": ["ms@2.1.3", "", {}, "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA=="],
@@ -577,8 +568,6 @@
 
     "tapable": ["tapable@2.3.3", "", {}, "sha512-uxc/zpqFg6x7C8vOE7lh6Lbda8eEL9zmVm/PLeTPBRhh1xCgdWaQ+J1CUieGpIfm2HdtsUpRv+HshiasBMcc6A=="],
 
-    "tar": ["tar@7.5.22", "", { "dependencies": { "@isaacs/fs-minipass": "^4.0.0", "chownr": "^3.0.0", "minipass": "^7.1.2", "minizlib": "^3.1.0", "yallist": "^5.0.0" } }, "sha512-MFO/QzvtAOmJbkhOaCTvbGcFN9L9b+JunIsDwaKljSOdcLMea3NJ1k9Usz/rjdfSXTq4dfzfeS7W4p4YOAAHeA=="],
-
     "tinyglobby": ["tinyglobby@0.2.17", "", { "dependencies": { "fdir": "^6.5.0", "picomatch": "^4.0.4" } }, "sha512-wXR/dYpcqKmfWpEdZjiKJOwCNFndD0DMnrW/cYjVGttEkBfVgcLFHoNrlj47mjOVic9yyNu65alsgF4NQyTa2g=="],
 
     "to-regex-range": ["to-regex-range@5.0.1", "", { "dependencies": { "is-number": "^7.0.0" } }, "sha512-65P7iz6X5yEr1cwcgvQxbbIw7Uk3gOy5dIdtZ4rDveLqhrdJP+Li/Hx6tyK0NEb+2GCyneCMJiGqrADCSNk8sQ=="],
@@ -623,8 +612,6 @@
 
     "y18n": ["y18n@5.0.8", "", {}, "sha512-0pfFzegeDWJHJIAmTLRP2DwHjdF5s7jo9tuztdQxAhINCdvS+3nGINqPd00AphqJR/0LhANUS6/+7SCb98YOfA=="],
 
-    "yallist": ["yallist@5.0.0", "", {}, "sha512-YgvUTfwqyc7UXVMrB+SImsVYSmTS8X/tSrtdNZMImM+n7+QTriRXyXim0mBrTXNeqzVF0KWGgHPeiyViFFrNDw=="],
-
     "yaml": ["yaml@2.9.1", "", { "bin": { "yaml": "bin.mjs" } }, "sha512-3NxN8+78OdzbT7C/WjGsyfPAtJaN3FNDsWxv7Y7mcDsT/oOmgW8BpyQQFFBnvZE3j9Y2Sdz1ULFLezL7Eb2yFw=="],
 
     "yargs": ["yargs@18.2.0", "", { "dependencies": { "cliui": "^9.0.1", "escalade": "^3.1.1", "get-caller-file": "^2.0.5", "string-width": "^8.2.1", "y18n": "^5.0.5", "yargs-parser": "^22.0.0" } }, "sha512-9OpKOLeaoNFecEp7P6iYbzze/5CqWoH7N3SMs/6y1XF4nMvFMspEGZzJ6uFi9MmQwXhyzqYoSEKO3tvA3Z/o2w=="],

package.json

@@ -29,8 +29,7 @@
     "@elysiajs/static": "^1.4.10",
     "@kitajs/html": "^4.2.13",
     "elysia": "1.4.30",
-    "sanitize-filename": "^1.6.4",
+    "sanitize-filename": "^1.6.4"
-    "tar": "^7.5.22"
   },
   "devDependencies": {
     "@eslint/js": "^10.0.1",

src/pages/download.tsx

@@ -1,14 +1,17 @@
+import { randomUUID } from "node:crypto";
+import { rename, unlink } from "node:fs/promises";
 import path from "node:path";
 
 import { Elysia } from "elysia";
 import sanitize from "sanitize-filename";
-import * as tar from "tar";
 
 import { outputDir } from "..";
 import db from "../db/db";
 import { WEBROOT } from "../helpers/env";
 import { userService } from "./user";
 
+const archiveBuilds = new Map<string, Promise<void>>();
+
 export const download = new Elysia()
   .use(userService)
   .get(
@@ -55,16 +58,39 @@ export const download = new Elysia()
       const outputPath = `${outputDir}${userId}/${jobId}`;
       const outputTar = path.join(outputPath, `converted_files_${jobId}.tar`);
 
-      await tar.create(
+      const archiveKey = `${userId}/${jobId}`;
-        {
+      let archiveBuild = archiveBuilds.get(archiveKey);
-          file: outputTar,
+      if (!archiveBuild) {
-          cwd: outputPath,
+        const build = async () => {
-          filter: (path) => {
+          const outputTarName = path.basename(outputTar);
-            return !path.match(".*\\.tar");
+          const outputTarTempName = `${outputTarName}.tmp-${randomUUID()}`;
-          },
+          const outputTarTemp = path.join(outputPath, outputTarTempName);
-        },
+
-        ["."],
+          try {
-      );
+            const files: Record<string, Uint8Array> = {};
+            const glob = new Bun.Glob("**/*");
+            for await (const file of glob.scan({ cwd: outputPath, onlyFiles: true, dot: true })) {
+              if (file !== outputTarName && !file.startsWith(`${outputTarName}.tmp-`)) {
+                files[file] = new Uint8Array(
+                  await Bun.file(path.join(outputPath, file)).arrayBuffer(),
+                );
+              }
+            }
+            await Bun.Archive.write(outputTarTemp, files);
+            await rename(outputTarTemp, outputTar);
+          } finally {
+            if (await Bun.file(outputTarTemp).exists()) {
+              await unlink(outputTarTemp);
+            }
+          }
+        };
+
+        archiveBuild = build().finally(() => {
+          archiveBuilds.delete(archiveKey);
+        });
+        archiveBuilds.set(archiveKey, archiveBuild);
+      }
+      await archiveBuild;
       return Bun.file(outputTar);
     },
     {

tests/pages/download.test.ts

@@ -1,13 +1,9 @@
 import { describe, expect, test } from "bun:test";
-import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs";
+import { mkdirSync, writeFileSync } from "node:fs";
-import { tmpdir } from "node:os";
-import { join } from "node:path";
-
-import * as tar from "tar";
 
 import { createJob, createUser, jobDirs, request } from "./helpers/app";
 
-async function userWithOutputs(files: Record<string, string>) {
+async function userWithOutputs(files: Record<string, string | Uint8Array>) {
   const user = await createUser();
   const job = createJob(user.id);
   const dirs = jobDirs(user.id, job);
@@ -19,16 +15,8 @@ async function userWithOutputs(files: Record<string, string>) {
 }
 
 async function tarEntries(response: Response) {
-  const dir = mkdtempSync(join(tmpdir(), "convertx-archive-"));
+  const archive = new Bun.Archive(await response.arrayBuffer());
-  try {
+  return [...(await archive.files()).keys()].sort();
-    const file = join(dir, "archive.tar");
-    writeFileSync(file, Buffer.from(await response.arrayBuffer()));
-    const entries: string[] = [];
-    await tar.list({ file, onReadEntry: (entry) => entries.push(entry.path) });
-    return entries.sort();
-  } finally {
-    rmSync(dir, { recursive: true, force: true });
-  }
 }
 
 describe("GET /download/:userId/:jobId/:fileName", () => {
@@ -142,7 +130,18 @@ describe("GET /archive/:jobId", () => {
 
     expect(response.status).toBe(200);
     expect(response.headers.get("content-type")).toBe("application/x-tar");
-    expect(await tarEntries(response)).toEqual(["./", "./a.csv", "./b.csv"]);
+    expect(await tarEntries(response)).toEqual(["a.csv", "b.csv"]);
+  });
+
+  test("preserves binary file contents", async () => {
+    const { user, job } = await userWithOutputs({ "image.png": new Uint8Array([0, 1, 2, 255]) });
+
+    const response = await request(`/archive/${job}`, { cookies: { auth: user.token } });
+    const archive = new Bun.Archive(await response.arrayBuffer());
+    const image = (await archive.files()).get("image.png");
+
+    expect(image).toBeDefined();
+    expect(await image?.arrayBuffer()).toEqual(new Uint8Array([0, 1, 2, 255]).buffer);
   });
 
   test("does not include earlier archives in a new archive", async () => {
@@ -151,7 +150,7 @@ describe("GET /archive/:jobId", () => {
     await request(`/archive/${job}`, { cookies: { auth: user.token } });
     const response = await request(`/archive/${job}`, { cookies: { auth: user.token } });
 
-    expect(await tarEntries(response)).toEqual(["./", "./a.csv"]);
+    expect(await tarEntries(response)).toEqual(["a.csv"]);
   });
 
   test("refuses another user's job", async () => {