From 64cd0130d3ae1fdc9154b77a050a2e8a37efc7aa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Emrik=20=C3=96stling?= Date: Sun, 25 Oct 2026 19:52:26 +0100 Subject: [PATCH] feature: use bun archive instead of tar package (#678) Co-authored-by: cubic-dev-ai[bot] <191113872+cubic-dev-ai[bot]@users.noreply.github.com> Co-authored-by: autofix-ci[bot] <114827586+autofix-ci[bot]@users.noreply.github.com> --- bun.lock | 13 ---------- package.json | 3 +-- src/pages/download.tsx | 48 +++++++++++++++++++++++++++--------- tests/pages/download.test.ts | 35 +++++++++++++------------- 4 files changed, 55 insertions(+), 44 deletions(-) diff --git a/bun.lock b/bun.lock index 5dcaf6e1..28ed1799 100644 --- a/bun.lock +++ b/bun.lock @@ -11,7 +11,6 @@ "@kitajs/html": "^4.2.13", "elysia": "1.4.30", "sanitize-filename": "^1.6.4", - "tar": "^7.5.22", }, "devDependencies": { "@eslint/js": "^10.0.1", @@ -89,8 +88,6 @@ "@humanwhocodes/retry": ["@humanwhocodes/retry@0.4.3", "", {}, "sha512-bV0Tgo9K4hfPCek+aMAn81RppFKv2ySDQeMoSZuvTASywNTnVJCArCZE2FWqpvIatKu7VMRLWlR1EazvVhDyhQ=="], - "@isaacs/fs-minipass": ["@isaacs/fs-minipass@4.0.1", "", { "dependencies": { "minipass": "^7.0.4" } }, "sha512-wgm9Ehl2jpeqP3zw/7mo3kRHFp5MEDhqAdwy1fTGkHAwnkGOVsgpvQhL8B5n1qlb01jV3n/bI0ZfZp5lWA1k4w=="], - "@jridgewell/gen-mapping": ["@jridgewell/gen-mapping@0.3.13", "", { "dependencies": { "@jridgewell/sourcemap-codec": "^1.5.0", "@jridgewell/trace-mapping": "^0.3.24" } }, "sha512-2kkt/7niJ6MgEPxF0bYdQ6etZaA+fQvDcLKckhy1yIQOzaoKjBBjSj63/aLVjYE3qhRt5dvM+uUyfCg6UKCBbA=="], "@jridgewell/remapping": ["@jridgewell/remapping@2.3.5", "", { "dependencies": { "@jridgewell/gen-mapping": "^0.3.5", "@jridgewell/trace-mapping": "^0.3.24" } }, "sha512-LI9u/+laYG4Ds1TDKSJW2YPrIlcVYOwi2fUC6xB43lueCjgxV4lffOCZCtYFiH6TNOX+tQKXx97T4IKHbhyHEQ=="], @@ -315,8 +312,6 @@ "chalk": ["chalk@5.6.2", "", {}, "sha512-7NzBL0rN6fMUW+f7A6Io4h40qQlG+xGmtMxfbnH/K7TAtt8JQWVQK+6g0UXKMeVJoyV5EkkNsErQ8pVD3bLHbA=="], - "chownr": ["chownr@3.0.0", "", {}, "sha512-+IxzY9BZOQd/XuYPRmrvEVjF/nqj5kgT4kEq7VofrDoM1MxoRjEWkrCC3EtLi59TVawxTAn+orJwFQcrqEN1+g=="], - "cliui": ["cliui@9.0.1", "", { "dependencies": { "string-width": "^7.2.0", "strip-ansi": "^7.1.0", "wrap-ansi": "^9.0.0" } }, "sha512-k7ndgKhwoQveBL+/1tqGJYNz097I7WOvwbmmU2AR5+magtbjPWQTS1C5vzGkBC8Ym8UWRzfKUzUUqFLypY4Q+w=="], "clsx": ["clsx@2.1.1", "", {}, "sha512-eYm0QWBtUrBWZWG0d386OGAw16Z995PiOVo2B7bjWSbHedGl5e0ZWaq65kOGgUSNesEIDkB9ISbTg/JK9dhCZA=="], @@ -483,10 +478,6 @@ "minimist": ["minimist@1.2.8", "", {}, "sha512-2yyAR8qBkN3YuheJanUpWC5U3bb5osDywNB8RzDVlDwDHbocAJveqqj1u8+SVD7jkWT4yvsHCpWqqWqAxb0zCA=="], - "minipass": ["minipass@7.1.3", "", {}, "sha512-tEBHqDnIoM/1rXME1zgka9g6Q2lcoCkxHLuc7ODJ5BxbP5d4c2Z5cGgtXAku59200Cx7diuHTOYfSBD8n6mm8A=="], - - "minizlib": ["minizlib@3.1.0", "", { "dependencies": { "minipass": "^7.1.2" } }, "sha512-KZxYo1BUkWD2TVFLr0MQoM8vUUigWD3LlD83a/75BqC+4qE0Hb1Vo5v1FgcfaNXvfXzr+5EhQ6ing/CaBijTlw=="], - "mri": ["mri@1.2.0", "", {}, "sha512-tzzskb3bG8LvYGFF/mDTpq3jpI6Q9wc3LEmBaghu+DdCssd1FakN7Bc0hVNmEyGq1bq3RgfkCb3cmQLpNPOroA=="], "ms": ["ms@2.1.3", "", {}, "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA=="], @@ -577,8 +568,6 @@ "tapable": ["tapable@2.3.3", "", {}, "sha512-uxc/zpqFg6x7C8vOE7lh6Lbda8eEL9zmVm/PLeTPBRhh1xCgdWaQ+J1CUieGpIfm2HdtsUpRv+HshiasBMcc6A=="], - "tar": ["tar@7.5.22", "", { "dependencies": { "@isaacs/fs-minipass": "^4.0.0", "chownr": "^3.0.0", "minipass": "^7.1.2", "minizlib": "^3.1.0", "yallist": "^5.0.0" } }, "sha512-MFO/QzvtAOmJbkhOaCTvbGcFN9L9b+JunIsDwaKljSOdcLMea3NJ1k9Usz/rjdfSXTq4dfzfeS7W4p4YOAAHeA=="], - "tinyglobby": ["tinyglobby@0.2.17", "", { "dependencies": { "fdir": "^6.5.0", "picomatch": "^4.0.4" } }, "sha512-wXR/dYpcqKmfWpEdZjiKJOwCNFndD0DMnrW/cYjVGttEkBfVgcLFHoNrlj47mjOVic9yyNu65alsgF4NQyTa2g=="], "to-regex-range": ["to-regex-range@5.0.1", "", { "dependencies": { "is-number": "^7.0.0" } }, "sha512-65P7iz6X5yEr1cwcgvQxbbIw7Uk3gOy5dIdtZ4rDveLqhrdJP+Li/Hx6tyK0NEb+2GCyneCMJiGqrADCSNk8sQ=="], @@ -623,8 +612,6 @@ "y18n": ["y18n@5.0.8", "", {}, "sha512-0pfFzegeDWJHJIAmTLRP2DwHjdF5s7jo9tuztdQxAhINCdvS+3nGINqPd00AphqJR/0LhANUS6/+7SCb98YOfA=="], - "yallist": ["yallist@5.0.0", "", {}, "sha512-YgvUTfwqyc7UXVMrB+SImsVYSmTS8X/tSrtdNZMImM+n7+QTriRXyXim0mBrTXNeqzVF0KWGgHPeiyViFFrNDw=="], - "yaml": ["yaml@2.9.1", "", { "bin": { "yaml": "bin.mjs" } }, "sha512-3NxN8+78OdzbT7C/WjGsyfPAtJaN3FNDsWxv7Y7mcDsT/oOmgW8BpyQQFFBnvZE3j9Y2Sdz1ULFLezL7Eb2yFw=="], "yargs": ["yargs@18.2.0", "", { "dependencies": { "cliui": "^9.0.1", "escalade": "^3.1.1", "get-caller-file": "^2.0.5", "string-width": "^8.2.1", "y18n": "^5.0.5", "yargs-parser": "^22.0.0" } }, "sha512-9OpKOLeaoNFecEp7P6iYbzze/5CqWoH7N3SMs/6y1XF4nMvFMspEGZzJ6uFi9MmQwXhyzqYoSEKO3tvA3Z/o2w=="], diff --git a/package.json b/package.json index 7c6f6aaf..62c7d96d 100644 --- a/package.json +++ b/package.json @@ -29,8 +29,7 @@ "@elysiajs/static": "^1.4.10", "@kitajs/html": "^4.2.13", "elysia": "1.4.30", - "sanitize-filename": "^1.6.4", - "tar": "^7.5.22" + "sanitize-filename": "^1.6.4" }, "devDependencies": { "@eslint/js": "^10.0.1", diff --git a/src/pages/download.tsx b/src/pages/download.tsx index e642bfa8..60cbf02c 100644 --- a/src/pages/download.tsx +++ b/src/pages/download.tsx @@ -1,14 +1,17 @@ +import { randomUUID } from "node:crypto"; +import { rename, unlink } from "node:fs/promises"; import path from "node:path"; import { Elysia } from "elysia"; import sanitize from "sanitize-filename"; -import * as tar from "tar"; import { outputDir } from ".."; import db from "../db/db"; import { WEBROOT } from "../helpers/env"; import { userService } from "./user"; +const archiveBuilds = new Map>(); + export const download = new Elysia() .use(userService) .get( @@ -55,16 +58,39 @@ export const download = new Elysia() const outputPath = `${outputDir}${userId}/${jobId}`; const outputTar = path.join(outputPath, `converted_files_${jobId}.tar`); - await tar.create( - { - file: outputTar, - cwd: outputPath, - filter: (path) => { - return !path.match(".*\\.tar"); - }, - }, - ["."], - ); + const archiveKey = `${userId}/${jobId}`; + let archiveBuild = archiveBuilds.get(archiveKey); + if (!archiveBuild) { + const build = async () => { + const outputTarName = path.basename(outputTar); + const outputTarTempName = `${outputTarName}.tmp-${randomUUID()}`; + const outputTarTemp = path.join(outputPath, outputTarTempName); + + try { + const files: Record = {}; + const glob = new Bun.Glob("**/*"); + for await (const file of glob.scan({ cwd: outputPath, onlyFiles: true, dot: true })) { + if (file !== outputTarName && !file.startsWith(`${outputTarName}.tmp-`)) { + files[file] = new Uint8Array( + await Bun.file(path.join(outputPath, file)).arrayBuffer(), + ); + } + } + await Bun.Archive.write(outputTarTemp, files); + await rename(outputTarTemp, outputTar); + } finally { + if (await Bun.file(outputTarTemp).exists()) { + await unlink(outputTarTemp); + } + } + }; + + archiveBuild = build().finally(() => { + archiveBuilds.delete(archiveKey); + }); + archiveBuilds.set(archiveKey, archiveBuild); + } + await archiveBuild; return Bun.file(outputTar); }, { diff --git a/tests/pages/download.test.ts b/tests/pages/download.test.ts index a8a8d553..be887f1b 100644 --- a/tests/pages/download.test.ts +++ b/tests/pages/download.test.ts @@ -1,13 +1,9 @@ import { describe, expect, test } from "bun:test"; -import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs"; -import { tmpdir } from "node:os"; -import { join } from "node:path"; - -import * as tar from "tar"; +import { mkdirSync, writeFileSync } from "node:fs"; import { createJob, createUser, jobDirs, request } from "./helpers/app"; -async function userWithOutputs(files: Record) { +async function userWithOutputs(files: Record) { const user = await createUser(); const job = createJob(user.id); const dirs = jobDirs(user.id, job); @@ -19,16 +15,8 @@ async function userWithOutputs(files: Record) { } async function tarEntries(response: Response) { - const dir = mkdtempSync(join(tmpdir(), "convertx-archive-")); - try { - const file = join(dir, "archive.tar"); - writeFileSync(file, Buffer.from(await response.arrayBuffer())); - const entries: string[] = []; - await tar.list({ file, onReadEntry: (entry) => entries.push(entry.path) }); - return entries.sort(); - } finally { - rmSync(dir, { recursive: true, force: true }); - } + const archive = new Bun.Archive(await response.arrayBuffer()); + return [...(await archive.files()).keys()].sort(); } describe("GET /download/:userId/:jobId/:fileName", () => { @@ -142,7 +130,18 @@ describe("GET /archive/:jobId", () => { expect(response.status).toBe(200); expect(response.headers.get("content-type")).toBe("application/x-tar"); - expect(await tarEntries(response)).toEqual(["./", "./a.csv", "./b.csv"]); + expect(await tarEntries(response)).toEqual(["a.csv", "b.csv"]); + }); + + test("preserves binary file contents", async () => { + const { user, job } = await userWithOutputs({ "image.png": new Uint8Array([0, 1, 2, 255]) }); + + const response = await request(`/archive/${job}`, { cookies: { auth: user.token } }); + const archive = new Bun.Archive(await response.arrayBuffer()); + const image = (await archive.files()).get("image.png"); + + expect(image).toBeDefined(); + expect(await image?.arrayBuffer()).toEqual(new Uint8Array([0, 1, 2, 255]).buffer); }); test("does not include earlier archives in a new archive", async () => { @@ -151,7 +150,7 @@ describe("GET /archive/:jobId", () => { await request(`/archive/${job}`, { cookies: { auth: user.token } }); const response = await request(`/archive/${job}`, { cookies: { auth: user.token } }); - expect(await tarEntries(response)).toEqual(["./", "./a.csv"]); + expect(await tarEntries(response)).toEqual(["a.csv"]); }); test("refuses another user's job", async () => {