starred/ConvertX
1
0
Fork 0
mirror of https://github.com/C4illin/ConvertX.git synced 2026-10-27 23:30:55 +01:00
Code Issues Packages Projects Releases 10 Wiki Activity

feature: use bun archive instead of tar package (#678)

Browse Source 
Co-authored-by: cubic-dev-ai[bot] <191113872+cubic-dev-ai[bot]@users.noreply.github.com>
Co-authored-by: autofix-ci[bot] <114827586+autofix-ci[bot]@users.noreply.github.com>
This commit is contained in:
Emrik Östling
2026-10-25 19:52:26 +01:00
committed by GitHub
parent 2135c6b9f1
commit 64cd0130d3
4 changed files with 55 additions and 44 deletions
Download Patch File Download Diff File Expand all files Collapse all files
bun.lock
-13
View File
@@ -11,7 +11,6 @@
"@kitajs/html": "^4.2.13",
"elysia": "1.4.30",
"sanitize-filename": "^1.6.4",
"tar": "^7.5.22",
},
"devDependencies": {
"@eslint/js": "^10.0.1",
@@ -89,8 +88,6 @@
"@humanwhocodes/retry": ["@humanwhocodes/retry@0.4.3", "", {}, "sha512-bV0Tgo9K4hfPCek+aMAn81RppFKv2ySDQeMoSZuvTASywNTnVJCArCZE2FWqpvIatKu7VMRLWlR1EazvVhDyhQ=="],
"@isaacs/fs-minipass": ["@isaacs/fs-minipass@4.0.1", "", { "dependencies": { "minipass": "^7.0.4" } }, "sha512-wgm9Ehl2jpeqP3zw/7mo3kRHFp5MEDhqAdwy1fTGkHAwnkGOVsgpvQhL8B5n1qlb01jV3n/bI0ZfZp5lWA1k4w=="],
"@jridgewell/gen-mapping": ["@jridgewell/gen-mapping@0.3.13", "", { "dependencies": { "@jridgewell/sourcemap-codec": "^1.5.0", "@jridgewell/trace-mapping": "^0.3.24" } }, "sha512-2kkt/7niJ6MgEPxF0bYdQ6etZaA+fQvDcLKckhy1yIQOzaoKjBBjSj63/aLVjYE3qhRt5dvM+uUyfCg6UKCBbA=="],
"@jridgewell/remapping": ["@jridgewell/remapping@2.3.5", "", { "dependencies": { "@jridgewell/gen-mapping": "^0.3.5", "@jridgewell/trace-mapping": "^0.3.24" } }, "sha512-LI9u/+laYG4Ds1TDKSJW2YPrIlcVYOwi2fUC6xB43lueCjgxV4lffOCZCtYFiH6TNOX+tQKXx97T4IKHbhyHEQ=="],
@@ -315,8 +312,6 @@
"chalk": ["chalk@5.6.2", "", {}, "sha512-7NzBL0rN6fMUW+f7A6Io4h40qQlG+xGmtMxfbnH/K7TAtt8JQWVQK+6g0UXKMeVJoyV5EkkNsErQ8pVD3bLHbA=="],
"chownr": ["chownr@3.0.0", "", {}, "sha512-+IxzY9BZOQd/XuYPRmrvEVjF/nqj5kgT4kEq7VofrDoM1MxoRjEWkrCC3EtLi59TVawxTAn+orJwFQcrqEN1+g=="],
"cliui": ["cliui@9.0.1", "", { "dependencies": { "string-width": "^7.2.0", "strip-ansi": "^7.1.0", "wrap-ansi": "^9.0.0" } }, "sha512-k7ndgKhwoQveBL+/1tqGJYNz097I7WOvwbmmU2AR5+magtbjPWQTS1C5vzGkBC8Ym8UWRzfKUzUUqFLypY4Q+w=="],
"clsx": ["clsx@2.1.1", "", {}, "sha512-eYm0QWBtUrBWZWG0d386OGAw16Z995PiOVo2B7bjWSbHedGl5e0ZWaq65kOGgUSNesEIDkB9ISbTg/JK9dhCZA=="],
@@ -483,10 +478,6 @@
"minimist": ["minimist@1.2.8", "", {}, "sha512-2yyAR8qBkN3YuheJanUpWC5U3bb5osDywNB8RzDVlDwDHbocAJveqqj1u8+SVD7jkWT4yvsHCpWqqWqAxb0zCA=="],
"minipass": ["minipass@7.1.3", "", {}, "sha512-tEBHqDnIoM/1rXME1zgka9g6Q2lcoCkxHLuc7ODJ5BxbP5d4c2Z5cGgtXAku59200Cx7diuHTOYfSBD8n6mm8A=="],
"minizlib": ["minizlib@3.1.0", "", { "dependencies": { "minipass": "^7.1.2" } }, "sha512-KZxYo1BUkWD2TVFLr0MQoM8vUUigWD3LlD83a/75BqC+4qE0Hb1Vo5v1FgcfaNXvfXzr+5EhQ6ing/CaBijTlw=="],
"mri": ["mri@1.2.0", "", {}, "sha512-tzzskb3bG8LvYGFF/mDTpq3jpI6Q9wc3LEmBaghu+DdCssd1FakN7Bc0hVNmEyGq1bq3RgfkCb3cmQLpNPOroA=="],
"ms": ["ms@2.1.3", "", {}, "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA=="],
@@ -577,8 +568,6 @@
"tapable": ["tapable@2.3.3", "", {}, "sha512-uxc/zpqFg6x7C8vOE7lh6Lbda8eEL9zmVm/PLeTPBRhh1xCgdWaQ+J1CUieGpIfm2HdtsUpRv+HshiasBMcc6A=="],
"tar": ["tar@7.5.22", "", { "dependencies": { "@isaacs/fs-minipass": "^4.0.0", "chownr": "^3.0.0", "minipass": "^7.1.2", "minizlib": "^3.1.0", "yallist": "^5.0.0" } }, "sha512-MFO/QzvtAOmJbkhOaCTvbGcFN9L9b+JunIsDwaKljSOdcLMea3NJ1k9Usz/rjdfSXTq4dfzfeS7W4p4YOAAHeA=="],
"tinyglobby": ["tinyglobby@0.2.17", "", { "dependencies": { "fdir": "^6.5.0", "picomatch": "^4.0.4" } }, "sha512-wXR/dYpcqKmfWpEdZjiKJOwCNFndD0DMnrW/cYjVGttEkBfVgcLFHoNrlj47mjOVic9yyNu65alsgF4NQyTa2g=="],
"to-regex-range": ["to-regex-range@5.0.1", "", { "dependencies": { "is-number": "^7.0.0" } }, "sha512-65P7iz6X5yEr1cwcgvQxbbIw7Uk3gOy5dIdtZ4rDveLqhrdJP+Li/Hx6tyK0NEb+2GCyneCMJiGqrADCSNk8sQ=="],
@@ -623,8 +612,6 @@
"y18n": ["y18n@5.0.8", "", {}, "sha512-0pfFzegeDWJHJIAmTLRP2DwHjdF5s7jo9tuztdQxAhINCdvS+3nGINqPd00AphqJR/0LhANUS6/+7SCb98YOfA=="],
"yallist": ["yallist@5.0.0", "", {}, "sha512-YgvUTfwqyc7UXVMrB+SImsVYSmTS8X/tSrtdNZMImM+n7+QTriRXyXim0mBrTXNeqzVF0KWGgHPeiyViFFrNDw=="],
"yaml": ["yaml@2.9.1", "", { "bin": { "yaml": "bin.mjs" } }, "sha512-3NxN8+78OdzbT7C/WjGsyfPAtJaN3FNDsWxv7Y7mcDsT/oOmgW8BpyQQFFBnvZE3j9Y2Sdz1ULFLezL7Eb2yFw=="],
"yargs": ["yargs@18.2.0", "", { "dependencies": { "cliui": "^9.0.1", "escalade": "^3.1.1", "get-caller-file": "^2.0.5", "string-width": "^8.2.1", "y18n": "^5.0.5", "yargs-parser": "^22.0.0" } }, "sha512-9OpKOLeaoNFecEp7P6iYbzze/5CqWoH7N3SMs/6y1XF4nMvFMspEGZzJ6uFi9MmQwXhyzqYoSEKO3tvA3Z/o2w=="],
package.json
+1 -2
View File
@@ -29,8 +29,7 @@
"@elysiajs/static": "^1.4.10",
"@kitajs/html": "^4.2.13",
"elysia": "1.4.30",
"sanitize-filename": "^1.6.4",
"tar": "^7.5.22"
"sanitize-filename": "^1.6.4"
},
"devDependencies": {
"@eslint/js": "^10.0.1",
src/pages/download.tsx
+37 -11
View File
@@ -1,14 +1,17 @@
import { randomUUID } from "node:crypto";
import { rename, unlink } from "node:fs/promises";
import path from "node:path";
import { Elysia } from "elysia";
import sanitize from "sanitize-filename";
import * as tar from "tar";
import { outputDir } from "..";
import db from "../db/db";
import { WEBROOT } from "../helpers/env";
import { userService } from "./user";
const archiveBuilds = new Map<string, Promise<void>>();
export const download = new Elysia()
.use(userService)
.get(
@@ -55,16 +58,39 @@ export const download = new Elysia()
const outputPath = `${outputDir}${userId}/${jobId}`;
const outputTar = path.join(outputPath, `converted_files_${jobId}.tar`);
await tar.create(
{
file: outputTar,
cwd: outputPath,
filter: (path) => {
return !path.match(".*\\.tar");
},
},
["."],
);
const archiveKey = `${userId}/${jobId}`;
let archiveBuild = archiveBuilds.get(archiveKey);
if (!archiveBuild) {
const build = async () => {
const outputTarName = path.basename(outputTar);
const outputTarTempName = `${outputTarName}.tmp-${randomUUID()}`;
const outputTarTemp = path.join(outputPath, outputTarTempName);
try {
const files: Record<string, Uint8Array> = {};
const glob = new Bun.Glob("**/*");
for await (const file of glob.scan({ cwd: outputPath, onlyFiles: true, dot: true })) {
if (file !== outputTarName && !file.startsWith(`${outputTarName}.tmp-`)) {
files[file] = new Uint8Array(
await Bun.file(path.join(outputPath, file)).arrayBuffer(),
);
}
}
await Bun.Archive.write(outputTarTemp, files);
await rename(outputTarTemp, outputTar);
} finally {
if (await Bun.file(outputTarTemp).exists()) {
await unlink(outputTarTemp);
}
}
};
archiveBuild = build().finally(() => {
archiveBuilds.delete(archiveKey);
});
archiveBuilds.set(archiveKey, archiveBuild);
}
await archiveBuild;
return Bun.file(outputTar);
},
{
tests/pages/download.test.ts
+17 -18
View File
@@ -1,13 +1,9 @@
import { describe, expect, test } from "bun:test";
import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs";
import { tmpdir } from "node:os";
import { join } from "node:path";
import * as tar from "tar";
import { mkdirSync, writeFileSync } from "node:fs";
import { createJob, createUser, jobDirs, request } from "./helpers/app";
async function userWithOutputs(files: Record<string, string>) {
async function userWithOutputs(files: Record<string, string | Uint8Array>) {
const user = await createUser();
const job = createJob(user.id);
const dirs = jobDirs(user.id, job);
@@ -19,16 +15,8 @@ async function userWithOutputs(files: Record<string, string>) {
}
async function tarEntries(response: Response) {
const dir = mkdtempSync(join(tmpdir(), "convertx-archive-"));
try {
const file = join(dir, "archive.tar");
writeFileSync(file, Buffer.from(await response.arrayBuffer()));
const entries: string[] = [];
await tar.list({ file, onReadEntry: (entry) => entries.push(entry.path) });
return entries.sort();
} finally {
rmSync(dir, { recursive: true, force: true });
}
const archive = new Bun.Archive(await response.arrayBuffer());
return [...(await archive.files()).keys()].sort();
}
describe("GET /download/:userId/:jobId/:fileName", () => {
@@ -142,7 +130,18 @@ describe("GET /archive/:jobId", () => {
expect(response.status).toBe(200);
expect(response.headers.get("content-type")).toBe("application/x-tar");
expect(await tarEntries(response)).toEqual(["./", "./a.csv", "./b.csv"]);
expect(await tarEntries(response)).toEqual(["a.csv", "b.csv"]);
});
test("preserves binary file contents", async () => {
const { user, job } = await userWithOutputs({ "image.png": new Uint8Array([0, 1, 2, 255]) });
const response = await request(`/archive/${job}`, { cookies: { auth: user.token } });
const archive = new Bun.Archive(await response.arrayBuffer());
const image = (await archive.files()).get("image.png");
expect(image).toBeDefined();
expect(await image?.arrayBuffer()).toEqual(new Uint8Array([0, 1, 2, 255]).buffer);
});
test("does not include earlier archives in a new archive", async () => {
@@ -151,7 +150,7 @@ describe("GET /archive/:jobId", () => {
await request(`/archive/${job}`, { cookies: { auth: user.token } });
const response = await request(`/archive/${job}`, { cookies: { auth: user.token } });
expect(await tarEntries(response)).toEqual(["./", "./a.csv"]);
expect(await tarEntries(response)).toEqual(["a.csv"]);
});
test("refuses another user's job", async () => {