sync commits from swe-productivity study (#616)

Co-authored-by: Jennifer Wang <jennifer123wang@gmail.com>
Co-authored-by: strawberry-raccoon <253751817+strawberry-raccoon@users.noreply.github.com>
This commit is contained in:
authored and GitHub committed 2026-08-19 22:48:25 +02:00
1 parent f444d84d4d
commit 866510373b
27 files changed
+836 -43

No files matched your search

+7
View File
@@ -0,0 +1,7 @@
#!/bin/bash
# Load project-local .env file
set -o allexport
source "$(dirname "$0")/../.env"
set +o allexport
# Output the API key to Claude Code
echo "$ANTHROPIC_API_KEY"
+3
View File
@@ -0,0 +1,3 @@
{
"apiKeyHelper": "./.claude/get-api-key.sh"
}
+114
View File
@@ -0,0 +1,114 @@
# AI-Blocking Git Hooks
Git hooks that prevent AI indicators from being committed to your repository.
## What These Hooks Block
The hooks scan for AI references including:
- AI tool names (Claude, Cursor, Copilot, ChatGPT, Gemini, etc.)
- AI-related phrases ("ai-generated", "ai-assisted", "generated with", etc.)
- Model names (GPT-4, Claude Sonnet, Claude Code, etc.)
- Company names (OpenAI, Anthropic, etc.)
- Co-authorship markers (Co-Authored-By: Claude, noreply@anthropic.com, etc.)
## Prerequisites
Install [pre-commit](https://pre-commit.com/):
```bash
pip install pre-commit
```
## Installation
### Single Repository
1. Navigate to your repository:
```bash
cd /path/to/your/repo
```
2. Create or update `.pre-commit-config.yaml`:
```yaml
repos:
- repo: local
hooks:
- id: ai-pre-commit
name: Block AI indicators in staged files
entry: python3 .config/ai-hooks/ai_guard.py pre-commit
language: system
pass_filenames: false
stages: [pre-commit]
- id: ai-commit-msg
name: Block AI indicators in commit messages
entry: python3 .config/ai-hooks/ai_guard.py commit-msg
language: system
stages: [commit-msg]
```
3. Install the hooks:
```bash
pre-commit install
```
## How It Works
### commit-msg Hook
- Runs when you create a commit
- Scans the commit message for AI indicators
- Blocks the commit if any indicators are found
### pre-commit Hook
- Runs before the commit is created
- Scans all staged files for AI indicators
- Blocks the commit if any indicators are found in the code
## Uninstallation
To remove the hooks from a repository:
```bash
cd /path/to/your/repo
rm .git/hooks/commit-msg .git/hooks/pre-commit .git/hooks/ai_guard.py
```
## Testing
To verify the hooks are working:
1. Create a test file with AI indicators:
```bash
echo "Generated with Claude Code" > test.txt
git add test.txt
git commit -m "test"
```
2. You should see an error message blocking the commit
3. Clean up:
```bash
git reset HEAD test.txt
rm test.txt
```
## Bypassing Hooks When Needed
In rare cases where you need to commit content with AI indicators, you can bypass the hooks using the `--no-verify` flag:
```bash
git commit --no-verify -m "your commit message"
```
## Notes
- These hooks are **repository-specific** and must be installed in each repo where you want them active
- Git hooks are not tracked by git itself (they live in `.git/hooks/`)
- If you have existing hooks, the installer will create `.backup` copies
- All pattern matching is case-insensitive
+59
View File
@@ -0,0 +1,59 @@
#!/usr/bin/env python3
"""
Python wrapper for AI indicator guard hooks.
Calls the appropriate bash script based on the hook type.
"""
import subprocess
import sys
from pathlib import Path
def main():
if len(sys.argv) < 2:
print("Usage: ai_guard.py <pre-commit|commit-msg>", file=sys.stderr)
sys.exit(1)
hook_type = sys.argv[1]
# Get the directory where this script is located
script_dir = Path(__file__).parent
# Determine which hook script to call
if hook_type == "pre-commit":
hook_script = script_dir / "pre-commit"
elif hook_type == "commit-msg":
hook_script = script_dir / "commit-msg"
# For commit-msg, we need to pass the commit message file
if len(sys.argv) < 3:
print("Error: commit-msg hook requires message file argument", file=sys.stderr)
sys.exit(1)
commit_msg_file = sys.argv[2]
else:
print(f"Unknown hook type: {hook_type}", file=sys.stderr)
sys.exit(1)
if not hook_script.exists():
print(f"Hook script not found: {hook_script}", file=sys.stderr)
sys.exit(1)
# Call the hook script
try:
if hook_type == "commit-msg":
result = subprocess.run(
[str(hook_script), commit_msg_file],
check=False
)
else:
result = subprocess.run(
[str(hook_script)],
check=False
)
sys.exit(result.returncode)
except Exception as e:
print(f"Error running hook: {e}", file=sys.stderr)
sys.exit(1)
if __name__ == "__main__":
main()
+118
View File
@@ -0,0 +1,118 @@
#!/usr/bin/env bash
# Guard commit messages against explicit AI usage indicators.
# The script rejects messages containing phrases that suggest the text
# was generated by an AI assistant or tool (as defined in onboarding docs).
set -euo pipefail
MSG_FILE=${1:?}
# Patterns to block; all matches are case-insensitive (grep -i).
declare -a PATTERNS=(
"generated by cursor"
"cursor ai"
"cursor wrote this"
"cursor suggestion"
"generated by claude"
"claude wrote this"
"claude ai"
"claude opus"
"claude sonnet"
"claude haiku"
"claude code"
"generated with claude code"
"generated with \\[claude code\\]"
"co-authored-by: claude"
"noreply@anthropic\\.com"
"generated by gemini"
"gemini wrote this"
"google gemini"
"gemini ultra"
"gemini pro"
"gemini nano"
"generated by bard"
"bard wrote this"
"google bard"
"as an ai assistant"
"as an ai"
"ai assistant"
"ai suggestion"
"ai suggestions"
"ai-generated"
"ai generated"
"ai-assisted"
"ai assisted"
"ai-powered"
"ai powered"
"powered by ai"
"with ai assistance"
"with the help of ai"
"assisted by ai"
"written with ai"
"written by ai"
"generated with ai"
"generated via ai"
"created with ai"
"created by ai"
"authored by ai"
"co-authored by ai"
"ai help"
"ai helped"
"ai copilot"
"ai co-pilot"
"chatgpt"
"gpt-4"
"gpt4"
"gpt-3"
"gpt3"
"openai"
"anthropic"
"google ai"
"google deepmind"
"deepmind"
"perplexity"
"perplexity ai"
"tabnine"
"tabnine ai"
"replit ghostwriter"
"ghostwriter ai"
"aws codewhisperer"
"amazon codewhisperer"
"code whisperer"
"codewhisperer"
"github copilot"
"copilot"
"amazon q"
"amazon q developer"
"meta ai"
"deepseek"
"phind"
"blackbox ai"
"magic ai"
"ai model"
"llm"
"large language model"
"language model"
"\\[ai\\]"
"\\[ai-generated\\]"
"\\[cursor\\]"
"\\[claude\\]"
"\\[gemini\\]"
"\\[copilot\\]"
"\\[chatgpt\\]"
"\\[llm\\]"
)
for pattern in "${PATTERNS[@]}"; do
if LC_ALL=C grep -qiE "$pattern" "$MSG_FILE"; then
cat >&2 <<EOF
✖ Commit message blocked: found AI indicator "$pattern".
Please rewrite the message without explicitly referencing AI tools or prompts.
EOF
exit 1
fi
done
exit 0
+171
View File
@@ -0,0 +1,171 @@
#!/usr/bin/env bash
# Block commits when staged files contain explicit AI usage indicators.
# Mirrors the onboarding guidance so reviewers remain blinded to AI tooling.
# Markdown (.md) files are explicitly scanned to prevent AI flag text in docs.
set -euo pipefail
declare -a PATTERNS=(
"generated by cursor"
"cursor ai"
"cursor wrote this"
"cursor suggestion"
"generated by claude"
"claude wrote this"
"claude ai"
"claude opus"
"claude sonnet"
"claude haiku"
"claude code"
"generated with claude code"
"generated with \\[claude code\\]"
"co-authored-by: claude"
"noreply@anthropic\\.com"
"generated by gemini"
"gemini wrote this"
"google gemini"
"gemini ultra"
"gemini pro"
"gemini nano"
"generated by bard"
"bard wrote this"
"google bard"
"as an ai assistant"
"as an ai"
"ai assistant"
"ai suggestion"
"ai suggestions"
"ai-generated"
"ai generated"
"ai-assisted"
"ai assisted"
"ai-powered"
"ai powered"
"powered by ai"
"with ai assistance"
"with the help of ai"
"assisted by ai"
"written with ai"
"written by ai"
"generated with ai"
"generated via ai"
"created with ai"
"created by ai"
"authored by ai"
"co-authored by ai"
"ai help"
"ai helped"
"ai copilot"
"ai co-pilot"
"chatgpt"
"gpt-4"
"gpt4"
"gpt-3"
"gpt3"
"openai"
"anthropic"
"google ai"
"google deepmind"
"deepmind"
"perplexity"
"perplexity ai"
"tabnine"
"tabnine ai"
"replit ghostwriter"
"ghostwriter ai"
"aws codewhisperer"
"amazon codewhisperer"
"code whisperer"
"codewhisperer"
"github copilot"
"copilot"
"amazon q"
"amazon q developer"
"meta ai"
"deepseek"
"phind"
"blackbox ai"
"magic ai"
"ai model"
"llm"
"large language model"
"language model"
"\\[ai\\]"
"\\[ai-generated\\]"
"\\[cursor\\]"
"\\[claude\\]"
"\\[gemini\\]"
"\\[copilot\\]"
"\\[chatgpt\\]"
"\\[llm\\]"
)
PATTERN_REGEX=$(printf '%s|' "${PATTERNS[@]}")
PATTERN_REGEX=${PATTERN_REGEX%|}
# Build a newline-separated list of binary file paths (bash 3.2 compatible)
BINARY_PATHS=""
while IFS=$'\t' read -r added deleted path; do
[[ -z "${path:-}" ]] && continue
if [[ "$path" == *" -> "* ]]; then
path=${path##* -> }
fi
if [[ "$added" == "-" || "$deleted" == "-" ]]; then
BINARY_PATHS="${BINARY_PATHS}${path}"$'\n'
fi
done < <(git diff --cached --numstat --diff-filter=ACMRT || true)
report=""
current_file=""
last_report_file=""
violations=0
while IFS= read -r line; do
case "$line" in
"diff --git "*)
current_file=""
;;
"+++ "*)
current_file=${line#+++ }
if [[ "$current_file" == b/* ]]; then
current_file=${current_file#b/}
elif [[ "$current_file" == a/* ]]; then
current_file=${current_file#a/}
fi
if [[ "$current_file" == "/dev/null" ]]; then
current_file=""
fi
;;
"Binary files "*)
current_file=""
;;
*)
if [[ "$line" == +* && "$line" != "+++"* ]]; then
[[ -z "$current_file" ]] && continue
if [[ -n "$BINARY_PATHS" ]] && echo "$BINARY_PATHS" | grep -qxF "$current_file"; then
continue
fi
content=${line:1}
if [[ -n "$content" ]] && LC_ALL=C grep -qiE "$PATTERN_REGEX" <<<"$content"; then
if [[ "$current_file" != "$last_report_file" ]]; then
report+=$'\n'"• $current_file"
last_report_file="$current_file"
fi
report+=$'\n'" ${line}"
violations=1
fi
fi
;;
esac
done < <(git diff --cached --diff-filter=ACMRT --unified=0 --no-color)
if [[ $violations -eq 1 ]]; then
cat >&2 <<EOF
✖ Commit blocked: staged changes contain AI usage indicators in the current diff.
Please remove or rephrase the highlighted additions before committing:$report
EOF
exit 1
fi
exit 0
+17
View File
@@ -0,0 +1,17 @@
# Configure pre-commit to reuse the AI indicator guard scripts.
# Both hooks run locally from .config/ai-hooks.
repos:
- repo: local
hooks:
- id: ai-pre-commit
name: Block AI indicators in staged files
entry: python3 .config/git-hooks/ai_guard.py pre-commit
language: system
pass_filenames: false
stages: [pre-commit]
- id: ai-commit-msg
name: Block AI indicators in commit messages
entry: python3 .config/git-hooks/ai_guard.py commit-msg
language: system
stages: [commit-msg]
+2 -1
View File
@@ -103,6 +103,7 @@ RUN ARCH=$(uname -m) && \
COPY --from=install /temp/prod/node_modules node_modules
COPY --from=prerelease /app/public/ /app/public/
COPY --from=prerelease /app/dist /app/dist
COPY entrypoint.sh /app/entrypoint.sh
# COPY . .
RUN mkdir data
@@ -111,4 +112,4 @@ EXPOSE 3000/tcp
# used for calibre
ENV QTWEBENGINE_CHROMIUM_FLAGS="--no-sandbox"
ENV NODE_ENV=production
ENTRYPOINT [ "bun", "run", "dist/src/index.js" ]
ENTRYPOINT [ "/app/entrypoint.sh" ]
+2
View File
@@ -101,6 +101,8 @@ All are optional, JWT_SECRET is recommended to be set.
| LANGUAGE | en | Language to format date strings in, specified as a [BCP 47 language tag](https://en.wikipedia.org/wiki/IETF_language_tag) |
| UNAUTHENTICATED_USER_SHARING | false | Shares conversion history between all unauthenticated users |
| MAX_CONVERT_PROCESS | 0 | Maximum number of concurrent conversion processes allowed. Set to 0 for unlimited. |
| MAGICK_MAX_WIDTH | 8000 | (Docker only) ImageMagick maximum image width in pixels |
| MAGICK_MAX_HEIGHT | 8000 | (Docker only) ImageMagick maximum image height in pixels |
| PORT | 3000 | Application listen port |
### Docker images
+6
View File
@@ -465,6 +465,9 @@
"micromatch": ["micromatch@4.0.8", "", { "dependencies": { "braces": "^3.0.3", "picomatch": "^2.3.1" } }, "sha512-PXwfBhYu0hBCPw8Dn0E+WDYb7af3dSLVWKi3HGv84IdF4TyFoC0ysxFd0Goxw7nSv4T/PzEJQxsYsEiFCKo2BA=="],
"mime": ["mime@4.1.0", "", { "bin": { "mime": "bin/cli.js" } }, "sha512-X5ju04+cAzsojXKes0B/S4tcYtFAJ6tTMuSPBEn9CPGlrWr8Fiw7qYeLT0XyH80HSoAoqWCaz+MWKh22P7G1cw=="],
"minimatch": ["minimatch@3.1.2", "", { "dependencies": { "brace-expansion": "^1.1.7" } }, "sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw=="],
"minimatch": ["minimatch@10.2.6", "", { "dependencies": { "brace-expansion": "^5.0.8" } }, "sha512-vpLQEs+VLCr1nU0BXS07maYoFwlDAH0gngQuuttxIwutDFEMHq2blX+8vpgxDdK3J1PwjCJiep77OitTZ4Ll1A=="],
"minimist": ["minimist@1.2.8", "", {}, "sha512-2yyAR8qBkN3YuheJanUpWC5U3bb5osDywNB8RzDVlDwDHbocAJveqqj1u8+SVD7jkWT4yvsHCpWqqWqAxb0zCA=="],
@@ -525,6 +528,7 @@
"resolve-pkg-maps": ["resolve-pkg-maps@1.0.0", "", {}, "sha512-seS2Tj26TBVOC2NIc2rOe2y2ZO7efxITtLZcGSOnHHNOQ7CkiUBfw0Iw2ck6xkIhPwLhKNLS8BO+hEpngQlqzw=="],
"semver": ["semver@7.7.3", "", { "bin": { "semver": "bin/semver.js" } }, "sha512-SdsKMrI9TdgjdweUSR9MweHA4EJ8YxHn8DFaDisvhVlUOe4BF1tLD7GAj0lIqWVl+dPb/rExr0Btby5loQm20Q=="],
"sanitize-filename": ["sanitize-filename@1.6.4", "", { "dependencies": { "truncate-utf8-bytes": "^1.0.0" } }, "sha512-9ZyI08PsvdQl2r/bBIGubpVdR3RR9sY6RDiWFPreA21C/EFlQhmgo20UZlNjZMMZNubusLhAQozkA0Od5J21Eg=="],
"semver": ["semver@7.7.4", "", { "bin": { "semver": "bin/semver.js" } }, "sha512-vFKC2IEtQnVhpT78h1Yp8wzwrf8CM+MzKMHGJZfBtzhZNycRFnXsHk6E5TxIkkMsgNS7mdX3AGB7x2QM2di4lA=="],
@@ -569,6 +573,7 @@
"token-types": ["token-types@6.1.2", "", { "dependencies": { "@borewit/text-codec": "^0.2.1", "@tokenizer/token": "^0.3.0", "ieee754": "^1.2.1" } }, "sha512-dRXchy+C0IgK8WPC6xvCHFRIWYUbqqdEIKPaKo/AcTUNzwLTK6AH7RjdLWsEZcAN/TBdtfUw3PYEgPr5VPr6ww=="],
"ts-api-utils": ["ts-api-utils@2.4.0", "", { "peerDependencies": { "typescript": ">=4.8.4" } }, "sha512-3TaVTaAv2gTiMB35i3FiGJaRfwb3Pyn/j3m/bfAvGe8FB7CF6u+LMYqYlDh7reQf7UNvoTvdfAqHGmPGOSsPmA=="],
"truncate-utf8-bytes": ["truncate-utf8-bytes@1.0.2", "", { "dependencies": { "utf8-byte-length": "^1.0.1" } }, "sha512-95Pu1QXQvruGEhv62XCMO3Mm90GscOCClvrIUwCM0PYOXK3kaF3l3sIHxx71ThJfcbM2O5Au6SO3AWCSEfW4mQ=="],
"ts-api-utils": ["ts-api-utils@2.5.0", "", { "peerDependencies": { "typescript": ">=4.8.4" } }, "sha512-OJ/ibxhPlqrMM0UiNHJ/0CKQkoKF243/AEmplt3qpRgkW8VG7IfOS41h7V8TjITqdByHzrjcS/2si+y4lIh8NA=="],
@@ -593,6 +598,7 @@
"uri-js": ["uri-js@4.4.1", "", { "dependencies": { "punycode": "^2.1.0" } }, "sha512-7rKUyy33Q1yc98pQ1DAmLtwX109F7TIfWlW1Ydo8Wl1ii1SeHieeh0HHfPeL2fMXK6z0s8ecKs9frCuLJvndBg=="],
"valibot": ["valibot@1.2.0", "", { "peerDependencies": { "typescript": ">=5" }, "optionalPeers": ["typescript"] }, "sha512-mm1rxUsmOxzrwnX5arGS+U4T25RdvpPjPN4yR0u9pUBov9+zGVtO84tif1eY4r6zWxVxu3KzIyknJy3rxfRZZg=="],
"utf8-byte-length": ["utf8-byte-length@1.0.5", "", {}, "sha512-Xn0w3MtiQ6zoz2vFyUVruaCL53O/DwUvkEeOvj+uulMm0BkUGYWmBYVyElqZaSLhY6ZD0ulfU3aBra2aVT4xfA=="],
"valibot": ["valibot@1.4.2", "", { "peerDependencies": { "typescript": ">=5" }, "optionalPeers": ["typescript"] }, "sha512-gjdCvJ6d3RyHAneqxMYMW9QMCwYMb3jpOO0IyHZV1bnRHFBHrX3VkIILt5XYR0WhwHiH7Mty8ovuPZ/O3gamrg=="],
Executable
+20
View File
@@ -0,0 +1,20 @@
#!/bin/bash
set -e
if [ -n "$MAGICK_MAX_WIDTH" ] || [ -n "$MAGICK_MAX_HEIGHT" ]; then
POLICY_DIR="/etc/ImageMagick-7"
POLICY_FILE="$POLICY_DIR/policy.xml"
mkdir -p "$POLICY_DIR"
echo "<policymap>" > "$POLICY_FILE"
if [ -n "$MAGICK_MAX_WIDTH" ]; then
echo " <policy domain=\"resource\" name=\"width\" value=\"$MAGICK_MAX_WIDTH\"/>" >> "$POLICY_FILE"
fi
if [ -n "$MAGICK_MAX_HEIGHT" ]; then
echo " <policy domain=\"resource\" name=\"height\" value=\"$MAGICK_MAX_HEIGHT\"/>" >> "$POLICY_FILE"
fi
echo "</policymap>" >> "$POLICY_FILE"
fi
exec bun run dist/src/index.js
+1
View File
@@ -51,6 +51,7 @@ export default defineConfig(
"target",
"convert_to_target",
"job-details-toggle",
"share-btn",
],
},
],
+1
View File
@@ -22,6 +22,7 @@
"@elysiajs/static": "^1.4.10",
"@kitajs/html": "^4.2.13",
"elysia": "1.4.22",
"mime": "^4.1.0",
"sanitize-filename": "^1.6.4",
"tar": "^7.5.22"
},
+38
View File
@@ -3,6 +3,42 @@ const jobId = window.location.pathname.split("/").pop();
const main = document.querySelector("main");
let progressElem = document.querySelector("progress");
const supportsWebShare = navigator.share && navigator.canShare;
const setupShareButtons = () => {
if (!supportsWebShare) {
return;
}
document.querySelectorAll(".share-btn").forEach((btn) => {
if (btn.dataset.setupComplete) return;
const filename = btn.dataset.filename;
const mimeType = btn.dataset.mimeType;
const fileUrl = btn.dataset.downloadUrl;
const dummyFile = new File([], filename, { type: mimeType });
if (!navigator.canShare({ files: [dummyFile] })) {
return;
}
btn.addEventListener("click", async (e) => {
e.preventDefault();
try {
const response = await fetch(fileUrl);
const blob = await response.blob();
const file = new File([blob], filename, { type: mimeType });
await navigator.share({ files: [file] });
} catch (err) {
if (err.name !== "AbortError") {
console.error("Error sharing:", err);
}
}
});
btn.style.display = "";
btn.dataset.setupComplete = true;
});
};
const refreshData = () => {
// console.log("Refreshing data...", progressElem.value, progressElem.max);
if (progressElem.value !== progressElem.max) {
@@ -12,6 +48,7 @@ const refreshData = () => {
.then((res) => res.text())
.then((html) => {
main.innerHTML = html;
setupShareButtons();
})
.catch((err) => console.log(err));
@@ -21,6 +58,7 @@ const refreshData = () => {
progressElem = document.querySelector("progress");
};
setupShareButtons();
refreshData();
window.downloadAll = function () {
+71 -11
View File
@@ -1,4 +1,6 @@
const webroot = document.querySelector("meta[name='webroot']").content;
const urlInput = document.querySelector("#url-input");
const urlSubmit = document.querySelector("#url-submit");
const fileInput = document.querySelector('input[type="file"]');
const dropZone = document.getElementById("dropzone");
const convertButton = document.querySelector("input[type='submit']");
@@ -16,37 +18,95 @@ dropZone.addEventListener("dragleave", () => {
dropZone.classList.remove("dragover");
});
dropZone.addEventListener("drop", (e) => {
const entryToFiles = async (entry) => {
if (entry.isFile) {
const file = await new Promise((resolve, reject) => entry.file(resolve, reject));
return [file];
}
const reader = entry.createReader();
const entries = await new Promise((resolve, reject) => reader.readEntries(resolve, reject));
const out = [];
for (const entry of entries) {
out.push(...(await entryToFiles(entry)));
}
return out;
};
const getFileNameWithSlashes = (file) => {
if (file.webkitRelativePath) {
return file.webkitRelativePath;
} else {
return file.name;
}
};
dropZone.addEventListener("drop", async (e) => {
e.preventDefault();
dropZone.classList.remove("dragover");
const files = e.dataTransfer.files;
const items = e.dataTransfer.items;
if (files.length === 0) {
if (items.length === 0) {
console.warn("No files dropped — likely a URL or unsupported source.");
return;
}
for (const file of files) {
console.log("Handling dropped file:", file.name);
handleFile(file);
for (const item of items) {
const entry = item.webkitGetAsEntry();
if (!entry) {
return;
}
const files = await entryToFiles(entry);
for (const file of files) {
console.log("Handling dropped file:", getFileNameWithSlashes(file));
handleFile(file);
}
}
});
urlSubmit.addEventListener("click", (e) => {
e.preventDefault();
handleUrl(urlInput.value);
});
function handleUrl(url) {
fetch(`${webroot}/url`, {
method: "POST",
body: JSON.stringify({ url }),
headers: { "Content-Type": "application/json" },
})
.then((res) => res.json())
.then((res) => {
const fileList = document.querySelector("#file-list");
const row = document.createElement("tr");
row.innerHTML = `
<td>${res.filename}</td>
<td></td>
<td>${(res.fileSizeBytes / 1024).toFixed(2)} kB</td>
<td><a onclick="deleteRow(this)">Remove</a></td>
`;
fileList.appendChild(row);
fileNames.push(res.filename);
})
.catch(console.error);
}
// Extracted handleFile function for reusability in drag-and-drop and file input
function handleFile(file) {
const fileList = document.querySelector("#file-list");
const row = document.createElement("tr");
row.innerHTML = `
<td>${file.name}</td>
<td>${getFileNameWithSlashes(file)}</td>
<td><progress max="100" class="inline-block h-2 appearance-none overflow-hidden rounded-full border-0 bg-neutral-700 bg-none text-accent-500 accent-accent-500 [&::-moz-progress-bar]:bg-accent-500 [&::-webkit-progress-value]:rounded-full [&::-webkit-progress-value]:[background:none] [&[value]::-webkit-progress-value]:bg-accent-500 [&[value]::-webkit-progress-value]:transition-[inline-size]"></progress></td>
<td>${(file.size / 1024).toFixed(2)} kB</td>
<td><button type="button" class="text-accent-500 hover:underline" onclick="deleteRow(this)">Remove</button></td>
`;
if (!fileType) {
fileType = file.name.split(".").pop();
fileType = getFileNameWithSlashes(file).split(".").pop();
fileInput.setAttribute("accept", `.${fileType}`);
setTitle();
@@ -65,7 +125,7 @@ function handleFile(file) {
fileList.appendChild(row);
file.htmlRow = row;
fileNames.push(file.name);
fileNames.push(getFileNameWithSlashes(file));
uploadFile(file);
}
@@ -202,7 +262,7 @@ const uploadFile = (file) => {
pendingFiles += 1;
const formData = new FormData();
formData.append("file", file, file.name);
formData.append("file", file, getFileNameWithSlashes(file));
let xhr = new XMLHttpRequest();
@@ -228,7 +288,7 @@ const uploadFile = (file) => {
xhr.upload.onprogress = (e) => {
let sent = e.loaded;
let total = e.total;
console.log(`upload progress (${file.name}):`, (100 * sent) / total);
console.log(`upload progress (${getFileNameWithSlashes(file)}):`, (100 * sent) / total);
let progressbar = file.htmlRow.getElementsByTagName("progress");
progressbar[0].value = (100 * sent) / total;
+29 -12
View File
@@ -1,6 +1,7 @@
import { Cookie } from "elysia";
import db from "../db/db";
import { MAX_CONVERT_PROCESS } from "../helpers/env";
import { isSafePath } from "../helpers/validatePath";
import { normalizeFiletype, normalizeOutputFiletype } from "../helpers/normalizeFiletype";
import { convert as convertassimp, properties as propertiesassimp } from "./assimp";
import { convert as convertCalibre, properties as propertiesCalibre } from "./calibre";
@@ -25,6 +26,8 @@ import { convert as convertVtracer, properties as propertiesVtracer } from "./vt
import { convert as convertVcf, properties as propertiesVcf } from "./vcf";
import { convert as convertxelatex, properties as propertiesxelatex } from "./xelatex";
import { convert as convertMarkitdown, properties as propertiesMarkitdown } from "./markitdown";
import path from "node:path";
import fs from "node:fs/promises";
// This should probably be reconstructed so that the functions are not imported instead the functions hook into this to make the converters more modular
@@ -163,23 +166,37 @@ export async function handleConvert(
for (const chunk of chunks(fileNames, MAX_CONVERT_PROCESS)) {
const toProcess: Promise<string>[] = [];
for (const fileName of chunk) {
const filePath = `${userUploadsDir}${fileName}`;
const fileTypeOrig = fileName.includes(".") ? (fileName.split(".").pop() ?? "") : "";
// `isSafePath` ensures the path as parsed by JS is safe, but if we pass the path to external programs, we don't
// want to be affected by a parser differential so we normalize with JS' parsing
const filePath = path.normalize(`${userUploadsDir}${fileName}`);
if (!isSafePath(userUploadsDir, filePath)) {
throw new Error("Unsafe filename");
}
const fileTypeOrig = fileName.split(".").pop() ?? "";
const fileType = normalizeFiletype(fileTypeOrig);
const newFileExt = normalizeOutputFiletype(convertTo);
let newFileName: string;
if (fileTypeOrig === "") {
newFileName = `${fileName}.${newFileExt}`;
} else {
newFileName = fileName.replace(
new RegExp(`${fileTypeOrig}(?!.*${fileTypeOrig})`),
newFileExt,
);
const newFileName = fileName.replace(
new RegExp(`${fileTypeOrig}(?!.*${fileTypeOrig})`),
newFileExt,
);
// `isSafePath` ensures the path as parsed by JS is safe, but if we pass the path to external programs, we don't
// want to be affected by a parser differential so we normalize with JS' parsing
// This shouldn't be unsafe as we've already validated the original filename is safe, but if a filename is
// provided which *attempts* to be malicious, `targetPath` could theoretically differ to `newFileName` due to
// normalization
const targetPath = path.normalize(`${userOutputDir}${newFileName}`);
if (!isSafePath(userOutputDir, targetPath)) {
throw new Error("Unsafe filename");
}
const targetPath = `${userOutputDir}${newFileName}`;
toProcess.push(
new Promise((resolve, reject) => {
mainConverter(filePath, fileType, convertTo, targetPath, {}, converterName)
const parent = path.dirname(targetPath);
fs.mkdir(parent, { recursive: true })
.then(() => {
return mainConverter(filePath, fileType, convertTo, targetPath, {}, converterName);
})
.then((r) => {
if (jobId.value) {
query.run(jobId.value, fileName, newFileName, r);
+6
View File
@@ -0,0 +1,6 @@
import path from "node:path";
export const isSafePath = (expectedParent: string, userInput: string): boolean => {
const relativePath = path.relative(expectedParent, userInput);
return !!relativePath && !relativePath.startsWith("..") && !path.isAbsolute(relativePath);
};
+18
View File
@@ -0,0 +1,18 @@
export function ShareIcon() {
return (
<svg
xmlns="http://www.w3.org/2000/svg"
fill="none"
viewBox="0 0 24 24"
strokeWidth="1.5"
stroke="currentColor"
class={`size-6`}
>
<path
stroke-linecap="round"
stroke-linejoin="round"
d="M7.217 10.907a2.25 2.25 0 1 0 0 2.186m0-2.186c.18.324.283.696.283 1.093s-.103.77-.283 1.093m0-2.186 9.566-5.314m-9.566 7.5 9.566 5.314m0 0a2.25 2.25 0 1 0 3.935 2.186 2.25 2.25 0 0 0-3.935-2.186Zm0-12.814a2.25 2.25 0 1 0 3.933-2.185 2.25 2.25 0 0 0-3.933 2.185Z"
/>
</svg>
);
}
+2
View File
@@ -16,6 +16,7 @@ import { listConverters } from "./pages/listConverters";
import { results } from "./pages/results";
import { root } from "./pages/root";
import { upload } from "./pages/upload";
import { url } from "./pages/url";
import { user } from "./pages/user";
import { healthcheck } from "./pages/healthcheck";
@@ -41,6 +42,7 @@ const app = new Elysia({
.use(user)
.use(root)
.use(upload)
.use(url)
.use(history)
.use(convert)
.use(download)
-5
View File
@@ -1,6 +1,5 @@
import { mkdir } from "node:fs/promises";
import { Elysia, t } from "elysia";
import sanitize from "sanitize-filename";
import { outputDir, uploadsDir } from "..";
import { handleConvert } from "../converters/main";
import db from "../db/db";
@@ -58,10 +57,6 @@ export const convert = new Elysia().use(userService).post(
const fileNames = JSON.parse(body.file_names) as string[];
for (let i = 0; i < fileNames.length; i++) {
fileNames[i] = sanitize(fileNames[i] || "");
}
if (!Array.isArray(fileNames) || fileNames.length === 0) {
return redirect(`${WEBROOT}/`, 302);
}
+5 -3
View File
@@ -3,8 +3,8 @@ import { Elysia, t } from "elysia";
import { uploadsDir } from "..";
import db from "../db/db";
import { WEBROOT } from "../helpers/env";
import { isSafePath } from "../helpers/validatePath";
import { userService } from "./user";
import sanitize from "sanitize-filename";
import path from "node:path";
export const deleteFile = new Elysia().use(userService).post(
@@ -24,8 +24,10 @@ export const deleteFile = new Elysia().use(userService).post(
const userUploadsDir = path.join(uploadsDir, user.id, jobId.value);
const sanitized = sanitize(body.filename);
const targetPath = path.join(userUploadsDir, sanitized);
const targetPath = path.join(userUploadsDir, body.filename);
if (!isSafePath(userUploadsDir, targetPath)) {
throw new Error("Unsafe filename");
}
await unlink(targetPath);
+7 -5
View File
@@ -1,16 +1,16 @@
import path from "node:path";
import { Elysia } from "elysia";
import sanitize from "sanitize-filename";
import * as tar from "tar";
import { outputDir } from "..";
import db from "../db/db";
import { WEBROOT } from "../helpers/env";
import { isSafePath } from "../helpers/validatePath";
import { userService } from "./user";
export const download = new Elysia()
.use(userService)
.get(
"/download/:userId/:jobId/:fileName",
"/download/:userId/:jobId/*",
async ({ params, redirect, user }) => {
const userId = user.id;
const job = await db
@@ -22,9 +22,11 @@ export const download = new Elysia()
}
// parse from URL encoded string
const jobId = decodeURIComponent(params.jobId);
const fileName = sanitize(decodeURIComponent(params.fileName));
const filePath = `${outputDir}${userId}/${jobId}/${fileName}`;
const jobPath = `${outputDir}${userId}/${jobId}/`;
const filePath = `${jobPath}${decodeURIComponent(params["*"])}`;
if (!isSafePath(jobPath, filePath)) {
throw new Error("Unsafe filename");
}
return Bun.file(filePath);
},
{
+14
View File
@@ -8,6 +8,8 @@ import { ALLOW_UNAUTHENTICATED, WEBROOT } from "../helpers/env";
import { DownloadIcon } from "../icons/download";
import { DeleteIcon } from "../icons/delete";
import { EyeIcon } from "../icons/eye";
import { ShareIcon } from "../icons/share";
import mime from "mime";
import { userService } from "./user";
function ResultsArticle({
@@ -122,6 +124,18 @@ function ResultsArticle({
>
<DownloadIcon />
</a>
<button
class={`
share-btn text-accent-500 underline
hover:text-accent-400
`}
data-filename={file.output_file_name}
data-download-url={`${WEBROOT}/download/${outputPath}${file.output_file_name}`}
data-mime-type={mime.getType(file.output_file_name) || "application/octet-stream"}
style="display: none;"
>
<ShareIcon />
</button>
</td>
</tr>
))}
+15 -1
View File
@@ -143,7 +143,7 @@ export const root = new Elysia().use(userService).get(
`}
>
<span>
<b>Choose a file</b> or drag it here
<b>Choose a file</b> or drag a file or folder here
</span>
<input
type="file"
@@ -152,6 +152,20 @@ export const root = new Elysia().use(userService).get(
class="absolute inset-0 size-full cursor-pointer opacity-0"
/>
</div>
<label class="mt-4 flex flex-col gap-1 text-neutral-400">
Or enter a URL
<div class="flex flex-row">
<input
name="url"
id="url-input"
class="mr-2 flex-auto rounded-sm bg-neutral-800 p-3"
placeholder="https://example.com/cat.jpg"
/>
<button id="url-submit" type="button" class="flex-initial btn-secondary">
Add URL
</button>
</div>
</label>
</article>
<form
method="post"
+11 -5
View File
@@ -1,9 +1,9 @@
import { Elysia, t } from "elysia";
import db from "../db/db";
import { WEBROOT } from "../helpers/env";
import { isSafePath } from "../helpers/validatePath";
import { uploadsDir } from "../index";
import { userService } from "./user";
import sanitize from "sanitize-filename";
export const upload = new Elysia().use(userService).post(
"/upload",
@@ -25,12 +25,18 @@ export const upload = new Elysia().use(userService).post(
if (body?.file) {
if (Array.isArray(body.file)) {
for (const file of body.file) {
const santizedFileName = sanitize(file.name);
await Bun.write(`${userUploadsDir}${santizedFileName}`, file);
const filePath = `${userUploadsDir}${file.name}`;
if (!isSafePath(userUploadsDir, filePath)) {
throw new Error("Unsafe filename");
}
await Bun.write(filePath, file);
}
} else {
const santizedFileName = sanitize(body.file["name"]);
await Bun.write(`${userUploadsDir}${santizedFileName}`, body.file);
const filePath = `${userUploadsDir}${body.file["name"]}`;
if (!isSafePath(userUploadsDir, filePath)) {
throw new Error("Unsafe filename");
}
await Bun.write(filePath, body.file);
}
}
+65
View File
@@ -0,0 +1,65 @@
import { Elysia, t } from "elysia";
import { uploadsDir } from "..";
import db from "../db/db";
import { WEBROOT } from "../helpers/env";
import { userService } from "./user";
import sanitize from "sanitize-filename";
import { randomUUID } from "node:crypto";
import mime from "mime";
const getFilename = (url: string, headers: Headers) => {
const contentDisposition = headers.get("Content-Disposition");
if (contentDisposition) {
const match = /filename="([^"]+)"/.exec(contentDisposition);
if (match && match[1]) {
return sanitize(match[1]);
}
}
const path = new URL(url).pathname;
const lastPart = path.split("/").at(-1);
const contentType = headers.get("content-type");
const extension = contentType ? mime.getExtension(contentType) : null;
if (!lastPart) {
if (extension) {
return `${randomUUID()}.${extension}`;
}
return randomUUID();
}
if (!lastPart.includes(".") && extension) {
return `${sanitize(lastPart)}.${extension}`;
}
return sanitize(lastPart);
};
export const url = new Elysia().use(userService).post(
"/url",
async ({ body, redirect, user, cookie: { jobId } }) => {
if (!jobId?.value) {
return redirect(`${WEBROOT}/`, 302);
}
const existingJob = await db
.query("SELECT * FROM jobs WHERE id = ? AND user_id = ?")
.get(jobId.value, user.id);
if (!existingJob) {
return redirect(`${WEBROOT}/`, 302);
}
const userUploadsDir = `${uploadsDir}${user.id}/${jobId.value}/`;
const res = await fetch(body.url);
if (!res.ok) {
throw new Error(`Failed to download URL, received ${res.status}`);
}
const filename = getFilename(body.url, res.headers);
const fileSizeBytes = await Bun.write(`${userUploadsDir}${filename}`, await res.blob());
return {
message: "Files downloaded successfully.",
filename,
fileSizeBytes,
};
},
{ body: t.Object({ url: t.String() }), auth: true },
);
+34
View File
@@ -0,0 +1,34 @@
import { expect, test, describe } from "bun:test";
import { isSafePath } from "../src/helpers/validatePath";
describe("isSafePath", () => {
test("should allow no slash", () => {
const path = "cat.jpg";
const result = isSafePath("./job/", `./job/${path}`);
expect(result).toEqual(true);
});
test("should allow safe slash", () => {
const path = "dir/cat.jpg";
const result = isSafePath("./job/", `./job/${path}`);
expect(result).toEqual(true);
});
test("should allow leading slash", () => {
const path = "/dir/cat.jpg";
const result = isSafePath("./job/", `./job/${path}`);
expect(result).toEqual(true);
});
test("should allow leading dots", () => {
const path = "dir/..cat.jpg";
const result = isSafePath("./job/", `./job/${path}`);
expect(result).toEqual(true);
});
test("should disallow parent", () => {
const path = "../cat.jpg";
const result = isSafePath("./job/", `./job/${path}`);
expect(result).toEqual(false);
});
});