feat: bilibili subtitle download

Also added backend/data/cookies.txt to .gitignore
This commit is contained in:
Peifan Li
2025-12-04 15:37:30 -05:00
parent 51e55bd0a5
commit 50e821784a
17 changed files with 408 additions and 4 deletions

3
.gitignore vendored
View File

@@ -55,3 +55,6 @@ backend/data/*.db
backend/data/*.db-journal
backend/data/status.json
backend/data/settings.json
# Sensitive data
backend/data/cookies.txt

View File

@@ -186,3 +186,27 @@ export const verifyPassword = async (req: Request, res: Response) => {
res.status(500).json({ error: 'Failed to verify password' });
}
};
export const uploadCookies = async (req: Request, res: Response) => {
try {
if (!req.file) {
return res.status(400).json({ error: 'No file uploaded' });
}
const { DATA_DIR } = require('../config/paths');
const targetPath = path.join(DATA_DIR, 'cookies.txt');
// Move the file to the target location
fs.moveSync(req.file.path, targetPath, { overwrite: true });
console.log(`Cookies uploaded and saved to ${targetPath}`);
res.json({ success: true, message: 'Cookies uploaded successfully' });
} catch (error: any) {
console.error('Error uploading cookies:', error);
// Clean up temp file if it exists
if (req.file && fs.existsSync(req.file.path)) {
fs.unlinkSync(req.file.path);
}
res.status(500).json({ error: 'Failed to upload cookies', details: error.message });
}
};

View File

@@ -1,12 +1,16 @@
import express from 'express';
import { deleteLegacyData, getSettings, migrateData, updateSettings, verifyPassword } from '../controllers/settingsController';
import multer from 'multer';
import os from 'os';
import { deleteLegacyData, getSettings, migrateData, updateSettings, uploadCookies, verifyPassword } from '../controllers/settingsController';
const router = express.Router();
const upload = multer({ dest: os.tmpdir() });
router.get('/', getSettings);
router.post('/', updateSettings);
router.post('/verify-password', verifyPassword);
router.post('/migrate', migrateData);
router.post('/delete-legacy', deleteLegacyData);
router.post('/upload-cookies', upload.single('file'), uploadCookies);
export default router;

View File

@@ -1,6 +1,9 @@
import fs from "fs-extra";
import { SUBTITLES_DIR } from "../config/paths";
import { BilibiliDownloader } from "../services/downloaders/BilibiliDownloader";
import * as storageService from "../services/storageService";
import { sanitizeFilename } from "../utils/helpers";
/**
* Scan subtitle directory and update video records with subtitle metadata
@@ -30,7 +33,31 @@ async function rescanSubtitles() {
continue;
}
// Look for subtitle files matching this video
// If it's a Bilibili video, try to download subtitles
if (video.source === 'bilibili' && video.sourceUrl) {
console.log(`Attempting to download subtitles for Bilibili video: ${video.title}`);
try {
// We need to reconstruct the base filename used during download
// Usually it's sanitizeFilename(title)_timestamp
// But we can just use the video ID (timestamp) as the base for subtitles
// to match the pattern expected by the system
const timestamp = video.id;
const safeBaseFilename = `${sanitizeFilename(video.title)}_${timestamp}`;
const downloadedSubtitles = await BilibiliDownloader.downloadSubtitles(video.sourceUrl, safeBaseFilename);
if (downloadedSubtitles.length > 0) {
storageService.updateVideo(video.id, { subtitles: downloadedSubtitles });
console.log(`Downloaded and linked ${downloadedSubtitles.length} subtitles for ${video.title}`);
updatedCount++;
continue; // Skip the local file check since we just downloaded them
}
} catch (e) {
console.error(`Failed to download subtitles for ${video.title}:`, e);
}
}
// Look for existing subtitle files matching this video (fallback)
const videoTimestamp = video.id;
const matchingSubtitles = subtitleFiles.filter((file) => file.includes(videoTimestamp));

View File

@@ -3,7 +3,8 @@ import fs from "fs-extra";
import path from "path";
// @ts-ignore
import { downloadByVedioPath } from "bilibili-save-nodejs";
import { IMAGES_DIR, VIDEOS_DIR } from "../../config/paths";
import { IMAGES_DIR, SUBTITLES_DIR, VIDEOS_DIR } from "../../config/paths";
import { bccToVtt } from "../../utils/bccToVtt";
import {
extractBilibiliVideoId,
sanitizeFilename
@@ -518,6 +519,16 @@ export class BilibiliDownloader {
console.error("Failed to get file size:", e);
}
// Download subtitles
let subtitles: Array<{ language: string; filename: string; path: string }> = [];
try {
console.log("Attempting to download subtitles...");
subtitles = await BilibiliDownloader.downloadSubtitles(url, newSafeBaseFilename);
console.log(`Downloaded ${subtitles.length} subtitles`);
} catch (e) {
console.error("Error downloading subtitles:", e);
}
// Create metadata for the video
const videoData: Video = {
id: timestamp.toString(),
@@ -528,6 +539,7 @@ export class BilibiliDownloader {
sourceUrl: url,
videoFilename: finalVideoFilename,
thumbnailFilename: thumbnailSaved ? finalThumbnailFilename : undefined,
subtitles: subtitles.length > 0 ? subtitles : undefined,
thumbnailUrl: thumbnailUrl || undefined,
videoPath: `/videos/${finalVideoFilename}`,
thumbnailPath: thumbnailSaved
@@ -749,4 +761,143 @@ export class BilibiliDownloader {
}
}
}
// Helper function to get cookies from cookies.txt
static getCookieHeader(): string {
try {
const { DATA_DIR } = require("../../config/paths");
const cookiesPath = path.join(DATA_DIR, "cookies.txt");
if (fs.existsSync(cookiesPath)) {
const content = fs.readFileSync(cookiesPath, "utf8");
const lines = content.split("\n");
const cookies = [];
for (const line of lines) {
if (line.startsWith("#") || !line.trim()) continue;
const parts = line.split("\t");
if (parts.length >= 7) {
const name = parts[5];
const value = parts[6].trim();
cookies.push(`${name}=${value}`);
}
}
return cookies.join("; ");
}
} catch (e) {
console.error("Error reading cookies.txt:", e);
}
return "";
}
// Helper function to download subtitles
static async downloadSubtitles(videoUrl: string, baseFilename: string): Promise<Array<{ language: string; filename: string; path: string }>> {
try {
const videoId = extractBilibiliVideoId(videoUrl);
if (!videoId) return [];
const cookieHeader = BilibiliDownloader.getCookieHeader();
if (!cookieHeader) {
console.warn("WARNING: No cookies found in cookies.txt. Bilibili subtitles usually require login.");
} else {
console.log(`Cookie header length: ${cookieHeader.length}`);
// Log first few chars to verify it's not empty/malformed
console.log(`Cookie header start: ${cookieHeader.substring(0, 20)}...`);
}
const headers = {
'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
'Referer': 'https://www.bilibili.com',
...(cookieHeader ? { 'Cookie': cookieHeader } : {})
};
// Get CID first
const viewApiUrl = `https://api.bilibili.com/x/web-interface/view?bvid=${videoId}`;
const viewResponse = await axios.get(viewApiUrl, { headers });
const cid = viewResponse.data?.data?.cid;
if (!cid) {
console.log("Could not find CID for video");
return [];
}
// Get subtitles
const playerApiUrl = `https://api.bilibili.com/x/player/v2?bvid=${videoId}&cid=${cid}`;
console.log(`Fetching subtitles from: ${playerApiUrl}`);
const playerResponse = await axios.get(playerApiUrl, { headers });
if (cookieHeader && !cookieHeader.includes("SESSDATA")) {
console.warn("WARNING: SESSDATA cookie not found! This is required for Bilibili authentication.");
}
let subtitlesData = playerResponse.data?.data?.subtitle?.subtitles;
// Fallback: Check if subtitles are in the view response (sometimes they are)
if (!subtitlesData || subtitlesData.length === 0) {
console.log("No subtitles in player API, checking view API response...");
// We already fetched viewResponse earlier to get CID
const viewSubtitles = viewResponse.data?.data?.subtitle?.list;
if (viewSubtitles && viewSubtitles.length > 0) {
console.log(`Found ${viewSubtitles.length} subtitles in view API`);
subtitlesData = viewSubtitles;
}
}
if (!subtitlesData) {
console.log("No subtitle field in response data");
} else if (!Array.isArray(subtitlesData)) {
console.log("Subtitles field is not an array");
} else {
console.log(`Found ${subtitlesData.length} subtitles`);
}
if (!subtitlesData || !Array.isArray(subtitlesData)) {
console.log("No subtitles found in API response");
return [];
}
const savedSubtitles = [];
// Ensure subtitles directory exists
fs.ensureDirSync(SUBTITLES_DIR);
for (const sub of subtitlesData) {
const lang = sub.lan;
const subUrl = sub.subtitle_url;
if (!subUrl) continue;
// Ensure URL is absolute (sometimes it starts with //)
const absoluteSubUrl = subUrl.startsWith('//') ? `https:${subUrl}` : subUrl;
console.log(`Downloading subtitle (${lang}): ${absoluteSubUrl}`);
// Do NOT send cookies to the subtitle CDN (hdslb.com) as it can cause 400 Bad Request (Header too large)
// and they are not needed for the CDN file itself.
const cdnHeaders = {
'User-Agent': headers['User-Agent'],
'Referer': headers['Referer']
};
const subResponse = await axios.get(absoluteSubUrl, { headers: cdnHeaders });
const vttContent = bccToVtt(subResponse.data);
if (vttContent) {
const subFilename = `${baseFilename}.${lang}.vtt`;
const subPath = path.join(SUBTITLES_DIR, subFilename);
fs.writeFileSync(subPath, vttContent);
savedSubtitles.push({
language: lang,
filename: subFilename,
path: `/subtitles/${subFilename}`
});
}
}
return savedSubtitles;
} catch (error) {
console.error("Error in downloadSubtitles:", error);
return [];
}
}
}

View File

@@ -0,0 +1,63 @@
/**
* Convert Bilibili BCC subtitle format to WebVTT
*/
interface BccItem {
from: number;
to: number;
location: number;
content: string;
}
interface BccBody {
font_size: number;
font_color: string;
background_alpha: number;
background_color: string;
Stroke: string;
type: string;
lang: string;
version: string;
body: BccItem[];
}
function formatTime(seconds: number): string {
const date = new Date(0);
date.setMilliseconds(seconds * 1000);
const hh = date.getUTCHours().toString().padStart(2, '0');
const mm = date.getUTCMinutes().toString().padStart(2, '0');
const ss = date.getUTCSeconds().toString().padStart(2, '0');
const ms = date.getUTCMilliseconds().toString().padStart(3, '0');
return `${hh}:${mm}:${ss}.${ms}`;
}
export function bccToVtt(bccContent: BccBody | string): string {
let bcc: BccBody;
if (typeof bccContent === 'string') {
try {
bcc = JSON.parse(bccContent);
} catch (e) {
console.error('Failed to parse BCC content', e);
return '';
}
} else {
bcc = bccContent;
}
if (!bcc.body || !Array.isArray(bcc.body)) {
return '';
}
let vtt = 'WEBVTT\n\n';
bcc.body.forEach((item) => {
const start = formatTime(item.from);
const end = formatTime(item.to);
vtt += `${start} --> ${end}\n`;
vtt += `${item.content}\n\n`;
});
return vtt;
}

View File

@@ -1,5 +1,6 @@
import {
ArrowBack,
CloudUpload,
Save
} from '@mui/icons-material';
import {
@@ -319,6 +320,57 @@ const SettingsPage: React.FC = () => {
<Grid size={12}><Divider /></Grid>
{/* Cookie Upload Settings */}
<Grid size={12}>
<Typography variant="h6" gutterBottom>{t('cookieSettings') || 'Cookie Settings'}</Typography>
<Typography variant="body2" color="text.secondary" paragraph>
{t('cookieUploadDescription') || 'Upload cookies.txt to pass YouTube bot checks and enable Bilibili subtitle downloads. The file will be renamed to cookies.txt automatically. (Example: use "Get cookies.txt LOCALLY" extension to export cookies)'}
</Typography>
<Box sx={{ display: 'flex', gap: 2, alignItems: 'center' }}>
<Button
variant="outlined"
component="label"
startIcon={<CloudUpload />}
>
{t('uploadCookies') || 'Upload Cookies'}
<input
type="file"
hidden
accept=".txt"
onChange={async (e) => {
const file = e.target.files?.[0];
if (!file) return;
if (!file.name.endsWith('.txt')) {
setMessage({ text: t('onlyTxtFilesAllowed') || 'Only .txt files are allowed', type: 'error' });
return;
}
const formData = new FormData();
formData.append('file', file);
try {
await axios.post(`${API_URL}/settings/upload-cookies`, formData, {
headers: {
'Content-Type': 'multipart/form-data'
}
});
setMessage({ text: t('cookiesUploadedSuccess') || 'Cookies uploaded successfully', type: 'success' });
} catch (error) {
console.error('Error uploading cookies:', error);
setMessage({ text: t('cookiesUploadFailed') || 'Failed to upload cookies', type: 'error' });
}
// Reset input
e.target.value = '';
}}
/>
</Button>
</Box>
</Grid>
<Grid size={12}><Divider /></Grid>
{/* Security Settings */}
<Grid size={12}>
<Typography variant="h6" gutterBottom>{t('security')}</Typography>
@@ -449,7 +501,7 @@ const SettingsPage: React.FC = () => {
{/* Cloud Drive Settings */}
<Grid size={12}>
<Typography variant="h6" gutterBottom>{t('cloudDriveSettings')}</Typography>
<Typography variant="h6" gutterBottom>{t('cloudDriveSettings')} (beta)</Typography>
<FormControlLabel
control={
<Switch

View File

@@ -87,6 +87,14 @@ export const ar = {
cleanupTempFilesSuccess: "تم حذف {count} ملف (ملفات) مؤقت بنجاح.",
cleanupTempFilesFailed: "فشل تنظيف الملفات المؤقتة",
// Cookie Settings
cookieSettings: "إعدادات ملفات تعريف الارتباط",
cookieUploadDescription: "قم بتحميل cookies.txt لتجاوز فحوصات الروبوت في YouTube وتمكين تنزيل ترجمات Bilibili. ستتم إعادة تسمية الملف تلقائيًا إلى cookies.txt. (مثال: استخدم إضافة \"Get cookies.txt LOCALLY\" لتصدير ملفات تعريف الارتباط)",
uploadCookies: "تحميل ملفات تعريف الارتباط",
onlyTxtFilesAllowed: "يسمح فقط بملفات .txt",
cookiesUploadedSuccess: "تم تحميل ملفات تعريف الارتباط بنجاح",
cookiesUploadFailed: "فشل تحميل ملفات تعريف الارتباط",
// Cloud Drive
cloudDriveSettings: "التخزين السحابي (OpenList)",
enableAutoSave: "تمكين الحفظ التلقائي في السحابة",

View File

@@ -45,6 +45,14 @@ export const de = {
cleanupTempFilesSuccess: "Erfolgreich {count} temporäre Datei(en) gelöscht.",
cleanupTempFilesFailed: "Fehler beim Bereinigen temporärer Dateien",
// Cookie Settings
cookieSettings: "Cookie-Einstellungen",
cookieUploadDescription: "Laden Sie cookies.txt hoch, um YouTube-Bot-Prüfungen zu bestehen und Bilibili-Untertitel-Downloads zu aktivieren. Die Datei wird automatisch in cookies.txt umbenannt. (Beispiel: Verwenden Sie die Erweiterung \"Get cookies.txt LOCALLY\" zum Exportieren von Cookies)",
uploadCookies: "Cookies hochladen",
onlyTxtFilesAllowed: "Nur .txt-Dateien erlaubt",
cookiesUploadedSuccess: "Cookies erfolgreich hochgeladen",
cookiesUploadFailed: "Fehler beim Hochladen der Cookies",
// Cloud Drive
cloudDriveSettings: "Cloud-Speicher (OpenList)",
enableAutoSave: "Automatisches Speichern in der Cloud aktivieren",

View File

@@ -88,6 +88,14 @@ export const en = {
cleanupTempFilesSuccess: "Successfully deleted {count} temporary file(s).",
cleanupTempFilesFailed: "Failed to clean up temporary files",
// Cookie Settings
cookieSettings: "Cookie Settings",
cookieUploadDescription: "Upload cookies.txt to pass YouTube bot checks and enable Bilibili subtitle downloads. The file will be renamed to cookies.txt automatically. (Example: use \"Get cookies.txt LOCALLY\" extension to export cookies)",
uploadCookies: "Upload Cookies",
onlyTxtFilesAllowed: "Only .txt files are allowed",
cookiesUploadedSuccess: "Cookies uploaded successfully",
cookiesUploadFailed: "Failed to upload cookies",
// Cloud Drive
cloudDriveSettings: "Cloud Drive (OpenList)",
enableAutoSave: "Enable Auto Save to Cloud",

View File

@@ -43,6 +43,14 @@ export const es = {
cleanupTempFilesSuccess: "Se eliminaron exitosamente {count} archivo(s) temporal(es).",
cleanupTempFilesFailed: "Error al limpiar archivos temporales",
// Cookie Settings
cookieSettings: "Configuración de Cookies",
cookieUploadDescription: "Sube cookies.txt para pasar las comprobaciones de bots de YouTube y habilitar la descarga de subtítulos de Bilibili. El archivo se renombrará automáticamente a cookies.txt. (Ejemplo: use la extensión \"Get cookies.txt LOCALLY\" para exportar cookies)",
uploadCookies: "Subir Cookies",
onlyTxtFilesAllowed: "Solo se permiten archivos .txt",
cookiesUploadedSuccess: "Cookies subidas con éxito",
cookiesUploadFailed: "Error al subir cookies",
// Cloud Drive
cloudDriveSettings: "Almacenamiento en la Nube (OpenList)",
enableAutoSave: "Habilitar guardado automático en la nube",

View File

@@ -87,6 +87,14 @@ export const fr = {
cleanupTempFilesSuccess: "{count} fichier(s) temporaire(s) supprimé(s) avec succès.",
cleanupTempFilesFailed: "Échec du nettoyage des fichiers temporaires",
// Cookie Settings
cookieSettings: "Paramètres des Cookies",
cookieUploadDescription: "Téléchargez cookies.txt pour passer les vérifications de robots YouTube et activer le téléchargement des sous-titres Bilibili. Le fichier sera automatiquement renommé en cookies.txt. (Exemple : utilisez l'extension \"Get cookies.txt LOCALLY\" pour exporter les cookies)",
uploadCookies: "Télécharger les Cookies",
onlyTxtFilesAllowed: "Seuls les fichiers .txt sont autorisés",
cookiesUploadedSuccess: "Cookies téléchargés avec succès",
cookiesUploadFailed: "Échec du téléchargement des cookies",
// Cloud Drive
cloudDriveSettings: "Stockage Cloud (OpenList)",
enableAutoSave: "Activer la sauvegarde automatique sur le Cloud",

View File

@@ -87,6 +87,14 @@ export const ja = {
cleanupTempFilesSuccess: "{count}個の一時ファイルを正常に削除しました。",
cleanupTempFilesFailed: "一時ファイルのクリーンアップに失敗しました",
// Cookie Settings
cookieSettings: "Cookie設定",
cookieUploadDescription: "YouTubeのボットチェックを通過し、Bilibiliの字幕ダウンロードを有効にするためにcookies.txtをアップロードしてください。ファイルは自動的にcookies.txtにリネームされます。(例:\"Get cookies.txt LOCALLY\" 拡張機能を使用してクッキーをエクスポート)",
uploadCookies: "Cookieをアップロード",
onlyTxtFilesAllowed: ".txtファイルのみ許可されています",
cookiesUploadedSuccess: "Cookieが正常にアップロードされました",
cookiesUploadFailed: "Cookieのアップロードに失敗しました",
// Cloud Drive
cloudDriveSettings: "クラウドストレージ (OpenList)",
enableAutoSave: "クラウドへの自動保存を有効にする",

View File

@@ -87,6 +87,14 @@ export const ko = {
cleanupTempFilesSuccess: "{count}개의 임시 파일을 성공적으로 삭제했습니다.",
cleanupTempFilesFailed: "임시 파일 정리 실패",
// Cookie Settings
cookieSettings: "쿠키 설정",
cookieUploadDescription: "YouTube 봇 확인을 통과하고 Bilibili 자막 다운로드를 활성화하려면 cookies.txt를 업로드하세요. 파일 이름은 자동으로 cookies.txt로 변경됩니다. (예: \"Get cookies.txt LOCALLY\" 확장 프로그램을 사용하여 쿠키 내보내기)",
uploadCookies: "쿠키 업로드",
onlyTxtFilesAllowed: ".txt 파일만 허용됩니다",
cookiesUploadedSuccess: "쿠키가 성공적으로 업로드되었습니다",
cookiesUploadFailed: "쿠키 업로드 실패",
// Cloud Drive
cloudDriveSettings: "클라우드 드라이브 (OpenList)",
enableAutoSave: "클라우드 자동 저장 활성화",

View File

@@ -87,6 +87,14 @@ export const pt = {
cleanupTempFilesSuccess: "{count} arquivo(s) temporário(s) excluído(s) com sucesso.",
cleanupTempFilesFailed: "Falha ao limpar arquivos temporários",
// Cookie Settings
cookieSettings: "Configurações de Cookies",
cookieUploadDescription: "Envie cookies.txt para passar nas verificações de bot do YouTube e ativar o download de legendas do Bilibili. O arquivo será renomeado automaticamente para cookies.txt. (Exemplo: use a extensão \"Get cookies.txt LOCALLY\" para exportar cookies)",
uploadCookies: "Enviar Cookies",
onlyTxtFilesAllowed: "Apenas arquivos .txt são permitidos",
cookiesUploadedSuccess: "Cookies enviados com sucesso",
cookiesUploadFailed: "Falha ao enviar cookies",
// Cloud Drive
cloudDriveSettings: "Armazenamento em Nuvem (OpenList)",
enableAutoSave: "Ativar salvamento automático na nuvem",

View File

@@ -87,6 +87,14 @@ export const ru = {
cleanupTempFilesSuccess: "Успешно удалено {count} временных файлов.",
cleanupTempFilesFailed: "Не удалось очистить временные файлы",
// Cookie Settings
cookieSettings: "Настройки Cookie",
cookieUploadDescription: "Загрузите cookies.txt, чтобы пройти проверку ботов YouTube и включить скачивание субтитров Bilibili. Файл будет автоматически переименован в cookies.txt. (Пример: используйте расширение \"Get cookies.txt LOCALLY\" для экспорта cookie)",
uploadCookies: "Загрузить Cookie",
onlyTxtFilesAllowed: "Разрешены только файлы .txt",
cookiesUploadedSuccess: "Cookie успешно загружены",
cookiesUploadFailed: "Не удалось загрузить cookie",
// Cloud Drive
cloudDriveSettings: "Облачное хранилище (OpenList)",
enableAutoSave: "Включить автосохранение в облако",

View File

@@ -88,6 +88,14 @@ export const zh = {
cleanupTempFilesSuccess: "成功删除了 {count} 个临时文件。",
cleanupTempFilesFailed: "清理临时文件失败",
// Cookie Settings
cookieSettings: "Cookie 设置",
cookieUploadDescription: "上传 cookies.txt 以通过 YouTube 机器人检测并启用 Bilibili 字幕下载。文件将自动重命名为 cookies.txt。(例如:使用 \"Get cookies.txt LOCALLY\" 扩展导出 cookies)",
uploadCookies: "上传 Cookie",
onlyTxtFilesAllowed: "仅允许 .txt 文件",
cookiesUploadedSuccess: "Cookie 上传成功",
cookiesUploadFailed: "Cookie 上传失败",
// Cloud Drive
cloudDriveSettings: "云端存储 (OpenList)",
enableAutoSave: "启用自动保存到云端",