davidl/MyTube
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

refactor: Update axios configuration for downloading subtitles

Browse Source
This commit is contained in:
Peifan Li
2026-01-04 12:32:28 -05:00
parent 8e533e3615
commit c995eb3637
10 changed files with 197 additions and 65 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
backend/src/__tests__/services/downloaders/file_location.test.ts
View File

@@ -70,10 +70,11 @@ vi.mock('fs-extra', () => {
}
}),
readdirSync: (...args: any[]) => mocks.readdirSync(...args),
readFileSync: (...args: any[]) => mocks.readFileSync(...args),
writeFileSync: (...args: any[]) => mocks.writeFileSync(...args),
unlinkSync: (...args: any[]) => mocks.unlinkSync(...args),
remove: (...args: any[]) => mocks.remove(...args),
readFileSync: (...args: any[]) => mocks.readFileSync(...args),
writeFileSync: (...args: any[]) => mocks.writeFileSync(...args),
copyFileSync: vi.fn(),
unlinkSync: (...args: any[]) => mocks.unlinkSync(...args),
remove: (...args: any[]) => mocks.remove(...args),
statSync: vi.fn().mockReturnValue({ size: 1000 }),
}
};
@@ -165,4 +166,3 @@ describe('File Location Logic', () => {
});
});
});

11
backend/src/services/downloaders/BaseDownloader.ts
View File

@@ -3,8 +3,8 @@ import fs from "fs-extra";
import path from "path";
import { DownloadCancelledError } from "../../errors/DownloadErrors";
import {
isCancellationError,
isDownloadActive,
isCancellationError,
isDownloadActive,
} from "../../utils/downloadUtils";
import { formatVideoFilename } from "../../utils/helpers";
import { logger } from "../../utils/logger";
@@ -46,10 +46,14 @@ export abstract class BaseDownloader implements IDownloader {
*/
protected async downloadThumbnail(
thumbnailUrl: string,
savePath: string
savePath: string,
axiosConfig: any = {}
): Promise<boolean> {
try {
logger.info("Downloading thumbnail from:", thumbnailUrl);
if (axiosConfig.proxy) {
logger.debug("Using proxy for thumbnail download");
}
// Ensure directory exists
fs.ensureDirSync(path.dirname(savePath));
@@ -58,6 +62,7 @@ export abstract class BaseDownloader implements IDownloader {
method: "GET",
url: thumbnailUrl,
responseType: "stream",
...axiosConfig,
});
const writer = fs.createWriteStream(savePath);

19
backend/src/services/downloaders/MissAVDownloader.ts
View File

@@ -10,6 +10,7 @@ import { logger } from "../../utils/logger";
import { ProgressTracker } from "../../utils/progressTracker";
import {
flagsToArgs,
getAxiosProxyConfig,
getNetworkConfigFromUserConfig,
getUserYtDlpConfig,
} from "../../utils/ytDlpUtils";
@@ -268,7 +269,13 @@ export class MissAVDownloader extends BaseDownloader {
const newThumbnailFilename = `${newSafeBaseFilename}.jpg`;
const newVideoPath = path.join(VIDEOS_DIR, newVideoFilename);
const newThumbnailPath = path.join(IMAGES_DIR, newThumbnailFilename);
const settings = storageService.getSettings();
const moveThumbnailsToVideoFolder =
settings.moveThumbnailsToVideoFolder || false;
const thumbnailDir = moveThumbnailsToVideoFolder
? VIDEOS_DIR
: IMAGES_DIR;
const newThumbnailPath = path.join(thumbnailDir, newThumbnailFilename);
// 7. Download the video using yt-dlp with the m3u8 URL
logger.info("Downloading video from m3u8 URL using yt-dlp:", m3u8Url);
@@ -418,10 +425,14 @@ export class MissAVDownloader extends BaseDownloader {
// 8. Download and save the thumbnail
if (thumbnailUrl) {
// Use base class method via temporary instance
const axiosConfig = userConfig.proxy
? getAxiosProxyConfig(userConfig.proxy)
: {};
const downloader = new MissAVDownloader();
thumbnailSaved = await downloader.downloadThumbnail(
thumbnailUrl,
newThumbnailPath
newThumbnailPath,
axiosConfig
);
}
@@ -463,7 +474,9 @@ export class MissAVDownloader extends BaseDownloader {
thumbnailUrl: thumbnailUrl || undefined,
videoPath: `/videos/${newVideoFilename}`,
thumbnailPath: thumbnailSaved
? `/images/${newThumbnailFilename}`
? moveThumbnailsToVideoFolder
? `/videos/${newThumbnailFilename}`
: `/images/${newThumbnailFilename}`
: null,
duration: duration,
fileSize: fileSize,

13
backend/src/services/downloaders/bilibili/bilibiliFileManager.ts
View File

@@ -47,7 +47,8 @@ export async function cleanupTempDir(tempDir: string): Promise<void> {
*/
export function prepareFilePaths(
mergeOutputFormat: string,
collectionName?: string
collectionName?: string,
moveThumbnailsToVideoFolder: boolean = false
): FilePaths {
// Create a safe base filename (without extension)
const timestamp = Date.now();
@@ -61,9 +62,13 @@ export function prepareFilePaths(
const videoDir = collectionName
? path.join(VIDEOS_DIR, collectionName)
: VIDEOS_DIR;
const imageDir = collectionName
? path.join(IMAGES_DIR, collectionName)
: IMAGES_DIR;
const imageDir = moveThumbnailsToVideoFolder
? collectionName
? path.join(VIDEOS_DIR, collectionName)
: VIDEOS_DIR
: collectionName
? path.join(IMAGES_DIR, collectionName)
: IMAGES_DIR;
// Ensure directories exist
fs.ensureDirSync(videoDir);

18
backend/src/services/downloaders/bilibili/bilibiliSubtitle.ts
View File

@@ -1,7 +1,6 @@
import axios from "axios";
import fs from "fs-extra";
import path from "path";
import { SUBTITLES_DIR } from "../../../config/paths";
import { bccToVtt } from "../../../utils/bccToVtt";
import { extractBilibiliVideoId } from "../../../utils/helpers";
import { logger } from "../../../utils/logger";
@@ -13,7 +12,9 @@ import { getCookieHeader } from "./bilibiliCookie";
export async function downloadSubtitles(
videoUrl: string,
baseFilename: string,
collectionName?: string
subtitleDir: string,
subtitlePathPrefix: string,
axiosConfig: any = {}
): Promise<Array<{ language: string; filename: string; path: string }>> {
try {
const videoId = extractBilibiliVideoId(videoUrl);
@@ -37,7 +38,7 @@ export async function downloadSubtitles(
const viewApiUrl = `https://api.bilibili.com/x/web-interface/view?bvid=${videoId}`;
let viewResponse;
try {
viewResponse = await axios.get(viewApiUrl, { headers });
viewResponse = await axios.get(viewApiUrl, { headers, ...axiosConfig });
} catch (viewError: any) {
logger.error(`Failed to fetch view API: ${viewError.message}`);
return [];
@@ -55,7 +56,7 @@ export async function downloadSubtitles(
logger.info(`Fetching subtitles from: ${playerApiUrl}`);
let playerResponse;
try {
playerResponse = await axios.get(playerApiUrl, { headers });
playerResponse = await axios.get(playerApiUrl, { headers, ...axiosConfig });
} catch (playerError: any) {
logger.warn(`Player API failed: ${playerError.message}`);
// Continue to check view API fallback
@@ -95,14 +96,6 @@ export async function downloadSubtitles(
const savedSubtitles = [];
// Determine subtitle directory based on collection name
const subtitleDir = collectionName
? path.join(SUBTITLES_DIR, collectionName)
: SUBTITLES_DIR;
const subtitlePathPrefix = collectionName
? `/subtitles/${collectionName}`
: `/subtitles`;
// Ensure subtitles directory exists
fs.ensureDirSync(subtitleDir);
@@ -131,6 +124,7 @@ export async function downloadSubtitles(
try {
const subResponse = await axios.get(absoluteSubUrl, {
headers: cdnHeaders,
...axiosConfig,
});
const vttContent = bccToVtt(subResponse.data);

50
backend/src/services/downloaders/bilibili/bilibiliVideo.ts
View File

@@ -1,5 +1,6 @@
import fs from "fs-extra";
import path from "path";
import { SUBTITLES_DIR } from "../../../config/paths";
import { DownloadCancelledError } from "../../../errors/DownloadErrors";
import { formatBytes } from "../../../utils/downloadUtils";
import { formatVideoFilename } from "../../../utils/helpers";
@@ -8,6 +9,7 @@ import { ProgressTracker } from "../../../utils/progressTracker";
import {
executeYtDlpJson,
executeYtDlpSpawn,
getAxiosProxyConfig,
getNetworkConfigFromUserConfig,
getUserYtDlpConfig,
} from "../../../utils/ytDlpUtils";
@@ -55,9 +57,10 @@ class BilibiliDownloaderHelper extends BaseDownloader {
public async downloadThumbnailPublic(
thumbnailUrl: string,
savePath: string
savePath: string,
axiosConfig: any = {}
): Promise<boolean> {
return this.downloadThumbnail(thumbnailUrl, savePath);
return this.downloadThumbnail(thumbnailUrl, savePath, axiosConfig);
}
}
@@ -255,10 +258,14 @@ export async function downloadVideo(
let thumbnailSaved = false;
if (thumbnailUrl) {
// Use base class method via temporary instance
const axiosConfig = userConfig.proxy
? getAxiosProxyConfig(userConfig.proxy)
: {};
const downloader = new BilibiliDownloaderHelper();
thumbnailSaved = await downloader.downloadThumbnailPublic(
thumbnailUrl,
thumbnailPath
thumbnailPath,
axiosConfig
);
}
@@ -308,6 +315,11 @@ export async function downloadSinglePart(
// Get user's yt-dlp configuration for merge output format
const userConfig = getUserYtDlpConfig(url);
const mergeOutputFormat = userConfig.mergeOutputFormat || "mp4";
const settings = storageService.getSettings();
const moveThumbnailsToVideoFolder =
settings.moveThumbnailsToVideoFolder || false;
const moveSubtitlesToVideoFolder =
settings.moveSubtitlesToVideoFolder || false;
// Create a safe base filename (without extension)
const timestamp = Date.now();
@@ -315,7 +327,8 @@ export async function downloadSinglePart(
// Prepare file paths using the file manager
const { videoPath, thumbnailPath, videoDir, imageDir } = prepareFilePaths(
mergeOutputFormat,
collectionName
collectionName,
moveThumbnailsToVideoFolder
);
let videoTitle,
@@ -420,10 +433,27 @@ export async function downloadSinglePart(
}> = [];
try {
logger.info("Attempting to download subtitles...");
const subtitleDir = moveSubtitlesToVideoFolder
? videoDir
: collectionName
? path.join(SUBTITLES_DIR, collectionName)
: SUBTITLES_DIR;
const subtitlePathPrefix = moveSubtitlesToVideoFolder
? collectionName
? `/videos/${collectionName}`
: `/videos`
: collectionName
? `/subtitles/${collectionName}`
: `/subtitles`;
const axiosConfig = userConfig.proxy
? getAxiosProxyConfig(userConfig.proxy)
: {};
subtitles = await downloadSubtitles(
url,
newSafeBaseFilename,
collectionName
subtitleDir,
subtitlePathPrefix,
axiosConfig
);
logger.info(`Downloaded ${subtitles.length} subtitles`);
} catch (e) {
@@ -462,9 +492,13 @@ export async function downloadSinglePart(
? `/videos/${collectionName}/${finalVideoFilename}`
: `/videos/${finalVideoFilename}`,
thumbnailPath: thumbnailSaved
? collectionName
? `/images/${collectionName}/${finalThumbnailFilename}`
: `/images/${finalThumbnailFilename}`
? moveThumbnailsToVideoFolder
? collectionName
? `/videos/${collectionName}/${finalThumbnailFilename}`
: `/videos/${finalThumbnailFilename}`
: collectionName
? `/images/${collectionName}/${finalThumbnailFilename}`
: `/images/${finalThumbnailFilename}`
: null,
duration: duration,
fileSize: fileSize,

70
backend/src/services/downloaders/ytdlp/ytdlpSubtitle.ts
View File

@@ -38,19 +38,45 @@ export async function processSubtitles(
const subtitles: Array<{ language: string; filename: string; path: string }> =
[];
logger.info(
`Processing subtitles for ${baseFilename}, move to video folder: ${moveSubtitlesToVideoFolder}`
);
const downloader = new YtDlpDownloaderHelper();
try {
const subtitleFiles = fs
.readdirSync(VIDEOS_DIR)
.filter(
(file: string) =>
file.startsWith(baseFilename) && file.endsWith(".vtt")
);
const subtitleExtensions = new Set([
".vtt",
".srt",
".ass",
".ssa",
".sub",
".ttml",
".dfxp",
".sbv",
]);
const searchDirs = [VIDEOS_DIR, SUBTITLES_DIR];
const subtitleFiles: Array<{ dir: string; file: string }> = [];
const seenFiles = new Set<string>();
for (const dir of searchDirs) {
const files = fs.readdirSync(dir).filter((file: string) => {
const ext = path.extname(file).toLowerCase();
return file.startsWith(baseFilename) && subtitleExtensions.has(ext);
});
for (const file of files) {
if (seenFiles.has(file)) {
continue;
}
seenFiles.add(file);
subtitleFiles.push({ dir, file });
}
}
logger.info(`Found ${subtitleFiles.length} subtitle files`);
for (const subtitleFile of subtitleFiles) {
for (const { dir, file: subtitleFile } of subtitleFiles) {
// Check if download was cancelled during subtitle processing
try {
downloader.throwIfCancelledPublic(downloadId);
@@ -61,13 +87,14 @@ export async function processSubtitles(
// Parse language from filename (e.g., video_123.en.vtt -> en)
const match = subtitleFile.match(
/\.([a-z]{2}(?:-[A-Z]{2})?)(?:\..*?)?\.vtt$/
/\.([a-z]{2}(?:-[A-Z]{2})?)(?:\..*?)?\.[^.]+$/
);
const language = match ? match[1] : "unknown";
const extension = path.extname(subtitleFile);
// Move subtitle to subtitles directory or keep in video directory if requested
const sourceSubPath = path.join(VIDEOS_DIR, subtitleFile);
const destSubFilename = `${baseFilename}.${language}.vtt`;
const sourceSubPath = path.join(dir, subtitleFile);
const destSubFilename = `${baseFilename}.${language}${extension}`;
let destSubPath: string;
let webPath: string;
@@ -78,16 +105,20 @@ export async function processSubtitles(
destSubPath = path.join(SUBTITLES_DIR, destSubFilename);
webPath = `/subtitles/${destSubFilename}`;
}
// Read VTT file and fix alignment for centering
let vttContent = fs.readFileSync(sourceSubPath, "utf-8");
// Replace align:start with align:middle for centered subtitles
// Also remove position:0% which forces left positioning
vttContent = vttContent.replace(/ align:start/g, " align:middle");
vttContent = vttContent.replace(/ position:0%/g, "");
// Write cleaned VTT to destination
fs.writeFileSync(destSubPath, vttContent, "utf-8");
if (extension.toLowerCase() === ".vtt") {
// Read VTT file and fix alignment for centering
let vttContent = fs.readFileSync(sourceSubPath, "utf-8");
// Replace align:start with align:middle for centered subtitles
// Also remove position:0% which forces left positioning
vttContent = vttContent.replace(/ align:start/g, " align:middle");
vttContent = vttContent.replace(/ position:0%/g, "");
// Write cleaned VTT to destination
fs.writeFileSync(destSubPath, vttContent, "utf-8");
} else if (sourceSubPath !== destSubPath) {
fs.copyFileSync(sourceSubPath, destSubPath);
}
// Remove original file if we moved it (if dest is different from source)
// If moveSubtitlesToVideoFolder is true, destSubPath might be same as sourceSubPath
@@ -116,4 +147,3 @@ export async function processSubtitles(
return subtitles;
}

34
backend/src/services/downloaders/ytdlp/ytdlpVideo.ts
View File

@@ -2,17 +2,17 @@ import fs from "fs-extra";
import path from "path";
import { IMAGES_DIR, VIDEOS_DIR } from "../../../config/paths";
import {
cleanupSubtitleFiles,
cleanupVideoArtifacts,
cleanupSubtitleFiles,
cleanupVideoArtifacts,
} from "../../../utils/downloadUtils";
import { formatVideoFilename } from "../../../utils/helpers";
import { logger } from "../../../utils/logger";
import { ProgressTracker } from "../../../utils/progressTracker";
import {
executeYtDlpJson,
executeYtDlpSpawn,
getNetworkConfigFromUserConfig,
getUserYtDlpConfig,
executeYtDlpSpawn,
getAxiosProxyConfig,
getNetworkConfigFromUserConfig,
getUserYtDlpConfig,
} from "../../../utils/ytDlpUtils";
import * as storageService from "../../storageService";
import { Video } from "../../storageService";
@@ -45,9 +45,10 @@ class YtDlpDownloaderHelper extends BaseDownloader {
public async downloadThumbnailPublic(
thumbnailUrl: string,
savePath: string
savePath: string,
axiosConfig: any = {}
): Promise<boolean> {
return this.downloadThumbnail(thumbnailUrl, savePath);
return this.downloadThumbnail(thumbnailUrl, savePath, axiosConfig);
}
}
@@ -142,6 +143,13 @@ export async function downloadVideo(
const moveSubtitlesToVideoFolder =
settings.moveSubtitlesToVideoFolder || false;
logger.info("File location settings:", {
moveThumbnailsToVideoFolder,
moveSubtitlesToVideoFolder,
videoDir: VIDEOS_DIR,
imageDir: IMAGES_DIR
});
const newVideoPath = path.join(VIDEOS_DIR, finalVideoFilename);
const newThumbnailPath = moveThumbnailsToVideoFolder
? path.join(VIDEOS_DIR, finalThumbnailFilename)
@@ -292,9 +300,17 @@ export async function downloadVideo(
thumbnailSaved = false;
if (thumbnailUrl) {
// Prepare axios config with proxy if available
let axiosConfig = {};
if (downloadUserConfig.proxy) {
axiosConfig = getAxiosProxyConfig(downloadUserConfig.proxy);
}
thumbnailSaved = await downloader.downloadThumbnailPublic(
thumbnailUrl,
newThumbnailPath
newThumbnailPath,
axiosConfig
);
}

35
backend/src/utils/ytDlpUtils.ts
View File

@@ -509,3 +509,38 @@ export function getNetworkConfigFromUserConfig(
return networkOptions;
}
/**
* Helper to convert a proxy URL string into an Axios config object
* Supports http/https proxies with authentication
* Format: http://user:pass@host:port
*/
export function getAxiosProxyConfig(proxyUrl: string): any {
if (!proxyUrl) return {};
try {
const url = new URL(proxyUrl);
const isHttps = url.protocol === "https:";
const defaultPort = isHttps ? 443 : 80;
// Axios proxy config structure
const proxyConfig: any = {
protocol: url.protocol.replace(":", ""),
host: url.hostname,
port: parseInt(url.port, 10) || defaultPort,
};
if (url.username || url.password) {
proxyConfig.auth = {
username: url.username,
password: url.password,
};
}
return { proxy: proxyConfig };
} catch (error) {
console.error("Invalid proxy URL:", proxyUrl);
return {};
}
}

2
frontend/src/components/Header/MobileMenu.tsx
View File

@@ -44,7 +44,7 @@ const MobileMenu: React.FC<MobileMenuProps> = ({
videos = [],
availableTags = [],
selectedTags = [],
onTagToggle
onTagToggle = () => { }
}) => {
const { t } = useLanguage();
const { logout, userRole } = useAuth();