6 Commits

Author SHA1 Message Date
Peifan Li
1d45692374 chore(release): v1.3.0 2025-11-28 20:50:17 -05:00
Peifan Li
fc070da102 refactor: Update YouTubeDownloader to YtDlpDownloader 2025-11-28 20:50:04 -05:00
Peifan Li
d1ceef9698 fix: Update backend and frontend package versions to 1.2.5 2025-11-27 20:57:31 -05:00
Peifan Li
bc9564f9bc chore(release): v1.2.5 2025-11-27 20:54:46 -05:00
Peifan Li
710e85ad5e style: Improve speed calculation and add version in footer 2025-11-27 20:54:44 -05:00
Peifan Li
bc3ab6f9ef fix: Update package versions to 1.2.4 2025-11-27 18:02:25 -05:00
17 changed files with 182 additions and 125 deletions

View File

@@ -1,12 +1,12 @@
{
"name": "backend",
"version": "1.2.3",
"version": "1.2.5",
"lockfileVersion": 3,
"requires": true,
"packages": {
"": {
"name": "backend",
"version": "1.2.3",
"version": "1.2.5",
"license": "ISC",
"dependencies": {
"axios": "^1.8.1",

View File

@@ -1,6 +1,6 @@
{
"name": "backend",
"version": "1.2.4",
"version": "1.3.0",
"main": "server.js",
"scripts": {
"start": "ts-node src/server.ts",

View File

@@ -2,10 +2,10 @@ import { beforeEach, describe, expect, it, vi } from 'vitest';
import * as downloadService from '../../services/downloadService';
import { BilibiliDownloader } from '../../services/downloaders/BilibiliDownloader';
import { MissAVDownloader } from '../../services/downloaders/MissAVDownloader';
import { YouTubeDownloader } from '../../services/downloaders/YouTubeDownloader';
import { YtDlpDownloader } from '../../services/downloaders/YtDlpDownloader';
vi.mock('../../services/downloaders/BilibiliDownloader');
vi.mock('../../services/downloaders/YouTubeDownloader');
vi.mock('../../services/downloaders/YtDlpDownloader');
vi.mock('../../services/downloaders/MissAVDownloader');
describe('DownloadService', () => {
@@ -56,22 +56,22 @@ describe('DownloadService', () => {
});
});
describe('YouTube', () => {
it('should call YouTubeDownloader.search', async () => {
describe('YouTube/Generic', () => {
it('should call YtDlpDownloader.search', async () => {
await downloadService.searchYouTube('query');
expect(YouTubeDownloader.search).toHaveBeenCalledWith('query');
expect(YtDlpDownloader.search).toHaveBeenCalledWith('query');
});
it('should call YouTubeDownloader.downloadVideo', async () => {
it('should call YtDlpDownloader.downloadVideo', async () => {
await downloadService.downloadYouTubeVideo('url', 'id');
expect(YouTubeDownloader.downloadVideo).toHaveBeenCalledWith('url', 'id');
expect(YtDlpDownloader.downloadVideo).toHaveBeenCalledWith('url', 'id', undefined);
});
});
describe('MissAV', () => {
it('should call MissAVDownloader.downloadVideo', async () => {
await downloadService.downloadMissAVVideo('url', 'id');
expect(MissAVDownloader.downloadVideo).toHaveBeenCalledWith('url', 'id');
expect(MissAVDownloader.downloadVideo).toHaveBeenCalledWith('url', 'id', undefined);
});
});
});

View File

@@ -9,12 +9,12 @@ import * as downloadService from "../services/downloadService";
import { getVideoDuration } from "../services/metadataService";
import * as storageService from "../services/storageService";
import {
extractBilibiliVideoId,
extractUrlFromText,
isBilibiliUrl,
isValidUrl,
resolveShortUrl,
trimBilibiliUrl
extractBilibiliVideoId,
extractUrlFromText,
isBilibiliUrl,
isValidUrl,
resolveShortUrl,
trimBilibiliUrl
} from "../utils/helpers";
// Configure Multer for file uploads
@@ -86,13 +86,12 @@ export const downloadVideo = async (req: Request, res: Response): Promise<any> =
console.log("Resolved shortened URL to:", videoUrl);
}
if (videoUrl.includes("youtube.com") || videoUrl.includes("youtu.be") || isBilibiliUrl(videoUrl) || videoUrl.includes("missav")) {
console.log("Fetching video info for title...");
const info = await downloadService.getVideoInfo(videoUrl);
if (info && info.title) {
initialTitle = info.title;
console.log("Fetched initial title:", initialTitle);
}
// Try to fetch video info for all URLs
console.log("Fetching video info for title...");
const info = await downloadService.getVideoInfo(videoUrl);
if (info && info.title) {
initialTitle = info.title;
console.log("Fetched initial title:", initialTitle);
}
} catch (err) {
console.warn("Failed to fetch video info for title, using default:", err);

View File

@@ -9,7 +9,7 @@ import {
DownloadResult
} from "./downloaders/BilibiliDownloader";
import { MissAVDownloader } from "./downloaders/MissAVDownloader";
import { YouTubeDownloader } from "./downloaders/YouTubeDownloader";
import { YtDlpDownloader } from "./downloaders/YtDlpDownloader";
import { Video } from "./storageService";
// Re-export types for compatibility
@@ -77,14 +77,14 @@ export async function downloadRemainingBilibiliParts(
return BilibiliDownloader.downloadRemainingParts(baseUrl, startPart, totalParts, seriesTitle, collectionId, downloadId);
}
// Search for videos on YouTube
// Search for videos on YouTube (using yt-dlp)
export async function searchYouTube(query: string): Promise<any[]> {
return YouTubeDownloader.search(query);
return YtDlpDownloader.search(query);
}
// Download YouTube video
// Download generic video (using yt-dlp)
export async function downloadYouTubeVideo(videoUrl: string, downloadId?: string, onStart?: (cancel: () => void) => void): Promise<Video> {
return YouTubeDownloader.downloadVideo(videoUrl, downloadId, onStart);
return YtDlpDownloader.downloadVideo(videoUrl, downloadId, onStart);
}
// Helper function to download MissAV video
@@ -99,19 +99,12 @@ export async function getVideoInfo(url: string): Promise<{ title: string; author
if (videoId) {
return BilibiliDownloader.getVideoInfo(videoId);
}
} else if (url.includes("youtube.com") || url.includes("youtu.be")) {
return YouTubeDownloader.getVideoInfo(url);
} else if (url.includes("missav")) {
return MissAVDownloader.getVideoInfo(url);
}
// Default fallback
return {
title: "Video",
author: "Unknown",
date: new Date().toISOString().slice(0, 10).replace(/-/g, ""),
thumbnailUrl: "",
};
// Default fallback to yt-dlp for everything else
return YtDlpDownloader.getVideoInfo(url);
}
// Factory function to create a download task
@@ -128,8 +121,8 @@ export function createDownloadTask(
// Complex collection handling would require persisting more state
return BilibiliDownloader.downloadSinglePart(url, 1, 1, "");
} else {
// Default to YouTube
return YouTubeDownloader.downloadVideo(url, downloadId, registerCancel);
// Default to yt-dlp
return YtDlpDownloader.downloadVideo(url, downloadId, registerCancel);
}
};
}

View File

@@ -215,7 +215,7 @@ export class MissAVDownloader {
// size= 12345kB time=00:01:23.45 bitrate= 1234.5kbits/s speed=1.23x
const timeMatch = output.match(/time=(\d{2}):(\d{2}):(\d{2})\.(\d{2})/);
const sizeMatch = output.match(/size=\s*(\d+)([kMG]?B)/);
const speedMatch = output.match(/speed=\s*(\d+\.?\d*)x/);
const bitrateMatch = output.match(/bitrate=\s*(\d+\.?\d*)kbits\/s/);
if (timeMatch && downloadId) {
const hours = parseInt(timeMatch[1]);
@@ -233,9 +233,17 @@ export class MissAVDownloader {
totalSizeStr = `${sizeMatch[1]}${sizeMatch[2]}`;
}
let speedStr = "0x";
if (speedMatch) {
speedStr = `${speedMatch[1]}x`;
let speedStr = "0 B/s";
if (bitrateMatch) {
const bitrateKbps = parseFloat(bitrateMatch[1]);
// Convert kbits/s to KB/s (approximate, usually bitrate is bits, so /8)
// But ffmpeg reports kbits/s. 1 byte = 8 bits.
const speedKBps = bitrateKbps / 8;
if (speedKBps > 1024) {
speedStr = `${(speedKBps / 1024).toFixed(2)} MB/s`;
} else {
speedStr = `${speedKBps.toFixed(2)} KB/s`;
}
}
storageService.updateActiveDownload(downloadId, {

View File

@@ -7,12 +7,48 @@ import { sanitizeFilename } from "../../utils/helpers";
import * as storageService from "../storageService";
import { Video } from "../storageService";
export class YouTubeDownloader {
// Search for videos on YouTube
// Helper function to extract author from XiaoHongShu page when yt-dlp doesn't provide it
async function extractXiaoHongShuAuthor(url: string): Promise<string | null> {
try {
console.log("Attempting to extract XiaoHongShu author from webpage...");
const response = await axios.get(url, {
headers: {
'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36'
},
timeout: 10000
});
const html = response.data;
// Try to find author name in the JSON data embedded in the page
// XiaoHongShu embeds data in window.__INITIAL_STATE__
const match = html.match(/"nickname":"([^"]+)"/);
if (match && match[1]) {
console.log("Found XiaoHongShu author:", match[1]);
return match[1];
}
// Alternative: try to find in user info
const userMatch = html.match(/"user":\{[^}]*"nickname":"([^"]+)"/);
if (userMatch && userMatch[1]) {
console.log("Found XiaoHongShu author (user):", userMatch[1]);
return userMatch[1];
}
console.log("Could not extract XiaoHongShu author from webpage");
return null;
} catch (error) {
console.error("Error extracting XiaoHongShu author:", error);
return null;
}
}
export class YtDlpDownloader {
// Search for videos (primarily for YouTube, but could be adapted)
static async search(query: string): Promise<any[]> {
console.log("Processing search request for query:", query);
// Use youtube-dl to search for videos
// Use ytsearch for searching
const searchResults = await youtubedl(`ytsearch5:${query}`, {
dumpSingleJson: true,
noWarnings: true,
@@ -33,7 +69,7 @@ export class YouTubeDownloader {
thumbnailUrl: entry.thumbnail,
duration: entry.duration,
viewCount: entry.view_count,
sourceUrl: `https://www.youtube.com/watch?v=${entry.id}`,
sourceUrl: `https://www.youtube.com/watch?v=${entry.id}`, // Default to YT for search results
source: "youtube",
}));
@@ -52,29 +88,29 @@ export class YouTubeDownloader {
noWarnings: true,
callHome: false,
preferFreeFormats: true,
youtubeSkipDashManifest: true,
// youtubeSkipDashManifest: true, // Specific to YT, might want to keep or make conditional
} as any);
return {
title: info.title || "YouTube Video",
author: info.uploader || "YouTube User",
title: info.title || "Video",
author: info.uploader || "Unknown",
date: info.upload_date || new Date().toISOString().slice(0, 10).replace(/-/g, ""),
thumbnailUrl: info.thumbnail,
};
} catch (error) {
console.error("Error fetching YouTube video info:", error);
console.error("Error fetching video info:", error);
return {
title: "YouTube Video",
author: "YouTube User",
title: "Video",
author: "Unknown",
date: new Date().toISOString().slice(0, 10).replace(/-/g, ""),
thumbnailUrl: "",
};
}
}
// Download YouTube video
// Download video
static async downloadVideo(videoUrl: string, downloadId?: string, onStart?: (cancel: () => void) => void): Promise<Video> {
console.log("Detected YouTube URL");
console.log("Detected URL:", videoUrl);
// Create a safe base filename (without extension)
const timestamp = Date.now();
@@ -84,36 +120,41 @@ export class YouTubeDownloader {
const videoFilename = `${safeBaseFilename}.mp4`;
const thumbnailFilename = `${safeBaseFilename}.jpg`;
// Set full paths for video and thumbnail
let videoTitle, videoAuthor, videoDate, thumbnailUrl, thumbnailSaved;
let videoTitle, videoAuthor, videoDate, thumbnailUrl, thumbnailSaved, source;
let finalVideoFilename = videoFilename;
let finalThumbnailFilename = thumbnailFilename;
try {
// Get YouTube video info first
// Get video info first
const info = await youtubedl(videoUrl, {
dumpSingleJson: true,
noWarnings: true,
callHome: false,
preferFreeFormats: true,
youtubeSkipDashManifest: true,
} as any);
console.log("YouTube video info:", {
console.log("Video info:", {
title: info.title,
uploader: info.uploader,
upload_date: info.upload_date,
extractor: info.extractor,
});
videoTitle = info.title || "YouTube Video";
videoAuthor = info.uploader || "YouTube User";
videoTitle = info.title || "Video";
videoAuthor = info.uploader || "Unknown";
// If author is unknown and it's a XiaoHongShu video, try custom extraction
if ((!info.uploader || info.uploader === "Unknown") && info.extractor === "XiaoHongShu") {
const customAuthor = await extractXiaoHongShuAuthor(videoUrl);
if (customAuthor) {
videoAuthor = customAuthor;
}
}
videoDate =
info.upload_date ||
new Date().toISOString().slice(0, 10).replace(/-/g, "");
thumbnailUrl = info.thumbnail;
source = info.extractor || "generic";
// Update the safe base filename with the actual title
const newSafeBaseFilename = `${sanitizeFilename(
@@ -130,8 +171,8 @@ export class YouTubeDownloader {
const newVideoPath = path.join(VIDEOS_DIR, finalVideoFilename);
const newThumbnailPath = path.join(IMAGES_DIR, finalThumbnailFilename);
// Download the YouTube video
console.log("Downloading YouTube video to:", newVideoPath);
// Download the video
console.log("Downloading video to:", newVideoPath);
if (downloadId) {
storageService.updateActiveDownload(downloadId, {
@@ -140,20 +181,25 @@ export class YouTubeDownloader {
});
}
// Use exec to capture stdout for progress
// Format selection prioritizes Safari-compatible codecs (H.264/AAC)
// avc1 is the H.264 variant that Safari supports best
// Use Android client to avoid SABR streaming issues and JS runtime requirements
const subprocess = youtubedl.exec(videoUrl, {
// Prepare flags
const flags: any = {
output: newVideoPath,
format: "bestvideo[ext=mp4][vcodec^=avc1]+bestaudio[ext=m4a][acodec=aac]/bestvideo[ext=mp4][vcodec=h264]+bestaudio[ext=m4a]/best[ext=mp4]/best",
format: "bestvideo[ext=mp4]+bestaudio[ext=m4a]/best[ext=mp4]/best",
mergeOutputFormat: "mp4",
'extractor-args': "youtube:player_client=android",
addHeader: [
};
// Add YouTube specific flags if it's a YouTube URL
if (videoUrl.includes("youtube.com") || videoUrl.includes("youtu.be")) {
flags.format = "bestvideo[ext=mp4][vcodec^=avc1]+bestaudio[ext=m4a][acodec=aac]/bestvideo[ext=mp4][vcodec=h264]+bestaudio[ext=m4a]/best[ext=mp4]/best";
flags['extractor-args'] = "youtube:player_client=android";
flags.addHeader = [
'Referer:https://www.youtube.com/',
'User-Agent:Mozilla/5.0 (Linux; Android 10; K) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Mobile Safari/537.36'
]
} as any);
];
}
// Use exec to capture stdout for progress
const subprocess = youtubedl.exec(videoUrl, flags);
if (onStart) {
onStart(() => {
@@ -163,7 +209,6 @@ export class YouTubeDownloader {
// Clean up partial files
console.log("Cleaning up partial files...");
try {
// youtube-dl creates .part files during download
const partVideoPath = `${newVideoPath}.part`;
const partThumbnailPath = `${newThumbnailPath}.part`;
@@ -209,12 +254,11 @@ export class YouTubeDownloader {
await subprocess;
console.log("YouTube video downloaded successfully");
console.log("Video downloaded successfully");
// Download and save the thumbnail
thumbnailSaved = false;
// Download the thumbnail image
if (thumbnailUrl) {
try {
console.log("Downloading thumbnail from:", thumbnailUrl);
@@ -242,9 +286,9 @@ export class YouTubeDownloader {
// Continue even if thumbnail download fails
}
}
} catch (youtubeError) {
console.error("Error in YouTube download process:", youtubeError);
throw youtubeError;
} catch (error) {
console.error("Error in download process:", error);
throw error;
}
// Create metadata for the video
@@ -254,7 +298,7 @@ export class YouTubeDownloader {
author: videoAuthor || "Unknown",
date:
videoDate || new Date().toISOString().slice(0, 10).replace(/-/g, ""),
source: "youtube",
source: source, // Use extracted source
sourceUrl: videoUrl,
videoFilename: finalVideoFilename,
thumbnailFilename: thumbnailSaved ? finalThumbnailFilename : undefined,
@@ -269,12 +313,9 @@ export class YouTubeDownloader {
};
// If duration is missing from info, try to extract it from file
// We need to reconstruct the path because newVideoPath is not in scope here if we are outside the try block
// But wait, finalVideoFilename is available.
const finalVideoPath = path.join(VIDEOS_DIR, finalVideoFilename);
try {
// Dynamic import to avoid circular dependency if any, though here it's fine
const { getVideoDuration } = await import("../../services/metadataService");
const duration = await getVideoDuration(finalVideoPath);
if (duration) {

View File

@@ -1,12 +1,12 @@
{
"name": "frontend",
"version": "1.2.3",
"version": "1.2.5",
"lockfileVersion": 3,
"requires": true,
"packages": {
"": {
"name": "frontend",
"version": "1.2.3",
"version": "1.2.5",
"dependencies": {
"@emotion/react": "^11.14.0",
"@emotion/styled": "^11.14.1",

View File

@@ -1,7 +1,7 @@
{
"name": "frontend",
"private": true,
"version": "1.2.4",
"version": "1.3.0",
"type": "module",
"scripts": {
"dev": "vite",

View File

@@ -22,6 +22,9 @@ const Footer = () => {
<Box sx={{ display: 'flex', flexDirection: { xs: 'column', sm: 'row' }, justifyContent: 'center', alignItems: 'center' }}>
<Box sx={{ display: 'flex', alignItems: 'center', mt: { xs: 1, sm: 0 } }}>
<Typography variant="body2" color="text.secondary" sx={{ mr: 2 }}>
v{import.meta.env.VITE_APP_VERSION}
</Typography>
<Typography variant="body2" color="text.secondary" sx={{ mr: 2 }}>
Created by franklioxygen
</Typography>

View File

@@ -123,10 +123,9 @@ const Header: React.FC<HeaderProps> = ({
return;
}
const youtubeRegex = /^(https?:\/\/)?(www\.)?(youtube\.com|youtu\.?be)\/.+$/;
const bilibiliRegex = /^(https?:\/\/)?(www\.)?(bilibili\.com|b23\.tv)\/.+$/;
const missavRegex = /^(https?:\/\/)?(www\.)?(missav\.(ai|ws|com))\/.+$/;
const isUrl = youtubeRegex.test(videoUrl) || bilibiliRegex.test(videoUrl) || missavRegex.test(videoUrl);
// Generic URL check
const urlRegex = /^(https?:\/\/[^\s]+)/;
const isUrl = urlRegex.test(videoUrl);
setError('');
setIsSubmitting(true);

View File

@@ -1,9 +1,6 @@
import {
Delete,
Folder,
Movie,
OndemandVideo,
YouTube
Folder
} from '@mui/icons-material';
import {
Box,
@@ -140,17 +137,7 @@ const VideoCard: React.FC<VideoCardProps> = ({
}
};
// Get source icon
const getSourceIcon = () => {
if (video.source === 'bilibili') {
return <OndemandVideo sx={{ color: '#23ade5' }} />; // Bilibili blue
} else if (video.source === 'local') {
return <Folder sx={{ color: '#4caf50' }} />; // Local green (using Folder as generic local icon, or maybe VideoFile if available)
} else if (video.source === 'missav') {
return <Movie sx={{ color: '#ff4081' }} />; // Pink for MissAV
}
return <YouTube sx={{ color: '#ff0000' }} />; // YouTube red
};
return (
<>
@@ -192,9 +179,7 @@ const VideoCard: React.FC<VideoCardProps> = ({
}}
/>
<Box sx={{ position: 'absolute', top: 8, right: 8, bgcolor: 'rgba(0,0,0,0.7)', borderRadius: '50%', p: 0.5, display: 'flex' }}>
{getSourceIcon()}
</Box>
{video.partNumber && video.totalParts && video.totalParts > 1 && (
<Chip
@@ -276,7 +261,7 @@ const VideoCard: React.FC<VideoCardProps> = ({
sx={{
position: 'absolute',
top: 8,
right: 40, // Positioned to the left of the source icon
right: 8,
bgcolor: 'rgba(0,0,0,0.6)',
color: 'white',
opacity: 0, // Hidden by default, shown on hover

View File

@@ -242,7 +242,7 @@ const VideoInfo: React.FC<VideoInfoProps> = ({
)}
<Typography variant="body2" sx={{ display: 'flex', alignItems: 'center' }}>
<VideoLibrary fontSize="small" sx={{ mr: 0.5 }} />
<strong>{t('source')}</strong> {video.source === 'bilibili' ? 'Bilibili' : (video.source === 'local' ? 'Local Upload' : 'YouTube')}
<strong>{t('source')}</strong> {video.source ? video.source.charAt(0).toUpperCase() + video.source.slice(1) : 'Unknown'}
</Typography>
{video.addedAt && (
<Typography variant="body2" sx={{ display: 'flex', alignItems: 'center' }}>

View File

@@ -93,15 +93,40 @@ const DownloadPage: React.FC = () => {
mutationFn: async (id: string) => {
await axios.post(`${API_URL}/downloads/cancel/${id}`);
},
onSuccess: () => {
showSnackbar(t('downloadCancelled') || 'Download cancelled');
// DownloadContext handles active/queued updates via its own polling
// But we might want to invalidate to be sure
onMutate: async (id: string) => {
// Cancel any outgoing refetches (so they don't overwrite our optimistic update)
await queryClient.cancelQueries({ queryKey: ['downloadStatus'] });
// Snapshot the previous value
const previousStatus = queryClient.getQueryData(['downloadStatus']);
// Optimistically update to the new value
queryClient.setQueryData(['downloadStatus'], (old: any) => {
if (!old) return old;
return {
...old,
activeDownloads: old.activeDownloads.filter((d: any) => d.id !== id),
queuedDownloads: old.queuedDownloads.filter((d: any) => d.id !== id),
};
});
// Return a context object with the snapshotted value
return { previousStatus };
},
onError: (_err, _id, context) => {
// If the mutation fails, use the context returned from onMutate to roll back
if (context?.previousStatus) {
queryClient.setQueryData(['downloadStatus'], context.previousStatus);
}
showSnackbar(t('error') || 'Error');
},
onSettled: () => {
// Always refetch after error or success:
queryClient.invalidateQueries({ queryKey: ['downloadStatus'] });
},
onError: () => {
showSnackbar(t('error') || 'Error');
}
onSuccess: () => {
showSnackbar(t('downloadCancelled') || 'Download cancelled');
},
});
const handleCancelDownload = (id: string) => {

View File

@@ -1,5 +1,6 @@
import react from "@vitejs/plugin-react";
import { defineConfig } from "vite";
import packageJson from './package.json';
// https://vite.dev/config/
export default defineConfig({
@@ -7,4 +8,7 @@ export default defineConfig({
server: {
port: 5556,
},
define: {
'import.meta.env.VITE_APP_VERSION': JSON.stringify(packageJson.version)
}
});

4
package-lock.json generated
View File

@@ -1,12 +1,12 @@
{
"name": "mytube",
"version": "1.2.3",
"version": "1.2.5",
"lockfileVersion": 3,
"requires": true,
"packages": {
"": {
"name": "mytube",
"version": "1.2.3",
"version": "1.2.5",
"license": "MIT",
"dependencies": {
"concurrently": "^8.2.2"

View File

@@ -1,6 +1,6 @@
{
"name": "mytube",
"version": "1.2.4",
"version": "1.3.0",
"description": "YouTube video downloader and player application",
"main": "index.js",
"scripts": {