davidl/MyTube
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: Add security measures and URL validation

Browse Source
This commit is contained in:
Peifan Li
2025-12-22 17:56:37 -05:00
parent fbeba5bca1
commit bbea5b3897
14 changed files with 420 additions and 109 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
.gitignore vendored
View File

@@ -58,3 +58,6 @@ backend/data/*
*.mp4
*.mkv
*.avi
# Snyk Security Extension - AI Rules (auto-generated)
.cursor/rules/snyk_rules.mdc

66
backend/src/controllers/videoController.ts
View File

@@ -1,4 +1,3 @@
import { exec } from "child_process";
import { Request, Response } from "express";
import fs from "fs-extra";
import multer from "multer";
@@ -9,6 +8,11 @@ import { getVideoDuration } from "../services/metadataService";
import * as storageService from "../services/storageService";
import { logger } from "../utils/logger";
import { successResponse } from "../utils/response";
import {
execFileSafe,
validateImagePath,
validateVideoPath,
} from "../utils/security";
// Configure Multer for file uploads
const storage = multer.diskStorage({
@@ -22,7 +26,13 @@ const storage = multer.diskStorage({
},
});
export const upload = multer({ storage: storage });
// Configure multer with large file size limit (100GB)
export const upload = multer({
storage: storage,
limits: {
fileSize: 100 * 1024 * 1024 * 1024, // 10GB in bytes
},
});
/**
* Get all videos
@@ -113,21 +123,25 @@ export const uploadVideo = async (
const videoPath = path.join(VIDEOS_DIR, videoFilename);
const thumbnailPath = path.join(IMAGES_DIR, thumbnailFilename);
// Generate thumbnail
await new Promise<void>((resolve, _reject) => {
exec(
`ffmpeg -i "${videoPath}" -ss 00:00:00 -vframes 1 "${thumbnailPath}"`,
(error) => {
if (error) {
logger.error("Error generating thumbnail:", error);
// We resolve anyway to not block the upload, just without a custom thumbnail
resolve();
} else {
resolve();
}
}
);
});
// Validate paths to prevent path traversal
const validatedVideoPath = validateVideoPath(videoPath);
const validatedThumbnailPath = validateImagePath(thumbnailPath);
// Generate thumbnail using execFileSafe to prevent command injection
try {
await execFileSafe("ffmpeg", [
"-i",
validatedVideoPath,
"-ss",
"00:00:00",
"-vframes",
"1",
validatedThumbnailPath,
]);
} catch (error) {
logger.error("Error generating thumbnail:", error);
// Continue without thumbnail - don't block the upload
}
// Get video duration
const duration = await getVideoDuration(videoPath);
@@ -226,7 +240,9 @@ export const getAuthorChannelUrl = async (
// First, check if we have the video in the database with a stored channelUrl
const existingVideo = storageService.getVideoBySourceUrl(sourceUrl);
if (existingVideo && existingVideo.channelUrl) {
res.status(200).json({ success: true, channelUrl: existingVideo.channelUrl });
res
.status(200)
.json({ success: true, channelUrl: existingVideo.channelUrl });
return;
}
@@ -261,10 +277,12 @@ export const getAuthorChannelUrl = async (
// If we have the video in database, try to get channelUrl from there first
// (already checked above, but this is for clarity)
if (existingVideo && existingVideo.channelUrl) {
res.status(200).json({ success: true, channelUrl: existingVideo.channelUrl });
res
.status(200)
.json({ success: true, channelUrl: existingVideo.channelUrl });
return;
}
const axios = (await import("axios")).default;
const { extractBilibiliVideoId } = await import("../utils/helpers");
@@ -295,12 +313,14 @@ export const getAuthorChannelUrl = async (
) {
const mid = response.data.data.owner.mid;
const spaceUrl = `https://space.bilibili.com/${mid}`;
// If we have the video in database, update it with the channelUrl
if (existingVideo) {
storageService.updateVideo(existingVideo.id, { channelUrl: spaceUrl });
storageService.updateVideo(existingVideo.id, {
channelUrl: spaceUrl,
});
}
res.status(200).json({ success: true, channelUrl: spaceUrl });
return;
}

34
backend/src/controllers/videoMetadataController.ts
View File

@@ -1,4 +1,3 @@
import { exec } from "child_process";
import { Request, Response } from "express";
import fs from "fs-extra";
import path from "path";
@@ -7,6 +6,7 @@ import { NotFoundError, ValidationError } from "../errors/DownloadErrors";
import * as storageService from "../services/storageService";
import { logger } from "../utils/logger";
import { successResponse } from "../utils/response";
import { execFileSafe, validateVideoPath, validateImagePath } from "../utils/security";
/**
* Rate video
@@ -91,21 +91,23 @@ export const refreshThumbnail = async (
// Ensure directory exists
fs.ensureDirSync(path.dirname(thumbnailAbsolutePath));
// Generate thumbnail
await new Promise<void>((resolve, reject) => {
// -y to overwrite existing file
exec(
`ffmpeg -i "${videoFilePath}" -ss 00:00:00 -vframes 1 "${thumbnailAbsolutePath}" -y`,
(error) => {
if (error) {
logger.error("Error generating thumbnail:", error);
reject(error);
} else {
resolve();
}
}
);
});
// Validate paths to prevent path traversal
const validatedVideoPath = validateVideoPath(videoFilePath);
const validatedThumbnailPath = validateImagePath(thumbnailAbsolutePath);
// Generate thumbnail using execFileSafe to prevent command injection
try {
await execFileSafe("ffmpeg", [
"-i", validatedVideoPath,
"-ss", "00:00:00",
"-vframes", "1",
validatedThumbnailPath,
"-y"
]);
} catch (error) {
logger.error("Error generating thumbnail:", error);
throw error;
}
// Update video record if needed (switching from remote to local, or creating new)
if (needsDbUpdate) {

12
backend/src/server.ts
View File

@@ -7,12 +7,12 @@ import cors from "cors";
import express from "express";
import { IMAGES_DIR, SUBTITLES_DIR, VIDEOS_DIR } from "./config/paths";
import { runMigrations } from "./db/migrate";
import { visitorModeMiddleware } from "./middleware/visitorModeMiddleware";
import { visitorModeSettingsMiddleware } from "./middleware/visitorModeSettingsMiddleware";
import apiRoutes from "./routes/api";
import settingsRoutes from "./routes/settingsRoutes";
import downloadManager from "./services/downloadManager";
import * as storageService from "./services/storageService";
import { visitorModeMiddleware } from "./middleware/visitorModeMiddleware";
import { visitorModeSettingsMiddleware } from "./middleware/visitorModeSettingsMiddleware";
import { logger } from "./utils/logger";
import { VERSION } from "./version";
@@ -22,10 +22,14 @@ VERSION.displayVersion();
const app = express();
const PORT = process.env.PORT || 5551;
// Security: Disable X-Powered-By header to prevent information disclosure
app.disable("x-powered-by");
// Middleware
app.use(cors());
app.use(express.json());
app.use(express.urlencoded({ extended: true }));
// Increase body size limits for large file uploads (10GB)
app.use(express.json({ limit: "100gb" }));
app.use(express.urlencoded({ extended: true, limit: "100gb" }));
// Initialize storage (create directories, etc.)
// storageService.initializeStorage(); // Moved inside startServer

62
backend/src/services/cloudStorage/cloudScanner.ts
View File

@@ -2,12 +2,12 @@
* Cloud storage scanning operations
*/
import { exec } from "child_process";
import fs from "fs-extra";
import path from "path";
import { IMAGES_DIR } from "../../config/paths";
import { formatVideoFilename } from "../../utils/helpers";
import { logger } from "../../utils/logger";
import { execFileSafe, validateImagePath, validateUrl } from "../../utils/security";
import { getVideos, saveVideo } from "../storageService";
import { clearFileListCache, getFilesRecursively } from "./fileLister";
import { uploadFile } from "./fileUploader";
@@ -238,25 +238,27 @@ export async function scanCloudFiles(
// Ensure directory exists
fs.ensureDirSync(path.dirname(tempThumbnailPath));
// Validate paths and URL to prevent command injection and SSRF
const validatedThumbnailPath = validateImagePath(tempThumbnailPath);
const validatedVideoUrl = validateUrl(videoSignedUrl);
// Generate thumbnail using ffmpeg with signed URL
// ffmpeg can work with HTTP URLs directly
await new Promise<void>((resolve, reject) => {
exec(
`ffmpeg -i "${videoSignedUrl}" -ss 00:00:00 -vframes 1 "${tempThumbnailPath}" -y`,
{ timeout: 30000 }, // 30 second timeout
(error) => {
if (error) {
logger.error(
`[CloudStorage] Error generating thumbnail for ${filename}:`,
error
);
reject(error);
} else {
resolve();
}
}
try {
await execFileSafe("ffmpeg", [
"-i", validatedVideoUrl,
"-ss", "00:00:00",
"-vframes", "1",
validatedThumbnailPath,
"-y"
], { timeout: 30000 });
} catch (error) {
logger.error(
`[CloudStorage] Error generating thumbnail for ${filename}:`,
error
);
});
throw error;
}
// Upload thumbnail to cloud storage (with correct filename and location)
// remoteThumbnailPath is a full absolute path (e.g., /a/电影/video/thumbnail.jpg)
@@ -271,21 +273,17 @@ export async function scanCloudFiles(
// Get duration
let duration: string | undefined = undefined;
try {
const durationOutput = await new Promise<string>(
(resolve, reject) => {
exec(
`ffprobe -v error -show_entries format=duration -of default=noprint_wrappers=1:nokey=1 "${videoSignedUrl}"`,
{ timeout: 10000 },
(error, stdout, _stderr) => {
if (error) {
reject(error);
} else {
resolve(stdout.trim());
}
}
);
}
);
// Validate URL to prevent SSRF
const validatedVideoUrlForDuration = validateUrl(videoSignedUrl);
const { stdout } = await execFileSafe("ffprobe", [
"-v", "error",
"-show_entries", "format=duration",
"-of", "default=noprint_wrappers=1:nokey=1",
validatedVideoUrlForDuration
], { timeout: 10000 });
const durationOutput = stdout.trim();
if (durationOutput) {
const durationSec = parseFloat(durationOutput);
if (!isNaN(durationSec)) {

31
backend/src/services/downloaders/ytdlp/ytdlpMetadata.ts
View File

@@ -18,16 +18,27 @@ export async function getVideoInfo(url: string): Promise<VideoInfo> {
const networkConfig = getNetworkConfigFromUserConfig(userConfig);
const PROVIDER_SCRIPT = getProviderScript();
const info = await executeYtDlpJson(url, {
...networkConfig,
noWarnings: true,
preferFreeFormats: true,
...(PROVIDER_SCRIPT
? {
extractorArgs: `youtubepot-bgutilscript:script_path=${PROVIDER_SCRIPT}`,
}
: {}),
});
// Explicitly exclude format-related options when fetching metadata
// Format restrictions should only apply during actual downloads, not metadata fetching
const info = await executeYtDlpJson(
url,
{
...networkConfig,
noWarnings: true,
preferFreeFormats: true,
...(PROVIDER_SCRIPT
? {
extractorArgs: `youtubepot-bgutilscript:script_path=${PROVIDER_SCRIPT}`,
}
: {}),
// Explicitly exclude format options to avoid format availability errors
formatSort: undefined,
format: undefined,
S: undefined,
f: undefined,
},
true // Enable retry without format restrictions if format error occurs
);
return {
title: info.title || "Video",

23
backend/src/services/metadataService.ts
View File

@@ -1,4 +1,3 @@
import { exec } from 'child_process';
import { eq } from 'drizzle-orm';
import fs from 'fs-extra';
import path from 'path';
@@ -9,6 +8,7 @@ import {
import { VIDEOS_DIR } from '../config/paths';
import { db } from '../db';
import { videos } from '../db/schema';
import { execFileSafe, validateVideoPath } from '../utils/security';
export const getVideoDuration = async (filePath: string): Promise<number | null> => {
try {
@@ -17,17 +17,18 @@ export const getVideoDuration = async (filePath: string): Promise<number | null>
throw FileError.notFound(filePath);
}
const command = `ffprobe -v error -show_entries format=duration -of default=noprint_wrappers=1:nokey=1 "${filePath}"`;
const duration = await new Promise<string>((resolve, reject) => {
exec(command, (error, stdout, stderr) => {
if (error) {
reject(ExecutionError.fromCommand(command, error, error.code || undefined));
} else {
resolve(stdout.trim());
}
});
});
// Validate path to prevent path traversal
const validatedPath = validateVideoPath(filePath);
// Use execFileSafe to prevent command injection
const { stdout } = await execFileSafe("ffprobe", [
"-v", "error",
"-show_entries", "format=duration",
"-of", "default=noprint_wrappers=1:nokey=1",
validatedPath
]);
const duration = stdout.trim();
if (duration) {
const durationSec = parseFloat(duration);
if (!isNaN(durationSec)) {

22
backend/src/utils/helpers.ts
View File

@@ -1,4 +1,5 @@
import axios from "axios";
import { validateUrl } from "./security";
// Helper function to check if a string is a valid URL
export function isValidUrl(string: string): boolean {
@@ -38,20 +39,29 @@ export async function resolveShortUrl(url: string): Promise<string> {
try {
console.log(`Resolving shortened URL: ${url}`);
// Validate URL to prevent SSRF attacks
const validatedUrl = validateUrl(url);
// Make a HEAD request to follow redirects
const response = await axios.head(url, {
const response = await axios.head(validatedUrl, {
maxRedirects: 5,
validateStatus: null,
});
// Get the final URL after redirects
const resolvedUrl = response.request.res.responseUrl || url;
console.log(`Resolved to: ${resolvedUrl}`);
// Get the final URL after redirects and validate it
const resolvedUrl = response.request.res.responseUrl || validatedUrl;
const validatedResolvedUrl = validateUrl(resolvedUrl);
console.log(`Resolved to: ${validatedResolvedUrl}`);
return resolvedUrl;
return validatedResolvedUrl;
} catch (error: any) {
console.error(`Error resolving shortened URL: ${error.message}`);
return url; // Return original URL if resolution fails
// If validation fails, return original URL only if it's already validated
try {
return validateUrl(url);
} catch {
throw new Error(`Invalid URL: ${url}`);
}
}
}

147
backend/src/utils/security.ts Normal file
View File

@@ -0,0 +1,147 @@
import { execFile } from "child_process";
import path from "path";
import { IMAGES_DIR, VIDEOS_DIR } from "../config/paths";
/**
* Validates that a file path is within an allowed directory
* Prevents path traversal attacks
*/
export function validatePathWithinDirectory(
filePath: string,
allowedDir: string
): boolean {
const resolvedPath = path.resolve(filePath);
const resolvedAllowedDir = path.resolve(allowedDir);
// Ensure the resolved path starts with the allowed directory
return (
resolvedPath.startsWith(resolvedAllowedDir + path.sep) ||
resolvedPath === resolvedAllowedDir
);
}
/**
* Safely resolves a file path within an allowed directory
* Throws an error if the path is outside the allowed directory
*/
export function resolveSafePath(filePath: string, allowedDir: string): string {
const resolvedPath = path.resolve(filePath);
const resolvedAllowedDir = path.resolve(allowedDir);
if (
!resolvedPath.startsWith(resolvedAllowedDir + path.sep) &&
resolvedPath !== resolvedAllowedDir
) {
throw new Error(
`Path traversal detected: ${filePath} is outside ${allowedDir}`
);
}
return resolvedPath;
}
/**
* Validates that a file path is within the videos directory
*/
export function validateVideoPath(filePath: string): string {
return resolveSafePath(filePath, VIDEOS_DIR);
}
/**
* Validates that a file path is within the images directory
*/
export function validateImagePath(filePath: string): string {
return resolveSafePath(filePath, IMAGES_DIR);
}
/**
* Safely execute a command with arguments
* Prevents command injection by using execFile instead of exec
*/
export function execFileSafe(
command: string,
args: string[],
options?: { cwd?: string; timeout?: number }
): Promise<{ stdout: string; stderr: string }> {
return new Promise((resolve, reject) => {
execFile(command, args, options, (error, stdout, stderr) => {
if (error) {
reject(error);
} else {
resolve({ stdout: stdout.toString(), stderr: stderr.toString() });
}
});
});
}
/**
* Validates a URL to prevent SSRF attacks
* Only allows http/https protocols and validates the hostname
*/
export function validateUrl(url: string): string {
try {
const urlObj = new URL(url);
// Only allow http and https protocols
if (urlObj.protocol !== "http:" && urlObj.protocol !== "https:") {
throw new Error(
`Invalid protocol: ${urlObj.protocol}. Only http and https are allowed.`
);
}
// Block private/internal IP addresses
const hostname = urlObj.hostname;
if (
hostname === "localhost" ||
hostname === "127.0.0.1" ||
hostname === "0.0.0.0" ||
hostname === "::1" ||
hostname.startsWith("192.168.") ||
hostname.startsWith("10.") ||
hostname.startsWith("172.16.") ||
hostname.startsWith("172.17.") ||
hostname.startsWith("172.18.") ||
hostname.startsWith("172.19.") ||
hostname.startsWith("172.20.") ||
hostname.startsWith("172.21.") ||
hostname.startsWith("172.22.") ||
hostname.startsWith("172.23.") ||
hostname.startsWith("172.24.") ||
hostname.startsWith("172.25.") ||
hostname.startsWith("172.26.") ||
hostname.startsWith("172.27.") ||
hostname.startsWith("172.28.") ||
hostname.startsWith("172.29.") ||
hostname.startsWith("172.30.") ||
hostname.startsWith("172.31.") ||
hostname === "[::1]"
) {
throw new Error(
`SSRF protection: Blocked access to private/internal IP: ${hostname}`
);
}
return url;
} catch (error) {
if (error instanceof TypeError) {
throw new Error(`Invalid URL format: ${url}`);
}
throw error;
}
}
/**
* Sanitizes a string for safe use in HTML
* Prevents XSS attacks
*/
export function sanitizeHtml(str: string): string {
const map: { [key: string]: string } = {
"&": "&amp;",
"<": "&lt;",
">": "&gt;",
'"': "&quot;",
"'": "&#x27;",
"/": "&#x2F;",
};
return str.replace(/[&<>"'/]/g, (s) => map[s]);
}

77
backend/src/utils/ytDlpUtils.ts
View File

@@ -106,10 +106,14 @@ export function flagsToArgs(flags: Record<string, any>): string[] {
/**
* Execute yt-dlp with JSON output and return parsed result
* @param url - Video URL
* @param flags - yt-dlp flags
* @param retryWithoutFormatRestrictions - If true, retry without format restrictions if format error occurs
*/
export async function executeYtDlpJson(
url: string,
flags: Record<string, any> = {}
flags: Record<string, any> = {},
retryWithoutFormatRestrictions: boolean = true
): Promise<any> {
const args = ["--dump-single-json", "--no-warnings", ...flagsToArgs(flags)];
@@ -144,8 +148,77 @@ export async function executeYtDlpJson(
stderr += data.toString();
});
subprocess.on("close", (code) => {
subprocess.on("close", async (code) => {
if (code !== 0) {
// Check if this is a format availability error
const isFormatError =
stderr.includes("Requested format is not available") ||
stderr.includes("format is not available") ||
stderr.includes("No video formats found");
// If it's a format error and we should retry, try again without format restrictions
if (isFormatError && retryWithoutFormatRestrictions) {
const hasFormatRestrictions =
(flags.formatSort !== undefined && flags.formatSort !== null) ||
(flags.format !== undefined && flags.format !== null) ||
(flags.S !== undefined && flags.S !== null) ||
(flags.f !== undefined && flags.f !== null);
if (hasFormatRestrictions) {
console.log(
"Format not available, retrying without format restrictions..."
);
try {
// Remove format-related flags
const retryFlags = { ...flags };
delete retryFlags.formatSort;
delete retryFlags.format;
delete retryFlags.S;
delete retryFlags.f;
// Retry without format restrictions (don't retry again to avoid infinite loop)
const result = await executeYtDlpJson(url, retryFlags, false);
resolve(result);
return;
} catch (retryError) {
// If retry also fails, reject with original error
const error = new Error(
`yt-dlp process exited with code ${code}`
);
(error as any).stderr = stderr;
reject(error);
return;
}
} else {
// Format error but no format restrictions in flags - might be from config file
// Try with explicit format override to bypass config file
console.log(
"Format not available (possibly from config file), retrying with explicit format override..."
);
try {
const retryFlags = {
...flags,
// Explicitly set format to "best" to override any config file settings
format: "best",
formatSort: undefined,
S: undefined,
f: undefined,
};
// Retry without format restrictions (don't retry again to avoid infinite loop)
const result = await executeYtDlpJson(url, retryFlags, false);
resolve(result);
return;
} catch (retryError) {
// If retry also fails, reject with original error
const error = new Error(
`yt-dlp process exited with code ${code}`
);
(error as any).stderr = stderr;
reject(error);
return;
}
}
}
const error = new Error(`yt-dlp process exited with code ${code}`);
(error as any).stderr = stderr;
reject(error);

7
frontend/nginx.conf
View File

@@ -1,6 +1,9 @@
server {
listen 5556;
# Allow large file uploads (10GB limit for video files)
client_max_body_size 100G;
location / {
root /usr/share/nginx/html;
index index.html index.htm;
@@ -13,6 +16,10 @@ server {
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Forwarded-Proto $scheme;
# Increase timeouts for large file uploads
proxy_read_timeout 300s;
proxy_connect_timeout 300s;
proxy_send_timeout 300s;
}
location ^~ /videos {

4
frontend/src/components/Header/SearchInput.tsx
View File

@@ -73,7 +73,7 @@ const SearchInput: React.FC<SearchInputProps> = ({
}}
slotProps={{
input: {
startAdornment: (
startAdornment: !isMobile ? (
<InputAdornment position="start">
<IconButton
onClick={handlePaste}
@@ -85,7 +85,7 @@ const SearchInput: React.FC<SearchInputProps> = ({
<ContentPaste />
</IconButton>
</InputAdornment>
),
) : null,
endAdornment: (
<InputAdornment position="end">
{isSearchMode && searchTerm && (

11
frontend/src/pages/VideoPlayer.tsx
View File

@@ -24,6 +24,7 @@ import { useVideo } from '../contexts/VideoContext';
import { useCloudStorageUrl } from '../hooks/useCloudStorageUrl';
import { Collection, Video } from '../types';
import { getRecommendations } from '../utils/recommendations';
import { validateUrlForOpen } from '../utils/urlValidation';
const API_URL = import.meta.env.VITE_API_URL;
const BACKEND_URL = import.meta.env.VITE_BACKEND_URL;
@@ -205,9 +206,13 @@ const VideoPlayer: React.FC = () => {
// If it's a YouTube or Bilibili video, try to get the channel URL
if (video.source === 'youtube' || video.source === 'bilibili') {
if (authorChannelUrl) {
// Open the channel URL in a new tab
window.open(authorChannelUrl, '_blank', 'noopener,noreferrer');
return;
// Validate URL to prevent open redirect attacks
const validatedUrl = validateUrlForOpen(authorChannelUrl);
if (validatedUrl) {
// Open the channel URL in a new tab
window.open(validatedUrl, '_blank', 'noopener,noreferrer');
return;
}
}
}

30
frontend/src/utils/urlValidation.ts Normal file
View File

@@ -0,0 +1,30 @@
/**
* Validates a URL to prevent open redirect attacks
* Only allows http/https protocols
*/
export function isValidUrl(url: string): boolean {
try {
const urlObj = new URL(url);
// Only allow http and https protocols
return urlObj.protocol === "http:" || urlObj.protocol === "https:";
} catch {
return false;
}
}
/**
* Validates and sanitizes a URL for safe use in window.open
* Returns null if URL is invalid
*/
export function validateUrlForOpen(
url: string | null | undefined
): string | null {
if (!url) return null;
if (!isValidUrl(url)) {
console.warn(`Invalid URL blocked: ${url}`);
return null;
}
return url;
}