davidl/MyTube
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: Add support for multiple scan paths in cloud storage

Browse Source
This commit is contained in:
Peifan Li
2025-12-22 16:34:09 -05:00
parent d96c785da9
commit 31b2d0569f
17 changed files with 191 additions and 55 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
backend/src/controllers/settingsController.ts
View File

@@ -28,6 +28,7 @@ interface Settings {
openListToken?: string;
openListPublicUrl?: string;
cloudDrivePath?: string;
cloudDriveScanPaths?: string;
homeSidebarOpen?: boolean;
subtitlesEnabled?: boolean;
websiteName?: string;
@@ -52,6 +53,7 @@ const defaultSettings: Settings = {
openListToken: "",
openListPublicUrl: "",
cloudDrivePath: "",
cloudDriveScanPaths: "",
homeSidebarOpen: true,
subtitlesEnabled: true,
websiteName: "MyTube",

156
backend/src/services/cloudStorage/cloudScanner.ts
View File

@@ -12,7 +12,7 @@ import { getVideos, saveVideo } from "../storageService";
import { clearFileListCache, getFilesRecursively } from "./fileLister";
import { uploadFile } from "./fileUploader";
import { normalizeUploadPath } from "./pathUtils";
import { CloudDriveConfig, ScanResult } from "./types";
import { CloudDriveConfig, FileWithPath, ScanResult } from "./types";
import { clearSignedUrlCache, getSignedUrl } from "./urlSigner";
/**
@@ -29,11 +29,40 @@ export async function scanCloudFiles(
onProgress?.("Scanning cloud storage for videos...");
try {
// Normalize upload path
const uploadPath = normalizeUploadPath(config.uploadPath);
// Determine which paths to scan
// Always scan the default uploadPath
// If scanPaths is provided, scan those as well
const uploadRoot = normalizeUploadPath(config.uploadPath);
const pathsToScan: string[] = [uploadRoot];
// Recursively get all files from cloud storage
const allCloudFiles = await getFilesRecursively(config, uploadPath);
if (config.scanPaths && config.scanPaths.length > 0) {
const additionalPaths = config.scanPaths.map((path) =>
normalizeUploadPath(path)
);
// Avoid duplicates
for (const path of additionalPaths) {
if (!pathsToScan.includes(path)) {
pathsToScan.push(path);
}
}
}
logger.info(
`[CloudStorage] Scanning ${
pathsToScan.length
} path(s): ${pathsToScan.join(", ")}`
);
// Recursively get all files from all scan paths
const allCloudFiles: FileWithPath[] = [];
for (const scanPath of pathsToScan) {
logger.info(`[CloudStorage] Scanning path: ${scanPath}`);
const filesFromPath = await getFilesRecursively(config, scanPath);
allCloudFiles.push(...filesFromPath);
logger.info(
`[CloudStorage] Found ${filesFromPath.length} files in ${scanPath}`
);
}
// Filter for video files
const videoExtensions = [".mp4", ".mkv", ".webm", ".avi", ".mov"];
@@ -77,6 +106,29 @@ export async function scanCloudFiles(
if (existingPaths.has(normalizedPath)) {
return false;
}
// Also check by calculated relative path (for backward compatibility with uploadPath files)
// This is important because for files in uploadPath, we store them relative to uploadPath
// But here filePath is absolute/relative to root
// Calculate what the storage path WOULD be for this file
let potentialStoragePath: string;
const absoluteFilePath = filePath.startsWith("/") ? filePath : "/" + filePath;
const absoluteUploadRoot = uploadRoot.startsWith("/") ? uploadRoot : "/" + uploadRoot;
if (absoluteFilePath.startsWith(absoluteUploadRoot)) {
// It's in the upload path, so it would be stored relative to that
const relativePath = path.relative(absoluteUploadRoot, absoluteFilePath);
potentialStoragePath = relativePath.replace(/\\/g, "/");
} else {
// It's NOT in the upload path, so it would be stored as full path (without leading slash)
potentialStoragePath = absoluteFilePath.substring(1);
}
if (existingPaths.has(potentialStoragePath)) {
return false;
}
// Also check by filename (for backward compatibility)
return !existingFilenames.has(file.name);
});
@@ -149,42 +201,39 @@ export async function scanCloudFiles(
);
// Determine remote thumbnail path (put it in the same folder as video)
// file.path is the full path from upload root (e.g., /a/b/Youtube/video/1.mp4) or relative path depending on alist version
// We normalize everything to be safe
// filePath is the full absolute path (e.g., /a/电影/video/1.mp4)
// We want to put the thumbnail in the same directory as the video
// 1. Get the upload root path
const uploadRoot = normalizeUploadPath(config.uploadPath);
// 2. Ensure filePath is absolute-like for path.relative calculation if it isn't already
// path.relative works best when both are absolute or both relative
// We treats them as absolute paths rooted at /
const absoluteUploadRoot = uploadRoot.startsWith("/")
? uploadRoot
: "/" + uploadRoot;
// 1. Normalize filePath to ensure it's an absolute path
const absoluteFilePath = filePath.startsWith("/")
? filePath
: "/" + filePath;
// 3. Calculate relative path from upload root to the file
// e.g. from "/a/b/Youtube" to "/a/b/Youtube/video/1.mp4" -> "video/1.mp4"
const relativeFilePath = path.relative(
absoluteUploadRoot,
absoluteFilePath
);
// 2. Get the directory of the video file
const videoDir = path.dirname(absoluteFilePath).replace(/\\/g, "/");
// 4. Get the directory of the video file relative to upload root
// e.g. "video"
const relativeVideoDir = path.dirname(relativeFilePath);
// 3. Construct thumbnail path in the same directory as video
const remoteThumbnailPath = videoDir.endsWith("/")
? `${videoDir}${newThumbnailFilename}`
: `${videoDir}/${newThumbnailFilename}`;
// 5. Construct thumbnail path
// If dir is root ".", relativeVideoDir is "."
const remoteThumbnailDir =
relativeVideoDir === "." ? "" : relativeVideoDir;
const remoteThumbnailPath = remoteThumbnailDir
? path
.join(remoteThumbnailDir, newThumbnailFilename)
.replace(/\\/g, "/")
: newThumbnailFilename;
// 4. Calculate relative path for video storage in database
// logic: if file is in uploadPath, use relative path; otherwise use full path
let relativeVideoPath: string;
// Ensure uploadRoot is absolute for comparison
const absoluteUploadRoot = uploadRoot.startsWith("/") ? uploadRoot : "/" + uploadRoot;
if (absoluteFilePath.startsWith(absoluteUploadRoot)) {
// It IS in the default upload path (or a subdir of it)
// Calculate relative path from uploadRoot
const relativePath = path.relative(absoluteUploadRoot, absoluteFilePath);
relativeVideoPath = relativePath.replace(/\\/g, "/");
} else {
// It is NOT in the default upload path (must be from one of the scanPaths)
// Use the full path relative to root
relativeVideoPath = absoluteFilePath.substring(1);
}
// Ensure directory exists
fs.ensureDirSync(path.dirname(tempThumbnailPath));
@@ -210,6 +259,8 @@ export async function scanCloudFiles(
});
// Upload thumbnail to cloud storage (with correct filename and location)
// remoteThumbnailPath is a full absolute path (e.g., /a/电影/video/thumbnail.jpg)
// uploadFile now supports absolute paths, so we can pass it directly
if (fs.existsSync(tempThumbnailPath)) {
await uploadFile(tempThumbnailPath, config, remoteThumbnailPath);
@@ -254,11 +305,14 @@ export async function scanCloudFiles(
Date.now() + Math.floor(Math.random() * 10000)
).toString();
// Store path relative to upload root
// relativeFilePath was already calculated above using path.relative
// e.g. "video/1.mp4" from "/a/b/Youtube" to "/a/b/Youtube/video/1.mp4"
// Just ensure it is normalized to forward slashes
const relativeVideoPath = relativeFilePath.replace(/\\/g, "/");
// relativeVideoPath was already calculated above
// For scan paths: full path without leading slash (e.g., "a/电影/video/1.mp4")
// For upload path: relative path (e.g., "video/1.mp4")
// For thumbnail path, store without leading slash to match video path format
const relativeThumbnailPath = remoteThumbnailPath.startsWith("/")
? remoteThumbnailPath.substring(1)
: remoteThumbnailPath;
const newVideo = {
id: videoId,
@@ -267,11 +321,11 @@ export async function scanCloudFiles(
source: "cloud",
sourceUrl: "",
videoFilename: filename, // Keep original filename
// Store path relative to upload root (e.g., video/1.mp4, not a/b/Youtube/video/1.mp4)
// Store path relative to root (e.g., "a/电影/video/1.mp4" or "video/1.mp4")
videoPath: `cloud:${relativeVideoPath}`,
thumbnailFilename: newThumbnailFilename,
thumbnailPath: `cloud:${remoteThumbnailPath}`, // Store relative path
thumbnailUrl: `cloud:${remoteThumbnailPath}`,
thumbnailPath: `cloud:${relativeThumbnailPath}`, // Store path without leading slash
thumbnailUrl: `cloud:${relativeThumbnailPath}`,
createdAt: file.modified
? new Date(file.modified).toISOString()
: new Date().toISOString(),
@@ -290,16 +344,18 @@ export async function scanCloudFiles(
// Clear cache for the new files
// Use relative paths (relative to upload root) for cache keys
clearSignedUrlCache(relativeVideoPath, "video");
clearSignedUrlCache(remoteThumbnailPath, "thumbnail");
// For thumbnail cache, use the directory path
const thumbnailDirForCache = path
.dirname(remoteThumbnailPath)
.replace(/\\/g, "/");
clearSignedUrlCache(thumbnailDirForCache, "thumbnail");
// Also clear file list cache for the directory where thumbnail was added
const baseUploadPath = normalizeUploadPath(config.uploadPath);
const dirPath = remoteThumbnailDir
? `${baseUploadPath}/${remoteThumbnailDir}`
: baseUploadPath;
// Normalize path (remove duplicate slashes)
const cleanDirPath = dirPath.replace(/\/+/g, "/");
clearFileListCache(cleanDirPath);
// remoteThumbnailPath is an absolute path, so we can use it directly
const thumbnailDir = path
.dirname(remoteThumbnailPath)
.replace(/\\/g, "/");
clearFileListCache(thumbnailDir);
} catch (error: any) {
const errorMessage =
error instanceof Error ? error.message : String(error);

16
backend/src/services/cloudStorage/config.ts
View File

@@ -10,12 +10,28 @@ import { CloudDriveConfig } from "./types";
*/
export function getConfig(): CloudDriveConfig {
const settings = getSettings();
// Parse scan paths from multi-line string
let scanPaths: string[] | undefined = undefined;
if (settings.cloudDriveScanPaths) {
scanPaths = settings.cloudDriveScanPaths
.split('\n')
.map((line: string) => line.trim())
.filter((line: string) => line.length > 0 && line.startsWith('/'));
// If no valid paths found, set to undefined
if (scanPaths && scanPaths.length === 0) {
scanPaths = undefined;
}
}
return {
enabled: settings.cloudDriveEnabled || false,
apiUrl: settings.openListApiUrl || "",
token: settings.openListToken || "",
publicUrl: settings.openListPublicUrl || undefined,
uploadPath: settings.cloudDrivePath || "/",
scanPaths: scanPaths,
};
}

16
backend/src/services/cloudStorage/fileUploader.ts
View File

@@ -93,12 +93,18 @@ export async function uploadFile(
let destinationPath = "";
if (remotePath) {
// If remotePath is provided, append it to uploadPath
// remotePath should be relative to uploadPath, e.g. "subdir/file.jpg" or "file.jpg"
// Check if remotePath is an absolute path (starts with /)
// If it's an absolute path, use it directly; otherwise, append to uploadPath
const normalizedRemotePath = remotePath.replace(/\\/g, "/");
destinationPath = normalizedUploadPath.endsWith("/")
? `${normalizedUploadPath}${normalizedRemotePath}`
: `${normalizedUploadPath}/${normalizedRemotePath}`;
if (normalizedRemotePath.startsWith("/")) {
// Absolute path - use it directly (e.g., /a/电影/video/thumbnail.jpg)
destinationPath = normalizedRemotePath;
} else {
// Relative path - append to uploadPath (e.g., "subdir/file.jpg" -> "/mytube-uploads/subdir/file.jpg")
destinationPath = normalizedUploadPath.endsWith("/")
? `${normalizedUploadPath}${normalizedRemotePath}`
: `${normalizedUploadPath}/${normalizedRemotePath}`;
}
} else {
// Default behavior: upload to root of uploadPath using source filename
destinationPath = normalizedUploadPath.endsWith("/")

1
backend/src/services/cloudStorage/types.ts
View File

@@ -8,6 +8,7 @@ export interface CloudDriveConfig {
token: string;
publicUrl?: string;
uploadPath: string;
scanPaths?: string[];
}
export interface CachedSignedUrl {

34
frontend/src/components/Settings/CloudDriveSettings.tsx
View File

@@ -79,6 +79,20 @@ const CloudDriveSettings: React.FC<CloudDriveSettingsProps> = ({ settings, onCha
return null;
};
// Validate scan paths (multi-line)
const validateScanPaths = (paths: string): string | null => {
if (!paths.trim()) {
return null; // Optional field
}
const lines = paths.split('\n').map(line => line.trim()).filter(line => line.length > 0);
for (const line of lines) {
if (!line.startsWith('/')) {
return 'Each path should start with / (e.g., /a/电影)';
}
}
return null;
};
const apiUrlError = settings.cloudDriveEnabled && settings.openListApiUrl
? validateApiUrl(settings.openListApiUrl)
: null;
@@ -88,6 +102,9 @@ const CloudDriveSettings: React.FC<CloudDriveSettingsProps> = ({ settings, onCha
const uploadPathError = settings.cloudDriveEnabled && settings.cloudDrivePath
? validateUploadPath(settings.cloudDrivePath)
: null;
const scanPathsError = settings.cloudDriveEnabled && settings.cloudDriveScanPaths
? validateScanPaths(settings.cloudDriveScanPaths)
: null;
const handleTestConnection = async () => {
if (!settings.openListApiUrl || !settings.openListToken) {
@@ -313,6 +330,23 @@ const CloudDriveSettings: React.FC<CloudDriveSettingsProps> = ({ settings, onCha
</Typography>
)}
<TextField
label={t('scanPaths')}
value={settings.cloudDriveScanPaths || ''}
onChange={(e) => onChange('cloudDriveScanPaths', e.target.value)}
helperText={t('scanPathsHelper')}
error={!!scanPathsError}
placeholder="/a/Movies&#10;/b/Documentaries"
multiline
rows={4}
fullWidth
/>
{scanPathsError && (
<Typography variant="caption" color="error" sx={{ mt: -1.5 }}>
{scanPathsError}
</Typography>
)}
<Box sx={{ display: 'flex', gap: 2, flexWrap: 'wrap' }}>
<Button
variant="outlined"

1
frontend/src/types.ts
View File

@@ -68,6 +68,7 @@ export interface Settings {
openListToken: string;
openListPublicUrl?: string;
cloudDrivePath: string;
cloudDriveScanPaths?: string;
homeSidebarOpen?: boolean;
subtitlesEnabled?: boolean;
websiteName?: string;

2
frontend/src/utils/locales/ar.ts
View File

@@ -144,6 +144,8 @@ export const ar = {
publicUrlHelper: "النطاق العام للوصول إلى الملفات (مثال: https://your-cloudflare-tunnel-domain.com). إذا تم تعيينه، سيتم استخدامه بدلاً من عنوان API للوصول إلى الملفات.",
uploadPath: "مسار التحميل",
cloudDrivePathHelper: "مسار الدليل في التخزين السحابي، مثال: /mytube-uploads",
scanPaths: "مسارات المسح",
scanPathsHelper: "مسار واحد في كل سطر. سيتم مسح مقاطع الفيديو من هذه المسارات. إذا كانت فارغة، سيتم استخدام مسار التحميل. مثال:\n/a/أفلام\n/b/وثائقيات",
cloudDriveNote:
"بعد تفعيل هذه الميزة، سيتم تحميل مقاطع الفيديو التي تم تنزيلها حديثًا تلقائيًا إلى التخزين السحابي وسيتم حذف الملفات المحلية. سيتم تشغيل مقاطع الفيديو من التخزين السحابي عبر الوكيل.",
testing: "جارٍ الاختبار...",

2
frontend/src/utils/locales/de.ts
View File

@@ -141,6 +141,8 @@ export const de = {
uploadPath: "Upload-Pfad",
cloudDrivePathHelper:
"Verzeichnispfad im Cloud-Speicher, z.B. /mytube-uploads",
scanPaths: "Scan-Pfade",
scanPathsHelper: "Ein Pfad pro Zeile. Videos werden von diesen Pfaden gescannt. Wenn leer, wird der Upload-Pfad verwendet. Beispiel:\n/a/Filme\n/b/Dokumentationen",
cloudDriveNote:
"Nach Aktivierung dieser Funktion werden neu heruntergeladene Videos automatisch in den Cloud-Speicher hochgeladen und lokale Dateien werden gelöscht. Videos werden über einen Proxy aus dem Cloud-Speicher abgespielt.",
testing: "Teste...",

2
frontend/src/utils/locales/en.ts
View File

@@ -141,6 +141,8 @@ export const en = {
publicUrlHelper: "Public domain for accessing files (e.g., https://your-cloudflare-tunnel-domain.com). If set, this will be used instead of the API URL for file access.",
uploadPath: "Upload Path",
cloudDrivePathHelper: "Directory path in cloud drive, e.g. /mytube-uploads",
scanPaths: "Scan Paths",
scanPathsHelper: "One path per line. Videos will be scanned from these paths. If empty, will use upload path. Example:\n/a/Movies\n/b/Documentaries",
cloudDriveNote:
"After enabling this feature, newly downloaded videos will be automatically uploaded to cloud storage and local files will be deleted. Videos will be played from cloud storage via proxy.",
testing: "Testing...",

2
frontend/src/utils/locales/es.ts
View File

@@ -155,6 +155,8 @@ export const es = {
publicUrlHelper: "Dominio público para acceder a archivos (ej. https://your-cloudflare-tunnel-domain.com). Si se establece, se usará en lugar de la URL de la API para acceder a archivos.",
uploadPath: "Ruta de carga",
cloudDrivePathHelper: "Ruta del directorio en la nube, ej. /mytube-uploads",
scanPaths: "Rutas de escaneo",
scanPathsHelper: "Una ruta por línea. Se escanearán videos de estas rutas. Si está vacío, se usará la ruta de carga. Ejemplo:\n/a/Peliculas\n/b/Documentales",
cloudDriveNote:
"Después de habilitar esta función, los videos recién descargados se subirán automáticamente al almacenamiento en la nube y se eliminarán los archivos locales. Los videos se reproducirán desde el almacenamiento en la nube a través de un proxy.",
testing: "Probando...",

2
frontend/src/utils/locales/fr.ts
View File

@@ -155,6 +155,8 @@ export const fr = {
uploadPath: "Chemin de téléchargement",
cloudDrivePathHelper:
"Chemin du répertoire dans le cloud, ex. /mytube-uploads",
scanPaths: "Chemins d'analyse",
scanPathsHelper: "Un chemin par ligne. Les vidéos seront analysées à partir de ces chemins. Si vide, le chemin de téléchargement sera utilisé. Exemple :\n/a/Films\n/b/Documentaires",
cloudDriveNote:
"Après avoir activé cette fonctionnalité, les vidéos nouvellement téléchargées seront automatiquement téléchargées vers le stockage cloud et les fichiers locaux seront supprimés. Les vidéos seront lues depuis le stockage cloud via un proxy.",
testing: "Test en cours...",

2
frontend/src/utils/locales/ja.ts
View File

@@ -150,6 +150,8 @@ export const ja = {
uploadPath: "アップロードパス",
cloudDrivePathHelper:
"クラウドドライブ内のディレクトリパス、例: /mytube-uploads",
scanPaths: "スキャンパス",
scanPathsHelper: "1行に1つのパスを入力してください。これらのパスから動画がスキャンされます。空の場合はアップロードパスが使用されます。例：\n/a/映画\n/b/ドキュメンタリー",
cloudDriveNote:
"この機能を有効にすると、新しくダウンロードされた動画は自動的にクラウドストレージにアップロードされ、ローカルファイルは削除されます。動画はプロキシ経由でクラウドストレージから再生されます。",
testing: "テスト中...",

2
frontend/src/utils/locales/ko.ts
View File

@@ -147,6 +147,8 @@ export const ko = {
uploadPath: "업로드 경로",
cloudDrivePathHelper:
"클라우드 드라이브 내 디렉토리 경로, 예: /mytube-uploads",
scanPaths: "스캔 경로",
scanPathsHelper: "줄당 하나의 경로. 이 경로에서 동영상을 스캔합니다. 비어 있으면 업로드 경로를 사용합니다. 예:\n/a/영화\n/b/다큐멘터리",
cloudDriveNote:
"이 기능을 활성화한 후 새로 다운로드된 비디오는 자동으로 클라우드 스토리지에 업로드되고 로컬 파일은 삭제됩니다. 비디오는 프록시를 통해 클라우드 스토리지에서 재생됩니다.",
testing: "테스트 중...",

2
frontend/src/utils/locales/pt.ts
View File

@@ -150,6 +150,8 @@ export const pt = {
publicUrlHelper: "Domínio público para acessar arquivos (ex. https://your-cloudflare-tunnel-domain.com). Se definido, será usado em vez da URL da API para acessar arquivos.",
uploadPath: "Caminho de upload",
cloudDrivePathHelper: "Caminho do diretório na nuvem, ex. /mytube-uploads",
scanPaths: "Caminhos de Varredura",
scanPathsHelper: "Um caminho por linha. Os vídeos serão verificados a partir desses caminhos. Se vazio, usará o caminho de upload. Exemplo:\n/a/Filmes\n/b/Documentários",
cloudDriveNote:
"Após habilitar este recurso, os vídeos recém-baixados serão automaticamente enviados para o armazenamento em nuvem e os arquivos locais serão excluídos. Os vídeos serão reproduzidos do armazenamento em nuvem via proxy.",
testing: "Testando...",

2
frontend/src/utils/locales/ru.ts
View File

@@ -158,6 +158,8 @@ export const ru = {
publicUrlHelper: "Публичный домен для доступа к файлам (напр. https://your-cloudflare-tunnel-domain.com). Если установлен, будет использоваться вместо URL API для доступа к файлам.",
uploadPath: "Путь загрузки",
cloudDrivePathHelper: "Путь к каталогу в облаке, напр. /mytube-uploads",
scanPaths: "Пути сканирования",
scanPathsHelper: "Один путь в строке. Видео будут сканироваться из этих путей. Если пусто, будет использоваться путь загрузки. Пример:\n/a/Фильмы\n/b/Документальные",
cloudDriveNote:
"После включения этой функции недавно загруженные видео будут автоматически загружены в облачное хранилище, а локальные файлы будут удалены. Видео будут воспроизводиться из облачного хранилища через прокси.",
testing: "Тестирование...",

2
frontend/src/utils/locales/zh.ts
View File

@@ -141,6 +141,8 @@ export const zh = {
publicUrlHelper: "用于访问文件的公开域名（例如：https://your-cloudflare-tunnel-domain.com）。如果设置，将使用此域名而不是 API 地址来访问文件。",
uploadPath: "上传路径",
cloudDrivePathHelper: "云端存储中的目录路径，例如：/mytube-uploads",
scanPaths: "扫描路径",
scanPathsHelper: "每行一个路径。系统将扫描这些路径下的视频。留空则使用默认上传路径。示例：\n/a/电影\n/b/纪录片",
cloudDriveNote:
"启用此功能后，新下载的视频将自动上传到云端存储，本地文件将被删除。视频将通过代理从云端存储播放。",
testing: "测试中...",