davidl/MyTube
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: Add function to check if file exists before upload

Browse Source
This commit is contained in:
Peifan Li
2025-12-22 15:37:46 -05:00
parent 216ee24677
commit 2816ea17f4
2 changed files with 165 additions and 67 deletions
Download Patch File Download Diff File Expand all files Collapse all files

85
backend/src/services/cloudStorage/fileUploader.ts
View File

@@ -7,20 +7,76 @@ import fs from "fs-extra";
import path from "path";
import { FileError, NetworkError } from "../../errors/DownloadErrors";
import { logger } from "../../utils/logger";
import { CloudDriveConfig } from "./types";
import { getFileList } from "./fileLister";
import { normalizeUploadPath } from "./pathUtils";
import { CloudDriveConfig } from "./types";
/**
* Upload result indicating whether file was actually uploaded or skipped
*/
export interface UploadResult {
uploaded: boolean;
skipped: boolean;
reason?: string;
}
/**
* Check if a file already exists in cloud storage
* @param fileName - Name of the file to check
* @param fileSize - Size of the file in bytes
* @param destinationPath - Full destination path in cloud storage
* @param config - Cloud drive configuration
* @returns true if file exists with same name and size
*/
async function fileExistsInCloud(
fileName: string,
fileSize: number,
destinationPath: string,
config: CloudDriveConfig
): Promise<boolean> {
try {
// Get the directory path from destination path
const dirPath = path.dirname(destinationPath);
const normalizedDirPath = normalizeUploadPath(dirPath);
// Get file list from OpenList
const files = await getFileList(config, normalizedDirPath);
// Check if file with same name and size exists
const existingFile = files.find(
(file) => file.name === fileName && file.size === fileSize && !file.is_dir
);
if (existingFile) {
logger.info(
`[CloudStorage] File ${fileName} already exists in cloud storage with same size (${fileSize} bytes), skipping upload`
);
return true;
}
return false;
} catch (error) {
logger.warn(
`[CloudStorage] Failed to check if file exists in cloud storage:`,
error instanceof Error ? error : new Error(String(error))
);
// If check fails, proceed with upload to be safe
return false;
}
}
/**
* Upload a file to cloud storage
* @param filePath - Local file path to upload
* @param config - Cloud drive configuration
* @param remotePath - Optional remote path (relative to uploadPath)
* @returns UploadResult indicating whether file was uploaded or skipped
*/
export async function uploadFile(
filePath: string,
config: CloudDriveConfig,
remotePath?: string
): Promise<void> {
): Promise<UploadResult> {
// 1. Get basic file information
const fileStat = fs.statSync(filePath);
const fileSize = fileStat.size;
@@ -55,13 +111,27 @@ export async function uploadFile(
? destinationPath
: `/${destinationPath}`;
// Check if file already exists in cloud storage before uploading
const exists = await fileExistsInCloud(
fileName,
fileSize,
destinationPath,
config
);
if (exists) {
return {
uploaded: false,
skipped: true,
reason: "File already exists in cloud storage with same size",
};
}
logger.info(
`[CloudStorage] Uploading ${fileName} to ${destinationPath} (${fileSize} bytes)...`
);
logger.debug(
`[CloudStorage] Destination path in header: ${destinationPath}`
);
logger.debug(`[CloudStorage] Destination path in header: ${destinationPath}`);
// 3. Prepare Headers
const headers = {
@@ -92,6 +162,10 @@ export async function uploadFile(
logger.info(
`[CloudStorage] Successfully uploaded ${fileName}. Server message: ${response.data.message}`
);
return {
uploaded: true,
skipped: false,
};
} else {
// Even if HTTP status code is 200, server may return business errors
const errorMessage = response.data
@@ -132,4 +206,3 @@ export async function uploadFile(
}
}
}

147
backend/src/services/cloudStorage/videoUploader.ts
View File

@@ -6,12 +6,15 @@ import fs from "fs-extra";
import path from "path";
import { logger } from "../../utils/logger";
import { updateVideo } from "../storageService";
import { CloudDriveConfig, FileType } from "./types";
import { resolveAbsolutePath, sanitizeFilename } from "./pathUtils";
import { uploadFile } from "./fileUploader";
import { clearSignedUrlCache } from "./urlSigner";
import { clearFileListCache } from "./fileLister";
import { normalizeUploadPath } from "./pathUtils";
import { uploadFile, UploadResult } from "./fileUploader";
import {
normalizeUploadPath,
resolveAbsolutePath,
sanitizeFilename,
} from "./pathUtils";
import { CloudDriveConfig } from "./types";
import { clearSignedUrlCache } from "./urlSigner";
/**
* Upload video, thumbnail, and metadata to cloud storage
@@ -25,14 +28,24 @@ export async function uploadVideo(
logger.info(`[CloudStorage] Starting upload for video: ${videoData.title}`);
const uploadedFiles: string[] = []; // Track successfully uploaded files for deletion
const filesToUpdate: string[] = []; // Track files that should update database (uploaded or skipped)
try {
// Upload Video File
if (videoData.videoPath) {
const absoluteVideoPath = resolveAbsolutePath(videoData.videoPath);
if (absoluteVideoPath && fs.existsSync(absoluteVideoPath)) {
await uploadFile(absoluteVideoPath, config);
uploadedFiles.push(absoluteVideoPath);
const uploadResult: UploadResult = await uploadFile(
absoluteVideoPath,
config
);
if (uploadResult.uploaded) {
uploadedFiles.push(absoluteVideoPath);
}
// Track file for database update whether uploaded or skipped
if (uploadResult.uploaded || uploadResult.skipped) {
filesToUpdate.push(absoluteVideoPath);
}
} else {
logger.error(
`[CloudStorage] Video file not found: ${videoData.videoPath}`
@@ -45,8 +58,17 @@ export async function uploadVideo(
if (videoData.thumbnailPath) {
const absoluteThumbPath = resolveAbsolutePath(videoData.thumbnailPath);
if (absoluteThumbPath && fs.existsSync(absoluteThumbPath)) {
await uploadFile(absoluteThumbPath, config);
uploadedFiles.push(absoluteThumbPath);
const uploadResult: UploadResult = await uploadFile(
absoluteThumbPath,
config
);
if (uploadResult.uploaded) {
uploadedFiles.push(absoluteThumbPath);
}
// Track file for database update whether uploaded or skipped
if (uploadResult.uploaded || uploadResult.skipped) {
filesToUpdate.push(absoluteThumbPath);
}
}
}
@@ -75,14 +97,17 @@ export async function uploadVideo(
fs.ensureDirSync(path.dirname(metadataPath));
fs.writeFileSync(metadataPath, JSON.stringify(metadata, null, 2));
await uploadFile(metadataPath, config);
const metadataUploadResult: UploadResult = await uploadFile(
metadataPath,
config
);
// Cleanup temp metadata (always delete temp file)
fs.unlinkSync(metadataPath);
logger.info(`[CloudStorage] Upload completed for: ${videoData.title}`);
// Delete local files after successful upload and update video record to point to cloud storage
// Delete local files after successful upload (only for files that were actually uploaded)
if (uploadedFiles.length > 0) {
logger.info(
`[CloudStorage] Deleting ${uploadedFiles.length} local file(s) after successful upload...`
@@ -108,59 +133,60 @@ export async function uploadVideo(
}
}
logger.info(`[CloudStorage] Local file cleanup completed`);
}
// Update video record to point to cloud storage (store only filename, not full URL with sign)
// Sign will be retrieved dynamically when needed
if (videoData.id && deletedFiles.length > 0) {
try {
const updates: any = {};
// Update video record to point to cloud storage (store only filename, not full URL with sign)
// Sign will be retrieved dynamically when needed
// Update database if files were uploaded OR skipped (already in cloud)
if (videoData.id && filesToUpdate.length > 0) {
try {
const updates: any = {};
// Store cloud storage indicator in path format: "cloud:filename"
// This allows us to identify cloud storage files and retrieve sign dynamically
const videoFilename =
videoData.videoFilename ||
(videoData.videoPath ? path.basename(videoData.videoPath) : null);
// Store cloud storage indicator in path format: "cloud:filename"
// This allows us to identify cloud storage files and retrieve sign dynamically
const videoFilename =
videoData.videoFilename ||
(videoData.videoPath ? path.basename(videoData.videoPath) : null);
if (videoFilename) {
updates.videoPath = `cloud:${videoFilename}`;
}
const thumbnailFilename =
videoData.thumbnailFilename ||
(videoData.thumbnailPath
? path.basename(videoData.thumbnailPath)
: null);
if (thumbnailFilename) {
updates.thumbnailPath = `cloud:${thumbnailFilename}`;
}
if (Object.keys(updates).length > 0) {
updateVideo(videoData.id, updates);
logger.info(
`[CloudStorage] Updated video record ${videoData.id} with cloud storage indicators`
);
// Clear cache for uploaded files to ensure fresh URLs
if (videoFilename) {
clearSignedUrlCache(videoFilename, "video");
}
if (thumbnailFilename) {
clearSignedUrlCache(thumbnailFilename, "thumbnail");
}
// Also clear file list cache since new files were added
const uploadPath = normalizeUploadPath(config.uploadPath);
clearFileListCache(uploadPath);
}
} catch (updateError: any) {
logger.error(
`[CloudStorage] Failed to update video record with cloud paths:`,
updateError instanceof Error
? updateError
: new Error(updateError.message)
);
// Don't throw - file deletion was successful
if (videoFilename) {
updates.videoPath = `cloud:${videoFilename}`;
}
const thumbnailFilename =
videoData.thumbnailFilename ||
(videoData.thumbnailPath
? path.basename(videoData.thumbnailPath)
: null);
if (thumbnailFilename) {
updates.thumbnailPath = `cloud:${thumbnailFilename}`;
}
if (Object.keys(updates).length > 0) {
updateVideo(videoData.id, updates);
logger.info(
`[CloudStorage] Updated video record ${videoData.id} with cloud storage indicators`
);
// Clear cache for uploaded files to ensure fresh URLs
if (videoFilename) {
clearSignedUrlCache(videoFilename, "video");
}
if (thumbnailFilename) {
clearSignedUrlCache(thumbnailFilename, "thumbnail");
}
// Also clear file list cache since new files were added
const uploadPath = normalizeUploadPath(config.uploadPath);
clearFileListCache(uploadPath);
}
} catch (updateError: any) {
logger.error(
`[CloudStorage] Failed to update video record with cloud paths:`,
updateError instanceof Error
? updateError
: new Error(updateError.message)
);
// Don't throw - file deletion was successful
}
}
} catch (error) {
@@ -171,4 +197,3 @@ export async function uploadVideo(
// If upload failed, don't delete local files
}
}