davidl/MyTube
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: migrate json file based DB to sqlite

Browse Source
This commit is contained in:
Peifan Li
2025-11-24 21:35:12 -05:00
parent 22214f26cd
commit d5a3ddb052
15 changed files with 2335 additions and 234 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
.gitignore vendored
View File

@@ -47,3 +47,7 @@ backend/uploads/images/*
# Ignore the videos database
backend/data/videos.json
backend/data/collections.json
backend/data/*.db
backend/data/*.db-journal
backend/data/status.json
backend/data/settings.json

10
backend/drizzle.config.ts Normal file
View File

@@ -0,0 +1,10 @@
import { defineConfig } from 'drizzle-kit';
export default defineConfig({
schema: './src/db/schema.ts',
out: './drizzle',
dialect: 'sqlite',
dbCredentials: {
url: './data/mytube.db',
},
});

1364
backend/package-lock.json generated
View File

File diff suppressed because it is too large Load Diff

4
backend/package.json
View File

@@ -15,10 +15,12 @@
"dependencies": {
"axios": "^1.8.1",
"bcryptjs": "^3.0.3",
"better-sqlite3": "^12.4.6",
"bilibili-save-nodejs": "^1.0.0",
"cheerio": "^1.1.2",
"cors": "^2.8.5",
"dotenv": "^16.4.7",
"drizzle-orm": "^0.44.7",
"express": "^4.18.2",
"fs-extra": "^11.2.0",
"multer": "^1.4.5-lts.1",
@@ -28,11 +30,13 @@
},
"devDependencies": {
"@types/bcryptjs": "^2.4.6",
"@types/better-sqlite3": "^7.6.13",
"@types/cors": "^2.8.19",
"@types/express": "^5.0.5",
"@types/fs-extra": "^11.0.4",
"@types/multer": "^2.0.0",
"@types/node": "^24.10.1",
"drizzle-kit": "^0.31.7",
"nodemon": "^3.0.3",
"ts-node": "^10.9.2",
"typescript": "^5.9.3"

177
backend/scripts/migrate-to-sqlite.ts Normal file
View File

@@ -0,0 +1,177 @@
import fs from 'fs-extra';
import path from 'path';
import { COLLECTIONS_DATA_PATH, STATUS_DATA_PATH, VIDEOS_DATA_PATH } from '../src/config/paths';
import { db } from '../src/db';
import { collections, collectionVideos, downloads, settings, videos } from '../src/db/schema';
// Hardcoded path for settings since it might not be exported from paths.ts
const SETTINGS_DATA_PATH = path.join(path.dirname(VIDEOS_DATA_PATH), 'settings.json');
async function migrate() {
console.log('Starting migration...');
// Migrate Videos
if (fs.existsSync(VIDEOS_DATA_PATH)) {
const videosData = fs.readJSONSync(VIDEOS_DATA_PATH);
console.log(`Found ${videosData.length} videos to migrate.`);
for (const video of videosData) {
try {
await db.insert(videos).values({
id: video.id,
title: video.title,
author: video.author,
date: video.date,
source: video.source,
sourceUrl: video.sourceUrl,
videoFilename: video.videoFilename,
thumbnailFilename: video.thumbnailFilename,
videoPath: video.videoPath,
thumbnailPath: video.thumbnailPath,
thumbnailUrl: video.thumbnailUrl,
addedAt: video.addedAt,
createdAt: video.createdAt,
updatedAt: video.updatedAt,
partNumber: video.partNumber,
totalParts: video.totalParts,
seriesTitle: video.seriesTitle,
rating: video.rating,
description: video.description,
viewCount: video.viewCount,
duration: video.duration,
}).onConflictDoNothing();
} catch (error) {
console.error(`Error migrating video ${video.id}:`, error);
}
}
console.log('Videos migration completed.');
} else {
console.log('No videos.json found.');
}
// Migrate Collections
if (fs.existsSync(COLLECTIONS_DATA_PATH)) {
const collectionsData = fs.readJSONSync(COLLECTIONS_DATA_PATH);
console.log(`Found ${collectionsData.length} collections to migrate.`);
for (const collection of collectionsData) {
try {
// Insert Collection
await db.insert(collections).values({
id: collection.id,
name: collection.name || collection.title || 'Untitled Collection',
title: collection.title,
createdAt: collection.createdAt || new Date().toISOString(),
updatedAt: collection.updatedAt,
}).onConflictDoNothing();
// Insert Collection Videos
if (collection.videos && collection.videos.length > 0) {
for (const videoId of collection.videos) {
try {
await db.insert(collectionVideos).values({
collectionId: collection.id,
videoId: videoId,
}).onConflictDoNothing();
} catch (err) {
console.error(`Error linking video ${videoId} to collection ${collection.id}:`, err);
}
}
}
} catch (error) {
console.error(`Error migrating collection ${collection.id}:`, error);
}
}
console.log('Collections migration completed.');
} else {
console.log('No collections.json found.');
}
// Migrate Settings
if (fs.existsSync(SETTINGS_DATA_PATH)) {
try {
const settingsData = fs.readJSONSync(SETTINGS_DATA_PATH);
console.log('Found settings.json to migrate.');
for (const [key, value] of Object.entries(settingsData)) {
await db.insert(settings).values({
key,
value: JSON.stringify(value),
}).onConflictDoUpdate({
target: settings.key,
set: { value: JSON.stringify(value) },
});
}
console.log('Settings migration completed.');
} catch (error) {
console.error('Error migrating settings:', error);
}
} else {
console.log('No settings.json found.');
}
// Migrate Status (Downloads)
if (fs.existsSync(STATUS_DATA_PATH)) {
try {
const statusData = fs.readJSONSync(STATUS_DATA_PATH);
console.log('Found status.json to migrate.');
// Migrate active downloads
if (statusData.activeDownloads && Array.isArray(statusData.activeDownloads)) {
for (const download of statusData.activeDownloads) {
await db.insert(downloads).values({
id: download.id,
title: download.title,
timestamp: download.timestamp,
filename: download.filename,
totalSize: download.totalSize,
downloadedSize: download.downloadedSize,
progress: download.progress,
speed: download.speed,
status: 'active',
}).onConflictDoUpdate({
target: downloads.id,
set: {
title: download.title,
timestamp: download.timestamp,
filename: download.filename,
totalSize: download.totalSize,
downloadedSize: download.downloadedSize,
progress: download.progress,
speed: download.speed,
status: 'active',
}
});
}
}
// Migrate queued downloads
if (statusData.queuedDownloads && Array.isArray(statusData.queuedDownloads)) {
for (const download of statusData.queuedDownloads) {
await db.insert(downloads).values({
id: download.id,
title: download.title,
timestamp: download.timestamp,
status: 'queued',
}).onConflictDoUpdate({
target: downloads.id,
set: {
title: download.title,
timestamp: download.timestamp,
status: 'queued',
}
});
}
}
console.log('Status migration completed.');
} catch (error) {
console.error('Error migrating status:', error);
}
} else {
console.log('No status.json found.');
}
console.log('Migration finished successfully.');
}
migrate().catch(console.error);

3
backend/scripts/test-import.ts Normal file
View File

@@ -0,0 +1,3 @@
import { getSettings } from '../src/services/storageService';
console.log('Imported getSettings:', typeof getSettings);

115
backend/scripts/verify-db.ts Normal file
View File

@@ -0,0 +1,115 @@
import {
addActiveDownload,
Collection,
deleteCollection,
deleteVideo,
getCollections,
getDownloadStatus,
getSettings,
getVideoById,
getVideos,
removeActiveDownload,
saveCollection,
saveSettings,
saveVideo,
Video
} from '../src/services/storageService';
async function verify() {
console.log('Starting verification...');
// 1. Get Videos (should be empty initially)
const videos = getVideos();
console.log(`Initial videos count: ${videos.length}`);
// 2. Save a Video
const newVideo: Video = {
id: 'test-video-1',
title: 'Test Video',
sourceUrl: 'http://example.com',
createdAt: new Date().toISOString(),
author: 'Test Author',
source: 'local'
};
saveVideo(newVideo);
console.log('Saved test video.');
// 3. Get Video by ID
const retrievedVideo = getVideoById('test-video-1');
if (retrievedVideo && retrievedVideo.title === 'Test Video') {
console.log('Retrieved video successfully.');
} else {
console.error('Failed to retrieve video.');
}
// 4. Save a Collection
const newCollection: Collection = {
id: 'test-collection-1',
title: 'Test Collection',
videos: ['test-video-1'],
createdAt: new Date().toISOString()
};
saveCollection(newCollection);
console.log('Saved test collection.');
// 5. Get Collections
const collections = getCollections();
console.log(`Collections count: ${collections.length}`);
const retrievedCollection = collections.find(c => c.id === 'test-collection-1');
if (retrievedCollection && retrievedCollection.videos.includes('test-video-1')) {
console.log('Retrieved collection with video link successfully.');
} else {
console.error('Failed to retrieve collection or video link.');
}
// 6. Delete Collection
deleteCollection('test-collection-1');
const collectionsAfterDelete = getCollections();
if (collectionsAfterDelete.find(c => c.id === 'test-collection-1')) {
console.error('Failed to delete collection.');
} else {
console.log('Deleted collection successfully.');
}
// 7. Delete Video
deleteVideo('test-video-1');
const videoAfterDelete = getVideoById('test-video-1');
if (videoAfterDelete) {
console.error('Failed to delete video.');
} else {
console.log('Deleted video successfully.');
}
// 8. Settings
const initialSettings = getSettings();
console.log('Initial settings:', initialSettings);
saveSettings({ ...initialSettings, testKey: 'testValue' });
const updatedSettings = getSettings();
if (updatedSettings.testKey === 'testValue') {
console.log('Settings saved and retrieved successfully.');
} else {
console.error('Failed to save/retrieve settings.');
}
// 9. Status (Active Downloads)
addActiveDownload('test-download-1', 'Test Download');
let status = getDownloadStatus();
if (status.activeDownloads.find(d => d.id === 'test-download-1')) {
console.log('Active download added successfully.');
} else {
console.error('Failed to add active download.');
}
removeActiveDownload('test-download-1');
status = getDownloadStatus();
if (status.activeDownloads.find(d => d.id === 'test-download-1')) {
console.error('Failed to remove active download.');
} else {
console.log('Active download removed successfully.');
}
console.log('Verification finished.');
}
verify().catch(console.error);

55
backend/src/controllers/settingsController.ts
View File

@@ -1,10 +1,7 @@
import bcrypt from 'bcryptjs';
import { Request, Response } from 'express';
import fs from 'fs-extra';
import path from 'path';
import downloadManager from '../services/downloadManager';
const SETTINGS_FILE = path.join(__dirname, '../../data/settings.json');
import * as storageService from '../services/storageService';
interface Settings {
loginEnabled: boolean;
@@ -26,14 +23,19 @@ const defaultSettings: Settings = {
export const getSettings = async (req: Request, res: Response) => {
try {
if (!fs.existsSync(SETTINGS_FILE)) {
await fs.writeJson(SETTINGS_FILE, defaultSettings, { spaces: 2 });
const settings = storageService.getSettings();
// If empty (first run), save defaults
if (Object.keys(settings).length === 0) {
storageService.saveSettings(defaultSettings);
return res.json(defaultSettings);
}
const settings = await fs.readJson(SETTINGS_FILE);
// Merge with defaults to ensure all fields exist
const mergedSettings = { ...defaultSettings, ...settings };
// Do not send the hashed password to the frontend
const { password, ...safeSettings } = settings;
const { password, ...safeSettings } = mergedSettings;
res.json({ ...safeSettings, isPasswordSet: !!password });
} catch (error) {
console.error('Error reading settings:', error);
@@ -41,6 +43,17 @@ export const getSettings = async (req: Request, res: Response) => {
}
};
export const migrateData = async (req: Request, res: Response) => {
try {
const { runMigration } = await import('../services/migrationService');
const results = await runMigration();
res.json({ success: true, results });
} catch (error: any) {
console.error('Error running migration:', error);
res.status(500).json({ error: 'Failed to run migration', details: error.message });
}
};
export const updateSettings = async (req: Request, res: Response) => {
try {
const newSettings: Settings = req.body;
@@ -56,14 +69,12 @@ export const updateSettings = async (req: Request, res: Response) => {
const salt = await bcrypt.genSalt(10);
newSettings.password = await bcrypt.hash(newSettings.password, salt);
} else {
// If password is empty/not provided, keep existing password if file exists
if (fs.existsSync(SETTINGS_FILE)) {
const existingSettings = await fs.readJson(SETTINGS_FILE);
newSettings.password = existingSettings.password;
}
// If password is empty/not provided, keep existing password
const existingSettings = storageService.getSettings();
newSettings.password = existingSettings.password;
}
await fs.writeJson(SETTINGS_FILE, newSettings, { spaces: 2 });
storageService.saveSettings(newSettings);
// Apply settings immediately where possible
downloadManager.setMaxConcurrentDownloads(newSettings.maxConcurrentDownloads);
@@ -79,23 +90,19 @@ export const verifyPassword = async (req: Request, res: Response) => {
try {
const { password } = req.body;
if (!fs.existsSync(SETTINGS_FILE)) {
return res.json({ success: true });
}
const settings = await fs.readJson(SETTINGS_FILE);
const settings = storageService.getSettings();
const mergedSettings = { ...defaultSettings, ...settings };
if (!settings.loginEnabled) {
if (!mergedSettings.loginEnabled) {
return res.json({ success: true });
}
if (!settings.password) {
// If no password set but login enabled, allow access (or force set password?)
// For now, allow access
if (!mergedSettings.password) {
// If no password set but login enabled, allow access
return res.json({ success: true });
}
const isMatch = await bcrypt.compare(password, settings.password);
const isMatch = await bcrypt.compare(password, mergedSettings.password);
if (isMatch) {
res.json({ success: true });

14
backend/src/db/index.ts Normal file
View File

@@ -0,0 +1,14 @@
import Database from 'better-sqlite3';
import { drizzle } from 'drizzle-orm/better-sqlite3';
import fs from 'fs-extra';
import path from 'path';
import { DATA_DIR } from '../config/paths';
import * as schema from './schema';
// Ensure data directory exists
fs.ensureDirSync(DATA_DIR);
const dbPath = path.join(DATA_DIR, 'mytube.db');
const sqlite = new Database(dbPath);
export const db = drizzle(sqlite, { schema });

88
backend/src/db/schema.ts Normal file
View File

@@ -0,0 +1,88 @@
import { relations } from 'drizzle-orm';
import { foreignKey, integer, primaryKey, sqliteTable, text } from 'drizzle-orm/sqlite-core';
export const videos = sqliteTable('videos', {
id: text('id').primaryKey(),
title: text('title').notNull(),
author: text('author'),
date: text('date'),
source: text('source'),
sourceUrl: text('source_url'),
videoFilename: text('video_filename'),
thumbnailFilename: text('thumbnail_filename'),
videoPath: text('video_path'),
thumbnailPath: text('thumbnail_path'),
thumbnailUrl: text('thumbnail_url'),
addedAt: text('added_at'),
createdAt: text('created_at').notNull(),
updatedAt: text('updated_at'),
partNumber: integer('part_number'),
totalParts: integer('total_parts'),
seriesTitle: text('series_title'),
rating: integer('rating'),
// Additional fields that might be present
description: text('description'),
viewCount: integer('view_count'),
duration: text('duration'),
});
export const collections = sqliteTable('collections', {
id: text('id').primaryKey(),
name: text('name').notNull(),
title: text('title'), // Keeping for backward compatibility/alias
createdAt: text('created_at').notNull(),
updatedAt: text('updated_at'),
});
export const collectionVideos = sqliteTable('collection_videos', {
collectionId: text('collection_id').notNull(),
videoId: text('video_id').notNull(),
order: integer('order'), // To maintain order if needed
}, (t) => ({
pk: primaryKey({ columns: [t.collectionId, t.videoId] }),
collectionFk: foreignKey({
columns: [t.collectionId],
foreignColumns: [collections.id],
}).onDelete('cascade'),
videoFk: foreignKey({
columns: [t.videoId],
foreignColumns: [videos.id],
}).onDelete('cascade'),
}));
// Relations
export const videosRelations = relations(videos, ({ many }) => ({
collections: many(collectionVideos),
}));
export const collectionsRelations = relations(collections, ({ many }) => ({
videos: many(collectionVideos),
}));
export const collectionVideosRelations = relations(collectionVideos, ({ one }) => ({
collection: one(collections, {
fields: [collectionVideos.collectionId],
references: [collections.id],
}),
video: one(videos, {
fields: [collectionVideos.videoId],
references: [videos.id],
}),
}));
export const settings = sqliteTable('settings', {
key: text('key').primaryKey(),
value: text('value').notNull(), // JSON stringified value
});
export const downloads = sqliteTable('downloads', {
id: text('id').primaryKey(),
title: text('title').notNull(),
timestamp: integer('timestamp'),
filename: text('filename'),
totalSize: text('total_size'),
downloadedSize: text('downloaded_size'),
progress: integer('progress'), // Using integer for percentage (0-100) or similar
speed: text('speed'),
status: text('status').notNull().default('active'), // 'active' or 'queued'
});

3
backend/src/routes/settingsRoutes.ts
View File

@@ -1,10 +1,11 @@
import express from 'express';
import { getSettings, updateSettings, verifyPassword } from '../controllers/settingsController';
import { getSettings, migrateData, updateSettings, verifyPassword } from '../controllers/settingsController';
const router = express.Router();
router.get('/', getSettings);
router.post('/', updateSettings);
router.post('/verify-password', verifyPassword);
router.post('/migrate', migrateData);
export default router;

189
backend/src/services/migrationService.ts Normal file
View File

@@ -0,0 +1,189 @@
import fs from 'fs-extra';
import path from 'path';
import { COLLECTIONS_DATA_PATH, STATUS_DATA_PATH, VIDEOS_DATA_PATH } from '../config/paths';
import { db } from '../db';
import { collections, collectionVideos, downloads, settings, videos } from '../db/schema';
// Hardcoded path for settings since it might not be exported from paths.ts
const SETTINGS_DATA_PATH = path.join(path.dirname(VIDEOS_DATA_PATH), 'settings.json');
export async function runMigration() {
console.log('Starting migration...');
const results = {
videos: 0,
collections: 0,
settings: 0,
downloads: 0,
errors: [] as string[]
};
// Migrate Videos
if (fs.existsSync(VIDEOS_DATA_PATH)) {
try {
const videosData = fs.readJSONSync(VIDEOS_DATA_PATH);
console.log(`Found ${videosData.length} videos to migrate.`);
for (const video of videosData) {
try {
await db.insert(videos).values({
id: video.id,
title: video.title,
author: video.author,
date: video.date,
source: video.source,
sourceUrl: video.sourceUrl,
videoFilename: video.videoFilename,
thumbnailFilename: video.thumbnailFilename,
videoPath: video.videoPath,
thumbnailPath: video.thumbnailPath,
thumbnailUrl: video.thumbnailUrl,
addedAt: video.addedAt,
createdAt: video.createdAt,
updatedAt: video.updatedAt,
partNumber: video.partNumber,
totalParts: video.totalParts,
seriesTitle: video.seriesTitle,
rating: video.rating,
description: video.description,
viewCount: video.viewCount,
duration: video.duration,
}).onConflictDoNothing();
results.videos++;
} catch (error: any) {
console.error(`Error migrating video ${video.id}:`, error);
results.errors.push(`Video ${video.id}: ${error.message}`);
}
}
} catch (error: any) {
results.errors.push(`Failed to read videos.json: ${error.message}`);
}
}
// Migrate Collections
if (fs.existsSync(COLLECTIONS_DATA_PATH)) {
try {
const collectionsData = fs.readJSONSync(COLLECTIONS_DATA_PATH);
console.log(`Found ${collectionsData.length} collections to migrate.`);
for (const collection of collectionsData) {
try {
// Insert Collection
await db.insert(collections).values({
id: collection.id,
name: collection.name || collection.title || 'Untitled Collection',
title: collection.title,
createdAt: collection.createdAt || new Date().toISOString(),
updatedAt: collection.updatedAt,
}).onConflictDoNothing();
results.collections++;
// Insert Collection Videos
if (collection.videos && collection.videos.length > 0) {
for (const videoId of collection.videos) {
try {
await db.insert(collectionVideos).values({
collectionId: collection.id,
videoId: videoId,
}).onConflictDoNothing();
} catch (err: any) {
console.error(`Error linking video ${videoId} to collection ${collection.id}:`, err);
results.errors.push(`Link ${videoId}->${collection.id}: ${err.message}`);
}
}
}
} catch (error: any) {
console.error(`Error migrating collection ${collection.id}:`, error);
results.errors.push(`Collection ${collection.id}: ${error.message}`);
}
}
} catch (error: any) {
results.errors.push(`Failed to read collections.json: ${error.message}`);
}
}
// Migrate Settings
if (fs.existsSync(SETTINGS_DATA_PATH)) {
try {
const settingsData = fs.readJSONSync(SETTINGS_DATA_PATH);
console.log('Found settings.json to migrate.');
for (const [key, value] of Object.entries(settingsData)) {
await db.insert(settings).values({
key,
value: JSON.stringify(value),
}).onConflictDoUpdate({
target: settings.key,
set: { value: JSON.stringify(value) },
});
results.settings++;
}
} catch (error: any) {
console.error('Error migrating settings:', error);
results.errors.push(`Settings: ${error.message}`);
}
}
// Migrate Status (Downloads)
if (fs.existsSync(STATUS_DATA_PATH)) {
try {
const statusData = fs.readJSONSync(STATUS_DATA_PATH);
console.log('Found status.json to migrate.');
// Migrate active downloads
if (statusData.activeDownloads && Array.isArray(statusData.activeDownloads)) {
for (const download of statusData.activeDownloads) {
await db.insert(downloads).values({
id: download.id,
title: download.title,
timestamp: download.timestamp,
filename: download.filename,
totalSize: download.totalSize,
downloadedSize: download.downloadedSize,
progress: download.progress,
speed: download.speed,
status: 'active',
}).onConflictDoUpdate({
target: downloads.id,
set: {
title: download.title,
timestamp: download.timestamp,
filename: download.filename,
totalSize: download.totalSize,
downloadedSize: download.downloadedSize,
progress: download.progress,
speed: download.speed,
status: 'active',
}
});
results.downloads++;
}
}
// Migrate queued downloads
if (statusData.queuedDownloads && Array.isArray(statusData.queuedDownloads)) {
for (const download of statusData.queuedDownloads) {
await db.insert(downloads).values({
id: download.id,
title: download.title,
timestamp: download.timestamp,
status: 'queued',
}).onConflictDoUpdate({
target: downloads.id,
set: {
title: download.title,
timestamp: download.timestamp,
status: 'queued',
}
});
results.downloads++;
}
}
} catch (error: any) {
console.error('Error migrating status:', error);
results.errors.push(`Status: ${error.message}`);
}
}
console.log('Migration finished successfully.');
return results;
}

503
backend/src/services/storageService.ts
View File

@@ -1,14 +1,15 @@
import { desc, eq, lt } from "drizzle-orm";
import fs from "fs-extra";
import path from "path";
import {
COLLECTIONS_DATA_PATH,
DATA_DIR,
IMAGES_DIR,
STATUS_DATA_PATH,
UPLOADS_DIR,
VIDEOS_DATA_PATH,
VIDEOS_DIR,
} from "../config/paths";
import { db } from "../db";
import { collections, collectionVideos, downloads, settings, videos } from "../db/schema";
export interface Video {
id: string;
@@ -25,7 +26,7 @@ export interface Collection {
title: string;
videos: string[];
updatedAt?: string;
name?: string; // Add name property
name?: string;
[key: string]: any;
}
@@ -52,27 +53,21 @@ export function initializeStorage(): void {
fs.ensureDirSync(IMAGES_DIR);
fs.ensureDirSync(DATA_DIR);
// Initialize status.json if it doesn't exist, or reset active downloads if it does
// Initialize status.json if it doesn't exist
if (!fs.existsSync(STATUS_DATA_PATH)) {
fs.writeFileSync(
STATUS_DATA_PATH,
JSON.stringify({ activeDownloads: [], queuedDownloads: [] }, null, 2)
);
} else {
// If it exists, we should clear active downloads because the server is just starting up
// so no downloads can be active yet.
try {
const status = JSON.parse(fs.readFileSync(STATUS_DATA_PATH, "utf8"));
status.activeDownloads = [];
// We keep queued downloads as they might still be valid to process later
// (though currently we don't auto-resume them, but we shouldn't delete them)
if (!status.queuedDownloads) status.queuedDownloads = [];
fs.writeFileSync(STATUS_DATA_PATH, JSON.stringify(status, null, 2));
console.log("Cleared active downloads on startup");
} catch (error) {
console.error("Error resetting active downloads:", error);
// Re-create if corrupt
fs.writeFileSync(
STATUS_DATA_PATH,
JSON.stringify({ activeDownloads: [], queuedDownloads: [] }, null, 2)
@@ -81,275 +76,390 @@ export function initializeStorage(): void {
}
}
// Add an active download
// --- Download Status ---
export function addActiveDownload(id: string, title: string): void {
try {
const status = getDownloadStatus();
const existingIndex = status.activeDownloads.findIndex((d) => d.id === id);
const downloadInfo: DownloadInfo = {
const now = Date.now();
db.insert(downloads).values({
id,
title,
timestamp: Date.now(),
};
if (existingIndex >= 0) {
// Preserve existing progress info if just updating title/timestamp
downloadInfo.filename = status.activeDownloads[existingIndex].filename;
downloadInfo.totalSize = status.activeDownloads[existingIndex].totalSize;
downloadInfo.downloadedSize = status.activeDownloads[existingIndex].downloadedSize;
downloadInfo.progress = status.activeDownloads[existingIndex].progress;
downloadInfo.speed = status.activeDownloads[existingIndex].speed;
status.activeDownloads[existingIndex] = downloadInfo;
} else {
status.activeDownloads.push(downloadInfo);
}
fs.writeFileSync(STATUS_DATA_PATH, JSON.stringify(status, null, 2));
timestamp: now,
status: 'active',
}).onConflictDoUpdate({
target: downloads.id,
set: {
title,
timestamp: now,
status: 'active',
}
}).run();
console.log(`Added/Updated active download: ${title} (${id})`);
} catch (error) {
console.error("Error adding active download:", error);
}
}
// Update an active download with partial info
export function updateActiveDownload(id: string, updates: Partial<DownloadInfo>): void {
try {
const status = getDownloadStatus();
const existingIndex = status.activeDownloads.findIndex((d) => d.id === id);
if (existingIndex >= 0) {
status.activeDownloads[existingIndex] = {
...status.activeDownloads[existingIndex],
...updates,
timestamp: Date.now() // Update timestamp to prevent stale removal
};
fs.writeFileSync(STATUS_DATA_PATH, JSON.stringify(status, null, 2));
}
const updateData: any = { ...updates, timestamp: Date.now() };
// Map fields to DB columns if necessary (though they match mostly)
if (updates.totalSize) updateData.totalSize = updates.totalSize;
if (updates.downloadedSize) updateData.downloadedSize = updates.downloadedSize;
db.update(downloads)
.set(updateData)
.where(eq(downloads.id, id))
.run();
} catch (error) {
console.error("Error updating active download:", error);
}
}
// Remove an active download
export function removeActiveDownload(id: string): void {
try {
const status = getDownloadStatus();
const initialLength = status.activeDownloads.length;
status.activeDownloads = status.activeDownloads.filter((d) => d.id !== id);
if (status.activeDownloads.length !== initialLength) {
fs.writeFileSync(STATUS_DATA_PATH, JSON.stringify(status, null, 2));
console.log(`Removed active download: ${id}`);
}
db.delete(downloads).where(eq(downloads.id, id)).run();
console.log(`Removed active download: ${id}`);
} catch (error) {
console.error("Error removing active download:", error);
}
}
// Set queued downloads
export function setQueuedDownloads(queuedDownloads: DownloadInfo[]): void {
try {
const status = getDownloadStatus();
status.queuedDownloads = queuedDownloads;
fs.writeFileSync(STATUS_DATA_PATH, JSON.stringify(status, null, 2));
// Transaction to clear old queued and add new ones
db.transaction(() => {
// First, remove all existing queued downloads
db.delete(downloads).where(eq(downloads.status, 'queued')).run();
// Then insert new ones
for (const download of queuedDownloads) {
db.insert(downloads).values({
id: download.id,
title: download.title,
timestamp: download.timestamp,
status: 'queued',
}).onConflictDoUpdate({
target: downloads.id,
set: {
title: download.title,
timestamp: download.timestamp,
status: 'queued'
}
}).run();
}
});
} catch (error) {
console.error("Error setting queued downloads:", error);
}
}
// Get download status
export function getDownloadStatus(): DownloadStatus {
if (!fs.existsSync(STATUS_DATA_PATH)) {
const initialStatus: DownloadStatus = { activeDownloads: [], queuedDownloads: [] };
fs.writeFileSync(STATUS_DATA_PATH, JSON.stringify(initialStatus, null, 2));
return initialStatus;
}
try {
const status: DownloadStatus = JSON.parse(
fs.readFileSync(STATUS_DATA_PATH, "utf8")
);
// Clean up stale downloads (older than 30 mins)
const thirtyMinsAgo = Date.now() - 30 * 60 * 1000;
db.delete(downloads)
.where(lt(downloads.timestamp, thirtyMinsAgo))
.run();
// Ensure arrays exist
if (!status.activeDownloads) status.activeDownloads = [];
if (!status.queuedDownloads) status.queuedDownloads = [];
const allDownloads = db.select().from(downloads).all();
const activeDownloads = allDownloads
.filter(d => d.status === 'active')
.map(d => ({
id: d.id,
title: d.title,
timestamp: d.timestamp || 0,
filename: d.filename || undefined,
totalSize: d.totalSize || undefined,
downloadedSize: d.downloadedSize || undefined,
progress: d.progress || undefined,
speed: d.speed || undefined,
}));
// Check for stale downloads (older than 30 minutes)
const now = Date.now();
const validDownloads = status.activeDownloads.filter((d) => {
return d.timestamp && now - d.timestamp < 30 * 60 * 1000;
});
const queuedDownloads = allDownloads
.filter(d => d.status === 'queued')
.map(d => ({
id: d.id,
title: d.title,
timestamp: d.timestamp || 0,
}));
if (validDownloads.length !== status.activeDownloads.length) {
console.log("Removed stale downloads");
status.activeDownloads = validDownloads;
fs.writeFileSync(STATUS_DATA_PATH, JSON.stringify(status, null, 2));
}
return status;
return { activeDownloads, queuedDownloads };
} catch (error) {
console.error("Error reading download status:", error);
return { activeDownloads: [], queuedDownloads: [] };
}
}
// Get all videos
// --- Settings ---
export function getSettings(): Record<string, any> {
try {
const allSettings = db.select().from(settings).all();
const settingsMap: Record<string, any> = {};
for (const setting of allSettings) {
try {
settingsMap[setting.key] = JSON.parse(setting.value);
} catch (e) {
settingsMap[setting.key] = setting.value;
}
}
return settingsMap;
} catch (error) {
console.error("Error getting settings:", error);
return {};
}
}
export function saveSettings(newSettings: Record<string, any>): void {
try {
db.transaction(() => {
for (const [key, value] of Object.entries(newSettings)) {
db.insert(settings).values({
key,
value: JSON.stringify(value),
}).onConflictDoUpdate({
target: settings.key,
set: { value: JSON.stringify(value) },
}).run();
}
});
} catch (error) {
console.error("Error saving settings:", error);
throw error;
}
}
// --- Videos ---
export function getVideos(): Video[] {
if (!fs.existsSync(VIDEOS_DATA_PATH)) {
try {
const allVideos = db.select().from(videos).orderBy(desc(videos.createdAt)).all();
return allVideos as Video[];
} catch (error) {
console.error("Error getting videos:", error);
return [];
}
return JSON.parse(fs.readFileSync(VIDEOS_DATA_PATH, "utf8"));
}
// Get video by ID
export function getVideoById(id: string): Video | undefined {
const videos = getVideos();
return videos.find((v) => v.id === id);
try {
const video = db.select().from(videos).where(eq(videos.id, id)).get();
return video as Video | undefined;
} catch (error) {
console.error("Error getting video by id:", error);
return undefined;
}
}
// Save a new video
export function saveVideo(videoData: Video): Video {
let videos = getVideos();
videos.unshift(videoData);
fs.writeFileSync(VIDEOS_DATA_PATH, JSON.stringify(videos, null, 2));
return videoData;
try {
db.insert(videos).values(videoData as any).onConflictDoUpdate({
target: videos.id,
set: videoData,
}).run();
return videoData;
} catch (error) {
console.error("Error saving video:", error);
throw error;
}
}
// Update a video
export function updateVideo(id: string, updates: Partial<Video>): Video | null {
let videos = getVideos();
const index = videos.findIndex((v) => v.id === id);
if (index === -1) {
try {
const result = db.update(videos).set(updates as any).where(eq(videos.id, id)).returning().get();
return (result as Video) || null;
} catch (error) {
console.error("Error updating video:", error);
return null;
}
const updatedVideo = { ...videos[index], ...updates };
videos[index] = updatedVideo;
fs.writeFileSync(VIDEOS_DATA_PATH, JSON.stringify(videos, null, 2));
return updatedVideo;
}
// Delete a video
export function deleteVideo(id: string): boolean {
let videos = getVideos();
const videoToDelete = videos.find((v) => v.id === id);
try {
const videoToDelete = getVideoById(id);
if (!videoToDelete) return false;
if (!videoToDelete) {
return false;
}
// Remove the video file
if (videoToDelete.videoFilename) {
const actualPath = findVideoFile(videoToDelete.videoFilename);
if (actualPath && fs.existsSync(actualPath)) {
// Remove files
if (videoToDelete.videoFilename) {
const actualPath = findVideoFile(videoToDelete.videoFilename);
if (actualPath && fs.existsSync(actualPath)) {
fs.unlinkSync(actualPath);
}
}
}
// Remove the thumbnail file
if (videoToDelete.thumbnailFilename) {
if (videoToDelete.thumbnailFilename) {
const actualPath = findImageFile(videoToDelete.thumbnailFilename);
if (actualPath && fs.existsSync(actualPath)) {
fs.unlinkSync(actualPath);
fs.unlinkSync(actualPath);
}
}
// Delete from DB
db.delete(videos).where(eq(videos.id, id)).run();
return true;
} catch (error) {
console.error("Error deleting video:", error);
return false;
}
// Filter out the deleted video from the videos array
videos = videos.filter((v) => v.id !== id);
// Save the updated videos array
fs.writeFileSync(VIDEOS_DATA_PATH, JSON.stringify(videos, null, 2));
return true;
}
// Get all collections
// --- Collections ---
export function getCollections(): Collection[] {
if (!fs.existsSync(COLLECTIONS_DATA_PATH)) {
try {
const rows = db.select({
c: collections,
cv: collectionVideos,
})
.from(collections)
.leftJoin(collectionVideos, eq(collections.id, collectionVideos.collectionId))
.all();
const map = new Map<string, Collection>();
for (const row of rows) {
if (!map.has(row.c.id)) {
map.set(row.c.id, {
...row.c,
title: row.c.title || row.c.name,
updatedAt: row.c.updatedAt || undefined,
videos: [],
});
}
if (row.cv) {
map.get(row.c.id)!.videos.push(row.cv.videoId);
}
}
return Array.from(map.values());
} catch (error) {
console.error("Error getting collections:", error);
return [];
}
return JSON.parse(fs.readFileSync(COLLECTIONS_DATA_PATH, "utf8"));
}
// Get collection by ID
export function getCollectionById(id: string): Collection | undefined {
const collections = getCollections();
return collections.find((c) => c.id === id);
try {
const rows = db.select({
c: collections,
cv: collectionVideos,
})
.from(collections)
.leftJoin(collectionVideos, eq(collections.id, collectionVideos.collectionId))
.where(eq(collections.id, id))
.all();
if (rows.length === 0) return undefined;
const collection: Collection = {
...rows[0].c,
title: rows[0].c.title || rows[0].c.name,
updatedAt: rows[0].c.updatedAt || undefined,
videos: [],
};
for (const row of rows) {
if (row.cv) {
collection.videos.push(row.cv.videoId);
}
}
return collection;
} catch (error) {
console.error("Error getting collection by id:", error);
return undefined;
}
}
// Save a new collection
export function saveCollection(collection: Collection): Collection {
let collections = getCollections();
collections.push(collection);
fs.writeFileSync(COLLECTIONS_DATA_PATH, JSON.stringify(collections, null, 2));
return collection;
try {
db.transaction(() => {
// Insert collection
db.insert(collections).values({
id: collection.id,
name: collection.name || collection.title,
title: collection.title,
createdAt: collection.createdAt || new Date().toISOString(),
updatedAt: collection.updatedAt,
}).onConflictDoUpdate({
target: collections.id,
set: {
name: collection.name || collection.title,
title: collection.title,
updatedAt: new Date().toISOString(),
}
}).run();
// Sync videos
// First delete existing links
db.delete(collectionVideos).where(eq(collectionVideos.collectionId, collection.id)).run();
// Then insert new links
if (collection.videos && collection.videos.length > 0) {
for (const videoId of collection.videos) {
// Check if video exists to avoid FK error
const videoExists = db.select({ id: videos.id }).from(videos).where(eq(videos.id, videoId)).get();
if (videoExists) {
db.insert(collectionVideos).values({
collectionId: collection.id,
videoId: videoId,
}).run();
}
}
}
});
return collection;
} catch (error) {
console.error("Error saving collection:", error);
throw error;
}
}
// Atomic update for a collection
export function atomicUpdateCollection(
id: string,
updateFn: (collection: Collection) => Collection | null
): Collection | null {
let collections = getCollections();
const index = collections.findIndex((c) => c.id === id);
try {
const collection = getCollectionById(id);
if (!collection) return null;
if (index === -1) {
// Deep copy not strictly needed as we reconstruct, but good for safety if updateFn mutates
const collectionCopy = JSON.parse(JSON.stringify(collection));
const updatedCollection = updateFn(collectionCopy);
if (!updatedCollection) return null;
updatedCollection.updatedAt = new Date().toISOString();
saveCollection(updatedCollection);
return updatedCollection;
} catch (error) {
console.error("Error atomic updating collection:", error);
return null;
}
// Create a deep copy of the collection to avoid reference issues
const originalCollection = collections[index];
const collectionCopy: Collection = JSON.parse(
JSON.stringify(originalCollection)
);
// Apply the update function
const updatedCollection = updateFn(collectionCopy);
// If the update function returned null or undefined, abort the update
if (!updatedCollection) {
return null;
}
updatedCollection.updatedAt = new Date().toISOString();
// Update the collection in the array
collections[index] = updatedCollection;
// Write back to file synchronously
fs.writeFileSync(COLLECTIONS_DATA_PATH, JSON.stringify(collections, null, 2));
return updatedCollection;
}
// Delete a collection
export function deleteCollection(id: string): boolean {
let collections = getCollections();
const updatedCollections = collections.filter((c) => c.id !== id);
if (updatedCollections.length === collections.length) {
try {
const result = db.delete(collections).where(eq(collections.id, id)).run();
return result.changes > 0;
} catch (error) {
console.error("Error deleting collection:", error);
return false;
}
fs.writeFileSync(
COLLECTIONS_DATA_PATH,
JSON.stringify(updatedCollections, null, 2)
);
return true;
}
// Helper to find where a video file currently resides
// --- File Management Helpers ---
function findVideoFile(filename: string): string | null {
// Check root
const rootPath = path.join(VIDEOS_DIR, filename);
if (fs.existsSync(rootPath)) return rootPath;
// Check collections
const collections = getCollections();
for (const collection of collections) {
const allCollections = getCollections();
for (const collection of allCollections) {
const collectionName = collection.name || collection.title;
if (collectionName) {
const collectionPath = path.join(VIDEOS_DIR, collectionName, filename);
@@ -359,15 +469,12 @@ function findVideoFile(filename: string): string | null {
return null;
}
// Helper to find where an image file currently resides
function findImageFile(filename: string): string | null {
// Check root
const rootPath = path.join(IMAGES_DIR, filename);
if (fs.existsSync(rootPath)) return rootPath;
// Check collections
const collections = getCollections();
for (const collection of collections) {
const allCollections = getCollections();
for (const collection of allCollections) {
const collectionName = collection.name || collection.title;
if (collectionName) {
const collectionPath = path.join(IMAGES_DIR, collectionName, filename);
@@ -377,7 +484,6 @@ function findImageFile(filename: string): string | null {
return null;
}
// Helper to move a file
function moveFile(sourcePath: string, destPath: string): void {
try {
if (fs.existsSync(sourcePath)) {
@@ -390,8 +496,10 @@ function moveFile(sourcePath: string, destPath: string): void {
}
}
// Add video to collection and move files
// --- Complex Operations ---
export function addVideoToCollection(collectionId: string, videoId: string): Collection | null {
// Use atomicUpdateCollection to handle DB update
const collection = atomicUpdateCollection(collectionId, (c) => {
if (!c.videos.includes(videoId)) {
c.videos.push(videoId);
@@ -407,7 +515,6 @@ export function addVideoToCollection(collectionId: string, videoId: string): Col
const updates: Partial<Video> = {};
let updated = false;
// Move video file
if (video.videoFilename) {
const currentVideoPath = findVideoFile(video.videoFilename);
const targetVideoPath = path.join(VIDEOS_DIR, collectionName, video.videoFilename);
@@ -419,7 +526,6 @@ export function addVideoToCollection(collectionId: string, videoId: string): Col
}
}
// Move image file
if (video.thumbnailFilename) {
const currentImagePath = findImageFile(video.thumbnailFilename);
const targetImagePath = path.join(IMAGES_DIR, collectionName, video.thumbnailFilename);
@@ -440,7 +546,6 @@ export function addVideoToCollection(collectionId: string, videoId: string): Col
return collection;
}
// Remove video from collection and move files
export function removeVideoFromCollection(collectionId: string, videoId: string): Collection | null {
const collection = atomicUpdateCollection(collectionId, (c) => {
c.videos = c.videos.filter((v) => v !== videoId);
@@ -473,7 +578,6 @@ export function removeVideoFromCollection(collectionId: string, videoId: string)
const updates: Partial<Video> = {};
let updated = false;
// Move video file
if (video.videoFilename) {
const currentVideoPath = findVideoFile(video.videoFilename);
const targetVideoPath = path.join(targetVideoDir, video.videoFilename);
@@ -485,7 +589,6 @@ export function removeVideoFromCollection(collectionId: string, videoId: string)
}
}
// Move image file
if (video.thumbnailFilename) {
const currentImagePath = findImageFile(video.thumbnailFilename);
const targetImagePath = path.join(targetImageDir, video.thumbnailFilename);
@@ -506,19 +609,16 @@ export function removeVideoFromCollection(collectionId: string, videoId: string)
return collection;
}
// Delete collection and move files back to root (or other collection)
export function deleteCollectionWithFiles(collectionId: string): boolean {
const collection = getCollectionById(collectionId);
if (!collection) return false;
const collectionName = collection.name || collection.title;
// Move files for all videos in the collection
if (collection.videos && collection.videos.length > 0) {
collection.videos.forEach(videoId => {
const video = getVideoById(videoId);
if (video) {
// Check if video is in any OTHER collection (excluding the one being deleted)
const allCollections = getCollections();
const otherCollection = allCollections.find(c => c.videos.includes(videoId) && c.id !== collectionId);
@@ -540,9 +640,7 @@ export function deleteCollectionWithFiles(collectionId: string): boolean {
const updates: Partial<Video> = {};
let updated = false;
// Move video file
if (video.videoFilename) {
// We know it should be in the collection folder being deleted, but use findVideoFile to be safe
const currentVideoPath = findVideoFile(video.videoFilename);
const targetVideoPath = path.join(targetVideoDir, video.videoFilename);
@@ -553,7 +651,6 @@ export function deleteCollectionWithFiles(collectionId: string): boolean {
}
}
// Move image file
if (video.thumbnailFilename) {
const currentImagePath = findImageFile(video.thumbnailFilename);
const targetImagePath = path.join(targetImageDir, video.thumbnailFilename);
@@ -572,10 +669,8 @@ export function deleteCollectionWithFiles(collectionId: string): boolean {
});
}
// Delete the collection from DB
const success = deleteCollection(collectionId);
// Remove the collection directories if empty
if (success && collectionName) {
try {
const videoCollectionDir = path.join(VIDEOS_DIR, collectionName);
@@ -595,26 +690,21 @@ export function deleteCollectionWithFiles(collectionId: string): boolean {
return success;
}
// Delete collection and all its videos
export function deleteCollectionAndVideos(collectionId: string): boolean {
const collection = getCollectionById(collectionId);
if (!collection) return false;
const collectionName = collection.name || collection.title;
// Delete all videos in the collection
if (collection.videos && collection.videos.length > 0) {
// Create a copy of the videos array to iterate over safely
const videosToDelete = [...collection.videos];
videosToDelete.forEach(videoId => {
deleteVideo(videoId);
});
}
// Delete the collection from DB
const success = deleteCollection(collectionId);
// Remove the collection directories
if (success && collectionName) {
try {
const videoCollectionDir = path.join(VIDEOS_DIR, collectionName);
@@ -633,3 +723,4 @@ export function deleteCollectionAndVideos(collectionId: string): boolean {
return success;
}

4
backend/tsconfig.json
View File

@@ -3,14 +3,14 @@
"target": "es2020",
"module": "commonjs",
"outDir": "./dist",
"rootDir": "./src",
"strict": true,
"esModuleInterop": true,
"skipLibCheck": true,
"forceConsistentCasingInFileNames": true
},
"include": [
"src/**/*"
"src/**/*",
"scripts/**/*"
],
"exclude": [
"node_modules"

36
frontend/src/pages/SettingsPage.tsx
View File

@@ -217,6 +217,42 @@ const SettingsPage: React.FC = () => {
</Box>
</Grid>
<Grid size={12}><Divider /></Grid>
{/* Database Settings */}
<Grid size={12}>
<Typography variant="h6" gutterBottom>Database</Typography>
<Typography variant="body2" color="text.secondary" paragraph>
Migrate data from legacy JSON files to the new SQLite database.
This action is safe to run multiple times (duplicates will be skipped).
</Typography>
<Button
variant="outlined"
color="warning"
onClick={async () => {
if (window.confirm('Are you sure you want to migrate data? This may take a few moments.')) {
setSaving(true);
try {
const res = await axios.post(`${API_URL}/settings/migrate`);
console.log('Migration results:', res.data.results);
setMessage({ text: 'Migration completed successfully!', type: 'success' });
} catch (error: any) {
console.error('Migration failed:', error);
setMessage({
text: `Migration failed: ${error.response?.data?.details || error.message}`,
type: 'error'
});
} finally {
setSaving(false);
}
}
}}
disabled={saving}
>
Migrate Data from JSON
</Button>
</Grid>
<Grid size={12}>
<Box sx={{ display: 'flex', justifyContent: 'flex-end', mt: 2 }}>
<Button