Add TMP persistent directory

To store files that must be preserved between peertube restarts
This commit is contained in:
Chocobozzz 2023-05-03 15:17:11 +02:00 committed by Chocobozzz
parent 3a0c2a77b1
commit 6a49056026
17 changed files with 148 additions and 62 deletions

View File

@ -120,6 +120,7 @@ defaults:
# From the project root directory # From the project root directory
storage: storage:
tmp: 'storage/tmp/' # Use to download data (imports etc), store uploaded files before and during processing... tmp: 'storage/tmp/' # Use to download data (imports etc), store uploaded files before and during processing...
tmp_persistent: 'storage/tmp-persistent/' # As tmp but the directory is not cleaned up between PeerTube restarts
bin: 'storage/bin/' bin: 'storage/bin/'
avatars: 'storage/avatars/' avatars: 'storage/avatars/'
videos: 'storage/videos/' videos: 'storage/videos/'

View File

@ -118,6 +118,7 @@ defaults:
# From the project root directory # From the project root directory
storage: storage:
tmp: '/var/www/peertube/storage/tmp/' # Use to download data (imports etc), store uploaded files before and during processing... tmp: '/var/www/peertube/storage/tmp/' # Use to download data (imports etc), store uploaded files before and during processing...
tmp_persistent: '/var/www/peertube/storage/tmp-persistent/' # As tmp but the directory is not cleaned up between PeerTube restarts
bin: '/var/www/peertube/storage/bin/' bin: '/var/www/peertube/storage/bin/'
avatars: '/var/www/peertube/storage/avatars/' avatars: '/var/www/peertube/storage/avatars/'
videos: '/var/www/peertube/storage/videos/' videos: '/var/www/peertube/storage/videos/'

View File

@ -10,6 +10,7 @@ database:
# From the project root directory # From the project root directory
storage: storage:
tmp: 'test1/tmp/' tmp: 'test1/tmp/'
tmp_persistent: 'test1/tmp-persistent/'
bin: 'test1/bin/' bin: 'test1/bin/'
avatars: 'test1/avatars/' avatars: 'test1/avatars/'
videos: 'test1/videos/' videos: 'test1/videos/'

View File

@ -10,6 +10,7 @@ database:
# From the project root directory # From the project root directory
storage: storage:
tmp: 'test2/tmp/' tmp: 'test2/tmp/'
tmp_persistent: 'test2/tmp-persistent/'
bin: 'test2/bin/' bin: 'test2/bin/'
avatars: 'test2/avatars/' avatars: 'test2/avatars/'
videos: 'test2/videos/' videos: 'test2/videos/'

View File

@ -10,6 +10,7 @@ database:
# From the project root directory # From the project root directory
storage: storage:
tmp: 'test3/tmp/' tmp: 'test3/tmp/'
tmp_persistent: 'test3/tmp-persistent/'
bin: 'test3/bin/' bin: 'test3/bin/'
avatars: 'test3/avatars/' avatars: 'test3/avatars/'
videos: 'test3/videos/' videos: 'test3/videos/'

View File

@ -10,6 +10,7 @@ database:
# From the project root directory # From the project root directory
storage: storage:
tmp: 'test4/tmp/' tmp: 'test4/tmp/'
tmp_persistent: 'test4/tmp-persistent/'
bin: 'test4/bin/' bin: 'test4/bin/'
avatars: 'test4/avatars/' avatars: 'test4/avatars/'
videos: 'test4/videos/' videos: 'test4/videos/'

View File

@ -10,6 +10,7 @@ database:
# From the project root directory # From the project root directory
storage: storage:
tmp: 'test5/tmp/' tmp: 'test5/tmp/'
tmp_persistent: 'test5/tmp-persistent/'
bin: 'test5/bin/' bin: 'test5/bin/'
avatars: 'test5/avatars/' avatars: 'test5/avatars/'
videos: 'test5/videos/' videos: 'test5/videos/'

View File

@ -10,6 +10,7 @@ database:
# From the project root directory # From the project root directory
storage: storage:
tmp: 'test6/tmp/' tmp: 'test6/tmp/'
tmp_persistent: 'test6/tmp-persistent/'
bin: 'test6/bin/' bin: 'test6/bin/'
avatars: 'test6/avatars/' avatars: 'test6/avatars/'
videos: 'test6/videos/' videos: 'test6/videos/'

View File

@ -1,8 +1,12 @@
import Bluebird from 'bluebird'
import express from 'express' import express from 'express'
import { move } from 'fs-extra'
import { basename, join } from 'path'
import { createAnyReqFiles } from '@server/helpers/express-utils' import { createAnyReqFiles } from '@server/helpers/express-utils'
import { CONFIG } from '@server/initializers/config'
import { MIMETYPES } from '@server/initializers/constants' import { MIMETYPES } from '@server/initializers/constants'
import { JobQueue } from '@server/lib/job-queue' import { JobQueue } from '@server/lib/job-queue'
import { buildTaskFileFieldname, getTaskFile } from '@server/lib/video-studio' import { buildTaskFileFieldname, getTaskFileFromReq } from '@server/lib/video-studio'
import { import {
HttpStatusCode, HttpStatusCode,
VideoState, VideoState,
@ -68,7 +72,7 @@ async function createEditionTasks (req: express.Request, res: express.Response)
const payload = { const payload = {
videoUUID: video.uuid, videoUUID: video.uuid,
tasks: body.tasks.map((t, i) => buildTaskPayload(t, i, files)) tasks: await Bluebird.mapSeries(body.tasks, (t, i) => buildTaskPayload(t, i, files))
} }
JobQueue.Instance.createJobAsync({ type: 'video-studio-edition', payload }) JobQueue.Instance.createJobAsync({ type: 'video-studio-edition', payload })
@ -77,7 +81,11 @@ async function createEditionTasks (req: express.Request, res: express.Response)
} }
const taskPayloadBuilders: { const taskPayloadBuilders: {
[id in VideoStudioTask['name']]: (task: VideoStudioTask, indice?: number, files?: Express.Multer.File[]) => VideoStudioTaskPayload [id in VideoStudioTask['name']]: (
task: VideoStudioTask,
indice?: number,
files?: Express.Multer.File[]
) => Promise<VideoStudioTaskPayload>
} = { } = {
'add-intro': buildIntroOutroTask, 'add-intro': buildIntroOutroTask,
'add-outro': buildIntroOutroTask, 'add-outro': buildIntroOutroTask,
@ -85,34 +93,46 @@ const taskPayloadBuilders: {
'add-watermark': buildWatermarkTask 'add-watermark': buildWatermarkTask
} }
function buildTaskPayload (task: VideoStudioTask, indice: number, files: Express.Multer.File[]): VideoStudioTaskPayload { function buildTaskPayload (task: VideoStudioTask, indice: number, files: Express.Multer.File[]): Promise<VideoStudioTaskPayload> {
return taskPayloadBuilders[task.name](task, indice, files) return taskPayloadBuilders[task.name](task, indice, files)
} }
function buildIntroOutroTask (task: VideoStudioTaskIntro | VideoStudioTaskOutro, indice: number, files: Express.Multer.File[]) { async function buildIntroOutroTask (task: VideoStudioTaskIntro | VideoStudioTaskOutro, indice: number, files: Express.Multer.File[]) {
const destination = await moveStudioFileToPersistentTMP(getTaskFileFromReq(files, indice).path)
return { return {
name: task.name, name: task.name,
options: { options: {
file: getTaskFile(files, indice).path file: destination
} }
} }
} }
function buildCutTask (task: VideoStudioTaskCut) { function buildCutTask (task: VideoStudioTaskCut) {
return { return Promise.resolve({
name: task.name, name: task.name,
options: { options: {
start: task.options.start, start: task.options.start,
end: task.options.end end: task.options.end
} }
} })
} }
function buildWatermarkTask (task: VideoStudioTaskWatermark, indice: number, files: Express.Multer.File[]) { async function buildWatermarkTask (task: VideoStudioTaskWatermark, indice: number, files: Express.Multer.File[]) {
const destination = await moveStudioFileToPersistentTMP(getTaskFileFromReq(files, indice).path)
return { return {
name: task.name, name: task.name,
options: { options: {
file: getTaskFile(files, indice).path file: destination
} }
} }
} }
async function moveStudioFileToPersistentTMP (file: string) {
const destination = join(CONFIG.STORAGE.TMP_PERSISTENT_DIR, basename(file))
await move(file, destination)
return destination
}

View File

@ -98,6 +98,7 @@ const CONFIG = {
STORAGE: { STORAGE: {
TMP_DIR: buildPath(config.get<string>('storage.tmp')), TMP_DIR: buildPath(config.get<string>('storage.tmp')),
TMP_PERSISTENT_DIR: buildPath(config.get<string>('storage.tmp_persistent')),
BIN_DIR: buildPath(config.get<string>('storage.bin')), BIN_DIR: buildPath(config.get<string>('storage.bin')),
ACTOR_IMAGES: buildPath(config.get<string>('storage.avatars')), ACTOR_IMAGES: buildPath(config.get<string>('storage.avatars')),
LOG_DIR: buildPath(config.get<string>('storage.logs')), LOG_DIR: buildPath(config.get<string>('storage.logs')),

View File

@ -12,7 +12,7 @@ import { VideoTranscodingProfilesManager } from '@server/lib/transcoding/default
import { isAbleToUploadVideo } from '@server/lib/user' import { isAbleToUploadVideo } from '@server/lib/user'
import { buildFileMetadata, removeHLSPlaylist, removeWebTorrentFile } from '@server/lib/video-file' import { buildFileMetadata, removeHLSPlaylist, removeWebTorrentFile } from '@server/lib/video-file'
import { VideoPathManager } from '@server/lib/video-path-manager' import { VideoPathManager } from '@server/lib/video-path-manager'
import { approximateIntroOutroAdditionalSize } from '@server/lib/video-studio' import { approximateIntroOutroAdditionalSize, safeCleanupStudioTMPFiles } from '@server/lib/video-studio'
import { UserModel } from '@server/models/user/user' import { UserModel } from '@server/models/user/user'
import { VideoModel } from '@server/models/video/video' import { VideoModel } from '@server/models/video/video'
import { VideoFileModel } from '@server/models/video/video-file' import { VideoFileModel } from '@server/models/video/video-file'
@ -39,63 +39,73 @@ async function processVideoStudioEdition (job: Job) {
logger.info('Process video studio edition of %s in job %s.', payload.videoUUID, job.id, lTags) logger.info('Process video studio edition of %s in job %s.', payload.videoUUID, job.id, lTags)
const video = await VideoModel.loadFull(payload.videoUUID) try {
const video = await VideoModel.loadFull(payload.videoUUID)
// No video, maybe deleted? // No video, maybe deleted?
if (!video) { if (!video) {
logger.info('Can\'t process job %d, video does not exist.', job.id, lTags) logger.info('Can\'t process job %d, video does not exist.', job.id, lTags)
return undefined
}
await checkUserQuotaOrThrow(video, payload) await safeCleanupStudioTMPFiles(payload)
return undefined
const inputFile = video.getMaxQualityFile()
const editionResultPath = await VideoPathManager.Instance.makeAvailableVideoFile(inputFile, async originalFilePath => {
let tmpInputFilePath: string
let outputPath: string
for (const task of payload.tasks) {
const outputFilename = buildUUID() + inputFile.extname
outputPath = join(CONFIG.STORAGE.TMP_DIR, outputFilename)
await processTask({
inputPath: tmpInputFilePath ?? originalFilePath,
video,
outputPath,
task,
lTags
})
if (tmpInputFilePath) await remove(tmpInputFilePath)
// For the next iteration
tmpInputFilePath = outputPath
} }
return outputPath await checkUserQuotaOrThrow(video, payload)
})
logger.info('Video edition ended for video %s.', video.uuid, lTags) const inputFile = video.getMaxQualityFile()
const newFile = await buildNewFile(video, editionResultPath) const editionResultPath = await VideoPathManager.Instance.makeAvailableVideoFile(inputFile, async originalFilePath => {
let tmpInputFilePath: string
let outputPath: string
const outputPath = VideoPathManager.Instance.getFSVideoFileOutputPath(video, newFile) for (const task of payload.tasks) {
await move(editionResultPath, outputPath) const outputFilename = buildUUID() + inputFile.extname
outputPath = join(CONFIG.STORAGE.TMP_DIR, outputFilename)
await createTorrentAndSetInfoHashFromPath(video, newFile, outputPath) await processTask({
await removeAllFiles(video, newFile) inputPath: tmpInputFilePath ?? originalFilePath,
video,
outputPath,
task,
lTags
})
await newFile.save() if (tmpInputFilePath) await remove(tmpInputFilePath)
video.duration = await getVideoStreamDuration(outputPath) // For the next iteration
await video.save() tmpInputFilePath = outputPath
}
await federateVideoIfNeeded(video, false, undefined) return outputPath
})
const user = await UserModel.loadByVideoId(video.id) logger.info('Video edition ended for video %s.', video.uuid, lTags)
await createOptimizeOrMergeAudioJobs({ video, videoFile: newFile, isNewVideo: false, user, videoFileAlreadyLocked: false }) const newFile = await buildNewFile(video, editionResultPath)
const outputPath = VideoPathManager.Instance.getFSVideoFileOutputPath(video, newFile)
await move(editionResultPath, outputPath)
await safeCleanupStudioTMPFiles(payload)
await createTorrentAndSetInfoHashFromPath(video, newFile, outputPath)
await removeAllFiles(video, newFile)
await newFile.save()
video.duration = await getVideoStreamDuration(outputPath)
await video.save()
await federateVideoIfNeeded(video, false, undefined)
const user = await UserModel.loadByVideoId(video.id)
await createOptimizeOrMergeAudioJobs({ video, videoFile: newFile, isNewVideo: false, user, videoFileAlreadyLocked: false })
} catch (err) {
await safeCleanupStudioTMPFiles(payload)
throw err
}
} }
// --------------------------------------------------------------------------- // ---------------------------------------------------------------------------

View File

@ -1,15 +1,31 @@
import { logger } from '@server/helpers/logger'
import { MVideoFullLight } from '@server/types/models' import { MVideoFullLight } from '@server/types/models'
import { getVideoStreamDuration } from '@shared/ffmpeg' import { getVideoStreamDuration } from '@shared/ffmpeg'
import { VideoStudioTask } from '@shared/models' import { VideoStudioEditionPayload, VideoStudioTask } from '@shared/models'
import { remove } from 'fs-extra'
function buildTaskFileFieldname (indice: number, fieldName = 'file') { function buildTaskFileFieldname (indice: number, fieldName = 'file') {
return `tasks[${indice}][options][${fieldName}]` return `tasks[${indice}][options][${fieldName}]`
} }
function getTaskFile (files: Express.Multer.File[], indice: number, fieldName = 'file') { function getTaskFileFromReq (files: Express.Multer.File[], indice: number, fieldName = 'file') {
return files.find(f => f.fieldname === buildTaskFileFieldname(indice, fieldName)) return files.find(f => f.fieldname === buildTaskFileFieldname(indice, fieldName))
} }
async function safeCleanupStudioTMPFiles (payload: VideoStudioEditionPayload) {
for (const task of payload.tasks) {
try {
if (task.name === 'add-intro' || task.name === 'add-outro') {
await remove(task.options.file)
} else if (task.name === 'add-watermark') {
await remove(task.options.file)
}
} catch (err) {
logger.error('Cannot remove studio file', { err })
}
}
}
async function approximateIntroOutroAdditionalSize (video: MVideoFullLight, tasks: VideoStudioTask[], fileFinder: (i: number) => string) { async function approximateIntroOutroAdditionalSize (video: MVideoFullLight, tasks: VideoStudioTask[], fileFinder: (i: number) => string) {
let additionalDuration = 0 let additionalDuration = 0
@ -28,5 +44,6 @@ async function approximateIntroOutroAdditionalSize (video: MVideoFullLight, task
export { export {
approximateIntroOutroAdditionalSize, approximateIntroOutroAdditionalSize,
buildTaskFileFieldname, buildTaskFileFieldname,
getTaskFile getTaskFileFromReq,
safeCleanupStudioTMPFiles
} }

View File

@ -9,7 +9,7 @@ import {
} from '@server/helpers/custom-validators/video-studio' } from '@server/helpers/custom-validators/video-studio'
import { cleanUpReqFiles } from '@server/helpers/express-utils' import { cleanUpReqFiles } from '@server/helpers/express-utils'
import { CONFIG } from '@server/initializers/config' import { CONFIG } from '@server/initializers/config'
import { approximateIntroOutroAdditionalSize, getTaskFile } from '@server/lib/video-studio' import { approximateIntroOutroAdditionalSize, getTaskFileFromReq } from '@server/lib/video-studio'
import { isAudioFile } from '@shared/ffmpeg' import { isAudioFile } from '@shared/ffmpeg'
import { HttpStatusCode, UserRight, VideoState, VideoStudioCreateEdition, VideoStudioTask } from '@shared/models' import { HttpStatusCode, UserRight, VideoState, VideoStudioCreateEdition, VideoStudioTask } from '@shared/models'
import { areValidationErrors, checkUserCanManageVideo, checkUserQuota, doesVideoExist } from '../shared' import { areValidationErrors, checkUserCanManageVideo, checkUserQuota, doesVideoExist } from '../shared'
@ -49,7 +49,7 @@ const videoStudioAddEditionValidator = [
} }
if (task.name === 'add-intro' || task.name === 'add-outro') { if (task.name === 'add-intro' || task.name === 'add-outro') {
const filePath = getTaskFile(files, i).path const filePath = getTaskFileFromReq(files, i).path
// Our concat filter needs a video stream // Our concat filter needs a video stream
if (await isAudioFile(filePath)) { if (await isAudioFile(filePath)) {
@ -79,7 +79,7 @@ const videoStudioAddEditionValidator = [
if (!checkUserCanManageVideo(user, video, UserRight.UPDATE_ANY_VIDEO, res)) return cleanUpReqFiles(req) if (!checkUserCanManageVideo(user, video, UserRight.UPDATE_ANY_VIDEO, res)) return cleanUpReqFiles(req)
// Try to make an approximation of bytes added by the intro/outro // Try to make an approximation of bytes added by the intro/outro
const additionalBytes = await approximateIntroOutroAdditionalSize(video, body.tasks, i => getTaskFile(files, i).path) const additionalBytes = await approximateIntroOutroAdditionalSize(video, body.tasks, i => getTaskFileFromReq(files, i).path)
if (await checkUserQuota(user, additionalBytes, res) === false) return cleanUpReqFiles(req) if (await checkUserQuota(user, additionalBytes, res) === false) return cleanUpReqFiles(req)
return next() return next()

View File

@ -1,5 +1,5 @@
import { expect } from 'chai' import { expect } from 'chai'
import { expectStartWith } from '@server/tests/shared' import { checkPersistentTmpIsEmpty, expectStartWith } from '@server/tests/shared'
import { areMockObjectStorageTestsDisabled, getAllFiles } from '@shared/core-utils' import { areMockObjectStorageTestsDisabled, getAllFiles } from '@shared/core-utils'
import { VideoStudioTask } from '@shared/models' import { VideoStudioTask } from '@shared/models'
import { import {
@ -356,6 +356,29 @@ describe('Test video studio', function () {
}) })
}) })
describe('Server restart', function () {
it('Should still be able to run video edition after a server restart', async function () {
this.timeout(240_000)
await renewVideo()
await servers[0].videoStudio.createEditionTasks({ videoId: videoUUID, tasks: VideoStudioCommand.getComplexTask() })
await servers[0].kill()
await servers[0].run()
await waitJobs(servers)
for (const server of servers) {
await checkDuration(server, 9)
}
})
it('Should have an empty persistent tmp directory', async function () {
await checkPersistentTmpIsEmpty(servers[0])
})
})
after(async function () { after(async function () {
await cleanupTests(servers) await cleanupTests(servers)
}) })

View File

@ -12,6 +12,10 @@ async function checkTmpIsEmpty (server: PeerTubeServer) {
} }
} }
async function checkPersistentTmpIsEmpty (server: PeerTubeServer) {
await checkDirectoryIsEmpty(server, 'tmp-persistent')
}
async function checkDirectoryIsEmpty (server: PeerTubeServer, directory: string, exceptions: string[] = []) { async function checkDirectoryIsEmpty (server: PeerTubeServer, directory: string, exceptions: string[] = []) {
const directoryPath = server.getDirectoryPath(directory) const directoryPath = server.getDirectoryPath(directory)
@ -26,5 +30,6 @@ async function checkDirectoryIsEmpty (server: PeerTubeServer, directory: string,
export { export {
checkTmpIsEmpty, checkTmpIsEmpty,
checkPersistentTmpIsEmpty,
checkDirectoryIsEmpty checkDirectoryIsEmpty
} }

View File

@ -364,6 +364,7 @@ export class PeerTubeServer {
}, },
storage: { storage: {
tmp: this.getDirectoryPath('tmp') + '/', tmp: this.getDirectoryPath('tmp') + '/',
tmp_persistent: this.getDirectoryPath('tmp-persistent') + '/',
bin: this.getDirectoryPath('bin') + '/', bin: this.getDirectoryPath('bin') + '/',
avatars: this.getDirectoryPath('avatars') + '/', avatars: this.getDirectoryPath('avatars') + '/',
videos: this.getDirectoryPath('videos') + '/', videos: this.getDirectoryPath('videos') + '/',

View File

@ -44,6 +44,7 @@ redis:
# From the project root directory # From the project root directory
storage: storage:
tmp: '../data/tmp/' # Use to download data (imports etc), store uploaded files before and during processing... tmp: '../data/tmp/' # Use to download data (imports etc), store uploaded files before and during processing...
tmp_persistent: '../data/tmp-persistent/' # As tmp but the directory is not cleaned up between PeerTube restarts
bin: '../data/bin/' bin: '../data/bin/'
avatars: '../data/avatars/' avatars: '../data/avatars/'
videos: '../data/videos/' videos: '../data/videos/'