PeerTube_original/server/initializers/installer.ts
kontrollanten f6d6e7f861
Resumable video uploads (#3933)
* WIP: resumable video uploads

relates to #324

* fix review comments

* video upload: error handling

* fix audio upload

* fixes after self review

* Update server/controllers/api/videos/index.ts

Co-authored-by: Rigel Kent <par@rigelk.eu>

* Update server/middlewares/validators/videos/videos.ts

Co-authored-by: Rigel Kent <par@rigelk.eu>

* Update server/controllers/api/videos/index.ts

Co-authored-by: Rigel Kent <par@rigelk.eu>

* update after code review

* refactor upload route

- restore multipart upload route
- move resumable to dedicated upload-resumable route
- move checks to middleware
- do not leak internal fs structure in response

* fix yarn.lock upon rebase

* factorize addVideo for reuse in both endpoints

* add resumable upload API to openapi spec

* add initial test and test helper for resumable upload

* typings for videoAddResumable middleware

* avoid including aws and google packages via node-uploadx, by only including uploadx/core

* rename ex-isAudioBg to more explicit name mentioning it is a preview file for audio

* add video-upload-tmp-folder-cleaner job

* stronger typing of video upload middleware

* reduce dependency to @uploadx/core

* add audio upload test

* refactor resumable uploads cleanup from job to scheduler

* refactor resumable uploads scheduler to compare to last execution time

* make resumable upload validator to always cleanup on failure

* move legacy upload request building outside of uploadVideo test helper

* filter upload-resumable middlewares down to POST, PUT, DELETE

also begin to type metadata

* merge add duration functions

* stronger typings and documentation for uploadx behaviour, move init validator up

* refactor(client/video-edit): options > uploadxOptions

* refactor(client/video-edit): remove obsolete else

* scheduler/remove-dangling-resum: rename tag

* refactor(server/video): add UploadVideoFiles type

* refactor(mw/validators): restructure eslint disable

* refactor(mw/validators/videos): rename import

* refactor(client/vid-upload): rename html elem id

* refactor(sched/remove-dangl): move fn to method

* refactor(mw/async): add method typing

* refactor(mw/vali/video): double quote > single

* refactor(server/upload-resum): express use > all

* proper http methud enum server/middlewares/async.ts

* properly type http methods

* factorize common video upload validation steps

* add check for maximum partially uploaded file size

* fix audioBg use

* fix extname(filename) in addVideo

* document parameters for uploadx's resumable protocol

* clear META files in scheduler

* last audio refactor before cramming preview in the initial POST form data

* refactor as mulitpart/form-data initial post request

this allows preview/thumbnail uploads alongside the initial request,
and cleans up the upload form

* Add more tests for resumable uploads

* Refactor remove dangling resumable uploads

* Prepare changelog

* Add more resumable upload tests

* Remove user quota check for resumable uploads

* Fix upload error handler

* Update nginx template for upload-resumable

* Cleanup comment

* Remove unused express methods

* Prefer to use got instead of raw http

* Don't retry on error 500

Co-authored-by: Rigel Kent <par@rigelk.eu>
Co-authored-by: Rigel Kent <sendmemail@rigelk.eu>
Co-authored-by: Chocobozzz <me@florianbigard.com>
2021-05-10 11:13:41 +02:00

170 lines
4.8 KiB
TypeScript

import * as passwordGenerator from 'password-generator'
import { UserRole } from '../../shared'
import { logger } from '../helpers/logger'
import { createApplicationActor, createUserAccountAndChannelAndPlaylist } from '../lib/user'
import { UserModel } from '../models/account/user'
import { ApplicationModel } from '../models/application/application'
import { OAuthClientModel } from '../models/oauth/oauth-client'
import { applicationExist, clientsExist, usersExist } from './checker-after-init'
import { FILES_CACHE, HLS_STREAMING_PLAYLIST_DIRECTORY, LAST_MIGRATION_VERSION, RESUMABLE_UPLOAD_DIRECTORY } from './constants'
import { sequelizeTypescript } from './database'
import { ensureDir, remove } from 'fs-extra'
import { CONFIG } from './config'
async function installApplication () {
try {
await Promise.all([
// Database related
sequelizeTypescript.sync()
.then(() => {
return Promise.all([
createApplicationIfNotExist(),
createOAuthClientIfNotExist(),
createOAuthAdminIfNotExist()
])
}),
// Directories
removeCacheAndTmpDirectories()
.then(() => createDirectoriesIfNotExist())
])
} catch (err) {
logger.error('Cannot install application.', { err })
process.exit(-1)
}
}
// ---------------------------------------------------------------------------
export {
installApplication
}
// ---------------------------------------------------------------------------
function removeCacheAndTmpDirectories () {
const cacheDirectories = Object.keys(FILES_CACHE)
.map(k => FILES_CACHE[k].DIRECTORY)
const tasks: Promise<any>[] = []
// Cache directories
for (const key of Object.keys(cacheDirectories)) {
const dir = cacheDirectories[key]
tasks.push(remove(dir))
}
tasks.push(remove(CONFIG.STORAGE.TMP_DIR))
return Promise.all(tasks)
}
function createDirectoriesIfNotExist () {
const storage = CONFIG.STORAGE
const cacheDirectories = Object.keys(FILES_CACHE)
.map(k => FILES_CACHE[k].DIRECTORY)
const tasks: Promise<void>[] = []
for (const key of Object.keys(storage)) {
const dir = storage[key]
tasks.push(ensureDir(dir))
}
// Cache directories
for (const key of Object.keys(cacheDirectories)) {
const dir = cacheDirectories[key]
tasks.push(ensureDir(dir))
}
// Playlist directories
tasks.push(ensureDir(HLS_STREAMING_PLAYLIST_DIRECTORY))
// Resumable upload directory
tasks.push(ensureDir(RESUMABLE_UPLOAD_DIRECTORY))
return Promise.all(tasks)
}
async function createOAuthClientIfNotExist () {
const exist = await clientsExist()
// Nothing to do, clients already exist
if (exist === true) return undefined
logger.info('Creating a default OAuth Client.')
const id = passwordGenerator(32, false, /[a-z0-9]/)
const secret = passwordGenerator(32, false, /[a-zA-Z0-9]/)
const client = new OAuthClientModel({
clientId: id,
clientSecret: secret,
grants: [ 'password', 'refresh_token' ],
redirectUris: null
})
const createdClient = await client.save()
logger.info('Client id: ' + createdClient.clientId)
logger.info('Client secret: ' + createdClient.clientSecret)
return undefined
}
async function createOAuthAdminIfNotExist () {
const exist = await usersExist()
// Nothing to do, users already exist
if (exist === true) return undefined
logger.info('Creating the administrator.')
const username = 'root'
const role = UserRole.ADMINISTRATOR
const email = CONFIG.ADMIN.EMAIL
let validatePassword = true
let password = ''
// Do not generate a random password for tests
if (process.env.NODE_ENV === 'test') {
password = 'test'
if (process.env.NODE_APP_INSTANCE) {
password += process.env.NODE_APP_INSTANCE
}
// Our password is weak so do not validate it
validatePassword = false
} else if (process.env.PT_INITIAL_ROOT_PASSWORD) {
password = process.env.PT_INITIAL_ROOT_PASSWORD
} else {
password = passwordGenerator(16, true)
}
const userData = {
username,
email,
password,
role,
verified: true,
nsfwPolicy: CONFIG.INSTANCE.DEFAULT_NSFW_POLICY,
videoQuota: -1,
videoQuotaDaily: -1
}
const user = new UserModel(userData)
await createUserAccountAndChannelAndPlaylist({ userToCreate: user, channelNames: undefined, validateUser: validatePassword })
logger.info('Username: ' + username)
logger.info('User password: ' + password)
}
async function createApplicationIfNotExist () {
const exist = await applicationExist()
// Nothing to do, application already exist
if (exist === true) return undefined
logger.info('Creating application account.')
const application = await ApplicationModel.create({
migrationVersion: LAST_MIGRATION_VERSION
})
return createApplicationActor(application.id)
}