Merge branch 'pr/2629' into develop
This commit is contained in:
commit
62068f4153
|
@ -12,6 +12,7 @@ import { FormValidatorService } from '@app/shared'
|
||||||
import { VideoCaptionService } from '@app/shared/video-caption'
|
import { VideoCaptionService } from '@app/shared/video-caption'
|
||||||
import { VideoImportService } from '@app/shared/video-import'
|
import { VideoImportService } from '@app/shared/video-import'
|
||||||
import { scrollToTop } from '@app/shared/misc/utils'
|
import { scrollToTop } from '@app/shared/misc/utils'
|
||||||
|
import { switchMap, map } from 'rxjs/operators'
|
||||||
|
|
||||||
@Component({
|
@Component({
|
||||||
selector: 'my-video-import-url',
|
selector: 'my-video-import-url',
|
||||||
|
@ -76,31 +77,44 @@ export class VideoImportUrlComponent extends VideoSend implements OnInit, CanCom
|
||||||
|
|
||||||
this.loadingBar.start()
|
this.loadingBar.start()
|
||||||
|
|
||||||
this.videoImportService.importVideoUrl(this.targetUrl, videoUpdate).subscribe(
|
this.videoImportService
|
||||||
res => {
|
.importVideoUrl(this.targetUrl, videoUpdate)
|
||||||
this.loadingBar.complete()
|
.pipe(
|
||||||
this.firstStepDone.emit(res.video.name)
|
switchMap(res => {
|
||||||
this.isImportingVideo = false
|
return this.videoCaptionService
|
||||||
this.hasImportedVideo = true
|
.listCaptions(res.video.id)
|
||||||
|
.pipe(
|
||||||
|
map(result => ({ video: res.video, videoCaptions: result.data }))
|
||||||
|
)
|
||||||
|
})
|
||||||
|
)
|
||||||
|
.subscribe(
|
||||||
|
({ video, videoCaptions }) => {
|
||||||
|
this.loadingBar.complete()
|
||||||
|
this.firstStepDone.emit(video.name)
|
||||||
|
this.isImportingVideo = false
|
||||||
|
this.hasImportedVideo = true
|
||||||
|
|
||||||
this.video = new VideoEdit(Object.assign(res.video, {
|
this.video = new VideoEdit(Object.assign(video, {
|
||||||
commentsEnabled: videoUpdate.commentsEnabled,
|
commentsEnabled: videoUpdate.commentsEnabled,
|
||||||
downloadEnabled: videoUpdate.downloadEnabled,
|
downloadEnabled: videoUpdate.downloadEnabled,
|
||||||
support: null,
|
support: null,
|
||||||
thumbnailUrl: null,
|
thumbnailUrl: null,
|
||||||
previewUrl: null
|
previewUrl: null
|
||||||
}))
|
}))
|
||||||
|
|
||||||
this.hydrateFormFromVideo()
|
this.videoCaptions = videoCaptions
|
||||||
},
|
|
||||||
|
|
||||||
err => {
|
this.hydrateFormFromVideo()
|
||||||
this.loadingBar.complete()
|
},
|
||||||
this.isImportingVideo = false
|
|
||||||
this.firstStepError.emit()
|
err => {
|
||||||
this.notifier.error(err.message)
|
this.loadingBar.complete()
|
||||||
}
|
this.isImportingVideo = false
|
||||||
)
|
this.firstStepError.emit()
|
||||||
|
this.notifier.error(err.message)
|
||||||
|
}
|
||||||
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
updateSecondStep () {
|
updateSecondStep () {
|
||||||
|
|
|
@ -3,11 +3,13 @@ import * as magnetUtil from 'magnet-uri'
|
||||||
import { auditLoggerFactory, getAuditIdFromRes, VideoImportAuditView } from '../../../helpers/audit-logger'
|
import { auditLoggerFactory, getAuditIdFromRes, VideoImportAuditView } from '../../../helpers/audit-logger'
|
||||||
import { asyncMiddleware, asyncRetryTransactionMiddleware, authenticate, videoImportAddValidator } from '../../../middlewares'
|
import { asyncMiddleware, asyncRetryTransactionMiddleware, authenticate, videoImportAddValidator } from '../../../middlewares'
|
||||||
import { MIMETYPES } from '../../../initializers/constants'
|
import { MIMETYPES } from '../../../initializers/constants'
|
||||||
import { getYoutubeDLInfo, YoutubeDLInfo } from '../../../helpers/youtube-dl'
|
import { getYoutubeDLInfo, YoutubeDLInfo, getYoutubeDLSubs } from '../../../helpers/youtube-dl'
|
||||||
import { createReqFiles } from '../../../helpers/express-utils'
|
import { createReqFiles } from '../../../helpers/express-utils'
|
||||||
import { logger } from '../../../helpers/logger'
|
import { logger } from '../../../helpers/logger'
|
||||||
import { VideoImportCreate, VideoImportState, VideoPrivacy, VideoState } from '../../../../shared'
|
import { VideoImportCreate, VideoImportState, VideoPrivacy, VideoState } from '../../../../shared'
|
||||||
import { VideoModel } from '../../../models/video/video'
|
import { VideoModel } from '../../../models/video/video'
|
||||||
|
import { VideoCaptionModel } from '../../../models/video/video-caption'
|
||||||
|
import { moveAndProcessCaptionFile } from '../../../helpers/captions-utils'
|
||||||
import { getVideoActivityPubUrl } from '../../../lib/activitypub'
|
import { getVideoActivityPubUrl } from '../../../lib/activitypub'
|
||||||
import { TagModel } from '../../../models/video/tag'
|
import { TagModel } from '../../../models/video/tag'
|
||||||
import { VideoImportModel } from '../../../models/video/video-import'
|
import { VideoImportModel } from '../../../models/video/video-import'
|
||||||
|
@ -28,6 +30,7 @@ import {
|
||||||
MThumbnail,
|
MThumbnail,
|
||||||
MUser,
|
MUser,
|
||||||
MVideoAccountDefault,
|
MVideoAccountDefault,
|
||||||
|
MVideoCaptionVideo,
|
||||||
MVideoTag,
|
MVideoTag,
|
||||||
MVideoThumbnailAccountDefault,
|
MVideoThumbnailAccountDefault,
|
||||||
MVideoWithBlacklistLight
|
MVideoWithBlacklistLight
|
||||||
|
@ -136,6 +139,7 @@ async function addYoutubeDLImport (req: express.Request, res: express.Response)
|
||||||
const targetUrl = body.targetUrl
|
const targetUrl = body.targetUrl
|
||||||
const user = res.locals.oauth.token.User
|
const user = res.locals.oauth.token.User
|
||||||
|
|
||||||
|
// Get video infos
|
||||||
let youtubeDLInfo: YoutubeDLInfo
|
let youtubeDLInfo: YoutubeDLInfo
|
||||||
try {
|
try {
|
||||||
youtubeDLInfo = await getYoutubeDLInfo(targetUrl)
|
youtubeDLInfo = await getYoutubeDLInfo(targetUrl)
|
||||||
|
@ -168,6 +172,30 @@ async function addYoutubeDLImport (req: express.Request, res: express.Response)
|
||||||
user
|
user
|
||||||
})
|
})
|
||||||
|
|
||||||
|
// Get video subtitles
|
||||||
|
try {
|
||||||
|
const subtitles = await getYoutubeDLSubs(targetUrl)
|
||||||
|
|
||||||
|
logger.info('Will create %s subtitles from youtube import %s.', subtitles.length, targetUrl)
|
||||||
|
|
||||||
|
for (const subtitle of subtitles) {
|
||||||
|
const videoCaption = new VideoCaptionModel({
|
||||||
|
videoId: video.id,
|
||||||
|
language: subtitle.language
|
||||||
|
}) as MVideoCaptionVideo
|
||||||
|
videoCaption.Video = video
|
||||||
|
|
||||||
|
// Move physical file
|
||||||
|
await moveAndProcessCaptionFile(subtitle, videoCaption)
|
||||||
|
|
||||||
|
await sequelizeTypescript.transaction(async t => {
|
||||||
|
await VideoCaptionModel.insertOrReplaceLanguage(video.id, subtitle.language, null, t)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
} catch (err) {
|
||||||
|
logger.warn('Cannot get video subtitles.', { err })
|
||||||
|
}
|
||||||
|
|
||||||
// Create job to import the video
|
// Create job to import the video
|
||||||
const payload = {
|
const payload = {
|
||||||
type: 'youtube-dl' as 'youtube-dl',
|
type: 'youtube-dl' as 'youtube-dl',
|
||||||
|
|
|
@ -20,6 +20,12 @@ export type YoutubeDLInfo = {
|
||||||
originallyPublishedAt?: Date
|
originallyPublishedAt?: Date
|
||||||
}
|
}
|
||||||
|
|
||||||
|
export type YoutubeDLSubs = {
|
||||||
|
language: string
|
||||||
|
filename: string
|
||||||
|
path: string
|
||||||
|
}[]
|
||||||
|
|
||||||
const processOptions = {
|
const processOptions = {
|
||||||
maxBuffer: 1024 * 1024 * 10 // 10MB
|
maxBuffer: 1024 * 1024 * 10 // 10MB
|
||||||
}
|
}
|
||||||
|
@ -45,6 +51,40 @@ function getYoutubeDLInfo (url: string, opts?: string[]): Promise<YoutubeDLInfo>
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function getYoutubeDLSubs (url: string, opts?: object): Promise<YoutubeDLSubs> {
|
||||||
|
return new Promise<YoutubeDLSubs>((res, rej) => {
|
||||||
|
const cwd = CONFIG.STORAGE.TMP_DIR
|
||||||
|
const options = opts || { all: true, format: 'vtt', cwd }
|
||||||
|
|
||||||
|
safeGetYoutubeDL()
|
||||||
|
.then(youtubeDL => {
|
||||||
|
youtubeDL.getSubs(url, options, (err, files) => {
|
||||||
|
if (err) return rej(err)
|
||||||
|
|
||||||
|
logger.debug('Get subtitles from youtube dl.', { url, files })
|
||||||
|
|
||||||
|
const subtitles = files.reduce((acc, filename) => {
|
||||||
|
const matched = filename.match(/\.([a-z]{2})\.(vtt|ttml)/i)
|
||||||
|
|
||||||
|
if (matched[1]) {
|
||||||
|
return [
|
||||||
|
...acc,
|
||||||
|
{
|
||||||
|
language: matched[1],
|
||||||
|
path: join(cwd, filename),
|
||||||
|
filename
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
||||||
|
}, [])
|
||||||
|
|
||||||
|
return res(subtitles)
|
||||||
|
})
|
||||||
|
})
|
||||||
|
.catch(err => rej(err))
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
function downloadYoutubeDLVideo (url: string, extension: string, timeout: number) {
|
function downloadYoutubeDLVideo (url: string, extension: string, timeout: number) {
|
||||||
const path = generateVideoImportTmpPath(url, extension)
|
const path = generateVideoImportTmpPath(url, extension)
|
||||||
let timer
|
let timer
|
||||||
|
@ -185,6 +225,7 @@ function buildOriginallyPublishedAt (obj: any) {
|
||||||
export {
|
export {
|
||||||
updateYoutubeDLBinary,
|
updateYoutubeDLBinary,
|
||||||
downloadYoutubeDLVideo,
|
downloadYoutubeDLVideo,
|
||||||
|
getYoutubeDLSubs,
|
||||||
getYoutubeDLInfo,
|
getYoutubeDLInfo,
|
||||||
safeGetYoutubeDL,
|
safeGetYoutubeDL,
|
||||||
buildOriginallyPublishedAt
|
buildOriginallyPublishedAt
|
||||||
|
|
|
@ -117,7 +117,7 @@ async function getOrCreateActorAndServerAndModel (
|
||||||
if (actor.VideoChannel) (actor as MActorAccountChannelIdActor).VideoChannel.Actor = actor
|
if (actor.VideoChannel) (actor as MActorAccountChannelIdActor).VideoChannel.Actor = actor
|
||||||
|
|
||||||
const { actor: actorRefreshed, refreshed } = await retryTransactionWrapper(refreshActorIfNeeded, actor, fetchType)
|
const { actor: actorRefreshed, refreshed } = await retryTransactionWrapper(refreshActorIfNeeded, actor, fetchType)
|
||||||
if (!actorRefreshed) throw new Error('Actor ' + actorRefreshed.url + ' does not exist anymore.')
|
if (!actorRefreshed) throw new Error('Actor ' + actor.url + ' does not exist anymore.')
|
||||||
|
|
||||||
if ((created === true || refreshed === true) && updateCollections === true) {
|
if ((created === true || refreshed === true) && updateCollections === true) {
|
||||||
const payload = { uri: actor.outboxUrl, type: 'activity' as 'activity' }
|
const payload = { uri: actor.outboxUrl, type: 'activity' as 'activity' }
|
||||||
|
|
|
@ -129,6 +129,7 @@ export class VideoImportModel extends Model<VideoImportModel> {
|
||||||
distinct: true,
|
distinct: true,
|
||||||
include: [
|
include: [
|
||||||
{
|
{
|
||||||
|
attributes: [ 'id' ],
|
||||||
model: UserModel.unscoped(), // FIXME: Without this, sequelize try to COUNT(DISTINCT(*)) which is an invalid SQL query
|
model: UserModel.unscoped(), // FIXME: Without this, sequelize try to COUNT(DISTINCT(*)) which is an invalid SQL query
|
||||||
required: true
|
required: true
|
||||||
}
|
}
|
||||||
|
|
|
@ -2,7 +2,7 @@
|
||||||
|
|
||||||
import * as chai from 'chai'
|
import * as chai from 'chai'
|
||||||
import 'mocha'
|
import 'mocha'
|
||||||
import { VideoDetails, VideoImport, VideoPrivacy } from '../../../../shared/models/videos'
|
import { VideoDetails, VideoImport, VideoPrivacy, VideoCaption } from '../../../../shared/models/videos'
|
||||||
import {
|
import {
|
||||||
cleanupTests,
|
cleanupTests,
|
||||||
doubleFollow,
|
doubleFollow,
|
||||||
|
@ -11,6 +11,8 @@ import {
|
||||||
getMyVideos,
|
getMyVideos,
|
||||||
getVideo,
|
getVideo,
|
||||||
getVideosList,
|
getVideosList,
|
||||||
|
listVideoCaptions,
|
||||||
|
testCaptionFile,
|
||||||
immutableAssign,
|
immutableAssign,
|
||||||
ServerInfo,
|
ServerInfo,
|
||||||
setAccessTokensToServers
|
setAccessTokensToServers
|
||||||
|
@ -60,11 +62,14 @@ describe('Test video imports', function () {
|
||||||
|
|
||||||
expect(videoTorrent.name).to.contain('你好 世界 720p.mp4')
|
expect(videoTorrent.name).to.contain('你好 世界 720p.mp4')
|
||||||
expect(videoMagnet.name).to.contain('super peertube2 video')
|
expect(videoMagnet.name).to.contain('super peertube2 video')
|
||||||
|
|
||||||
|
const resCaptions = await listVideoCaptions(url, idHttp)
|
||||||
|
expect(resCaptions.body.total).to.equal(2)
|
||||||
}
|
}
|
||||||
|
|
||||||
async function checkVideoServer2 (url: string, id: number | string) {
|
async function checkVideoServer2 (url: string, id: number | string) {
|
||||||
const res = await getVideo(url, id)
|
const res = await getVideo(url, id)
|
||||||
const video = res.body
|
const video: VideoDetails = res.body
|
||||||
|
|
||||||
expect(video.name).to.equal('my super name')
|
expect(video.name).to.equal('my super name')
|
||||||
expect(video.category.label).to.equal('Entertainment')
|
expect(video.category.label).to.equal('Entertainment')
|
||||||
|
@ -75,6 +80,9 @@ describe('Test video imports', function () {
|
||||||
expect(video.tags).to.deep.equal([ 'supertag1', 'supertag2' ])
|
expect(video.tags).to.deep.equal([ 'supertag1', 'supertag2' ])
|
||||||
|
|
||||||
expect(video.files).to.have.lengthOf(1)
|
expect(video.files).to.have.lengthOf(1)
|
||||||
|
|
||||||
|
const resCaptions = await listVideoCaptions(url, id)
|
||||||
|
expect(resCaptions.body.total).to.equal(2)
|
||||||
}
|
}
|
||||||
|
|
||||||
before(async function () {
|
before(async function () {
|
||||||
|
@ -110,6 +118,44 @@ describe('Test video imports', function () {
|
||||||
const attributes = immutableAssign(baseAttributes, { targetUrl: getYoutubeVideoUrl() })
|
const attributes = immutableAssign(baseAttributes, { targetUrl: getYoutubeVideoUrl() })
|
||||||
const res = await importVideo(servers[0].url, servers[0].accessToken, attributes)
|
const res = await importVideo(servers[0].url, servers[0].accessToken, attributes)
|
||||||
expect(res.body.video.name).to.equal('small video - youtube')
|
expect(res.body.video.name).to.equal('small video - youtube')
|
||||||
|
|
||||||
|
const resCaptions = await listVideoCaptions(servers[0].url, res.body.video.id)
|
||||||
|
const videoCaptions: VideoCaption[] = resCaptions.body.data
|
||||||
|
expect(videoCaptions).to.have.lengthOf(2)
|
||||||
|
|
||||||
|
const enCaption = videoCaptions.find(caption => caption.language.id === 'en')
|
||||||
|
expect(enCaption).to.exist
|
||||||
|
expect(enCaption.language.label).to.equal('English')
|
||||||
|
expect(enCaption.captionPath).to.equal(`/static/video-captions/${res.body.video.uuid}-en.vtt`)
|
||||||
|
await testCaptionFile(servers[0].url, enCaption.captionPath, `WEBVTT
|
||||||
|
Kind: captions
|
||||||
|
Language: en
|
||||||
|
|
||||||
|
00:00:01.600 --> 00:00:04.200
|
||||||
|
English (US)
|
||||||
|
|
||||||
|
00:00:05.900 --> 00:00:07.999
|
||||||
|
This is a subtitle in American English
|
||||||
|
|
||||||
|
00:00:10.000 --> 00:00:14.000
|
||||||
|
Adding subtitles is very easy to do`)
|
||||||
|
|
||||||
|
const frCaption = videoCaptions.find(caption => caption.language.id === 'fr')
|
||||||
|
expect(frCaption).to.exist
|
||||||
|
expect(frCaption.language.label).to.equal('French')
|
||||||
|
expect(frCaption.captionPath).to.equal(`/static/video-captions/${res.body.video.uuid}-fr.vtt`)
|
||||||
|
await testCaptionFile(servers[0].url, frCaption.captionPath, `WEBVTT
|
||||||
|
Kind: captions
|
||||||
|
Language: fr
|
||||||
|
|
||||||
|
00:00:01.600 --> 00:00:04.200
|
||||||
|
Français (FR)
|
||||||
|
|
||||||
|
00:00:05.900 --> 00:00:07.999
|
||||||
|
C'est un sous-titre français
|
||||||
|
|
||||||
|
00:00:10.000 --> 00:00:14.000
|
||||||
|
Ajouter un sous-titre est vraiment facile`)
|
||||||
}
|
}
|
||||||
|
|
||||||
{
|
{
|
||||||
|
|
|
@ -18,6 +18,7 @@ export * from './users/users'
|
||||||
export * from './users/accounts'
|
export * from './users/accounts'
|
||||||
export * from './videos/video-abuses'
|
export * from './videos/video-abuses'
|
||||||
export * from './videos/video-blacklist'
|
export * from './videos/video-blacklist'
|
||||||
|
export * from './videos/video-captions'
|
||||||
export * from './videos/video-channels'
|
export * from './videos/video-channels'
|
||||||
export * from './videos/video-comments'
|
export * from './videos/video-comments'
|
||||||
export * from './videos/video-streaming-playlists'
|
export * from './videos/video-streaming-playlists'
|
||||||
|
|
Loading…
Reference in New Issue
Block a user