Add refresh video on search
This commit is contained in:
parent
f6eebcb336
commit
1297eb5db6
|
@ -36,8 +36,9 @@ before_script:
|
||||||
matrix:
|
matrix:
|
||||||
include:
|
include:
|
||||||
- env: TEST_SUITE=misc
|
- env: TEST_SUITE=misc
|
||||||
- env: TEST_SUITE=api-fast
|
- env: TEST_SUITE=api-1
|
||||||
- env: TEST_SUITE=api-slow
|
- env: TEST_SUITE=api-2
|
||||||
|
- env: TEST_SUITE=api-3
|
||||||
- env: TEST_SUITE=cli
|
- env: TEST_SUITE=cli
|
||||||
- env: TEST_SUITE=lint
|
- env: TEST_SUITE=lint
|
||||||
|
|
||||||
|
|
|
@ -57,6 +57,11 @@ storage:
|
||||||
log:
|
log:
|
||||||
level: 'info' # debug/info/warning/error
|
level: 'info' # debug/info/warning/error
|
||||||
|
|
||||||
|
search:
|
||||||
|
remote_uri: # Add ability to fetch remote videos/actors by their URI, that may not be federated with your instance
|
||||||
|
users: true
|
||||||
|
anonymous: false
|
||||||
|
|
||||||
cache:
|
cache:
|
||||||
previews:
|
previews:
|
||||||
size: 500 # Max number of previews you want to cache
|
size: 500 # Max number of previews you want to cache
|
||||||
|
|
|
@ -58,6 +58,10 @@ storage:
|
||||||
log:
|
log:
|
||||||
level: 'info' # debug/info/warning/error
|
level: 'info' # debug/info/warning/error
|
||||||
|
|
||||||
|
search:
|
||||||
|
remote_uri: # Add ability to search remote videos/actors by URI, that may not be federated with your instance
|
||||||
|
users: true
|
||||||
|
anonymous: false
|
||||||
|
|
||||||
###############################################################################
|
###############################################################################
|
||||||
#
|
#
|
||||||
|
|
|
@ -12,19 +12,22 @@ killall -q peertube || true
|
||||||
if [ "$1" = "misc" ]; then
|
if [ "$1" = "misc" ]; then
|
||||||
npm run build
|
npm run build
|
||||||
mocha --timeout 5000 --exit --require ts-node/register/type-check --bail server/tests/client.ts server/tests/activitypub.ts \
|
mocha --timeout 5000 --exit --require ts-node/register/type-check --bail server/tests/client.ts server/tests/activitypub.ts \
|
||||||
server/tests/feeds/feeds.ts
|
server/tests/feeds/index.ts
|
||||||
elif [ "$1" = "api" ]; then
|
elif [ "$1" = "api" ]; then
|
||||||
npm run build:server
|
npm run build:server
|
||||||
mocha --timeout 5000 --exit --require ts-node/register/type-check --bail server/tests/api/index.ts
|
mocha --timeout 5000 --exit --require ts-node/register/type-check --bail server/tests/api/index.ts
|
||||||
elif [ "$1" = "cli" ]; then
|
elif [ "$1" = "cli" ]; then
|
||||||
npm run build:server
|
npm run build:server
|
||||||
mocha --timeout 5000 --exit --require ts-node/register/type-check --bail server/tests/cli/index.ts
|
mocha --timeout 5000 --exit --require ts-node/register/type-check --bail server/tests/cli/index.ts
|
||||||
elif [ "$1" = "api-fast" ]; then
|
elif [ "$1" = "api-1" ]; then
|
||||||
npm run build:server
|
npm run build:server
|
||||||
mocha --timeout 5000 --exit --require ts-node/register/type-check --bail server/tests/api/index-fast.ts
|
mocha --timeout 5000 --exit --require ts-node/register/type-check --bail server/tests/api/index-1.ts
|
||||||
elif [ "$1" = "api-slow" ]; then
|
elif [ "$1" = "api-2" ]; then
|
||||||
npm run build:server
|
npm run build:server
|
||||||
mocha --timeout 5000 --exit --require ts-node/register/type-check --bail server/tests/api/index-slow.ts
|
mocha --timeout 5000 --exit --require ts-node/register/type-check --bail server/tests/api/index-2.ts
|
||||||
|
elif [ "$1" = "api-3" ]; then
|
||||||
|
npm run build:server
|
||||||
|
mocha --timeout 5000 --exit --require ts-node/register/type-check --bail server/tests/api/index-3.ts
|
||||||
elif [ "$1" = "lint" ]; then
|
elif [ "$1" = "lint" ]; then
|
||||||
( cd client
|
( cd client
|
||||||
npm run lint
|
npm run lint
|
||||||
|
|
|
@ -13,8 +13,10 @@ import {
|
||||||
videosSearchSortValidator
|
videosSearchSortValidator
|
||||||
} from '../../middlewares'
|
} from '../../middlewares'
|
||||||
import { VideosSearchQuery } from '../../../shared/models/search'
|
import { VideosSearchQuery } from '../../../shared/models/search'
|
||||||
import { getOrCreateAccountAndVideoAndChannel } from '../../lib/activitypub'
|
import { getOrCreateVideoAndAccountAndChannel } from '../../lib/activitypub'
|
||||||
import { logger } from '../../helpers/logger'
|
import { logger } from '../../helpers/logger'
|
||||||
|
import { User } from '../../../shared/models/users'
|
||||||
|
import { CONFIG } from '../../initializers/constants'
|
||||||
|
|
||||||
const searchRouter = express.Router()
|
const searchRouter = express.Router()
|
||||||
|
|
||||||
|
@ -56,20 +58,30 @@ async function searchVideosDB (query: VideosSearchQuery, res: express.Response)
|
||||||
|
|
||||||
async function searchVideoUrl (url: string, res: express.Response) {
|
async function searchVideoUrl (url: string, res: express.Response) {
|
||||||
let video: VideoModel
|
let video: VideoModel
|
||||||
|
const user: User = res.locals.oauth ? res.locals.oauth.token.User : undefined
|
||||||
|
|
||||||
try {
|
// Check if we can fetch a remote video with the URL
|
||||||
const syncParam = {
|
if (
|
||||||
likes: false,
|
CONFIG.SEARCH.REMOTE_URI.ANONYMOUS === true ||
|
||||||
dislikes: false,
|
(CONFIG.SEARCH.REMOTE_URI.USERS === true && user !== undefined)
|
||||||
shares: false,
|
) {
|
||||||
comments: false,
|
try {
|
||||||
thumbnail: true
|
const syncParam = {
|
||||||
|
likes: false,
|
||||||
|
dislikes: false,
|
||||||
|
shares: false,
|
||||||
|
comments: false,
|
||||||
|
thumbnail: true,
|
||||||
|
refreshVideo: false
|
||||||
|
}
|
||||||
|
|
||||||
|
const res = await getOrCreateVideoAndAccountAndChannel(url, syncParam)
|
||||||
|
video = res ? res.video : undefined
|
||||||
|
} catch (err) {
|
||||||
|
logger.info('Cannot search remote video %s.', url)
|
||||||
}
|
}
|
||||||
|
} else {
|
||||||
const res = await getOrCreateAccountAndVideoAndChannel(url, syncParam)
|
video = await VideoModel.loadByUrlAndPopulateAccount(url)
|
||||||
video = res ? res.video : undefined
|
|
||||||
} catch (err) {
|
|
||||||
logger.info('Cannot search remote video %s.', url)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
return res.json({
|
return res.json({
|
||||||
|
|
|
@ -181,6 +181,12 @@ const CONFIG = {
|
||||||
LOG: {
|
LOG: {
|
||||||
LEVEL: config.get<string>('log.level')
|
LEVEL: config.get<string>('log.level')
|
||||||
},
|
},
|
||||||
|
SEARCH: {
|
||||||
|
REMOTE_URI: {
|
||||||
|
USERS: config.get<boolean>('search.remote_uri.users'),
|
||||||
|
ANONYMOUS: config.get<boolean>('search.remote_uri.anonymous')
|
||||||
|
}
|
||||||
|
},
|
||||||
ADMIN: {
|
ADMIN: {
|
||||||
get EMAIL () { return config.get<string>('admin.email') }
|
get EMAIL () { return config.get<string>('admin.email') }
|
||||||
},
|
},
|
||||||
|
@ -462,7 +468,8 @@ const ACTIVITY_PUB = {
|
||||||
MAGNET: [ 'application/x-bittorrent;x-scheme-handler/magnet' ]
|
MAGNET: [ 'application/x-bittorrent;x-scheme-handler/magnet' ]
|
||||||
},
|
},
|
||||||
MAX_RECURSION_COMMENTS: 100,
|
MAX_RECURSION_COMMENTS: 100,
|
||||||
ACTOR_REFRESH_INTERVAL: 3600 * 24 * 1000 // 1 day
|
ACTOR_REFRESH_INTERVAL: 3600 * 24 * 1000, // 1 day
|
||||||
|
VIDEO_REFRESH_INTERVAL: 3600 * 24 * 1000 // 1 day
|
||||||
}
|
}
|
||||||
|
|
||||||
const ACTIVITY_PUB_ACTOR_TYPES: { [ id: string ]: ActivityPubActorType } = {
|
const ACTIVITY_PUB_ACTOR_TYPES: { [ id: string ]: ActivityPubActorType } = {
|
||||||
|
@ -574,6 +581,7 @@ if (isTestInstance() === true) {
|
||||||
|
|
||||||
ACTIVITY_PUB.COLLECTION_ITEMS_PER_PAGE = 2
|
ACTIVITY_PUB.COLLECTION_ITEMS_PER_PAGE = 2
|
||||||
ACTIVITY_PUB.ACTOR_REFRESH_INTERVAL = 10 * 1000 // 10 seconds
|
ACTIVITY_PUB.ACTOR_REFRESH_INTERVAL = 10 * 1000 // 10 seconds
|
||||||
|
ACTIVITY_PUB.VIDEO_REFRESH_INTERVAL = 10 * 1000 // 10 seconds
|
||||||
|
|
||||||
CONSTRAINTS_FIELDS.ACTORS.AVATAR.FILE_SIZE.max = 100 * 1024 // 100KB
|
CONSTRAINTS_FIELDS.ACTORS.AVATAR.FILE_SIZE.max = 100 * 1024 // 100KB
|
||||||
|
|
||||||
|
|
|
@ -6,7 +6,7 @@ import { VideoModel } from '../../../models/video/video'
|
||||||
import { VideoShareModel } from '../../../models/video/video-share'
|
import { VideoShareModel } from '../../../models/video/video-share'
|
||||||
import { getOrCreateActorAndServerAndModel } from '../actor'
|
import { getOrCreateActorAndServerAndModel } from '../actor'
|
||||||
import { forwardVideoRelatedActivity } from '../send/utils'
|
import { forwardVideoRelatedActivity } from '../send/utils'
|
||||||
import { getOrCreateAccountAndVideoAndChannel } from '../videos'
|
import { getOrCreateVideoAndAccountAndChannel } from '../videos'
|
||||||
|
|
||||||
async function processAnnounceActivity (activity: ActivityAnnounce) {
|
async function processAnnounceActivity (activity: ActivityAnnounce) {
|
||||||
const actorAnnouncer = await getOrCreateActorAndServerAndModel(activity.actor)
|
const actorAnnouncer = await getOrCreateActorAndServerAndModel(activity.actor)
|
||||||
|
@ -25,7 +25,7 @@ export {
|
||||||
async function processVideoShare (actorAnnouncer: ActorModel, activity: ActivityAnnounce) {
|
async function processVideoShare (actorAnnouncer: ActorModel, activity: ActivityAnnounce) {
|
||||||
const objectUri = typeof activity.object === 'string' ? activity.object : activity.object.id
|
const objectUri = typeof activity.object === 'string' ? activity.object : activity.object.id
|
||||||
|
|
||||||
const { video } = await getOrCreateAccountAndVideoAndChannel(objectUri)
|
const { video } = await getOrCreateVideoAndAccountAndChannel(objectUri)
|
||||||
|
|
||||||
return sequelizeTypescript.transaction(async t => {
|
return sequelizeTypescript.transaction(async t => {
|
||||||
// Add share entry
|
// Add share entry
|
||||||
|
|
|
@ -10,7 +10,7 @@ import { VideoAbuseModel } from '../../../models/video/video-abuse'
|
||||||
import { VideoCommentModel } from '../../../models/video/video-comment'
|
import { VideoCommentModel } from '../../../models/video/video-comment'
|
||||||
import { getOrCreateActorAndServerAndModel } from '../actor'
|
import { getOrCreateActorAndServerAndModel } from '../actor'
|
||||||
import { resolveThread } from '../video-comments'
|
import { resolveThread } from '../video-comments'
|
||||||
import { getOrCreateAccountAndVideoAndChannel } from '../videos'
|
import { getOrCreateVideoAndAccountAndChannel } from '../videos'
|
||||||
import { forwardActivity, forwardVideoRelatedActivity } from '../send/utils'
|
import { forwardActivity, forwardVideoRelatedActivity } from '../send/utils'
|
||||||
|
|
||||||
async function processCreateActivity (activity: ActivityCreate) {
|
async function processCreateActivity (activity: ActivityCreate) {
|
||||||
|
@ -45,7 +45,7 @@ export {
|
||||||
async function processCreateVideo (activity: ActivityCreate) {
|
async function processCreateVideo (activity: ActivityCreate) {
|
||||||
const videoToCreateData = activity.object as VideoTorrentObject
|
const videoToCreateData = activity.object as VideoTorrentObject
|
||||||
|
|
||||||
const { video } = await getOrCreateAccountAndVideoAndChannel(videoToCreateData)
|
const { video } = await getOrCreateVideoAndAccountAndChannel(videoToCreateData)
|
||||||
|
|
||||||
return video
|
return video
|
||||||
}
|
}
|
||||||
|
@ -56,7 +56,7 @@ async function processCreateDislike (byActor: ActorModel, activity: ActivityCrea
|
||||||
|
|
||||||
if (!byAccount) throw new Error('Cannot create dislike with the non account actor ' + byActor.url)
|
if (!byAccount) throw new Error('Cannot create dislike with the non account actor ' + byActor.url)
|
||||||
|
|
||||||
const { video } = await getOrCreateAccountAndVideoAndChannel(dislike.object)
|
const { video } = await getOrCreateVideoAndAccountAndChannel(dislike.object)
|
||||||
|
|
||||||
return sequelizeTypescript.transaction(async t => {
|
return sequelizeTypescript.transaction(async t => {
|
||||||
const rate = {
|
const rate = {
|
||||||
|
@ -83,7 +83,7 @@ async function processCreateDislike (byActor: ActorModel, activity: ActivityCrea
|
||||||
async function processCreateView (byActor: ActorModel, activity: ActivityCreate) {
|
async function processCreateView (byActor: ActorModel, activity: ActivityCreate) {
|
||||||
const view = activity.object as ViewObject
|
const view = activity.object as ViewObject
|
||||||
|
|
||||||
const { video } = await getOrCreateAccountAndVideoAndChannel(view.object)
|
const { video } = await getOrCreateVideoAndAccountAndChannel(view.object)
|
||||||
|
|
||||||
const actor = await ActorModel.loadByUrl(view.actor)
|
const actor = await ActorModel.loadByUrl(view.actor)
|
||||||
if (!actor) throw new Error('Unknown actor ' + view.actor)
|
if (!actor) throw new Error('Unknown actor ' + view.actor)
|
||||||
|
@ -103,7 +103,7 @@ async function processCreateVideoAbuse (actor: ActorModel, videoAbuseToCreateDat
|
||||||
const account = actor.Account
|
const account = actor.Account
|
||||||
if (!account) throw new Error('Cannot create dislike with the non account actor ' + actor.url)
|
if (!account) throw new Error('Cannot create dislike with the non account actor ' + actor.url)
|
||||||
|
|
||||||
const { video } = await getOrCreateAccountAndVideoAndChannel(videoAbuseToCreateData.object)
|
const { video } = await getOrCreateVideoAndAccountAndChannel(videoAbuseToCreateData.object)
|
||||||
|
|
||||||
return sequelizeTypescript.transaction(async t => {
|
return sequelizeTypescript.transaction(async t => {
|
||||||
const videoAbuseData = {
|
const videoAbuseData = {
|
||||||
|
|
|
@ -5,7 +5,7 @@ import { AccountVideoRateModel } from '../../../models/account/account-video-rat
|
||||||
import { ActorModel } from '../../../models/activitypub/actor'
|
import { ActorModel } from '../../../models/activitypub/actor'
|
||||||
import { getOrCreateActorAndServerAndModel } from '../actor'
|
import { getOrCreateActorAndServerAndModel } from '../actor'
|
||||||
import { forwardVideoRelatedActivity } from '../send/utils'
|
import { forwardVideoRelatedActivity } from '../send/utils'
|
||||||
import { getOrCreateAccountAndVideoAndChannel } from '../videos'
|
import { getOrCreateVideoAndAccountAndChannel } from '../videos'
|
||||||
|
|
||||||
async function processLikeActivity (activity: ActivityLike) {
|
async function processLikeActivity (activity: ActivityLike) {
|
||||||
const actor = await getOrCreateActorAndServerAndModel(activity.actor)
|
const actor = await getOrCreateActorAndServerAndModel(activity.actor)
|
||||||
|
@ -27,7 +27,7 @@ async function processLikeVideo (byActor: ActorModel, activity: ActivityLike) {
|
||||||
const byAccount = byActor.Account
|
const byAccount = byActor.Account
|
||||||
if (!byAccount) throw new Error('Cannot create like with the non account actor ' + byActor.url)
|
if (!byAccount) throw new Error('Cannot create like with the non account actor ' + byActor.url)
|
||||||
|
|
||||||
const { video } = await getOrCreateAccountAndVideoAndChannel(videoUrl)
|
const { video } = await getOrCreateVideoAndAccountAndChannel(videoUrl)
|
||||||
|
|
||||||
return sequelizeTypescript.transaction(async t => {
|
return sequelizeTypescript.transaction(async t => {
|
||||||
const rate = {
|
const rate = {
|
||||||
|
|
|
@ -9,7 +9,7 @@ import { AccountVideoRateModel } from '../../../models/account/account-video-rat
|
||||||
import { ActorModel } from '../../../models/activitypub/actor'
|
import { ActorModel } from '../../../models/activitypub/actor'
|
||||||
import { ActorFollowModel } from '../../../models/activitypub/actor-follow'
|
import { ActorFollowModel } from '../../../models/activitypub/actor-follow'
|
||||||
import { forwardVideoRelatedActivity } from '../send/utils'
|
import { forwardVideoRelatedActivity } from '../send/utils'
|
||||||
import { getOrCreateAccountAndVideoAndChannel } from '../videos'
|
import { getOrCreateVideoAndAccountAndChannel } from '../videos'
|
||||||
import { VideoShareModel } from '../../../models/video/video-share'
|
import { VideoShareModel } from '../../../models/video/video-share'
|
||||||
|
|
||||||
async function processUndoActivity (activity: ActivityUndo) {
|
async function processUndoActivity (activity: ActivityUndo) {
|
||||||
|
@ -43,7 +43,7 @@ export {
|
||||||
async function processUndoLike (actorUrl: string, activity: ActivityUndo) {
|
async function processUndoLike (actorUrl: string, activity: ActivityUndo) {
|
||||||
const likeActivity = activity.object as ActivityLike
|
const likeActivity = activity.object as ActivityLike
|
||||||
|
|
||||||
const { video } = await getOrCreateAccountAndVideoAndChannel(likeActivity.object)
|
const { video } = await getOrCreateVideoAndAccountAndChannel(likeActivity.object)
|
||||||
|
|
||||||
return sequelizeTypescript.transaction(async t => {
|
return sequelizeTypescript.transaction(async t => {
|
||||||
const byAccount = await AccountModel.loadByUrl(actorUrl, t)
|
const byAccount = await AccountModel.loadByUrl(actorUrl, t)
|
||||||
|
@ -67,7 +67,7 @@ async function processUndoLike (actorUrl: string, activity: ActivityUndo) {
|
||||||
async function processUndoDislike (actorUrl: string, activity: ActivityUndo) {
|
async function processUndoDislike (actorUrl: string, activity: ActivityUndo) {
|
||||||
const dislike = activity.object.object as DislikeObject
|
const dislike = activity.object.object as DislikeObject
|
||||||
|
|
||||||
const { video } = await getOrCreateAccountAndVideoAndChannel(dislike.object)
|
const { video } = await getOrCreateVideoAndAccountAndChannel(dislike.object)
|
||||||
|
|
||||||
return sequelizeTypescript.transaction(async t => {
|
return sequelizeTypescript.transaction(async t => {
|
||||||
const byAccount = await AccountModel.loadByUrl(actorUrl, t)
|
const byAccount = await AccountModel.loadByUrl(actorUrl, t)
|
||||||
|
|
|
@ -1,4 +1,3 @@
|
||||||
import * as Bluebird from 'bluebird'
|
|
||||||
import { ActivityUpdate, VideoTorrentObject } from '../../../../shared/models/activitypub'
|
import { ActivityUpdate, VideoTorrentObject } from '../../../../shared/models/activitypub'
|
||||||
import { ActivityPubActor } from '../../../../shared/models/activitypub/activitypub-actor'
|
import { ActivityPubActor } from '../../../../shared/models/activitypub/activitypub-actor'
|
||||||
import { resetSequelizeInstance, retryTransactionWrapper } from '../../../helpers/database-utils'
|
import { resetSequelizeInstance, retryTransactionWrapper } from '../../../helpers/database-utils'
|
||||||
|
@ -6,19 +5,10 @@ import { logger } from '../../../helpers/logger'
|
||||||
import { sequelizeTypescript } from '../../../initializers'
|
import { sequelizeTypescript } from '../../../initializers'
|
||||||
import { AccountModel } from '../../../models/account/account'
|
import { AccountModel } from '../../../models/account/account'
|
||||||
import { ActorModel } from '../../../models/activitypub/actor'
|
import { ActorModel } from '../../../models/activitypub/actor'
|
||||||
import { TagModel } from '../../../models/video/tag'
|
|
||||||
import { VideoChannelModel } from '../../../models/video/video-channel'
|
import { VideoChannelModel } from '../../../models/video/video-channel'
|
||||||
import { VideoFileModel } from '../../../models/video/video-file'
|
|
||||||
import { fetchAvatarIfExists, getOrCreateActorAndServerAndModel, updateActorAvatarInstance, updateActorInstance } from '../actor'
|
import { fetchAvatarIfExists, getOrCreateActorAndServerAndModel, updateActorAvatarInstance, updateActorInstance } from '../actor'
|
||||||
import {
|
import { getOrCreateVideoAndAccountAndChannel, getOrCreateVideoChannel, updateVideoFromAP } from '../videos'
|
||||||
generateThumbnailFromUrl,
|
|
||||||
getOrCreateAccountAndVideoAndChannel,
|
|
||||||
getOrCreateVideoChannel,
|
|
||||||
videoActivityObjectToDBAttributes,
|
|
||||||
videoFileActivityUrlToDBAttributes
|
|
||||||
} from '../videos'
|
|
||||||
import { sanitizeAndCheckVideoTorrentObject } from '../../../helpers/custom-validators/activitypub/videos'
|
import { sanitizeAndCheckVideoTorrentObject } from '../../../helpers/custom-validators/activitypub/videos'
|
||||||
import { VideoCaptionModel } from '../../../models/video/video-caption'
|
|
||||||
|
|
||||||
async function processUpdateActivity (activity: ActivityUpdate) {
|
async function processUpdateActivity (activity: ActivityUpdate) {
|
||||||
const actor = await getOrCreateActorAndServerAndModel(activity.actor)
|
const actor = await getOrCreateActorAndServerAndModel(activity.actor)
|
||||||
|
@ -49,91 +39,10 @@ async function processUpdateVideo (actor: ActorModel, activity: ActivityUpdate)
|
||||||
return undefined
|
return undefined
|
||||||
}
|
}
|
||||||
|
|
||||||
const res = await getOrCreateAccountAndVideoAndChannel(videoObject.id)
|
const { video } = await getOrCreateVideoAndAccountAndChannel(videoObject.id)
|
||||||
|
const channelActor = await getOrCreateVideoChannel(videoObject)
|
||||||
|
|
||||||
// Fetch video channel outside the transaction
|
return updateVideoFromAP(video, videoObject, actor, channelActor, activity.to)
|
||||||
const newVideoChannelActor = await getOrCreateVideoChannel(videoObject)
|
|
||||||
const newVideoChannel = newVideoChannelActor.VideoChannel
|
|
||||||
|
|
||||||
logger.debug('Updating remote video "%s".', videoObject.uuid)
|
|
||||||
let videoInstance = res.video
|
|
||||||
let videoFieldsSave: any
|
|
||||||
|
|
||||||
try {
|
|
||||||
await sequelizeTypescript.transaction(async t => {
|
|
||||||
const sequelizeOptions = {
|
|
||||||
transaction: t
|
|
||||||
}
|
|
||||||
|
|
||||||
videoFieldsSave = videoInstance.toJSON()
|
|
||||||
|
|
||||||
// Check actor has the right to update the video
|
|
||||||
const videoChannel = videoInstance.VideoChannel
|
|
||||||
if (videoChannel.Account.Actor.id !== actor.id) {
|
|
||||||
throw new Error('Account ' + actor.url + ' does not own video channel ' + videoChannel.Actor.url)
|
|
||||||
}
|
|
||||||
|
|
||||||
const videoData = await videoActivityObjectToDBAttributes(newVideoChannel, videoObject, activity.to)
|
|
||||||
videoInstance.set('name', videoData.name)
|
|
||||||
videoInstance.set('uuid', videoData.uuid)
|
|
||||||
videoInstance.set('url', videoData.url)
|
|
||||||
videoInstance.set('category', videoData.category)
|
|
||||||
videoInstance.set('licence', videoData.licence)
|
|
||||||
videoInstance.set('language', videoData.language)
|
|
||||||
videoInstance.set('description', videoData.description)
|
|
||||||
videoInstance.set('support', videoData.support)
|
|
||||||
videoInstance.set('nsfw', videoData.nsfw)
|
|
||||||
videoInstance.set('commentsEnabled', videoData.commentsEnabled)
|
|
||||||
videoInstance.set('waitTranscoding', videoData.waitTranscoding)
|
|
||||||
videoInstance.set('state', videoData.state)
|
|
||||||
videoInstance.set('duration', videoData.duration)
|
|
||||||
videoInstance.set('createdAt', videoData.createdAt)
|
|
||||||
videoInstance.set('updatedAt', videoData.updatedAt)
|
|
||||||
videoInstance.set('views', videoData.views)
|
|
||||||
videoInstance.set('privacy', videoData.privacy)
|
|
||||||
videoInstance.set('channelId', videoData.channelId)
|
|
||||||
|
|
||||||
await videoInstance.save(sequelizeOptions)
|
|
||||||
|
|
||||||
// Don't block on request
|
|
||||||
generateThumbnailFromUrl(videoInstance, videoObject.icon)
|
|
||||||
.catch(err => logger.warn('Cannot generate thumbnail of %s.', videoObject.id, { err }))
|
|
||||||
|
|
||||||
// Remove old video files
|
|
||||||
const videoFileDestroyTasks: Bluebird<void>[] = []
|
|
||||||
for (const videoFile of videoInstance.VideoFiles) {
|
|
||||||
videoFileDestroyTasks.push(videoFile.destroy(sequelizeOptions))
|
|
||||||
}
|
|
||||||
await Promise.all(videoFileDestroyTasks)
|
|
||||||
|
|
||||||
const videoFileAttributes = videoFileActivityUrlToDBAttributes(videoInstance, videoObject)
|
|
||||||
const tasks = videoFileAttributes.map(f => VideoFileModel.create(f, sequelizeOptions))
|
|
||||||
await Promise.all(tasks)
|
|
||||||
|
|
||||||
// Update Tags
|
|
||||||
const tags = videoObject.tag.map(tag => tag.name)
|
|
||||||
const tagInstances = await TagModel.findOrCreateTags(tags, t)
|
|
||||||
await videoInstance.$set('Tags', tagInstances, sequelizeOptions)
|
|
||||||
|
|
||||||
// Update captions
|
|
||||||
await VideoCaptionModel.deleteAllCaptionsOfRemoteVideo(videoInstance.id, t)
|
|
||||||
|
|
||||||
const videoCaptionsPromises = videoObject.subtitleLanguage.map(c => {
|
|
||||||
return VideoCaptionModel.insertOrReplaceLanguage(videoInstance.id, c.identifier, t)
|
|
||||||
})
|
|
||||||
await Promise.all(videoCaptionsPromises)
|
|
||||||
})
|
|
||||||
|
|
||||||
logger.info('Remote video with uuid %s updated', videoObject.uuid)
|
|
||||||
} catch (err) {
|
|
||||||
if (videoInstance !== undefined && videoFieldsSave !== undefined) {
|
|
||||||
resetSequelizeInstance(videoInstance, videoFieldsSave)
|
|
||||||
}
|
|
||||||
|
|
||||||
// This is just a debug because we will retry the insert
|
|
||||||
logger.debug('Cannot update the remote video.', { err })
|
|
||||||
throw err
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
async function processUpdateActor (actor: ActorModel, activity: ActivityUpdate) {
|
async function processUpdateActor (actor: ActorModel, activity: ActivityUpdate) {
|
||||||
|
|
|
@ -6,6 +6,11 @@ import { VideoShareModel } from '../../models/video/video-share'
|
||||||
import { sendUndoAnnounce, sendVideoAnnounce } from './send'
|
import { sendUndoAnnounce, sendVideoAnnounce } from './send'
|
||||||
import { getAnnounceActivityPubUrl } from './url'
|
import { getAnnounceActivityPubUrl } from './url'
|
||||||
import { VideoChannelModel } from '../../models/video/video-channel'
|
import { VideoChannelModel } from '../../models/video/video-channel'
|
||||||
|
import * as Bluebird from 'bluebird'
|
||||||
|
import { doRequest } from '../../helpers/requests'
|
||||||
|
import { getOrCreateActorAndServerAndModel } from './actor'
|
||||||
|
import { logger } from '../../helpers/logger'
|
||||||
|
import { CRAWL_REQUEST_CONCURRENCY } from '../../initializers'
|
||||||
|
|
||||||
async function shareVideoByServerAndChannel (video: VideoModel, t: Transaction) {
|
async function shareVideoByServerAndChannel (video: VideoModel, t: Transaction) {
|
||||||
if (video.privacy === VideoPrivacy.PRIVATE) return undefined
|
if (video.privacy === VideoPrivacy.PRIVATE) return undefined
|
||||||
|
@ -22,8 +27,41 @@ async function changeVideoChannelShare (video: VideoModel, oldVideoChannel: Vide
|
||||||
await shareByVideoChannel(video, t)
|
await shareByVideoChannel(video, t)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
async function addVideoShares (shareUrls: string[], instance: VideoModel) {
|
||||||
|
await Bluebird.map(shareUrls, async shareUrl => {
|
||||||
|
try {
|
||||||
|
// Fetch url
|
||||||
|
const { body } = await doRequest({
|
||||||
|
uri: shareUrl,
|
||||||
|
json: true,
|
||||||
|
activityPub: true
|
||||||
|
})
|
||||||
|
if (!body || !body.actor) throw new Error('Body of body actor is invalid')
|
||||||
|
|
||||||
|
const actorUrl = body.actor
|
||||||
|
const actor = await getOrCreateActorAndServerAndModel(actorUrl)
|
||||||
|
|
||||||
|
const entry = {
|
||||||
|
actorId: actor.id,
|
||||||
|
videoId: instance.id,
|
||||||
|
url: shareUrl
|
||||||
|
}
|
||||||
|
|
||||||
|
await VideoShareModel.findOrCreate({
|
||||||
|
where: {
|
||||||
|
url: shareUrl
|
||||||
|
},
|
||||||
|
defaults: entry
|
||||||
|
})
|
||||||
|
} catch (err) {
|
||||||
|
logger.warn('Cannot add share %s.', shareUrl, { err })
|
||||||
|
}
|
||||||
|
}, { concurrency: CRAWL_REQUEST_CONCURRENCY })
|
||||||
|
}
|
||||||
|
|
||||||
export {
|
export {
|
||||||
changeVideoChannelShare,
|
changeVideoChannelShare,
|
||||||
|
addVideoShares,
|
||||||
shareVideoByServerAndChannel
|
shareVideoByServerAndChannel
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -7,7 +7,7 @@ import { ActorModel } from '../../models/activitypub/actor'
|
||||||
import { VideoModel } from '../../models/video/video'
|
import { VideoModel } from '../../models/video/video'
|
||||||
import { VideoCommentModel } from '../../models/video/video-comment'
|
import { VideoCommentModel } from '../../models/video/video-comment'
|
||||||
import { getOrCreateActorAndServerAndModel } from './actor'
|
import { getOrCreateActorAndServerAndModel } from './actor'
|
||||||
import { getOrCreateAccountAndVideoAndChannel } from './videos'
|
import { getOrCreateVideoAndAccountAndChannel } from './videos'
|
||||||
import * as Bluebird from 'bluebird'
|
import * as Bluebird from 'bluebird'
|
||||||
|
|
||||||
async function videoCommentActivityObjectToDBAttributes (video: VideoModel, actor: ActorModel, comment: VideoCommentObject) {
|
async function videoCommentActivityObjectToDBAttributes (video: VideoModel, actor: ActorModel, comment: VideoCommentObject) {
|
||||||
|
@ -91,7 +91,7 @@ async function resolveThread (url: string, comments: VideoCommentModel[] = []) {
|
||||||
|
|
||||||
try {
|
try {
|
||||||
// Maybe it's a reply to a video?
|
// Maybe it's a reply to a video?
|
||||||
const { video } = await getOrCreateAccountAndVideoAndChannel(url)
|
const { video } = await getOrCreateVideoAndAccountAndChannel(url)
|
||||||
|
|
||||||
if (comments.length !== 0) {
|
if (comments.length !== 0) {
|
||||||
const firstReply = comments[ comments.length - 1 ]
|
const firstReply = comments[ comments.length - 1 ]
|
||||||
|
|
|
@ -2,6 +2,45 @@ import { Transaction } from 'sequelize'
|
||||||
import { AccountModel } from '../../models/account/account'
|
import { AccountModel } from '../../models/account/account'
|
||||||
import { VideoModel } from '../../models/video/video'
|
import { VideoModel } from '../../models/video/video'
|
||||||
import { sendCreateDislike, sendLike, sendUndoDislike, sendUndoLike } from './send'
|
import { sendCreateDislike, sendLike, sendUndoDislike, sendUndoLike } from './send'
|
||||||
|
import { VideoRateType } from '../../../shared/models/videos'
|
||||||
|
import * as Bluebird from 'bluebird'
|
||||||
|
import { getOrCreateActorAndServerAndModel } from './actor'
|
||||||
|
import { AccountVideoRateModel } from '../../models/account/account-video-rate'
|
||||||
|
import { logger } from '../../helpers/logger'
|
||||||
|
import { CRAWL_REQUEST_CONCURRENCY } from '../../initializers'
|
||||||
|
|
||||||
|
async function createRates (actorUrls: string[], video: VideoModel, rate: VideoRateType) {
|
||||||
|
let rateCounts = 0
|
||||||
|
|
||||||
|
await Bluebird.map(actorUrls, async actorUrl => {
|
||||||
|
try {
|
||||||
|
const actor = await getOrCreateActorAndServerAndModel(actorUrl)
|
||||||
|
const [ , created ] = await AccountVideoRateModel
|
||||||
|
.findOrCreate({
|
||||||
|
where: {
|
||||||
|
videoId: video.id,
|
||||||
|
accountId: actor.Account.id
|
||||||
|
},
|
||||||
|
defaults: {
|
||||||
|
videoId: video.id,
|
||||||
|
accountId: actor.Account.id,
|
||||||
|
type: rate
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
if (created) rateCounts += 1
|
||||||
|
} catch (err) {
|
||||||
|
logger.warn('Cannot add rate %s for actor %s.', rate, actorUrl, { err })
|
||||||
|
}
|
||||||
|
}, { concurrency: CRAWL_REQUEST_CONCURRENCY })
|
||||||
|
|
||||||
|
logger.info('Adding %d %s to video %s.', rateCounts, rate, video.uuid)
|
||||||
|
|
||||||
|
// This is "likes" and "dislikes"
|
||||||
|
if (rateCounts !== 0) await video.increment(rate + 's', { by: rateCounts })
|
||||||
|
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
async function sendVideoRateChange (account: AccountModel,
|
async function sendVideoRateChange (account: AccountModel,
|
||||||
video: VideoModel,
|
video: VideoModel,
|
||||||
|
@ -24,5 +63,6 @@ async function sendVideoRateChange (account: AccountModel,
|
||||||
}
|
}
|
||||||
|
|
||||||
export {
|
export {
|
||||||
|
createRates,
|
||||||
sendVideoRateChange
|
sendVideoRateChange
|
||||||
}
|
}
|
||||||
|
|
|
@ -5,29 +5,30 @@ import { join } from 'path'
|
||||||
import * as request from 'request'
|
import * as request from 'request'
|
||||||
import { ActivityIconObject, VideoState } from '../../../shared/index'
|
import { ActivityIconObject, VideoState } from '../../../shared/index'
|
||||||
import { VideoTorrentObject } from '../../../shared/models/activitypub/objects'
|
import { VideoTorrentObject } from '../../../shared/models/activitypub/objects'
|
||||||
import { VideoPrivacy, VideoRateType } from '../../../shared/models/videos'
|
import { VideoPrivacy } from '../../../shared/models/videos'
|
||||||
import { sanitizeAndCheckVideoTorrentObject } from '../../helpers/custom-validators/activitypub/videos'
|
import { sanitizeAndCheckVideoTorrentObject } from '../../helpers/custom-validators/activitypub/videos'
|
||||||
import { isVideoFileInfoHashValid } from '../../helpers/custom-validators/videos'
|
import { isVideoFileInfoHashValid } from '../../helpers/custom-validators/videos'
|
||||||
import { retryTransactionWrapper } from '../../helpers/database-utils'
|
import { resetSequelizeInstance, retryTransactionWrapper, updateInstanceWithAnother } from '../../helpers/database-utils'
|
||||||
import { logger } from '../../helpers/logger'
|
import { logger } from '../../helpers/logger'
|
||||||
import { doRequest, doRequestAndSaveToFile } from '../../helpers/requests'
|
import { doRequest, doRequestAndSaveToFile } from '../../helpers/requests'
|
||||||
import { ACTIVITY_PUB, CONFIG, CRAWL_REQUEST_CONCURRENCY, REMOTE_SCHEME, sequelizeTypescript, VIDEO_MIMETYPE_EXT } from '../../initializers'
|
import { ACTIVITY_PUB, CONFIG, REMOTE_SCHEME, sequelizeTypescript, VIDEO_MIMETYPE_EXT } from '../../initializers'
|
||||||
import { AccountVideoRateModel } from '../../models/account/account-video-rate'
|
|
||||||
import { ActorModel } from '../../models/activitypub/actor'
|
import { ActorModel } from '../../models/activitypub/actor'
|
||||||
import { TagModel } from '../../models/video/tag'
|
import { TagModel } from '../../models/video/tag'
|
||||||
import { VideoModel } from '../../models/video/video'
|
import { VideoModel } from '../../models/video/video'
|
||||||
import { VideoChannelModel } from '../../models/video/video-channel'
|
import { VideoChannelModel } from '../../models/video/video-channel'
|
||||||
import { VideoFileModel } from '../../models/video/video-file'
|
import { VideoFileModel } from '../../models/video/video-file'
|
||||||
import { VideoShareModel } from '../../models/video/video-share'
|
import { getOrCreateActorAndServerAndModel, updateActorAvatarInstance } from './actor'
|
||||||
import { getOrCreateActorAndServerAndModel } from './actor'
|
|
||||||
import { addVideoComments } from './video-comments'
|
import { addVideoComments } from './video-comments'
|
||||||
import { crawlCollectionPage } from './crawl'
|
import { crawlCollectionPage } from './crawl'
|
||||||
import { sendCreateVideo, sendUpdateVideo } from './send'
|
import { sendCreateVideo, sendUpdateVideo } from './send'
|
||||||
import { shareVideoByServerAndChannel } from './index'
|
|
||||||
import { isArray } from '../../helpers/custom-validators/misc'
|
import { isArray } from '../../helpers/custom-validators/misc'
|
||||||
import { VideoCaptionModel } from '../../models/video/video-caption'
|
import { VideoCaptionModel } from '../../models/video/video-caption'
|
||||||
import { JobQueue } from '../job-queue'
|
import { JobQueue } from '../job-queue'
|
||||||
import { ActivitypubHttpFetcherPayload } from '../job-queue/handlers/activitypub-http-fetcher'
|
import { ActivitypubHttpFetcherPayload } from '../job-queue/handlers/activitypub-http-fetcher'
|
||||||
|
import { getUrlFromWebfinger } from '../../helpers/webfinger'
|
||||||
|
import { createRates } from './video-rates'
|
||||||
|
import { addVideoShares, shareVideoByServerAndChannel } from './share'
|
||||||
|
import { AccountModel } from '../../models/account/account'
|
||||||
|
|
||||||
async function federateVideoIfNeeded (video: VideoModel, isNewVideo: boolean, transaction?: sequelize.Transaction) {
|
async function federateVideoIfNeeded (video: VideoModel, isNewVideo: boolean, transaction?: sequelize.Transaction) {
|
||||||
// If the video is not private and published, we federate it
|
// If the video is not private and published, we federate it
|
||||||
|
@ -180,15 +181,11 @@ function getOrCreateVideoChannel (videoObject: VideoTorrentObject) {
|
||||||
return getOrCreateActorAndServerAndModel(channel.id)
|
return getOrCreateActorAndServerAndModel(channel.id)
|
||||||
}
|
}
|
||||||
|
|
||||||
async function getOrCreateVideo (videoObject: VideoTorrentObject, channelActor: ActorModel, waitThumbnail = false) {
|
async function createVideo (videoObject: VideoTorrentObject, channelActor: ActorModel, waitThumbnail = false) {
|
||||||
logger.debug('Adding remote video %s.', videoObject.id)
|
logger.debug('Adding remote video %s.', videoObject.id)
|
||||||
|
|
||||||
const videoCreated: VideoModel = await sequelizeTypescript.transaction(async t => {
|
const videoCreated: VideoModel = await sequelizeTypescript.transaction(async t => {
|
||||||
const sequelizeOptions = {
|
const sequelizeOptions = { transaction: t }
|
||||||
transaction: t
|
|
||||||
}
|
|
||||||
const videoFromDatabase = await VideoModel.loadByUUIDOrURLAndPopulateAccount(videoObject.uuid, videoObject.id, t)
|
|
||||||
if (videoFromDatabase) return videoFromDatabase
|
|
||||||
|
|
||||||
const videoData = await videoActivityObjectToDBAttributes(channelActor.VideoChannel, videoObject, videoObject.to)
|
const videoData = await videoActivityObjectToDBAttributes(channelActor.VideoChannel, videoObject, videoObject.to)
|
||||||
const video = VideoModel.build(videoData)
|
const video = VideoModel.build(videoData)
|
||||||
|
@ -230,26 +227,32 @@ async function getOrCreateVideo (videoObject: VideoTorrentObject, channelActor:
|
||||||
}
|
}
|
||||||
|
|
||||||
type SyncParam = {
|
type SyncParam = {
|
||||||
likes: boolean,
|
likes: boolean
|
||||||
dislikes: boolean,
|
dislikes: boolean
|
||||||
shares: boolean,
|
shares: boolean
|
||||||
comments: boolean,
|
comments: boolean
|
||||||
thumbnail: boolean
|
thumbnail: boolean
|
||||||
|
refreshVideo: boolean
|
||||||
}
|
}
|
||||||
async function getOrCreateAccountAndVideoAndChannel (
|
async function getOrCreateVideoAndAccountAndChannel (
|
||||||
videoObject: VideoTorrentObject | string,
|
videoObject: VideoTorrentObject | string,
|
||||||
syncParam: SyncParam = { likes: true, dislikes: true, shares: true, comments: true, thumbnail: true }
|
syncParam: SyncParam = { likes: true, dislikes: true, shares: true, comments: true, thumbnail: true, refreshVideo: false }
|
||||||
) {
|
) {
|
||||||
const videoUrl = typeof videoObject === 'string' ? videoObject : videoObject.id
|
const videoUrl = typeof videoObject === 'string' ? videoObject : videoObject.id
|
||||||
|
|
||||||
const videoFromDatabase = await VideoModel.loadByUrlAndPopulateAccount(videoUrl)
|
let videoFromDatabase = await VideoModel.loadByUrlAndPopulateAccount(videoUrl)
|
||||||
if (videoFromDatabase) return { video: videoFromDatabase }
|
if (videoFromDatabase) {
|
||||||
|
const p = retryTransactionWrapper(refreshVideoIfNeeded, videoFromDatabase)
|
||||||
|
if (syncParam.refreshVideo === true) videoFromDatabase = await p
|
||||||
|
|
||||||
const fetchedVideo = await fetchRemoteVideo(videoUrl)
|
return { video: videoFromDatabase }
|
||||||
|
}
|
||||||
|
|
||||||
|
const { videoObject: fetchedVideo } = await fetchRemoteVideo(videoUrl)
|
||||||
if (!fetchedVideo) throw new Error('Cannot fetch remote video with url: ' + videoUrl)
|
if (!fetchedVideo) throw new Error('Cannot fetch remote video with url: ' + videoUrl)
|
||||||
|
|
||||||
const channelActor = await getOrCreateVideoChannel(fetchedVideo)
|
const channelActor = await getOrCreateVideoChannel(fetchedVideo)
|
||||||
const video = await retryTransactionWrapper(getOrCreateVideo, fetchedVideo, channelActor, syncParam.thumbnail)
|
const video = await retryTransactionWrapper(createVideo, fetchedVideo, channelActor, syncParam.thumbnail)
|
||||||
|
|
||||||
// Process outside the transaction because we could fetch remote data
|
// Process outside the transaction because we could fetch remote data
|
||||||
|
|
||||||
|
@ -290,72 +293,7 @@ async function getOrCreateAccountAndVideoAndChannel (
|
||||||
return { video }
|
return { video }
|
||||||
}
|
}
|
||||||
|
|
||||||
async function createRates (actorUrls: string[], video: VideoModel, rate: VideoRateType) {
|
async function fetchRemoteVideo (videoUrl: string): Promise<{ response: request.RequestResponse, videoObject: VideoTorrentObject }> {
|
||||||
let rateCounts = 0
|
|
||||||
|
|
||||||
await Bluebird.map(actorUrls, async actorUrl => {
|
|
||||||
try {
|
|
||||||
const actor = await getOrCreateActorAndServerAndModel(actorUrl)
|
|
||||||
const [ , created ] = await AccountVideoRateModel
|
|
||||||
.findOrCreate({
|
|
||||||
where: {
|
|
||||||
videoId: video.id,
|
|
||||||
accountId: actor.Account.id
|
|
||||||
},
|
|
||||||
defaults: {
|
|
||||||
videoId: video.id,
|
|
||||||
accountId: actor.Account.id,
|
|
||||||
type: rate
|
|
||||||
}
|
|
||||||
})
|
|
||||||
|
|
||||||
if (created) rateCounts += 1
|
|
||||||
} catch (err) {
|
|
||||||
logger.warn('Cannot add rate %s for actor %s.', rate, actorUrl, { err })
|
|
||||||
}
|
|
||||||
}, { concurrency: CRAWL_REQUEST_CONCURRENCY })
|
|
||||||
|
|
||||||
logger.info('Adding %d %s to video %s.', rateCounts, rate, video.uuid)
|
|
||||||
|
|
||||||
// This is "likes" and "dislikes"
|
|
||||||
if (rateCounts !== 0) await video.increment(rate + 's', { by: rateCounts })
|
|
||||||
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
async function addVideoShares (shareUrls: string[], instance: VideoModel) {
|
|
||||||
await Bluebird.map(shareUrls, async shareUrl => {
|
|
||||||
try {
|
|
||||||
// Fetch url
|
|
||||||
const { body } = await doRequest({
|
|
||||||
uri: shareUrl,
|
|
||||||
json: true,
|
|
||||||
activityPub: true
|
|
||||||
})
|
|
||||||
if (!body || !body.actor) throw new Error('Body of body actor is invalid')
|
|
||||||
|
|
||||||
const actorUrl = body.actor
|
|
||||||
const actor = await getOrCreateActorAndServerAndModel(actorUrl)
|
|
||||||
|
|
||||||
const entry = {
|
|
||||||
actorId: actor.id,
|
|
||||||
videoId: instance.id,
|
|
||||||
url: shareUrl
|
|
||||||
}
|
|
||||||
|
|
||||||
await VideoShareModel.findOrCreate({
|
|
||||||
where: {
|
|
||||||
url: shareUrl
|
|
||||||
},
|
|
||||||
defaults: entry
|
|
||||||
})
|
|
||||||
} catch (err) {
|
|
||||||
logger.warn('Cannot add share %s.', shareUrl, { err })
|
|
||||||
}
|
|
||||||
}, { concurrency: CRAWL_REQUEST_CONCURRENCY })
|
|
||||||
}
|
|
||||||
|
|
||||||
async function fetchRemoteVideo (videoUrl: string): Promise<VideoTorrentObject> {
|
|
||||||
const options = {
|
const options = {
|
||||||
uri: videoUrl,
|
uri: videoUrl,
|
||||||
method: 'GET',
|
method: 'GET',
|
||||||
|
@ -365,26 +303,143 @@ async function fetchRemoteVideo (videoUrl: string): Promise<VideoTorrentObject>
|
||||||
|
|
||||||
logger.info('Fetching remote video %s.', videoUrl)
|
logger.info('Fetching remote video %s.', videoUrl)
|
||||||
|
|
||||||
const { body } = await doRequest(options)
|
const { response, body } = await doRequest(options)
|
||||||
|
|
||||||
if (sanitizeAndCheckVideoTorrentObject(body) === false) {
|
if (sanitizeAndCheckVideoTorrentObject(body) === false) {
|
||||||
logger.debug('Remote video JSON is not valid.', { body })
|
logger.debug('Remote video JSON is not valid.', { body })
|
||||||
return undefined
|
return { response, videoObject: undefined }
|
||||||
}
|
}
|
||||||
|
|
||||||
return body
|
return { response, videoObject: body }
|
||||||
|
}
|
||||||
|
|
||||||
|
async function refreshVideoIfNeeded (video: VideoModel): Promise<VideoModel> {
|
||||||
|
if (!video.isOutdated()) return video
|
||||||
|
|
||||||
|
try {
|
||||||
|
const { response, videoObject } = await fetchRemoteVideo(video.url)
|
||||||
|
if (response.statusCode === 404) {
|
||||||
|
// Video does not exist anymore
|
||||||
|
await video.destroy()
|
||||||
|
return undefined
|
||||||
|
}
|
||||||
|
|
||||||
|
if (videoObject === undefined) {
|
||||||
|
logger.warn('Cannot refresh remote video: invalid body.')
|
||||||
|
return video
|
||||||
|
}
|
||||||
|
|
||||||
|
const channelActor = await getOrCreateVideoChannel(videoObject)
|
||||||
|
const account = await AccountModel.load(channelActor.VideoChannel.accountId)
|
||||||
|
return updateVideoFromAP(video, videoObject, account.Actor, channelActor)
|
||||||
|
|
||||||
|
} catch (err) {
|
||||||
|
logger.warn('Cannot refresh video.', { err })
|
||||||
|
return video
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async function updateVideoFromAP (
|
||||||
|
video: VideoModel,
|
||||||
|
videoObject: VideoTorrentObject,
|
||||||
|
accountActor: ActorModel,
|
||||||
|
channelActor: ActorModel,
|
||||||
|
overrideTo?: string[]
|
||||||
|
) {
|
||||||
|
logger.debug('Updating remote video "%s".', videoObject.uuid)
|
||||||
|
let videoFieldsSave: any
|
||||||
|
|
||||||
|
try {
|
||||||
|
const updatedVideo: VideoModel = await sequelizeTypescript.transaction(async t => {
|
||||||
|
const sequelizeOptions = {
|
||||||
|
transaction: t
|
||||||
|
}
|
||||||
|
|
||||||
|
videoFieldsSave = video.toJSON()
|
||||||
|
|
||||||
|
// Check actor has the right to update the video
|
||||||
|
const videoChannel = video.VideoChannel
|
||||||
|
if (videoChannel.Account.Actor.id !== accountActor.id) {
|
||||||
|
throw new Error('Account ' + accountActor.url + ' does not own video channel ' + videoChannel.Actor.url)
|
||||||
|
}
|
||||||
|
|
||||||
|
const to = overrideTo ? overrideTo : videoObject.to
|
||||||
|
const videoData = await videoActivityObjectToDBAttributes(channelActor.VideoChannel, videoObject, to)
|
||||||
|
video.set('name', videoData.name)
|
||||||
|
video.set('uuid', videoData.uuid)
|
||||||
|
video.set('url', videoData.url)
|
||||||
|
video.set('category', videoData.category)
|
||||||
|
video.set('licence', videoData.licence)
|
||||||
|
video.set('language', videoData.language)
|
||||||
|
video.set('description', videoData.description)
|
||||||
|
video.set('support', videoData.support)
|
||||||
|
video.set('nsfw', videoData.nsfw)
|
||||||
|
video.set('commentsEnabled', videoData.commentsEnabled)
|
||||||
|
video.set('waitTranscoding', videoData.waitTranscoding)
|
||||||
|
video.set('state', videoData.state)
|
||||||
|
video.set('duration', videoData.duration)
|
||||||
|
video.set('createdAt', videoData.createdAt)
|
||||||
|
video.set('publishedAt', videoData.publishedAt)
|
||||||
|
video.set('views', videoData.views)
|
||||||
|
video.set('privacy', videoData.privacy)
|
||||||
|
video.set('channelId', videoData.channelId)
|
||||||
|
|
||||||
|
await video.save(sequelizeOptions)
|
||||||
|
|
||||||
|
// Don't block on request
|
||||||
|
generateThumbnailFromUrl(video, videoObject.icon)
|
||||||
|
.catch(err => logger.warn('Cannot generate thumbnail of %s.', videoObject.id, { err }))
|
||||||
|
|
||||||
|
// Remove old video files
|
||||||
|
const videoFileDestroyTasks: Bluebird<void>[] = []
|
||||||
|
for (const videoFile of video.VideoFiles) {
|
||||||
|
videoFileDestroyTasks.push(videoFile.destroy(sequelizeOptions))
|
||||||
|
}
|
||||||
|
await Promise.all(videoFileDestroyTasks)
|
||||||
|
|
||||||
|
const videoFileAttributes = videoFileActivityUrlToDBAttributes(video, videoObject)
|
||||||
|
const tasks = videoFileAttributes.map(f => VideoFileModel.create(f, sequelizeOptions))
|
||||||
|
await Promise.all(tasks)
|
||||||
|
|
||||||
|
// Update Tags
|
||||||
|
const tags = videoObject.tag.map(tag => tag.name)
|
||||||
|
const tagInstances = await TagModel.findOrCreateTags(tags, t)
|
||||||
|
await video.$set('Tags', tagInstances, sequelizeOptions)
|
||||||
|
|
||||||
|
// Update captions
|
||||||
|
await VideoCaptionModel.deleteAllCaptionsOfRemoteVideo(video.id, t)
|
||||||
|
|
||||||
|
const videoCaptionsPromises = videoObject.subtitleLanguage.map(c => {
|
||||||
|
return VideoCaptionModel.insertOrReplaceLanguage(video.id, c.identifier, t)
|
||||||
|
})
|
||||||
|
await Promise.all(videoCaptionsPromises)
|
||||||
|
})
|
||||||
|
|
||||||
|
logger.info('Remote video with uuid %s updated', videoObject.uuid)
|
||||||
|
|
||||||
|
return updatedVideo
|
||||||
|
} catch (err) {
|
||||||
|
if (video !== undefined && videoFieldsSave !== undefined) {
|
||||||
|
resetSequelizeInstance(video, videoFieldsSave)
|
||||||
|
}
|
||||||
|
|
||||||
|
// This is just a debug because we will retry the insert
|
||||||
|
logger.debug('Cannot update the remote video.', { err })
|
||||||
|
throw err
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
export {
|
export {
|
||||||
|
updateVideoFromAP,
|
||||||
federateVideoIfNeeded,
|
federateVideoIfNeeded,
|
||||||
fetchRemoteVideo,
|
fetchRemoteVideo,
|
||||||
getOrCreateAccountAndVideoAndChannel,
|
getOrCreateVideoAndAccountAndChannel,
|
||||||
fetchRemoteVideoStaticFile,
|
fetchRemoteVideoStaticFile,
|
||||||
fetchRemoteVideoDescription,
|
fetchRemoteVideoDescription,
|
||||||
generateThumbnailFromUrl,
|
generateThumbnailFromUrl,
|
||||||
videoActivityObjectToDBAttributes,
|
videoActivityObjectToDBAttributes,
|
||||||
videoFileActivityUrlToDBAttributes,
|
videoFileActivityUrlToDBAttributes,
|
||||||
getOrCreateVideo,
|
createVideo,
|
||||||
getOrCreateVideoChannel,
|
getOrCreateVideoChannel,
|
||||||
addVideoShares,
|
addVideoShares,
|
||||||
createRates
|
createRates
|
||||||
|
|
|
@ -56,6 +56,7 @@ import { generateImageFromVideoFile, getVideoFileFPS, getVideoFileResolution, tr
|
||||||
import { logger } from '../../helpers/logger'
|
import { logger } from '../../helpers/logger'
|
||||||
import { getServerActor } from '../../helpers/utils'
|
import { getServerActor } from '../../helpers/utils'
|
||||||
import {
|
import {
|
||||||
|
ACTIVITY_PUB,
|
||||||
API_VERSION,
|
API_VERSION,
|
||||||
CONFIG,
|
CONFIG,
|
||||||
CONSTRAINTS_FIELDS,
|
CONSTRAINTS_FIELDS,
|
||||||
|
@ -1004,21 +1005,6 @@ export class VideoModel extends Model<VideoModel> {
|
||||||
return VideoModel.scope([ ScopeNames.WITH_ACCOUNT_DETAILS, ScopeNames.WITH_FILES ]).findOne(query)
|
return VideoModel.scope([ ScopeNames.WITH_ACCOUNT_DETAILS, ScopeNames.WITH_FILES ]).findOne(query)
|
||||||
}
|
}
|
||||||
|
|
||||||
static loadByUUIDOrURLAndPopulateAccount (uuid: string, url: string, t?: Sequelize.Transaction) {
|
|
||||||
const query: IFindOptions<VideoModel> = {
|
|
||||||
where: {
|
|
||||||
[Sequelize.Op.or]: [
|
|
||||||
{ uuid },
|
|
||||||
{ url }
|
|
||||||
]
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
if (t !== undefined) query.transaction = t
|
|
||||||
|
|
||||||
return VideoModel.scope([ ScopeNames.WITH_ACCOUNT_DETAILS, ScopeNames.WITH_FILES ]).findOne(query)
|
|
||||||
}
|
|
||||||
|
|
||||||
static loadAndPopulateAccountAndServerAndTags (id: number) {
|
static loadAndPopulateAccountAndServerAndTags (id: number) {
|
||||||
const options = {
|
const options = {
|
||||||
order: [ [ 'Tags', 'name', 'ASC' ] ]
|
order: [ [ 'Tags', 'name', 'ASC' ] ]
|
||||||
|
@ -1646,6 +1632,17 @@ export class VideoModel extends Model<VideoModel> {
|
||||||
return 'PT' + this.duration + 'S'
|
return 'PT' + this.duration + 'S'
|
||||||
}
|
}
|
||||||
|
|
||||||
|
isOutdated () {
|
||||||
|
if (this.isOwned()) return false
|
||||||
|
|
||||||
|
const now = Date.now()
|
||||||
|
const createdAtTime = this.createdAt.getTime()
|
||||||
|
const updatedAtTime = this.updatedAt.getTime()
|
||||||
|
|
||||||
|
return (now - createdAtTime) > ACTIVITY_PUB.VIDEO_REFRESH_INTERVAL &&
|
||||||
|
(now - updatedAtTime) > ACTIVITY_PUB.VIDEO_REFRESH_INTERVAL
|
||||||
|
}
|
||||||
|
|
||||||
private getBaseUrls () {
|
private getBaseUrls () {
|
||||||
let baseUrlHttp
|
let baseUrlHttp
|
||||||
let baseUrlWs
|
let baseUrlWs
|
||||||
|
|
2
server/tests/api/index-1.ts
Normal file
2
server/tests/api/index-1.ts
Normal file
|
@ -0,0 +1,2 @@
|
||||||
|
import './check-params'
|
||||||
|
import './search'
|
2
server/tests/api/index-2.ts
Normal file
2
server/tests/api/index-2.ts
Normal file
|
@ -0,0 +1,2 @@
|
||||||
|
import './server'
|
||||||
|
import './users'
|
1
server/tests/api/index-3.ts
Normal file
1
server/tests/api/index-3.ts
Normal file
|
@ -0,0 +1 @@
|
||||||
|
import './videos'
|
|
@ -1,18 +0,0 @@
|
||||||
// Order of the tests we want to execute
|
|
||||||
import './server/stats'
|
|
||||||
import './check-params'
|
|
||||||
import './users/users'
|
|
||||||
import './videos/single-server'
|
|
||||||
import './videos/video-abuse'
|
|
||||||
import './videos/video-captions'
|
|
||||||
import './videos/video-blacklist'
|
|
||||||
import './videos/video-blacklist-management'
|
|
||||||
import './videos/video-description'
|
|
||||||
import './videos/video-nsfw'
|
|
||||||
import './videos/video-privacy'
|
|
||||||
import './videos/services'
|
|
||||||
import './server/email'
|
|
||||||
import './server/config'
|
|
||||||
import './server/reverse-proxy'
|
|
||||||
import './search/search-videos'
|
|
||||||
import './server/tracker'
|
|
|
@ -1,12 +0,0 @@
|
||||||
// Order of the tests we want to execute
|
|
||||||
import './videos/video-channels'
|
|
||||||
import './videos/video-transcoder'
|
|
||||||
import './videos/multiple-servers'
|
|
||||||
import './server/follows'
|
|
||||||
import './server/jobs'
|
|
||||||
import './videos/video-comments'
|
|
||||||
import './users/users-multiple-servers'
|
|
||||||
import './users/user-subscriptions'
|
|
||||||
import './server/handle-down'
|
|
||||||
import './videos/video-schedule-update'
|
|
||||||
import './videos/video-imports'
|
|
|
@ -1,3 +1,4 @@
|
||||||
// Order of the tests we want to execute
|
// Order of the tests we want to execute
|
||||||
import './index-fast'
|
import './index-1'
|
||||||
import './index-slow'
|
import './index-2'
|
||||||
|
import './index-3'
|
||||||
|
|
2
server/tests/api/search/index.ts
Normal file
2
server/tests/api/search/index.ts
Normal file
|
@ -0,0 +1,2 @@
|
||||||
|
import './search-activitypub-videos'
|
||||||
|
import './search-videos'
|
161
server/tests/api/search/search-activitypub-videos.ts
Normal file
161
server/tests/api/search/search-activitypub-videos.ts
Normal file
|
@ -0,0 +1,161 @@
|
||||||
|
/* tslint:disable:no-unused-expression */
|
||||||
|
|
||||||
|
import * as chai from 'chai'
|
||||||
|
import 'mocha'
|
||||||
|
import {
|
||||||
|
addVideoChannel,
|
||||||
|
flushAndRunMultipleServers,
|
||||||
|
flushTests,
|
||||||
|
getVideosList,
|
||||||
|
killallServers,
|
||||||
|
removeVideo,
|
||||||
|
searchVideoWithToken,
|
||||||
|
ServerInfo,
|
||||||
|
setAccessTokensToServers,
|
||||||
|
updateVideo,
|
||||||
|
uploadVideo,
|
||||||
|
wait,
|
||||||
|
searchVideo
|
||||||
|
} from '../../utils'
|
||||||
|
import { waitJobs } from '../../utils/server/jobs'
|
||||||
|
import { Video, VideoPrivacy } from '../../../../shared/models/videos'
|
||||||
|
|
||||||
|
const expect = chai.expect
|
||||||
|
|
||||||
|
describe('Test a ActivityPub videos search', function () {
|
||||||
|
let servers: ServerInfo[]
|
||||||
|
let videoServer1UUID: string
|
||||||
|
let videoServer2UUID: string
|
||||||
|
|
||||||
|
before(async function () {
|
||||||
|
this.timeout(120000)
|
||||||
|
|
||||||
|
await flushTests()
|
||||||
|
|
||||||
|
servers = await flushAndRunMultipleServers(2)
|
||||||
|
|
||||||
|
await setAccessTokensToServers(servers)
|
||||||
|
|
||||||
|
{
|
||||||
|
const res = await uploadVideo(servers[ 0 ].url, servers[ 0 ].accessToken, { name: 'video 1 on server 1' })
|
||||||
|
videoServer1UUID = res.body.video.uuid
|
||||||
|
}
|
||||||
|
|
||||||
|
{
|
||||||
|
const res = await uploadVideo(servers[ 1 ].url, servers[ 1 ].accessToken, { name: 'video 1 on server 2' })
|
||||||
|
videoServer2UUID = res.body.video.uuid
|
||||||
|
}
|
||||||
|
|
||||||
|
await waitJobs(servers)
|
||||||
|
})
|
||||||
|
|
||||||
|
it('Should not find a remote video', async function () {
|
||||||
|
{
|
||||||
|
const res = await searchVideoWithToken(servers[ 0 ].url, 'http://localhost:9002/videos/watch/43', servers[ 0 ].accessToken)
|
||||||
|
|
||||||
|
expect(res.body.total).to.equal(0)
|
||||||
|
expect(res.body.data).to.be.an('array')
|
||||||
|
expect(res.body.data).to.have.lengthOf(0)
|
||||||
|
}
|
||||||
|
|
||||||
|
{
|
||||||
|
const res = await searchVideo(servers[0].url, 'http://localhost:9002/videos/watch/' + videoServer2UUID)
|
||||||
|
|
||||||
|
expect(res.body.total).to.equal(0)
|
||||||
|
expect(res.body.data).to.be.an('array')
|
||||||
|
expect(res.body.data).to.have.lengthOf(0)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
it('Should search a local video', async function () {
|
||||||
|
const res = await searchVideo(servers[0].url, 'http://localhost:9001/videos/watch/' + videoServer1UUID)
|
||||||
|
|
||||||
|
expect(res.body.total).to.equal(1)
|
||||||
|
expect(res.body.data).to.be.an('array')
|
||||||
|
expect(res.body.data).to.have.lengthOf(1)
|
||||||
|
expect(res.body.data[0].name).to.equal('video 1 on server 1')
|
||||||
|
})
|
||||||
|
|
||||||
|
it('Should search a remote video', async function () {
|
||||||
|
const res = await searchVideoWithToken(servers[0].url, 'http://localhost:9002/videos/watch/' + videoServer2UUID, servers[0].accessToken)
|
||||||
|
|
||||||
|
expect(res.body.total).to.equal(1)
|
||||||
|
expect(res.body.data).to.be.an('array')
|
||||||
|
expect(res.body.data).to.have.lengthOf(1)
|
||||||
|
expect(res.body.data[0].name).to.equal('video 1 on server 2')
|
||||||
|
})
|
||||||
|
|
||||||
|
it('Should not list this remote video', async function () {
|
||||||
|
const res = await getVideosList(servers[0].url)
|
||||||
|
expect(res.body.total).to.equal(1)
|
||||||
|
expect(res.body.data).to.have.lengthOf(1)
|
||||||
|
expect(res.body.data[0].name).to.equal('video 1 on server 1')
|
||||||
|
})
|
||||||
|
|
||||||
|
it('Should update video of server 2, and refresh it on server 1', async function () {
|
||||||
|
this.timeout(60000)
|
||||||
|
|
||||||
|
const channelAttributes = {
|
||||||
|
name: 'super_channel',
|
||||||
|
displayName: 'super channel'
|
||||||
|
}
|
||||||
|
const resChannel = await addVideoChannel(servers[1].url, servers[1].accessToken, channelAttributes)
|
||||||
|
const videoChannelId = resChannel.body.videoChannel.id
|
||||||
|
|
||||||
|
const attributes = {
|
||||||
|
name: 'updated',
|
||||||
|
tag: [ 'tag1', 'tag2' ],
|
||||||
|
privacy: VideoPrivacy.UNLISTED,
|
||||||
|
channelId: videoChannelId
|
||||||
|
}
|
||||||
|
await updateVideo(servers[1].url, servers[1].accessToken, videoServer2UUID, attributes)
|
||||||
|
|
||||||
|
await waitJobs(servers)
|
||||||
|
// Expire video
|
||||||
|
await wait(10000)
|
||||||
|
|
||||||
|
// Will run refresh async
|
||||||
|
await searchVideoWithToken(servers[0].url, 'http://localhost:9002/videos/watch/' + videoServer2UUID, servers[0].accessToken)
|
||||||
|
|
||||||
|
// Wait refresh
|
||||||
|
await wait(5000)
|
||||||
|
|
||||||
|
const res = await searchVideoWithToken(servers[0].url, 'http://localhost:9002/videos/watch/' + videoServer2UUID, servers[0].accessToken)
|
||||||
|
expect(res.body.total).to.equal(1)
|
||||||
|
expect(res.body.data).to.have.lengthOf(1)
|
||||||
|
|
||||||
|
const video: Video = res.body.data[0]
|
||||||
|
expect(video.name).to.equal('updated')
|
||||||
|
expect(video.channel.name).to.equal('super_channel')
|
||||||
|
expect(video.privacy.id).to.equal(VideoPrivacy.UNLISTED)
|
||||||
|
})
|
||||||
|
|
||||||
|
it('Should delete video of server 2, and delete it on server 1', async function () {
|
||||||
|
this.timeout(60000)
|
||||||
|
|
||||||
|
await removeVideo(servers[1].url, servers[1].accessToken, videoServer2UUID)
|
||||||
|
|
||||||
|
await waitJobs(servers)
|
||||||
|
// Expire video
|
||||||
|
await wait(10000)
|
||||||
|
|
||||||
|
// Will run refresh async
|
||||||
|
await searchVideoWithToken(servers[0].url, 'http://localhost:9002/videos/watch/' + videoServer2UUID, servers[0].accessToken)
|
||||||
|
|
||||||
|
// Wait refresh
|
||||||
|
await wait(5000)
|
||||||
|
|
||||||
|
const res = await searchVideoWithToken(servers[0].url, 'http://localhost:9002/videos/watch/' + videoServer2UUID, servers[0].accessToken)
|
||||||
|
expect(res.body.total).to.equal(0)
|
||||||
|
expect(res.body.data).to.have.lengthOf(0)
|
||||||
|
})
|
||||||
|
|
||||||
|
after(async function () {
|
||||||
|
killallServers(servers)
|
||||||
|
|
||||||
|
// Keep the logs if the test failed
|
||||||
|
if (this['ok']) {
|
||||||
|
await flushTests()
|
||||||
|
}
|
||||||
|
})
|
||||||
|
})
|
8
server/tests/api/server/index.ts
Normal file
8
server/tests/api/server/index.ts
Normal file
|
@ -0,0 +1,8 @@
|
||||||
|
import './config'
|
||||||
|
import './email'
|
||||||
|
import './follows'
|
||||||
|
import './handle-down'
|
||||||
|
import './jobs'
|
||||||
|
import './reverse-proxy'
|
||||||
|
import './stats'
|
||||||
|
import './tracker'
|
3
server/tests/api/users/index.ts
Normal file
3
server/tests/api/users/index.ts
Normal file
|
@ -0,0 +1,3 @@
|
||||||
|
import './user-subscriptions'
|
||||||
|
import './users'
|
||||||
|
import './users-multiple-servers'
|
15
server/tests/api/videos/index.ts
Normal file
15
server/tests/api/videos/index.ts
Normal file
|
@ -0,0 +1,15 @@
|
||||||
|
import './multiple-servers'
|
||||||
|
import './services'
|
||||||
|
import './single-server'
|
||||||
|
import './video-abuse'
|
||||||
|
import './video-blacklist'
|
||||||
|
import './video-blacklist-management'
|
||||||
|
import './video-captions'
|
||||||
|
import './video-channels'
|
||||||
|
import './video-comme'
|
||||||
|
import './video-description'
|
||||||
|
import './video-impo'
|
||||||
|
import './video-nsfw'
|
||||||
|
import './video-privacy'
|
||||||
|
import './video-schedule-update'
|
||||||
|
import './video-transcoder'
|
1
server/tests/feeds/index.ts
Normal file
1
server/tests/feeds/index.ts
Normal file
|
@ -0,0 +1 @@
|
||||||
|
import './feeds'
|
|
@ -1,5 +1,6 @@
|
||||||
// Order of the tests we want to execute
|
// Order of the tests we want to execute
|
||||||
import './client'
|
import './client'
|
||||||
import './activitypub'
|
import './activitypub'
|
||||||
import './api/'
|
import './feeds/'
|
||||||
import './cli/'
|
import './cli/'
|
||||||
|
import './api/'
|
||||||
|
|
Loading…
Reference in New Issue
Block a user