Add refresh video on search

.travis.yml

@@ -36,8 +36,9 @@ before_script:
 matrix:
   include:
   - env: TEST_SUITE=misc
-  - env: TEST_SUITE=api-fast
+  - env: TEST_SUITE=api-1
-  - env: TEST_SUITE=api-slow
+  - env: TEST_SUITE=api-2
+  - env: TEST_SUITE=api-3
   - env: TEST_SUITE=cli
   - env: TEST_SUITE=lint
 

config/default.yaml

@@ -57,6 +57,11 @@ storage:
 log:
   level: 'info' # debug/info/warning/error
 
+search:
+  remote_uri: # Add ability to fetch remote videos/actors by their URI, that may not be federated with your instance
+    users: true
+    anonymous: false
+
 cache:
   previews:
     size: 500 # Max number of previews you want to cache

config/production.yaml.example

@@ -58,6 +58,10 @@ storage:
 log:
   level: 'info' # debug/info/warning/error
 
+search:
+  remote_uri: # Add ability to search remote videos/actors by URI, that may not be federated with your instance
+    users: true
+    anonymous: false
 
 ###############################################################################
 #

scripts/travis.sh

@@ -12,19 +12,22 @@ killall -q peertube || true
 if [ "$1" = "misc" ]; then
     npm run build
     mocha --timeout 5000 --exit --require ts-node/register/type-check --bail server/tests/client.ts server/tests/activitypub.ts \
-        server/tests/feeds/feeds.ts
+        server/tests/feeds/index.ts
 elif [ "$1" = "api" ]; then
     npm run build:server
     mocha --timeout 5000 --exit --require ts-node/register/type-check --bail server/tests/api/index.ts
 elif [ "$1" = "cli" ]; then
     npm run build:server
     mocha --timeout 5000 --exit --require ts-node/register/type-check --bail server/tests/cli/index.ts
-elif [ "$1" = "api-fast" ]; then
+elif [ "$1" = "api-1" ]; then
     npm run build:server
-    mocha --timeout 5000 --exit --require ts-node/register/type-check --bail server/tests/api/index-fast.ts
+    mocha --timeout 5000 --exit --require ts-node/register/type-check --bail server/tests/api/index-1.ts
-elif [ "$1" = "api-slow" ]; then
+elif [ "$1" = "api-2" ]; then
     npm run build:server
-    mocha --timeout 5000 --exit --require ts-node/register/type-check --bail server/tests/api/index-slow.ts
+    mocha --timeout 5000 --exit --require ts-node/register/type-check --bail server/tests/api/index-2.ts
+elif [ "$1" = "api-3" ]; then
+    npm run build:server
+    mocha --timeout 5000 --exit --require ts-node/register/type-check --bail server/tests/api/index-3.ts
 elif [ "$1" = "lint" ]; then
     ( cd client
       npm run lint

server/controllers/api/search.ts

@@ -13,8 +13,10 @@ import {
   videosSearchSortValidator
 } from '../../middlewares'
 import { VideosSearchQuery } from '../../../shared/models/search'
-import { getOrCreateAccountAndVideoAndChannel } from '../../lib/activitypub'
+import { getOrCreateVideoAndAccountAndChannel } from '../../lib/activitypub'
 import { logger } from '../../helpers/logger'
+import { User } from '../../../shared/models/users'
+import { CONFIG } from '../../initializers/constants'
 
 const searchRouter = express.Router()
 
@@ -56,20 +58,30 @@ async function searchVideosDB (query: VideosSearchQuery, res: express.Response)
 
 async function searchVideoUrl (url: string, res: express.Response) {
   let video: VideoModel
+  const user: User = res.locals.oauth ? res.locals.oauth.token.User : undefined
 
-  try {
+  // Check if we can fetch a remote video with the URL
-    const syncParam = {
+  if (
-      likes: false,
+    CONFIG.SEARCH.REMOTE_URI.ANONYMOUS === true ||
-      dislikes: false,
+    (CONFIG.SEARCH.REMOTE_URI.USERS === true && user !== undefined)
-      shares: false,
+  ) {
-      comments: false,
+    try {
-      thumbnail: true
+      const syncParam = {
+        likes: false,
+        dislikes: false,
+        shares: false,
+        comments: false,
+        thumbnail: true,
+        refreshVideo: false
+      }
+
+      const res = await getOrCreateVideoAndAccountAndChannel(url, syncParam)
+      video = res ? res.video : undefined
+    } catch (err) {
+      logger.info('Cannot search remote video %s.', url)
     }
-
+  } else {
-    const res = await getOrCreateAccountAndVideoAndChannel(url, syncParam)
+    video = await VideoModel.loadByUrlAndPopulateAccount(url)
-    video = res ? res.video : undefined
-  } catch (err) {
-    logger.info('Cannot search remote video %s.', url)
   }
 
   return res.json({

server/initializers/constants.ts

@@ -181,6 +181,12 @@ const CONFIG = {
   LOG: {
     LEVEL: config.get<string>('log.level')
   },
+  SEARCH: {
+    REMOTE_URI: {
+      USERS: config.get<boolean>('search.remote_uri.users'),
+      ANONYMOUS: config.get<boolean>('search.remote_uri.anonymous')
+    }
+  },
   ADMIN: {
     get EMAIL () { return config.get<string>('admin.email') }
   },
@@ -462,7 +468,8 @@ const ACTIVITY_PUB = {
     MAGNET: [ 'application/x-bittorrent;x-scheme-handler/magnet' ]
   },
   MAX_RECURSION_COMMENTS: 100,
-  ACTOR_REFRESH_INTERVAL: 3600 * 24 * 1000 // 1 day
+  ACTOR_REFRESH_INTERVAL: 3600 * 24 * 1000, // 1 day
+  VIDEO_REFRESH_INTERVAL: 3600 * 24 * 1000 // 1 day
 }
 
 const ACTIVITY_PUB_ACTOR_TYPES: { [ id: string ]: ActivityPubActorType } = {
@@ -574,6 +581,7 @@ if (isTestInstance() === true) {
 
   ACTIVITY_PUB.COLLECTION_ITEMS_PER_PAGE = 2
   ACTIVITY_PUB.ACTOR_REFRESH_INTERVAL = 10 * 1000 // 10 seconds
+  ACTIVITY_PUB.VIDEO_REFRESH_INTERVAL = 10 * 1000 // 10 seconds
 
   CONSTRAINTS_FIELDS.ACTORS.AVATAR.FILE_SIZE.max = 100 * 1024 // 100KB
 

server/lib/activitypub/process/process-announce.ts

@@ -6,7 +6,7 @@ import { VideoModel } from '../../../models/video/video'
 import { VideoShareModel } from '../../../models/video/video-share'
 import { getOrCreateActorAndServerAndModel } from '../actor'
 import { forwardVideoRelatedActivity } from '../send/utils'
-import { getOrCreateAccountAndVideoAndChannel } from '../videos'
+import { getOrCreateVideoAndAccountAndChannel } from '../videos'
 
 async function processAnnounceActivity (activity: ActivityAnnounce) {
   const actorAnnouncer = await getOrCreateActorAndServerAndModel(activity.actor)
@@ -25,7 +25,7 @@ export {
 async function processVideoShare (actorAnnouncer: ActorModel, activity: ActivityAnnounce) {
   const objectUri = typeof activity.object === 'string' ? activity.object : activity.object.id
 
-  const { video } = await getOrCreateAccountAndVideoAndChannel(objectUri)
+  const { video } = await getOrCreateVideoAndAccountAndChannel(objectUri)
 
   return sequelizeTypescript.transaction(async t => {
     // Add share entry

server/lib/activitypub/process/process-create.ts

@@ -10,7 +10,7 @@ import { VideoAbuseModel } from '../../../models/video/video-abuse'
 import { VideoCommentModel } from '../../../models/video/video-comment'
 import { getOrCreateActorAndServerAndModel } from '../actor'
 import { resolveThread } from '../video-comments'
-import { getOrCreateAccountAndVideoAndChannel } from '../videos'
+import { getOrCreateVideoAndAccountAndChannel } from '../videos'
 import { forwardActivity, forwardVideoRelatedActivity } from '../send/utils'
 
 async function processCreateActivity (activity: ActivityCreate) {
@@ -45,7 +45,7 @@ export {
 async function processCreateVideo (activity: ActivityCreate) {
   const videoToCreateData = activity.object as VideoTorrentObject
 
-  const { video } = await getOrCreateAccountAndVideoAndChannel(videoToCreateData)
+  const { video } = await getOrCreateVideoAndAccountAndChannel(videoToCreateData)
 
   return video
 }
@@ -56,7 +56,7 @@ async function processCreateDislike (byActor: ActorModel, activity: ActivityCrea
 
   if (!byAccount) throw new Error('Cannot create dislike with the non account actor ' + byActor.url)
 
-  const { video } = await getOrCreateAccountAndVideoAndChannel(dislike.object)
+  const { video } = await getOrCreateVideoAndAccountAndChannel(dislike.object)
 
   return sequelizeTypescript.transaction(async t => {
     const rate = {
@@ -83,7 +83,7 @@ async function processCreateDislike (byActor: ActorModel, activity: ActivityCrea
 async function processCreateView (byActor: ActorModel, activity: ActivityCreate) {
   const view = activity.object as ViewObject
 
-  const { video } = await getOrCreateAccountAndVideoAndChannel(view.object)
+  const { video } = await getOrCreateVideoAndAccountAndChannel(view.object)
 
   const actor = await ActorModel.loadByUrl(view.actor)
   if (!actor) throw new Error('Unknown actor ' + view.actor)
@@ -103,7 +103,7 @@ async function processCreateVideoAbuse (actor: ActorModel, videoAbuseToCreateDat
   const account = actor.Account
   if (!account) throw new Error('Cannot create dislike with the non account actor ' + actor.url)
 
-  const { video } = await getOrCreateAccountAndVideoAndChannel(videoAbuseToCreateData.object)
+  const { video } = await getOrCreateVideoAndAccountAndChannel(videoAbuseToCreateData.object)
 
   return sequelizeTypescript.transaction(async t => {
     const videoAbuseData = {

server/lib/activitypub/process/process-like.ts

@@ -5,7 +5,7 @@ import { AccountVideoRateModel } from '../../../models/account/account-video-rat
 import { ActorModel } from '../../../models/activitypub/actor'
 import { getOrCreateActorAndServerAndModel } from '../actor'
 import { forwardVideoRelatedActivity } from '../send/utils'
-import { getOrCreateAccountAndVideoAndChannel } from '../videos'
+import { getOrCreateVideoAndAccountAndChannel } from '../videos'
 
 async function processLikeActivity (activity: ActivityLike) {
   const actor = await getOrCreateActorAndServerAndModel(activity.actor)
@@ -27,7 +27,7 @@ async function processLikeVideo (byActor: ActorModel, activity: ActivityLike) {
   const byAccount = byActor.Account
   if (!byAccount) throw new Error('Cannot create like with the non account actor ' + byActor.url)
 
-  const { video } = await getOrCreateAccountAndVideoAndChannel(videoUrl)
+  const { video } = await getOrCreateVideoAndAccountAndChannel(videoUrl)
 
   return sequelizeTypescript.transaction(async t => {
     const rate = {

server/lib/activitypub/process/process-undo.ts

@@ -9,7 +9,7 @@ import { AccountVideoRateModel } from '../../../models/account/account-video-rat
 import { ActorModel } from '../../../models/activitypub/actor'
 import { ActorFollowModel } from '../../../models/activitypub/actor-follow'
 import { forwardVideoRelatedActivity } from '../send/utils'
-import { getOrCreateAccountAndVideoAndChannel } from '../videos'
+import { getOrCreateVideoAndAccountAndChannel } from '../videos'
 import { VideoShareModel } from '../../../models/video/video-share'
 
 async function processUndoActivity (activity: ActivityUndo) {
@@ -43,7 +43,7 @@ export {
 async function processUndoLike (actorUrl: string, activity: ActivityUndo) {
   const likeActivity = activity.object as ActivityLike
 
-  const { video } = await getOrCreateAccountAndVideoAndChannel(likeActivity.object)
+  const { video } = await getOrCreateVideoAndAccountAndChannel(likeActivity.object)
 
   return sequelizeTypescript.transaction(async t => {
     const byAccount = await AccountModel.loadByUrl(actorUrl, t)
@@ -67,7 +67,7 @@ async function processUndoLike (actorUrl: string, activity: ActivityUndo) {
 async function processUndoDislike (actorUrl: string, activity: ActivityUndo) {
   const dislike = activity.object.object as DislikeObject
 
-  const { video } = await getOrCreateAccountAndVideoAndChannel(dislike.object)
+  const { video } = await getOrCreateVideoAndAccountAndChannel(dislike.object)
 
   return sequelizeTypescript.transaction(async t => {
     const byAccount = await AccountModel.loadByUrl(actorUrl, t)

server/lib/activitypub/process/process-update.ts

@@ -1,4 +1,3 @@
-import * as Bluebird from 'bluebird'
 import { ActivityUpdate, VideoTorrentObject } from '../../../../shared/models/activitypub'
 import { ActivityPubActor } from '../../../../shared/models/activitypub/activitypub-actor'
 import { resetSequelizeInstance, retryTransactionWrapper } from '../../../helpers/database-utils'
@@ -6,19 +5,10 @@ import { logger } from '../../../helpers/logger'
 import { sequelizeTypescript } from '../../../initializers'
 import { AccountModel } from '../../../models/account/account'
 import { ActorModel } from '../../../models/activitypub/actor'
-import { TagModel } from '../../../models/video/tag'
 import { VideoChannelModel } from '../../../models/video/video-channel'
-import { VideoFileModel } from '../../../models/video/video-file'
 import { fetchAvatarIfExists, getOrCreateActorAndServerAndModel, updateActorAvatarInstance, updateActorInstance } from '../actor'
-import {
+import { getOrCreateVideoAndAccountAndChannel, getOrCreateVideoChannel, updateVideoFromAP } from '../videos'
-  generateThumbnailFromUrl,
-  getOrCreateAccountAndVideoAndChannel,
-  getOrCreateVideoChannel,
-  videoActivityObjectToDBAttributes,
-  videoFileActivityUrlToDBAttributes
-} from '../videos'
 import { sanitizeAndCheckVideoTorrentObject } from '../../../helpers/custom-validators/activitypub/videos'
-import { VideoCaptionModel } from '../../../models/video/video-caption'
 
 async function processUpdateActivity (activity: ActivityUpdate) {
   const actor = await getOrCreateActorAndServerAndModel(activity.actor)
@@ -49,91 +39,10 @@ async function processUpdateVideo (actor: ActorModel, activity: ActivityUpdate)
     return undefined
   }
 
-  const res = await getOrCreateAccountAndVideoAndChannel(videoObject.id)
+  const { video } = await getOrCreateVideoAndAccountAndChannel(videoObject.id)
+  const channelActor = await getOrCreateVideoChannel(videoObject)
 
-  // Fetch video channel outside the transaction
+  return updateVideoFromAP(video, videoObject, actor, channelActor, activity.to)
-  const newVideoChannelActor = await getOrCreateVideoChannel(videoObject)
-  const newVideoChannel = newVideoChannelActor.VideoChannel
-
-  logger.debug('Updating remote video "%s".', videoObject.uuid)
-  let videoInstance = res.video
-  let videoFieldsSave: any
-
-  try {
-    await sequelizeTypescript.transaction(async t => {
-      const sequelizeOptions = {
-        transaction: t
-      }
-
-      videoFieldsSave = videoInstance.toJSON()
-
-      // Check actor has the right to update the video
-      const videoChannel = videoInstance.VideoChannel
-      if (videoChannel.Account.Actor.id !== actor.id) {
-        throw new Error('Account ' + actor.url + ' does not own video channel ' + videoChannel.Actor.url)
-      }
-
-      const videoData = await videoActivityObjectToDBAttributes(newVideoChannel, videoObject, activity.to)
-      videoInstance.set('name', videoData.name)
-      videoInstance.set('uuid', videoData.uuid)
-      videoInstance.set('url', videoData.url)
-      videoInstance.set('category', videoData.category)
-      videoInstance.set('licence', videoData.licence)
-      videoInstance.set('language', videoData.language)
-      videoInstance.set('description', videoData.description)
-      videoInstance.set('support', videoData.support)
-      videoInstance.set('nsfw', videoData.nsfw)
-      videoInstance.set('commentsEnabled', videoData.commentsEnabled)
-      videoInstance.set('waitTranscoding', videoData.waitTranscoding)
-      videoInstance.set('state', videoData.state)
-      videoInstance.set('duration', videoData.duration)
-      videoInstance.set('createdAt', videoData.createdAt)
-      videoInstance.set('updatedAt', videoData.updatedAt)
-      videoInstance.set('views', videoData.views)
-      videoInstance.set('privacy', videoData.privacy)
-      videoInstance.set('channelId', videoData.channelId)
-
-      await videoInstance.save(sequelizeOptions)
-
-      // Don't block on request
-      generateThumbnailFromUrl(videoInstance, videoObject.icon)
-        .catch(err => logger.warn('Cannot generate thumbnail of %s.', videoObject.id, { err }))
-
-      // Remove old video files
-      const videoFileDestroyTasks: Bluebird<void>[] = []
-      for (const videoFile of videoInstance.VideoFiles) {
-        videoFileDestroyTasks.push(videoFile.destroy(sequelizeOptions))
-      }
-      await Promise.all(videoFileDestroyTasks)
-
-      const videoFileAttributes = videoFileActivityUrlToDBAttributes(videoInstance, videoObject)
-      const tasks = videoFileAttributes.map(f => VideoFileModel.create(f, sequelizeOptions))
-      await Promise.all(tasks)
-
-      // Update Tags
-      const tags = videoObject.tag.map(tag => tag.name)
-      const tagInstances = await TagModel.findOrCreateTags(tags, t)
-      await videoInstance.$set('Tags', tagInstances, sequelizeOptions)
-
-      // Update captions
-      await VideoCaptionModel.deleteAllCaptionsOfRemoteVideo(videoInstance.id, t)
-
-      const videoCaptionsPromises = videoObject.subtitleLanguage.map(c => {
-        return VideoCaptionModel.insertOrReplaceLanguage(videoInstance.id, c.identifier, t)
-      })
-      await Promise.all(videoCaptionsPromises)
-    })
-
-    logger.info('Remote video with uuid %s updated', videoObject.uuid)
-  } catch (err) {
-    if (videoInstance !== undefined && videoFieldsSave !== undefined) {
-      resetSequelizeInstance(videoInstance, videoFieldsSave)
-    }
-
-    // This is just a debug because we will retry the insert
-    logger.debug('Cannot update the remote video.', { err })
-    throw err
-  }
 }
 
 async function processUpdateActor (actor: ActorModel, activity: ActivityUpdate) {

server/lib/activitypub/share.ts

@@ -6,6 +6,11 @@ import { VideoShareModel } from '../../models/video/video-share'
 import { sendUndoAnnounce, sendVideoAnnounce } from './send'
 import { getAnnounceActivityPubUrl } from './url'
 import { VideoChannelModel } from '../../models/video/video-channel'
+import * as Bluebird from 'bluebird'
+import { doRequest } from '../../helpers/requests'
+import { getOrCreateActorAndServerAndModel } from './actor'
+import { logger } from '../../helpers/logger'
+import { CRAWL_REQUEST_CONCURRENCY } from '../../initializers'
 
 async function shareVideoByServerAndChannel (video: VideoModel, t: Transaction) {
   if (video.privacy === VideoPrivacy.PRIVATE) return undefined
@@ -22,8 +27,41 @@ async function changeVideoChannelShare (video: VideoModel, oldVideoChannel: Vide
   await shareByVideoChannel(video, t)
 }
 
+async function addVideoShares (shareUrls: string[], instance: VideoModel) {
+  await Bluebird.map(shareUrls, async shareUrl => {
+    try {
+      // Fetch url
+      const { body } = await doRequest({
+        uri: shareUrl,
+        json: true,
+        activityPub: true
+      })
+      if (!body || !body.actor) throw new Error('Body of body actor is invalid')
+
+      const actorUrl = body.actor
+      const actor = await getOrCreateActorAndServerAndModel(actorUrl)
+
+      const entry = {
+        actorId: actor.id,
+        videoId: instance.id,
+        url: shareUrl
+      }
+
+      await VideoShareModel.findOrCreate({
+        where: {
+          url: shareUrl
+        },
+        defaults: entry
+      })
+    } catch (err) {
+      logger.warn('Cannot add share %s.', shareUrl, { err })
+    }
+  }, { concurrency: CRAWL_REQUEST_CONCURRENCY })
+}
+
 export {
   changeVideoChannelShare,
+  addVideoShares,
   shareVideoByServerAndChannel
 }
 

server/lib/activitypub/video-comments.ts

@@ -7,7 +7,7 @@ import { ActorModel } from '../../models/activitypub/actor'
 import { VideoModel } from '../../models/video/video'
 import { VideoCommentModel } from '../../models/video/video-comment'
 import { getOrCreateActorAndServerAndModel } from './actor'
-import { getOrCreateAccountAndVideoAndChannel } from './videos'
+import { getOrCreateVideoAndAccountAndChannel } from './videos'
 import * as Bluebird from 'bluebird'
 
 async function videoCommentActivityObjectToDBAttributes (video: VideoModel, actor: ActorModel, comment: VideoCommentObject) {
@@ -91,7 +91,7 @@ async function resolveThread (url: string, comments: VideoCommentModel[] = []) {
 
   try {
     // Maybe it's a reply to a video?
-    const { video } = await getOrCreateAccountAndVideoAndChannel(url)
+    const { video } = await getOrCreateVideoAndAccountAndChannel(url)
 
     if (comments.length !== 0) {
       const firstReply = comments[ comments.length - 1 ]

server/lib/activitypub/video-rates.ts

@@ -2,6 +2,45 @@ import { Transaction } from 'sequelize'
 import { AccountModel } from '../../models/account/account'
 import { VideoModel } from '../../models/video/video'
 import { sendCreateDislike, sendLike, sendUndoDislike, sendUndoLike } from './send'
+import { VideoRateType } from '../../../shared/models/videos'
+import * as Bluebird from 'bluebird'
+import { getOrCreateActorAndServerAndModel } from './actor'
+import { AccountVideoRateModel } from '../../models/account/account-video-rate'
+import { logger } from '../../helpers/logger'
+import { CRAWL_REQUEST_CONCURRENCY } from '../../initializers'
+
+async function createRates (actorUrls: string[], video: VideoModel, rate: VideoRateType) {
+  let rateCounts = 0
+
+  await Bluebird.map(actorUrls, async actorUrl => {
+    try {
+      const actor = await getOrCreateActorAndServerAndModel(actorUrl)
+      const [ , created ] = await AccountVideoRateModel
+        .findOrCreate({
+          where: {
+            videoId: video.id,
+            accountId: actor.Account.id
+          },
+          defaults: {
+            videoId: video.id,
+            accountId: actor.Account.id,
+            type: rate
+          }
+        })
+
+      if (created) rateCounts += 1
+    } catch (err) {
+      logger.warn('Cannot add rate %s for actor %s.', rate, actorUrl, { err })
+    }
+  }, { concurrency: CRAWL_REQUEST_CONCURRENCY })
+
+  logger.info('Adding %d %s to video %s.', rateCounts, rate, video.uuid)
+
+  // This is "likes" and "dislikes"
+  if (rateCounts !== 0) await video.increment(rate + 's', { by: rateCounts })
+
+  return
+}
 
 async function sendVideoRateChange (account: AccountModel,
                               video: VideoModel,
@@ -24,5 +63,6 @@ async function sendVideoRateChange (account: AccountModel,
 }
 
 export {
+  createRates,
   sendVideoRateChange
 }

server/lib/activitypub/videos.ts

@@ -5,29 +5,30 @@ import { join } from 'path'
 import * as request from 'request'
 import { ActivityIconObject, VideoState } from '../../../shared/index'
 import { VideoTorrentObject } from '../../../shared/models/activitypub/objects'
-import { VideoPrivacy, VideoRateType } from '../../../shared/models/videos'
+import { VideoPrivacy } from '../../../shared/models/videos'
 import { sanitizeAndCheckVideoTorrentObject } from '../../helpers/custom-validators/activitypub/videos'
 import { isVideoFileInfoHashValid } from '../../helpers/custom-validators/videos'
-import { retryTransactionWrapper } from '../../helpers/database-utils'
+import { resetSequelizeInstance, retryTransactionWrapper, updateInstanceWithAnother } from '../../helpers/database-utils'
 import { logger } from '../../helpers/logger'
 import { doRequest, doRequestAndSaveToFile } from '../../helpers/requests'
-import { ACTIVITY_PUB, CONFIG, CRAWL_REQUEST_CONCURRENCY, REMOTE_SCHEME, sequelizeTypescript, VIDEO_MIMETYPE_EXT } from '../../initializers'
+import { ACTIVITY_PUB, CONFIG, REMOTE_SCHEME, sequelizeTypescript, VIDEO_MIMETYPE_EXT } from '../../initializers'
-import { AccountVideoRateModel } from '../../models/account/account-video-rate'
 import { ActorModel } from '../../models/activitypub/actor'
 import { TagModel } from '../../models/video/tag'
 import { VideoModel } from '../../models/video/video'
 import { VideoChannelModel } from '../../models/video/video-channel'
 import { VideoFileModel } from '../../models/video/video-file'
-import { VideoShareModel } from '../../models/video/video-share'
+import { getOrCreateActorAndServerAndModel, updateActorAvatarInstance } from './actor'
-import { getOrCreateActorAndServerAndModel } from './actor'
 import { addVideoComments } from './video-comments'
 import { crawlCollectionPage } from './crawl'
 import { sendCreateVideo, sendUpdateVideo } from './send'
-import { shareVideoByServerAndChannel } from './index'
 import { isArray } from '../../helpers/custom-validators/misc'
 import { VideoCaptionModel } from '../../models/video/video-caption'
 import { JobQueue } from '../job-queue'
 import { ActivitypubHttpFetcherPayload } from '../job-queue/handlers/activitypub-http-fetcher'
+import { getUrlFromWebfinger } from '../../helpers/webfinger'
+import { createRates } from './video-rates'
+import { addVideoShares, shareVideoByServerAndChannel } from './share'
+import { AccountModel } from '../../models/account/account'
 
 async function federateVideoIfNeeded (video: VideoModel, isNewVideo: boolean, transaction?: sequelize.Transaction) {
   // If the video is not private and published, we federate it
@@ -180,15 +181,11 @@ function getOrCreateVideoChannel (videoObject: VideoTorrentObject) {
   return getOrCreateActorAndServerAndModel(channel.id)
 }
 
-async function getOrCreateVideo (videoObject: VideoTorrentObject, channelActor: ActorModel, waitThumbnail = false) {
+async function createVideo (videoObject: VideoTorrentObject, channelActor: ActorModel, waitThumbnail = false) {
   logger.debug('Adding remote video %s.', videoObject.id)
 
   const videoCreated: VideoModel = await sequelizeTypescript.transaction(async t => {
-    const sequelizeOptions = {
+    const sequelizeOptions = { transaction: t }
-      transaction: t
-    }
-    const videoFromDatabase = await VideoModel.loadByUUIDOrURLAndPopulateAccount(videoObject.uuid, videoObject.id, t)
-    if (videoFromDatabase) return videoFromDatabase
 
     const videoData = await videoActivityObjectToDBAttributes(channelActor.VideoChannel, videoObject, videoObject.to)
     const video = VideoModel.build(videoData)
@@ -230,26 +227,32 @@ async function getOrCreateVideo (videoObject: VideoTorrentObject, channelActor:
 }
 
 type SyncParam = {
-  likes: boolean,
+  likes: boolean
-  dislikes: boolean,
+  dislikes: boolean
-  shares: boolean,
+  shares: boolean
-  comments: boolean,
+  comments: boolean
   thumbnail: boolean
+  refreshVideo: boolean
 }
-async function getOrCreateAccountAndVideoAndChannel (
+async function getOrCreateVideoAndAccountAndChannel (
   videoObject: VideoTorrentObject | string,
-  syncParam: SyncParam = { likes: true, dislikes: true, shares: true, comments: true, thumbnail: true }
+  syncParam: SyncParam = { likes: true, dislikes: true, shares: true, comments: true, thumbnail: true, refreshVideo: false }
 ) {
   const videoUrl = typeof videoObject === 'string' ? videoObject : videoObject.id
 
-  const videoFromDatabase = await VideoModel.loadByUrlAndPopulateAccount(videoUrl)
+  let videoFromDatabase = await VideoModel.loadByUrlAndPopulateAccount(videoUrl)
-  if (videoFromDatabase) return { video: videoFromDatabase }
+  if (videoFromDatabase) {
+    const p = retryTransactionWrapper(refreshVideoIfNeeded, videoFromDatabase)
+    if (syncParam.refreshVideo === true) videoFromDatabase = await p
 
-  const fetchedVideo = await fetchRemoteVideo(videoUrl)
+    return { video: videoFromDatabase }
+  }
+
+  const { videoObject: fetchedVideo } = await fetchRemoteVideo(videoUrl)
   if (!fetchedVideo) throw new Error('Cannot fetch remote video with url: ' + videoUrl)
 
   const channelActor = await getOrCreateVideoChannel(fetchedVideo)
-  const video = await retryTransactionWrapper(getOrCreateVideo, fetchedVideo, channelActor, syncParam.thumbnail)
+  const video = await retryTransactionWrapper(createVideo, fetchedVideo, channelActor, syncParam.thumbnail)
 
   // Process outside the transaction because we could fetch remote data
 
@@ -290,72 +293,7 @@ async function getOrCreateAccountAndVideoAndChannel (
   return { video }
 }
 
-async function createRates (actorUrls: string[], video: VideoModel, rate: VideoRateType) {
+async function fetchRemoteVideo (videoUrl: string): Promise<{ response: request.RequestResponse, videoObject: VideoTorrentObject }> {
-  let rateCounts = 0
-
-  await Bluebird.map(actorUrls, async actorUrl => {
-    try {
-      const actor = await getOrCreateActorAndServerAndModel(actorUrl)
-      const [ , created ] = await AccountVideoRateModel
-        .findOrCreate({
-          where: {
-            videoId: video.id,
-            accountId: actor.Account.id
-          },
-          defaults: {
-            videoId: video.id,
-            accountId: actor.Account.id,
-            type: rate
-          }
-        })
-
-      if (created) rateCounts += 1
-    } catch (err) {
-      logger.warn('Cannot add rate %s for actor %s.', rate, actorUrl, { err })
-    }
-  }, { concurrency: CRAWL_REQUEST_CONCURRENCY })
-
-  logger.info('Adding %d %s to video %s.', rateCounts, rate, video.uuid)
-
-  // This is "likes" and "dislikes"
-  if (rateCounts !== 0) await video.increment(rate + 's', { by: rateCounts })
-
-  return
-}
-
-async function addVideoShares (shareUrls: string[], instance: VideoModel) {
-  await Bluebird.map(shareUrls, async shareUrl => {
-    try {
-      // Fetch url
-      const { body } = await doRequest({
-        uri: shareUrl,
-        json: true,
-        activityPub: true
-      })
-      if (!body || !body.actor) throw new Error('Body of body actor is invalid')
-
-      const actorUrl = body.actor
-      const actor = await getOrCreateActorAndServerAndModel(actorUrl)
-
-      const entry = {
-        actorId: actor.id,
-        videoId: instance.id,
-        url: shareUrl
-      }
-
-      await VideoShareModel.findOrCreate({
-        where: {
-          url: shareUrl
-        },
-        defaults: entry
-      })
-    } catch (err) {
-      logger.warn('Cannot add share %s.', shareUrl, { err })
-    }
-  }, { concurrency: CRAWL_REQUEST_CONCURRENCY })
-}
-
-async function fetchRemoteVideo (videoUrl: string): Promise<VideoTorrentObject> {
   const options = {
     uri: videoUrl,
     method: 'GET',
@@ -365,26 +303,143 @@ async function fetchRemoteVideo (videoUrl: string): Promise<VideoTorrentObject>
 
   logger.info('Fetching remote video %s.', videoUrl)
 
-  const { body } = await doRequest(options)
+  const { response, body } = await doRequest(options)
 
   if (sanitizeAndCheckVideoTorrentObject(body) === false) {
     logger.debug('Remote video JSON is not valid.', { body })
-    return undefined
+    return { response, videoObject: undefined }
   }
 
-  return body
+  return { response, videoObject: body }
+}
+
+async function refreshVideoIfNeeded (video: VideoModel): Promise<VideoModel> {
+  if (!video.isOutdated()) return video
+
+  try {
+    const { response, videoObject } = await fetchRemoteVideo(video.url)
+    if (response.statusCode === 404) {
+      // Video does not exist anymore
+      await video.destroy()
+      return undefined
+    }
+
+    if (videoObject === undefined) {
+      logger.warn('Cannot refresh remote video: invalid body.')
+      return video
+    }
+
+    const channelActor = await getOrCreateVideoChannel(videoObject)
+    const account = await AccountModel.load(channelActor.VideoChannel.accountId)
+    return updateVideoFromAP(video, videoObject, account.Actor, channelActor)
+
+  } catch (err) {
+    logger.warn('Cannot refresh video.', { err })
+    return video
+  }
+}
+
+async function updateVideoFromAP (
+  video: VideoModel,
+  videoObject: VideoTorrentObject,
+  accountActor: ActorModel,
+  channelActor: ActorModel,
+  overrideTo?: string[]
+) {
+  logger.debug('Updating remote video "%s".', videoObject.uuid)
+  let videoFieldsSave: any
+
+  try {
+    const updatedVideo: VideoModel = await sequelizeTypescript.transaction(async t => {
+      const sequelizeOptions = {
+        transaction: t
+      }
+
+      videoFieldsSave = video.toJSON()
+
+      // Check actor has the right to update the video
+      const videoChannel = video.VideoChannel
+      if (videoChannel.Account.Actor.id !== accountActor.id) {
+        throw new Error('Account ' + accountActor.url + ' does not own video channel ' + videoChannel.Actor.url)
+      }
+
+      const to = overrideTo ? overrideTo : videoObject.to
+      const videoData = await videoActivityObjectToDBAttributes(channelActor.VideoChannel, videoObject, to)
+      video.set('name', videoData.name)
+      video.set('uuid', videoData.uuid)
+      video.set('url', videoData.url)
+      video.set('category', videoData.category)
+      video.set('licence', videoData.licence)
+      video.set('language', videoData.language)
+      video.set('description', videoData.description)
+      video.set('support', videoData.support)
+      video.set('nsfw', videoData.nsfw)
+      video.set('commentsEnabled', videoData.commentsEnabled)
+      video.set('waitTranscoding', videoData.waitTranscoding)
+      video.set('state', videoData.state)
+      video.set('duration', videoData.duration)
+      video.set('createdAt', videoData.createdAt)
+      video.set('publishedAt', videoData.publishedAt)
+      video.set('views', videoData.views)
+      video.set('privacy', videoData.privacy)
+      video.set('channelId', videoData.channelId)
+
+      await video.save(sequelizeOptions)
+
+      // Don't block on request
+      generateThumbnailFromUrl(video, videoObject.icon)
+        .catch(err => logger.warn('Cannot generate thumbnail of %s.', videoObject.id, { err }))
+
+      // Remove old video files
+      const videoFileDestroyTasks: Bluebird<void>[] = []
+      for (const videoFile of video.VideoFiles) {
+        videoFileDestroyTasks.push(videoFile.destroy(sequelizeOptions))
+      }
+      await Promise.all(videoFileDestroyTasks)
+
+      const videoFileAttributes = videoFileActivityUrlToDBAttributes(video, videoObject)
+      const tasks = videoFileAttributes.map(f => VideoFileModel.create(f, sequelizeOptions))
+      await Promise.all(tasks)
+
+      // Update Tags
+      const tags = videoObject.tag.map(tag => tag.name)
+      const tagInstances = await TagModel.findOrCreateTags(tags, t)
+      await video.$set('Tags', tagInstances, sequelizeOptions)
+
+      // Update captions
+      await VideoCaptionModel.deleteAllCaptionsOfRemoteVideo(video.id, t)
+
+      const videoCaptionsPromises = videoObject.subtitleLanguage.map(c => {
+        return VideoCaptionModel.insertOrReplaceLanguage(video.id, c.identifier, t)
+      })
+      await Promise.all(videoCaptionsPromises)
+    })
+
+    logger.info('Remote video with uuid %s updated', videoObject.uuid)
+
+    return updatedVideo
+  } catch (err) {
+    if (video !== undefined && videoFieldsSave !== undefined) {
+      resetSequelizeInstance(video, videoFieldsSave)
+    }
+
+    // This is just a debug because we will retry the insert
+    logger.debug('Cannot update the remote video.', { err })
+    throw err
+  }
 }
 
 export {
+  updateVideoFromAP,
   federateVideoIfNeeded,
   fetchRemoteVideo,
-  getOrCreateAccountAndVideoAndChannel,
+  getOrCreateVideoAndAccountAndChannel,
   fetchRemoteVideoStaticFile,
   fetchRemoteVideoDescription,
   generateThumbnailFromUrl,
   videoActivityObjectToDBAttributes,
   videoFileActivityUrlToDBAttributes,
-  getOrCreateVideo,
+  createVideo,
   getOrCreateVideoChannel,
   addVideoShares,
   createRates

server/models/video/video.ts

@@ -56,6 +56,7 @@ import { generateImageFromVideoFile, getVideoFileFPS, getVideoFileResolution, tr
 import { logger } from '../../helpers/logger'
 import { getServerActor } from '../../helpers/utils'
 import {
+  ACTIVITY_PUB,
   API_VERSION,
   CONFIG,
   CONSTRAINTS_FIELDS,
@@ -1004,21 +1005,6 @@ export class VideoModel extends Model<VideoModel> {
     return VideoModel.scope([ ScopeNames.WITH_ACCOUNT_DETAILS, ScopeNames.WITH_FILES ]).findOne(query)
   }
 
-  static loadByUUIDOrURLAndPopulateAccount (uuid: string, url: string, t?: Sequelize.Transaction) {
-    const query: IFindOptions<VideoModel> = {
-      where: {
-        [Sequelize.Op.or]: [
-          { uuid },
-          { url }
-        ]
-      }
-    }
-
-    if (t !== undefined) query.transaction = t
-
-    return VideoModel.scope([ ScopeNames.WITH_ACCOUNT_DETAILS, ScopeNames.WITH_FILES ]).findOne(query)
-  }
-
   static loadAndPopulateAccountAndServerAndTags (id: number) {
     const options = {
       order: [ [ 'Tags', 'name', 'ASC' ] ]
@@ -1646,6 +1632,17 @@ export class VideoModel extends Model<VideoModel> {
     return 'PT' + this.duration + 'S'
   }
 
+  isOutdated () {
+    if (this.isOwned()) return false
+
+    const now = Date.now()
+    const createdAtTime = this.createdAt.getTime()
+    const updatedAtTime = this.updatedAt.getTime()
+
+    return (now - createdAtTime) > ACTIVITY_PUB.VIDEO_REFRESH_INTERVAL &&
+      (now - updatedAtTime) > ACTIVITY_PUB.VIDEO_REFRESH_INTERVAL
+  }
+
   private getBaseUrls () {
     let baseUrlHttp
     let baseUrlWs

server/tests/api/index-1.ts

@@ -0,0 +1,2 @@
+import './check-params'
+import './search'

server/tests/api/index-2.ts

@@ -0,0 +1,2 @@
+import './server'
+import './users'

server/tests/api/index-3.ts

@@ -0,0 +1 @@
+import './videos'

server/tests/api/index-fast.ts

@@ -1,18 +0,0 @@
-// Order of the tests we want to execute
-import './server/stats'
-import './check-params'
-import './users/users'
-import './videos/single-server'
-import './videos/video-abuse'
-import './videos/video-captions'
-import './videos/video-blacklist'
-import './videos/video-blacklist-management'
-import './videos/video-description'
-import './videos/video-nsfw'
-import './videos/video-privacy'
-import './videos/services'
-import './server/email'
-import './server/config'
-import './server/reverse-proxy'
-import './search/search-videos'
-import './server/tracker'

server/tests/api/index-slow.ts

@@ -1,12 +0,0 @@
-// Order of the tests we want to execute
-import './videos/video-channels'
-import './videos/video-transcoder'
-import './videos/multiple-servers'
-import './server/follows'
-import './server/jobs'
-import './videos/video-comments'
-import './users/users-multiple-servers'
-import './users/user-subscriptions'
-import './server/handle-down'
-import './videos/video-schedule-update'
-import './videos/video-imports'

server/tests/api/index.ts

@@ -1,3 +1,4 @@
 // Order of the tests we want to execute
-import './index-fast'
+import './index-1'
-import './index-slow'
+import './index-2'
+import './index-3'

server/tests/api/search/index.ts

@@ -0,0 +1,2 @@
+import './search-activitypub-videos'
+import './search-videos'

server/tests/api/search/search-activitypub-videos.ts

@@ -0,0 +1,161 @@
+/* tslint:disable:no-unused-expression */
+
+import * as chai from 'chai'
+import 'mocha'
+import {
+  addVideoChannel,
+  flushAndRunMultipleServers,
+  flushTests,
+  getVideosList,
+  killallServers,
+  removeVideo,
+  searchVideoWithToken,
+  ServerInfo,
+  setAccessTokensToServers,
+  updateVideo,
+  uploadVideo,
+  wait,
+  searchVideo
+} from '../../utils'
+import { waitJobs } from '../../utils/server/jobs'
+import { Video, VideoPrivacy } from '../../../../shared/models/videos'
+
+const expect = chai.expect
+
+describe('Test a ActivityPub videos search', function () {
+  let servers: ServerInfo[]
+  let videoServer1UUID: string
+  let videoServer2UUID: string
+
+  before(async function () {
+    this.timeout(120000)
+
+    await flushTests()
+
+    servers = await flushAndRunMultipleServers(2)
+
+    await setAccessTokensToServers(servers)
+
+    {
+      const res = await uploadVideo(servers[ 0 ].url, servers[ 0 ].accessToken, { name: 'video 1 on server 1' })
+      videoServer1UUID = res.body.video.uuid
+    }
+
+    {
+      const res = await uploadVideo(servers[ 1 ].url, servers[ 1 ].accessToken, { name: 'video 1 on server 2' })
+      videoServer2UUID = res.body.video.uuid
+    }
+
+    await waitJobs(servers)
+  })
+
+  it('Should not find a remote video', async function () {
+    {
+      const res = await searchVideoWithToken(servers[ 0 ].url, 'http://localhost:9002/videos/watch/43', servers[ 0 ].accessToken)
+
+      expect(res.body.total).to.equal(0)
+      expect(res.body.data).to.be.an('array')
+      expect(res.body.data).to.have.lengthOf(0)
+    }
+
+    {
+      const res = await searchVideo(servers[0].url, 'http://localhost:9002/videos/watch/' + videoServer2UUID)
+
+      expect(res.body.total).to.equal(0)
+      expect(res.body.data).to.be.an('array')
+      expect(res.body.data).to.have.lengthOf(0)
+    }
+  })
+
+  it('Should search a local video', async function () {
+    const res = await searchVideo(servers[0].url, 'http://localhost:9001/videos/watch/' + videoServer1UUID)
+
+    expect(res.body.total).to.equal(1)
+    expect(res.body.data).to.be.an('array')
+    expect(res.body.data).to.have.lengthOf(1)
+    expect(res.body.data[0].name).to.equal('video 1 on server 1')
+  })
+
+  it('Should search a remote video', async function () {
+    const res = await searchVideoWithToken(servers[0].url, 'http://localhost:9002/videos/watch/' + videoServer2UUID, servers[0].accessToken)
+
+    expect(res.body.total).to.equal(1)
+    expect(res.body.data).to.be.an('array')
+    expect(res.body.data).to.have.lengthOf(1)
+    expect(res.body.data[0].name).to.equal('video 1 on server 2')
+  })
+
+  it('Should not list this remote video', async function () {
+    const res = await getVideosList(servers[0].url)
+    expect(res.body.total).to.equal(1)
+    expect(res.body.data).to.have.lengthOf(1)
+    expect(res.body.data[0].name).to.equal('video 1 on server 1')
+  })
+
+  it('Should update video of server 2, and refresh it on server 1', async function () {
+    this.timeout(60000)
+
+    const channelAttributes = {
+      name: 'super_channel',
+      displayName: 'super channel'
+    }
+    const resChannel = await addVideoChannel(servers[1].url, servers[1].accessToken, channelAttributes)
+    const videoChannelId = resChannel.body.videoChannel.id
+
+    const attributes = {
+      name: 'updated',
+      tag: [ 'tag1', 'tag2' ],
+      privacy: VideoPrivacy.UNLISTED,
+      channelId: videoChannelId
+    }
+    await updateVideo(servers[1].url, servers[1].accessToken, videoServer2UUID, attributes)
+
+    await waitJobs(servers)
+    // Expire video
+    await wait(10000)
+
+    // Will run refresh async
+    await searchVideoWithToken(servers[0].url, 'http://localhost:9002/videos/watch/' + videoServer2UUID, servers[0].accessToken)
+
+    // Wait refresh
+    await wait(5000)
+
+    const res = await searchVideoWithToken(servers[0].url, 'http://localhost:9002/videos/watch/' + videoServer2UUID, servers[0].accessToken)
+    expect(res.body.total).to.equal(1)
+    expect(res.body.data).to.have.lengthOf(1)
+
+    const video: Video = res.body.data[0]
+    expect(video.name).to.equal('updated')
+    expect(video.channel.name).to.equal('super_channel')
+    expect(video.privacy.id).to.equal(VideoPrivacy.UNLISTED)
+  })
+
+  it('Should delete video of server 2, and delete it on server 1', async function () {
+    this.timeout(60000)
+
+    await removeVideo(servers[1].url, servers[1].accessToken, videoServer2UUID)
+
+    await waitJobs(servers)
+    // Expire video
+    await wait(10000)
+
+    // Will run refresh async
+    await searchVideoWithToken(servers[0].url, 'http://localhost:9002/videos/watch/' + videoServer2UUID, servers[0].accessToken)
+
+    // Wait refresh
+    await wait(5000)
+
+    const res = await searchVideoWithToken(servers[0].url, 'http://localhost:9002/videos/watch/' + videoServer2UUID, servers[0].accessToken)
+    expect(res.body.total).to.equal(0)
+    expect(res.body.data).to.have.lengthOf(0)
+  })
+
+  after(async function () {
+    killallServers(servers)
+
+    // Keep the logs if the test failed
+    if (this['ok']) {
+      await flushTests()
+    }
+  })
+})

server/tests/api/server/index.ts

@@ -0,0 +1,8 @@
+import './config'
+import './email'
+import './follows'
+import './handle-down'
+import './jobs'
+import './reverse-proxy'
+import './stats'
+import './tracker'

server/tests/api/users/index.ts

@@ -0,0 +1,3 @@
+import './user-subscriptions'
+import './users'
+import './users-multiple-servers'

server/tests/api/videos/index.ts

@@ -0,0 +1,15 @@
+import './multiple-servers'
+import './services'
+import './single-server'
+import './video-abuse'
+import './video-blacklist'
+import './video-blacklist-management'
+import './video-captions'
+import './video-channels'
+import './video-comme'
+import './video-description'
+import './video-impo'
+import './video-nsfw'
+import './video-privacy'
+import './video-schedule-update'
+import './video-transcoder'

server/tests/feeds/index.ts

@@ -0,0 +1 @@
+import './feeds'

server/tests/index.ts

@@ -1,5 +1,6 @@
 // Order of the tests we want to execute
 import './client'
 import './activitypub'
-import './api/'
+import './feeds/'
 import './cli/'
+import './api/'