Refresh playlists

server/controllers/api/video-playlist.ts

@@ -40,6 +40,7 @@ import { VideoPlaylistElementUpdate } from '../../../shared/models/videos/playli
 import { copy, pathExists } from 'fs-extra'
 import { AccountModel } from '../../models/account/account'
 import { VideoPlaylistReorder } from '../../../shared/models/videos/playlist/video-playlist-reorder.model'
+import { JobQueue } from '../../lib/job-queue'
 
 const reqThumbnailFile = createReqFiles([ 'thumbnailfile' ], MIMETYPES.IMAGE.MIMETYPE_EXT, { thumbnailfile: CONFIG.STORAGE.TMP_DIR })
 
@@ -142,6 +143,11 @@ async function listVideoPlaylists (req: express.Request, res: express.Response)
 function getVideoPlaylist (req: express.Request, res: express.Response) {
   const videoPlaylist = res.locals.videoPlaylist
 
+  if (videoPlaylist.isOutdated()) {
+    JobQueue.Instance.createJob({ type: 'activitypub-refresher', payload: { type: 'video-playlist', url: videoPlaylist.url } })
+            .catch(err => logger.error('Cannot create AP refresher job for playlist %s.', videoPlaylist.url, { err }))
+  }
+
   return res.json(videoPlaylist.toFormattedJSON())
 }
 

server/initializers/constants.ts

@@ -584,7 +584,8 @@ const ACTIVITY_PUB = {
   },
   MAX_RECURSION_COMMENTS: 100,
   ACTOR_REFRESH_INTERVAL: 3600 * 24 * 1000 * 2, // 2 days
-  VIDEO_REFRESH_INTERVAL: 3600 * 24 * 1000 * 2 // 2 days
+  VIDEO_REFRESH_INTERVAL: 3600 * 24 * 1000 * 2, // 2 days
+  VIDEO_PLAYLIST_REFRESH_INTERVAL: 3600 * 24 * 1000 * 2 // 2 days
 }
 
 const ACTIVITY_PUB_ACTOR_TYPES: { [ id: string ]: ActivityPubActorType } = {
@@ -724,6 +725,7 @@ if (isTestInstance() === true) {
   ACTIVITY_PUB.COLLECTION_ITEMS_PER_PAGE = 2
   ACTIVITY_PUB.ACTOR_REFRESH_INTERVAL = 10 * 1000 // 10 seconds
   ACTIVITY_PUB.VIDEO_REFRESH_INTERVAL = 10 * 1000 // 10 seconds
+  ACTIVITY_PUB.VIDEO_PLAYLIST_REFRESH_INTERVAL = 10 * 1000 // 10 seconds
 
   CONSTRAINTS_FIELDS.ACTORS.AVATAR.FILE_SIZE.max = 100 * 1024 // 100KB
 

server/lib/activitypub/actor.ts

@@ -375,7 +375,8 @@ async function fetchRemoteActor (actorUrl: string): Promise<{ statusCode?: numbe
   }
 
   if (checkUrlsSameHost(actorJSON.id, actorUrl) !== true) {
-    throw new Error('Actor url ' + actorUrl + ' has not the same host than its AP id ' + actorJSON.id)
+    logger.warn('Actor url %s has not the same host than its AP id %s', actorUrl, actorJSON.id)
+    return { result: undefined, statusCode: requestResult.response.statusCode }
   }
 
   const followersCount = await fetchActorTotalItems(actorJSON.followers)

server/lib/activitypub/playlist.ts

@@ -95,7 +95,7 @@ async function createOrUpdateVideoPlaylist (playlistObject: PlaylistObject, byAc
     return Promise.resolve()
   })
 
-  // Empty playlists generally do not have a miniature, so skip it
+  // Empty playlists generally do not have a miniature, so skip this
   if (accItems.length !== 0) {
     try {
       await generateThumbnailFromUrl(playlist, playlistObject.icon)
@@ -107,13 +107,45 @@ async function createOrUpdateVideoPlaylist (playlistObject: PlaylistObject, byAc
   return resetVideoPlaylistElements(accItems, playlist)
 }
 
+async function refreshVideoPlaylistIfNeeded (videoPlaylist: VideoPlaylistModel): Promise<VideoPlaylistModel> {
+  if (!videoPlaylist.isOutdated()) return videoPlaylist
+
+  try {
+    const { statusCode, playlistObject } = await fetchRemoteVideoPlaylist(videoPlaylist.url)
+    if (statusCode === 404) {
+      logger.info('Cannot refresh remote video playlist %s: it does not exist anymore. Deleting it.', videoPlaylist.url)
+
+      await videoPlaylist.destroy()
+      return undefined
+    }
+
+    if (playlistObject === undefined) {
+      logger.warn('Cannot refresh remote playlist %s: invalid body.', videoPlaylist.url)
+
+      await videoPlaylist.setAsRefreshed()
+      return videoPlaylist
+    }
+
+    const byAccount = videoPlaylist.OwnerAccount
+    await createOrUpdateVideoPlaylist(playlistObject, byAccount, playlistObject.to)
+
+    return videoPlaylist
+  } catch (err) {
+    logger.warn('Cannot refresh video playlist %s.', videoPlaylist.url, { err })
+
+    await videoPlaylist.setAsRefreshed()
+    return videoPlaylist
+  }
+}
+
 // ---------------------------------------------------------------------------
 
 export {
   createAccountPlaylists,
   playlistObjectToDBAttributes,
   playlistElementObjectToDBAttributes,
-  createOrUpdateVideoPlaylist
+  createOrUpdateVideoPlaylist,
+  refreshVideoPlaylistIfNeeded
 }
 
 // ---------------------------------------------------------------------------
@@ -162,3 +194,23 @@ function generateThumbnailFromUrl (playlist: VideoPlaylistModel, icon: ActivityI
 
   return downloadImage(icon.url, CONFIG.STORAGE.THUMBNAILS_DIR, thumbnailName, THUMBNAILS_SIZE)
 }
+
+async function fetchRemoteVideoPlaylist (playlistUrl: string): Promise<{ statusCode: number, playlistObject: PlaylistObject }> {
+  const options = {
+    uri: playlistUrl,
+    method: 'GET',
+    json: true,
+    activityPub: true
+  }
+
+  logger.info('Fetching remote playlist %s.', playlistUrl)
+
+  const { response, body } = await doRequest(options)
+
+  if (isPlaylistObjectValid(body) === false || checkUrlsSameHost(body.id, playlistUrl) !== true) {
+    logger.debug('Remote video playlist JSON is not valid.', { body })
+    return { statusCode: response.statusCode, playlistObject: undefined }
+  }
+
+  return { statusCode: response.statusCode, playlistObject: body }
+}

server/lib/job-queue/handlers/activitypub-refresher.ts

@@ -1,11 +1,12 @@
 import * as Bull from 'bull'
 import { logger } from '../../../helpers/logger'
 import { fetchVideoByUrl } from '../../../helpers/video'
-import { refreshVideoIfNeeded, refreshActorIfNeeded } from '../../activitypub'
+import { refreshActorIfNeeded, refreshVideoIfNeeded, refreshVideoPlaylistIfNeeded } from '../../activitypub'
 import { ActorModel } from '../../../models/activitypub/actor'
+import { VideoPlaylistModel } from '../../../models/video/video-playlist'
 
 export type RefreshPayload = {
-  type: 'video' | 'actor'
+  type: 'video' | 'video-playlist' | 'actor'
   url: string
 }
 
@@ -15,13 +16,13 @@ async function refreshAPObject (job: Bull.Job) {
   logger.info('Processing AP refresher in job %d for %s.', job.id, payload.url)
 
   if (payload.type === 'video') return refreshVideo(payload.url)
+  if (payload.type === 'video-playlist') return refreshVideoPlaylist(payload.url)
   if (payload.type === 'actor') return refreshActor(payload.url)
 }
 
 // ---------------------------------------------------------------------------
 
 export {
-  refreshActor,
   refreshAPObject
 }
 
@@ -50,5 +51,12 @@ async function refreshActor (actorUrl: string) {
   if (actor) {
     await refreshActorIfNeeded(actor, fetchType)
   }
-
+}
+
+async function refreshVideoPlaylist (playlistUrl: string) {
+  const playlist = await VideoPlaylistModel.loadByUrlAndPopulateAccount(playlistUrl)
+
+  if (playlist) {
+    await refreshVideoPlaylistIfNeeded(playlist)
+  }
 }

server/middlewares/validators/videos/video-playlists.ts

@@ -8,8 +8,8 @@ import { doesVideoExist, isVideoImage } from '../../../helpers/custom-validators
 import { CONSTRAINTS_FIELDS } from '../../../initializers'
 import { isArrayOf, isIdOrUUIDValid, isIdValid, isUUIDValid, toIntArray, toValueOrNull } from '../../../helpers/custom-validators/misc'
 import {
-  isVideoPlaylistDescriptionValid,
   doesVideoPlaylistExist,
+  isVideoPlaylistDescriptionValid,
   isVideoPlaylistNameValid,
   isVideoPlaylistPrivacyValid,
   isVideoPlaylistTimestampValid,
@@ -19,7 +19,6 @@ import { VideoPlaylistModel } from '../../../models/video/video-playlist'
 import { cleanUpReqFiles } from '../../../helpers/express-utils'
 import { doesVideoChannelIdExist } from '../../../helpers/custom-validators/video-channels'
 import { VideoPlaylistElementModel } from '../../../models/video/video-playlist-element'
-import { VideoModel } from '../../../models/video/video'
 import { authenticatePromiseIfNeeded } from '../../oauth'
 import { VideoPlaylistPrivacy } from '../../../../shared/models/videos/playlist/video-playlist-privacy.model'
 import { VideoPlaylistType } from '../../../../shared/models/videos/playlist/video-playlist-type.model'

server/models/activitypub/actor.ts

@@ -34,7 +34,7 @@ import { ACTIVITY_PUB, ACTIVITY_PUB_ACTOR_TYPES, CONFIG, CONSTRAINTS_FIELDS } fr
 import { AccountModel } from '../account/account'
 import { AvatarModel } from '../avatar/avatar'
 import { ServerModel } from '../server/server'
-import { throwIfNotValid } from '../utils'
+import { isOutdated, throwIfNotValid } from '../utils'
 import { VideoChannelModel } from '../video/video-channel'
 import { ActorFollowModel } from './actor-follow'
 import { VideoModel } from '../video/video'
@@ -532,11 +532,6 @@ export class ActorModel extends Model<ActorModel> {
   isOutdated () {
     if (this.isOwned()) return false
 
-    const now = Date.now()
+    return isOutdated(this, ACTIVITY_PUB.ACTOR_REFRESH_INTERVAL)
-    const createdAtTime = this.createdAt.getTime()
-    const updatedAtTime = this.updatedAt.getTime()
-
-    return (now - createdAtTime) > ACTIVITY_PUB.ACTOR_REFRESH_INTERVAL &&
-      (now - updatedAtTime) > ACTIVITY_PUB.ACTOR_REFRESH_INTERVAL
   }
 }

server/models/utils.ts

@@ -1,5 +1,6 @@
 import { Sequelize } from 'sequelize-typescript'
 import * as validator from 'validator'
+import { ACTIVITY_PUB } from '../initializers'
 
 type SortType = { sortModel: any, sortValue: string }
 
@@ -44,6 +45,14 @@ function getSortOnModel (model: any, value: string, lastSort: string[] = [ 'id',
   return [ firstSort, lastSort ]
 }
 
+function isOutdated (model: { createdAt: Date, updatedAt: Date }, refreshInterval: number) {
+  const now = Date.now()
+  const createdAtTime = model.createdAt.getTime()
+  const updatedAtTime = model.updatedAt.getTime()
+
+  return (now - createdAtTime) > refreshInterval && (now - updatedAtTime) > refreshInterval
+}
+
 function throwIfNotValid (value: any, validator: (value: any) => boolean, fieldName = 'value') {
   if (validator(value) === false) {
     throw new Error(`"${value}" is not a valid ${fieldName}.`)
@@ -108,7 +117,8 @@ export {
   throwIfNotValid,
   buildServerIdsFollowedBy,
   buildTrigramSearchIndex,
-  buildWhereIdOrUUID
+  buildWhereIdOrUUID,
+  isOutdated
 }
 
 // ---------------------------------------------------------------------------

server/models/video/video-playlist.ts

@@ -17,7 +17,7 @@ import {
 } from 'sequelize-typescript'
 import * as Sequelize from 'sequelize'
 import { VideoPlaylistPrivacy } from '../../../shared/models/videos/playlist/video-playlist-privacy.model'
-import { buildServerIdsFollowedBy, buildWhereIdOrUUID, getSort, throwIfNotValid } from '../utils'
+import { buildServerIdsFollowedBy, buildWhereIdOrUUID, getSort, isOutdated, throwIfNotValid } from '../utils'
 import {
   isVideoPlaylistDescriptionValid,
   isVideoPlaylistNameValid,
@@ -25,6 +25,7 @@ import {
 } from '../../helpers/custom-validators/video-playlists'
 import { isActivityPubUrlValid } from '../../helpers/custom-validators/activitypub/misc'
 import {
+  ACTIVITY_PUB,
   CONFIG,
   CONSTRAINTS_FIELDS,
   STATIC_PATHS,
@@ -429,10 +430,22 @@ export class VideoPlaylistModel extends Model<VideoPlaylistModel> {
       .catch(err => logger.warn('Cannot delete thumbnail %s.', thumbnailPath, { err }))
   }
 
+  setAsRefreshed () {
+    this.changed('updatedAt', true)
+
+    return this.save()
+  }
+
   isOwned () {
     return this.OwnerAccount.isOwned()
   }
 
+  isOutdated () {
+    if (this.isOwned()) return false
+
+    return isOutdated(this, ACTIVITY_PUB.VIDEO_PLAYLIST_REFRESH_INTERVAL)
+  }
+
   toFormattedJSON (): VideoPlaylist {
     return {
       id: this.id,

server/models/video/video.ts

@@ -77,7 +77,7 @@ import {
   buildTrigramSearchIndex,
   buildWhereIdOrUUID,
   createSimilarityAttribute,
-  getVideoSort,
+  getVideoSort, isOutdated,
   throwIfNotValid
 } from '../utils'
 import { TagModel } from './tag'
@@ -1547,7 +1547,7 @@ export class VideoModel extends Model<VideoModel> {
       attributes: query.attributes,
       order: [ // Keep original order
         Sequelize.literal(
-          ids.map(id => `"VideoModel".id = ${id} DESC`).join(', ')
+          ids.map(id => `"VideoModel".id = ${id}`).join(', ')
         )
       ]
     }
@@ -1767,12 +1767,7 @@ export class VideoModel extends Model<VideoModel> {
   isOutdated () {
     if (this.isOwned()) return false
 
-    const now = Date.now()
+    return isOutdated(this, ACTIVITY_PUB.VIDEO_REFRESH_INTERVAL)
-    const createdAtTime = this.createdAt.getTime()
-    const updatedAtTime = this.updatedAt.getTime()
-
-    return (now - createdAtTime) > ACTIVITY_PUB.VIDEO_REFRESH_INTERVAL &&
-      (now - updatedAtTime) > ACTIVITY_PUB.VIDEO_REFRESH_INTERVAL
   }
 
   setAsRefreshed () {

server/tests/api/activitypub/refresher.ts

@@ -2,89 +2,155 @@
 
 import 'mocha'
 import {
+  createVideoPlaylist,
   doubleFollow,
   flushAndRunMultipleServers,
+  generateUserAccessToken,
   getVideo,
+  getVideoPlaylist,
   killallServers,
   reRunServer,
   ServerInfo,
   setAccessTokensToServers,
-  uploadVideo,
+  setActorField,
-  wait,
+  setDefaultVideoChannel,
+  setPlaylistField,
   setVideoField,
+  uploadVideo,
+  uploadVideoAndGetId,
+  wait,
   waitJobs
 } from '../../../../shared/utils'
+import { getAccount } from '../../../../shared/utils/users/accounts'
+import { VideoPlaylistPrivacy } from '../../../../shared/models/videos'
 
 describe('Test AP refresher', function () {
   let servers: ServerInfo[] = []
   let videoUUID1: string
   let videoUUID2: string
   let videoUUID3: string
+  let playlistUUID1: string
+  let playlistUUID2: string
 
   before(async function () {
     this.timeout(60000)
 
-    servers = await flushAndRunMultipleServers(2)
+    servers = await flushAndRunMultipleServers(2, { transcoding: { enabled: false } })
 
     // Get the access tokens
     await setAccessTokensToServers(servers)
+    await setDefaultVideoChannel(servers)
 
     {
-      const res = await uploadVideo(servers[1].url, servers[1].accessToken, { name: 'video1' })
+      videoUUID1 = (await uploadVideoAndGetId({ server: servers[ 1 ], videoName: 'video1' })).uuid
-      videoUUID1 = res.body.video.uuid
+      videoUUID2 = (await uploadVideoAndGetId({ server: servers[ 1 ], videoName: 'video2' })).uuid
+      videoUUID3 = (await uploadVideoAndGetId({ server: servers[ 1 ], videoName: 'video3' })).uuid
     }
 
     {
-      const res = await uploadVideo(servers[1].url, servers[1].accessToken, { name: 'video2' })
+      const a1 = await generateUserAccessToken(servers[1], 'user1')
-      videoUUID2 = res.body.video.uuid
+      await uploadVideo(servers[1].url, a1, { name: 'video4' })
+
+      const a2 = await generateUserAccessToken(servers[1], 'user2')
+      await uploadVideo(servers[1].url, a2, { name: 'video5' })
     }
 
     {
-      const res = await uploadVideo(servers[1].url, servers[1].accessToken, { name: 'video3' })
+      const playlistAttrs = { displayName: 'playlist1', privacy: VideoPlaylistPrivacy.PUBLIC, videoChannelId: servers[1].videoChannel.id }
-      videoUUID3 = res.body.video.uuid
+      const res = await createVideoPlaylist({ url: servers[1].url, token: servers[1].accessToken, playlistAttrs })
+      playlistUUID1 = res.body.videoPlaylist.uuid
+    }
+
+    {
+      const playlistAttrs = { displayName: 'playlist2', privacy: VideoPlaylistPrivacy.PUBLIC, videoChannelId: servers[1].videoChannel.id }
+      const res = await createVideoPlaylist({ url: servers[1].url, token: servers[1].accessToken, playlistAttrs })
+      playlistUUID2 = res.body.videoPlaylist.uuid
     }
 
     await doubleFollow(servers[0], servers[1])
   })
 
-  it('Should remove a deleted remote video', async function () {
+  describe('Videos refresher', function () {
-    this.timeout(60000)
 
-    await wait(10000)
+    it('Should remove a deleted remote video', async function () {
+      this.timeout(60000)
 
-    // Change UUID so the remote server returns a 404
+      await wait(10000)
-    await setVideoField(2, videoUUID1, 'uuid', '304afe4f-39f9-4d49-8ed7-ac57b86b174f')
 
-    await getVideo(servers[0].url, videoUUID1)
+      // Change UUID so the remote server returns a 404
-    await getVideo(servers[0].url, videoUUID2)
+      await setVideoField(2, videoUUID1, 'uuid', '304afe4f-39f9-4d49-8ed7-ac57b86b174f')
 
-    await waitJobs(servers)
+      await getVideo(servers[ 0 ].url, videoUUID1)
+      await getVideo(servers[ 0 ].url, videoUUID2)
 
-    await getVideo(servers[0].url, videoUUID1, 404)
+      await waitJobs(servers)
-    await getVideo(servers[0].url, videoUUID2, 200)
+
+      await getVideo(servers[ 0 ].url, videoUUID1, 404)
+      await getVideo(servers[ 0 ].url, videoUUID2, 200)
+    })
+
+    it('Should not update a remote video if the remote instance is down', async function () {
+      this.timeout(60000)
+
+      killallServers([ servers[ 1 ] ])
+
+      await setVideoField(2, videoUUID3, 'uuid', '304afe4f-39f9-4d49-8ed7-ac57b86b174e')
+
+      // Video will need a refresh
+      await wait(10000)
+
+      await getVideo(servers[ 0 ].url, videoUUID3)
+      // The refresh should fail
+      await waitJobs([ servers[ 0 ] ])
+
+      await reRunServer(servers[ 1 ])
+
+      // Should not refresh the video, even if the last refresh failed (to avoir a loop on dead instances)
+      await getVideo(servers[ 0 ].url, videoUUID3)
+      await waitJobs(servers)
+
+      await getVideo(servers[ 0 ].url, videoUUID3, 200)
+    })
   })
 
-  it('Should not update a remote video if the remote instance is down', async function () {
+  describe('Actors refresher', function () {
-    this.timeout(60000)
 
-    killallServers([ servers[1] ])
+    it('Should remove a deleted actor', async function () {
+      this.timeout(60000)
 
-    await setVideoField(2, videoUUID3, 'uuid', '304afe4f-39f9-4d49-8ed7-ac57b86b174e')
+      await wait(10000)
 
-    // Video will need a refresh
+      // Change actor name so the remote server returns a 404
-    await wait(10000)
+      await setActorField(2, 'http://localhost:9002/accounts/user2', 'preferredUsername', 'toto')
 
-    await getVideo(servers[0].url, videoUUID3)
+      await getAccount(servers[ 0 ].url, 'user1@localhost:9002')
-    // The refresh should fail
+      await getAccount(servers[ 0 ].url, 'user2@localhost:9002')
-    await waitJobs([ servers[0] ])
 
-    await reRunServer(servers[1])
+      await waitJobs(servers)
 
-    // Should not refresh the video, even if the last refresh failed (to avoir a loop on dead instances)
+      await getAccount(servers[ 0 ].url, 'user1@localhost:9002', 200)
-    await getVideo(servers[0].url, videoUUID3)
+      await getAccount(servers[ 0 ].url, 'user2@localhost:9002', 404)
-    await waitJobs(servers)
+    })
+  })
 
-    await getVideo(servers[0].url, videoUUID3, 200)
+  describe('Playlist refresher', function () {
+
+    it('Should remove a deleted playlist', async function () {
+      this.timeout(60000)
+
+      await wait(10000)
+
+      // Change UUID so the remote server returns a 404
+      await setPlaylistField(2, playlistUUID2, 'uuid', '304afe4f-39f9-4d49-8ed7-ac57b86b178e')
+
+      await getVideoPlaylist(servers[ 0 ].url, playlistUUID1)
+      await getVideoPlaylist(servers[ 0 ].url, playlistUUID2)
+
+      await waitJobs(servers)
+
+      await getVideoPlaylist(servers[ 0 ].url, playlistUUID1, 200)
+      await getVideoPlaylist(servers[ 0 ].url, playlistUUID2, 404)
+    })
   })
 
   after(async function () {

shared/utils/miscs/sql.ts

@@ -40,6 +40,14 @@ function setVideoField (serverNumber: number, uuid: string, field: string, value
   return seq.query(`UPDATE video SET "${field}" = '${value}' WHERE uuid = '${uuid}'`, options)
 }
 
+function setPlaylistField (serverNumber: number, uuid: string, field: string, value: string) {
+  const seq = getSequelize(serverNumber)
+
+  const options = { type: Sequelize.QueryTypes.UPDATE }
+
+  return seq.query(`UPDATE "videoPlaylist" SET "${field}" = '${value}' WHERE uuid = '${uuid}'`, options)
+}
+
 async function closeAllSequelize (servers: any[]) {
   for (let i = 1; i <= servers.length; i++) {
     if (sequelizes[ i ]) {
@@ -51,6 +59,7 @@ async function closeAllSequelize (servers: any[]) {
 
 export {
   setVideoField,
+  setPlaylistField,
   setActorField,
   closeAllSequelize
 }

shared/utils/videos/video-playlists.ts

@@ -127,7 +127,7 @@ function createVideoPlaylist (options: {
   playlistAttrs: VideoPlaylistCreate,
   expectedStatus?: number
 }) {
-  const path = '/api/v1/video-playlists/'
+  const path = '/api/v1/video-playlists'
 
   const fields = omit(options.playlistAttrs, 'thumbnailfile')