Compare commits

..

3 Commits

Author SHA1 Message Date
Alejandro
cf843c3f12 Text corrections 2023-12-06 08:45:06 +01:00
Alejandro
6966f37c4b Corrected reference to production.yaml 2023-12-06 08:45:06 +01:00
Chocobozzz
4fd8d34175
Ensure user is owned by plugin before updating it 2023-12-06 08:43:19 +01:00
26 changed files with 223 additions and 408 deletions

View File

@ -43,7 +43,7 @@ We have many important notes in this release. We know it's a pain for sysadmin,
* Directory on filesystem must be **renamed** from `videos/` to `web-videos/` to represent the value of `storage.web_videos` * Directory on filesystem must be **renamed** from `videos/` to `web-videos/` to represent the value of `storage.web_videos`
* Classic installation: `sudo -u peertube mv '/var/www/peertube/storage/videos/' '/var/www/peertube/storage/web-videos/'` * Classic installation: `sudo -u peertube mv '/var/www/peertube/storage/videos/' '/var/www/peertube/storage/web-videos/'`
* Docker installation: `mv '/path-to-docker-installation/docker-volume/data/videos/' '/path-to-docker-installation/docker-volume/data/web-videos/'` * Docker installation: `mv '/path-to-docker-installation/docker-volume/data/videos/' '/path-to-docker-installation/docker-volume/data/web-videos/'`
* `transcoding.webtorrent` must be **renamed** to `transcoding.web_videos`: https://github.com/Chocobozzz/PeerTube/blob/develop/config/production.yaml.example#L522 * `transcoding.webtorrent` must be **renamed** to `transcoding.web_videos`: https://github.com/Chocobozzz/PeerTube/blob/develop/config/production.yaml.example#L532
* `object_storage.videos` must be **renamed** to `object_storage.web_videos`. The value of `object_storage.web_videos.bucket_name` doesn't need to be changed: https://github.com/Chocobozzz/PeerTube/blob/develop/config/production.yaml.example#L223 * `object_storage.videos` must be **renamed** to `object_storage.web_videos`. The value of `object_storage.web_videos.bucket_name` doesn't need to be changed: https://github.com/Chocobozzz/PeerTube/blob/develop/config/production.yaml.example#L223
* `storage.storyboards` must be **added**: https://github.com/Chocobozzz/PeerTube/blob/develop/config/production.yaml.example#L157 * `storage.storyboards` must be **added**: https://github.com/Chocobozzz/PeerTube/blob/develop/config/production.yaml.example#L157
@ -61,7 +61,7 @@ We have many important notes in this release. We know it's a pain for sysadmin,
* `location ~ ^(/static/(webseed|streaming-playlists)/private/)|^/download {` must be updated to `location ~ ^(/static/(webseed|web-videos|streaming-playlists)/private/)|^/download {` * `location ~ ^(/static/(webseed|streaming-playlists)/private/)|^/download {` must be updated to `location ~ ^(/static/(webseed|web-videos|streaming-playlists)/private/)|^/download {`
* `location ~ ^/static/(webseed|redundancy|streaming-playlists)/ {` must be updated to `location ~ ^/static/(webseed|web-videos|redundancy|streaming-playlists)/ {` * `location ~ ^/static/(webseed|redundancy|streaming-playlists)/ {` must be updated to `location ~ ^/static/(webseed|web-videos|redundancy|streaming-playlists)/ {`
* Tracing requires `--experimental-loader=@opentelemetry/instrumentation/hook.mjs` node option: https://github.com/Chocobozzz/PeerTube/blob/develop/config/production.yaml.example#L263 * Tracing requires `--experimental-loader=@opentelemetry/instrumentation/hook.mjs` node option: https://github.com/Chocobozzz/PeerTube/blob/develop/config/production.yaml.example#L264
#### Developers important notes #### Developers important notes

View File

@ -116,11 +116,6 @@ export interface ActivityView extends BaseActivity {
// If sending a "viewer" event // If sending a "viewer" event
expires?: string expires?: string
result?: {
type: 'InteractionCounter'
interactionType: 'WatchAction'
userInteractionCount: number
}
} }
export interface ActivityDislike extends BaseActivity { export interface ActivityDislike extends BaseActivity {

View File

@ -18,7 +18,6 @@ export interface VideoObject {
licence: ActivityIdentifierObject licence: ActivityIdentifierObject
language: ActivityIdentifierObject language: ActivityIdentifierObject
subtitleLanguage: ActivityIdentifierObject[] subtitleLanguage: ActivityIdentifierObject[]
views: number views: number
sensitive: boolean sensitive: boolean

View File

@ -56,7 +56,3 @@ export function isProdInstance () {
export function getAppNumber () { export function getAppNumber () {
return process.env.NODE_APP_INSTANCE || '' return process.env.NODE_APP_INSTANCE || ''
} }
export function isUsingViewersFederationV2 () {
return process.env.USE_VIEWERS_FEDERATION_V2 === 'true'
}

View File

@ -21,7 +21,12 @@ describe('Test video views/viewers counters', function () {
} }
} }
function runTests () { before(async function () {
this.timeout(120000)
servers = await prepareViewsServers()
})
describe('Test views counter on VOD', function () { describe('Test views counter on VOD', function () {
let videoUUID: string let videoUUID: string
@ -87,17 +92,12 @@ describe('Test video views/viewers counters', function () {
it('Should view twice and display 1 view/viewer', async function () { it('Should view twice and display 1 view/viewer', async function () {
this.timeout(30000) this.timeout(30000)
for (let i = 0; i < 3; i++) {
await servers[0].views.simulateViewer({ id: liveVideoId, currentTimes: [ 0, 35 ] }) await servers[0].views.simulateViewer({ id: liveVideoId, currentTimes: [ 0, 35 ] })
await servers[0].views.simulateViewer({ id: liveVideoId, currentTimes: [ 0, 35 ] }) await servers[0].views.simulateViewer({ id: liveVideoId, currentTimes: [ 0, 35 ] })
await servers[0].views.simulateViewer({ id: vodVideoId, currentTimes: [ 0, 5 ] }) await servers[0].views.simulateViewer({ id: vodVideoId, currentTimes: [ 0, 5 ] })
await servers[0].views.simulateViewer({ id: vodVideoId, currentTimes: [ 0, 5 ] }) await servers[0].views.simulateViewer({ id: vodVideoId, currentTimes: [ 0, 5 ] })
await wait(1000)
}
await waitJobs(servers) await waitJobs(servers)
await checkCounter('viewers', liveVideoId, 1) await checkCounter('viewers', liveVideoId, 1)
await checkCounter('viewers', vodVideoId, 1) await checkCounter('viewers', vodVideoId, 1)
@ -108,31 +108,21 @@ describe('Test video views/viewers counters', function () {
}) })
it('Should wait and display 0 viewers but still have 1 view', async function () { it('Should wait and display 0 viewers but still have 1 view', async function () {
this.timeout(45000) this.timeout(30000)
let error = false await wait(12000)
await waitJobs(servers)
do {
try {
await checkCounter('views', liveVideoId, 1) await checkCounter('views', liveVideoId, 1)
await checkCounter('viewers', liveVideoId, 0) await checkCounter('viewers', liveVideoId, 0)
await checkCounter('views', vodVideoId, 1) await checkCounter('views', vodVideoId, 1)
await checkCounter('viewers', vodVideoId, 0) await checkCounter('viewers', vodVideoId, 0)
error = false
await wait(2500)
} catch {
error = true
}
} while (error)
}) })
it('Should view on a remote and on local and display appropriate views/viewers', async function () { it('Should view on a remote and on local and display 2 viewers and 3 views', async function () {
this.timeout(30000) this.timeout(30000)
await servers[0].views.simulateViewer({ id: vodVideoId, xForwardedFor: '0.0.0.1,127.0.0.1', currentTimes: [ 0, 5 ] })
await servers[0].views.simulateViewer({ id: vodVideoId, xForwardedFor: '0.0.0.1,127.0.0.1', currentTimes: [ 0, 5 ] })
await servers[0].views.simulateViewer({ id: vodVideoId, currentTimes: [ 0, 5 ] }) await servers[0].views.simulateViewer({ id: vodVideoId, currentTimes: [ 0, 5 ] })
await servers[1].views.simulateViewer({ id: vodVideoId, currentTimes: [ 0, 5 ] }) await servers[1].views.simulateViewer({ id: vodVideoId, currentTimes: [ 0, 5 ] })
await servers[1].views.simulateViewer({ id: vodVideoId, currentTimes: [ 0, 5 ] }) await servers[1].views.simulateViewer({ id: vodVideoId, currentTimes: [ 0, 5 ] })
@ -141,51 +131,23 @@ describe('Test video views/viewers counters', function () {
await servers[1].views.simulateViewer({ id: liveVideoId, currentTimes: [ 0, 35 ] }) await servers[1].views.simulateViewer({ id: liveVideoId, currentTimes: [ 0, 35 ] })
await servers[1].views.simulateViewer({ id: liveVideoId, currentTimes: [ 0, 35 ] }) await servers[1].views.simulateViewer({ id: liveVideoId, currentTimes: [ 0, 35 ] })
await wait(3000) // Throttled federation
await waitJobs(servers) await waitJobs(servers)
await checkCounter('viewers', liveVideoId, 2) await checkCounter('viewers', liveVideoId, 2)
await checkCounter('viewers', vodVideoId, 3) await checkCounter('viewers', vodVideoId, 2)
await processViewsBuffer(servers) await processViewsBuffer(servers)
await checkCounter('views', liveVideoId, 3) await checkCounter('views', liveVideoId, 3)
await checkCounter('views', vodVideoId, 4) await checkCounter('views', vodVideoId, 3)
}) })
after(async function () { after(async function () {
await stopFfmpeg(command) await stopFfmpeg(command)
}) })
}) })
}
describe('Federation V1', function () {
before(async function () {
this.timeout(120000)
servers = await prepareViewsServers({ viewExpiration: '5 seconds', viewersFederationV2: false })
})
runTests()
after(async function () { after(async function () {
await cleanupTests(servers) await cleanupTests(servers)
}) })
}) })
describe('Federation V2', function () {
before(async function () {
this.timeout(120000)
servers = await prepareViewsServers({ viewExpiration: '5 seconds', viewersFederationV2: true })
})
runTests()
after(async function () {
await cleanupTests(servers)
})
})
})

View File

@ -3,7 +3,6 @@
import { expect } from 'chai' import { expect } from 'chai'
import { prepareViewsServers, prepareViewsVideos, processViewersStats } from '@tests/shared/views.js' import { prepareViewsServers, prepareViewsVideos, processViewersStats } from '@tests/shared/views.js'
import { cleanupTests, PeerTubeServer } from '@peertube/peertube-server-commands' import { cleanupTests, PeerTubeServer } from '@peertube/peertube-server-commands'
import { wait } from '@peertube/peertube-core-utils'
describe('Test views retention stats', function () { describe('Test views retention stats', function () {
let servers: PeerTubeServer[] let servers: PeerTubeServer[]
@ -46,28 +45,6 @@ describe('Test views retention stats', function () {
expect(data.map(d => d.retentionPercent)).to.deep.equal([ 50, 75, 25, 25, 25, 0 ]) expect(data.map(d => d.retentionPercent)).to.deep.equal([ 50, 75, 25, 25, 25, 0 ])
}) })
it('Should display appropriate retention metrics after a server restart', async function () {
this.timeout(240000)
const newVideo = await servers[0].videos.quickUpload({ name: 'video 2' })
await servers[0].views.simulateViewer({ xForwardedFor: '127.0.0.2,127.0.0.1', id: newVideo.id, currentTimes: [ 0, 1 ] })
await servers[0].views.simulateViewer({ xForwardedFor: '127.0.0.3,127.0.0.1', id: newVideo.id, currentTimes: [ 1, 3 ] })
await wait(2500)
await servers[0].kill()
await servers[0].run()
await processViewersStats(servers)
const { data } = await servers[0].videoStats.getRetentionStats({ videoId: newVideo.id })
expect(data).to.have.lengthOf(6)
expect(data.map(d => d.retentionPercent)).to.deep.equal([ 50, 100, 50, 50, 0, 0 ])
})
}) })
after(async function () { after(async function () {

View File

@ -242,6 +242,29 @@ describe('Test id and pass auth plugins', function () {
expect(laguna.pluginAuth).to.equal('peertube-plugin-test-id-pass-auth-two') expect(laguna.pluginAuth).to.equal('peertube-plugin-test-id-pass-auth-two')
}) })
it('Should not update a user if not owned by the plugin auth', async function () {
{
await server.users.update({ userId: lagunaId, videoQuota: 43000, password: 'coucou', pluginAuth: null })
const body = await server.users.get({ userId: lagunaId })
expect(body.videoQuota).to.equal(43000)
expect(body.pluginAuth).to.be.null
}
{
await server.login.login({
user: { username: 'laguna', password: 'laguna password' },
expectedStatus: HttpStatusCode.BAD_REQUEST_400
})
}
{
const body = await server.users.get({ userId: lagunaId })
expect(body.videoQuota).to.equal(43000)
expect(body.pluginAuth).to.be.null
}
})
after(async function () { after(async function () {
await cleanupTests([ server ]) await cleanupTests([ server ])
}) })

View File

@ -30,17 +30,8 @@ async function processViewsBuffer (servers: PeerTubeServer[]) {
await waitJobs(servers) await waitJobs(servers)
} }
async function prepareViewsServers (options: { async function prepareViewsServers () {
viewersFederationV2?: boolean const servers = await createMultipleServers(2)
viewExpiration?: string // default 1 second
} = {}) {
const { viewExpiration = '1 second' } = options
const env = options?.viewersFederationV2 === true
? { USE_VIEWERS_FEDERATION_V2: 'true' }
: undefined
const servers = await createMultipleServers(2, { views: { videos: { ip_view_expiration: viewExpiration } } }, { env })
await setAccessTokensToServers(servers) await setAccessTokensToServers(servers)
await setDefaultVideoChannel(servers) await setDefaultVideoChannel(servers)

View File

@ -196,17 +196,11 @@ const contextStore: { [ id in ContextType ]: (string | { [ id: string ]: string
uuid: 'sc:identifier' uuid: 'sc:identifier'
}), }),
View: buildContext({
WatchAction: 'sc:WatchAction',
InteractionCounter: 'sc:InteractionCounter',
interactionType: 'sc:interactionType',
userInteractionCount: 'sc:userInteractionCount'
}),
Collection: buildContext(), Collection: buildContext(),
Follow: buildContext(), Follow: buildContext(),
Reject: buildContext(), Reject: buildContext(),
Accept: buildContext(), Accept: buildContext(),
View: buildContext(),
Announce: buildContext(), Announce: buildContext(),
Comment: buildContext(), Comment: buildContext(),
Delete: buildContext(), Delete: buildContext(),

View File

@ -9,6 +9,7 @@ export function Debounce (config: { timeoutMS: number }) {
timeoutRef = setTimeout(() => { timeoutRef = setTimeout(() => {
original.apply(this, args) original.apply(this, args)
}, config.timeoutMS) }, config.timeoutMS)
} }
} }

View File

@ -480,7 +480,6 @@ const VIEW_LIFETIME = {
VIEWER_COUNTER: 60000 * 2, // 2 minutes VIEWER_COUNTER: 60000 * 2, // 2 minutes
VIEWER_STATS: 60000 * 60 // 1 hour VIEWER_STATS: 60000 * 60 // 1 hour
} }
let VIEWER_SYNC_REDIS = 30000 // Sync viewer into redis
const MAX_LOCAL_VIEWER_WATCH_SECTIONS = 100 const MAX_LOCAL_VIEWER_WATCH_SECTIONS = 100
@ -899,9 +898,6 @@ const LRU_CACHE = {
USER_TOKENS: { USER_TOKENS: {
MAX_SIZE: 1000 MAX_SIZE: 1000
}, },
LOCAL_VIDEO_VIEWERS_FEDERATION: {
MAX_SIZE: 1000
},
FILENAME_TO_PATH_PERMANENT_FILE_CACHE: { FILENAME_TO_PATH_PERMANENT_FILE_CACHE: {
MAX_SIZE: 1000 MAX_SIZE: 1000
}, },
@ -1106,8 +1102,6 @@ if (process.env.PRODUCTION_CONSTANTS !== 'true') {
PLUGIN_EXTERNAL_AUTH_TOKEN_LIFETIME = 5000 PLUGIN_EXTERNAL_AUTH_TOKEN_LIFETIME = 5000
JOB_REMOVAL_OPTIONS.SUCCESS['videos-views-stats'] = 10000 JOB_REMOVAL_OPTIONS.SUCCESS['videos-views-stats'] = 10000
VIEWER_SYNC_REDIS = 1000
} }
if (isTestInstance()) { if (isTestInstance()) {
@ -1208,7 +1202,6 @@ export {
DEFAULT_THEME_NAME, DEFAULT_THEME_NAME,
NSFW_POLICY_TYPES, NSFW_POLICY_TYPES,
STATIC_MAX_AGE, STATIC_MAX_AGE,
VIEWER_SYNC_REDIS,
STATIC_PATHS, STATIC_PATHS,
VIDEO_IMPORT_TIMEOUT, VIDEO_IMPORT_TIMEOUT,
VIDEO_PLAYLIST_TYPES, VIDEO_PLAYLIST_TYPES,

View File

@ -28,15 +28,11 @@ async function processCreateView (activity: ActivityView, byActor: MActorSignatu
allowRefresh: false allowRefresh: false
}) })
await VideoViewsManager.Instance.processRemoteView({ const viewerExpires = activity.expires
video,
viewerId: activity.id,
viewerExpires: activity.expires
? new Date(activity.expires) ? new Date(activity.expires)
: undefined, : undefined
viewerResultCounter: getViewerResultCounter(activity)
}) await VideoViewsManager.Instance.processRemoteView({ video, viewerId: activity.id, viewerExpires })
if (video.isOwned()) { if (video.isOwned()) {
// Forward the view but don't resend the activity to the sender // Forward the view but don't resend the activity to the sender
@ -44,15 +40,3 @@ async function processCreateView (activity: ActivityView, byActor: MActorSignatu
await forwardVideoRelatedActivity(activity, undefined, exceptions, video) await forwardVideoRelatedActivity(activity, undefined, exceptions, video)
} }
} }
// Viewer protocol V2
function getViewerResultCounter (activity: ActivityView) {
const result = activity.result
if (!activity.expires || result?.interactionType !== 'WatchAction' || result?.type !== 'InteractionCounter') return undefined
const counter = parseInt(result.userInteractionCount + '')
if (isNaN(counter)) return undefined
return counter
}

View File

@ -6,23 +6,24 @@ import { logger } from '../../../helpers/logger.js'
import { audiencify, getAudience } from '../audience.js' import { audiencify, getAudience } from '../audience.js'
import { getLocalVideoViewActivityPubUrl } from '../url.js' import { getLocalVideoViewActivityPubUrl } from '../url.js'
import { sendVideoRelatedActivity } from './shared/send-utils.js' import { sendVideoRelatedActivity } from './shared/send-utils.js'
import { isUsingViewersFederationV2 } from '@peertube/peertube-node-utils'
type ViewType = 'view' | 'viewer'
async function sendView (options: { async function sendView (options: {
byActor: MActorLight byActor: MActorLight
type: ViewType
video: MVideoImmutable video: MVideoImmutable
viewerIdentifier: string viewerIdentifier: string
viewersCount?: number
transaction?: Transaction transaction?: Transaction
}) { }) {
const { byActor, viewersCount, video, viewerIdentifier, transaction } = options const { byActor, type, video, viewerIdentifier, transaction } = options
logger.info('Creating job to send %s of %s.', viewersCount !== undefined ? 'viewer' : 'view', video.url) logger.info('Creating job to send %s of %s.', type, video.url)
const activityBuilder = (audience: ActivityAudience) => { const activityBuilder = (audience: ActivityAudience) => {
const url = getLocalVideoViewActivityPubUrl(byActor, video, viewerIdentifier) const url = getLocalVideoViewActivityPubUrl(byActor, video, viewerIdentifier)
return buildViewActivity({ url, byActor, video, audience, viewersCount }) return buildViewActivity({ url, byActor, video, audience, type })
} }
return sendVideoRelatedActivity(activityBuilder, { byActor, video, transaction, contextType: 'View', parallelizable: true }) return sendVideoRelatedActivity(activityBuilder, { byActor, video, transaction, contextType: 'View', parallelizable: true })
@ -40,33 +41,22 @@ function buildViewActivity (options: {
url: string url: string
byActor: MActorAudience byActor: MActorAudience
video: MVideoUrl video: MVideoUrl
viewersCount?: number type: ViewType
audience?: ActivityAudience audience?: ActivityAudience
}): ActivityView { }): ActivityView {
const { url, byActor, viewersCount, video, audience = getAudience(byActor) } = options const { url, byActor, type, video, audience = getAudience(byActor) } = options
const base = { return audiencify(
{
id: url, id: url,
type: 'View' as 'View', type: 'View' as 'View',
actor: byActor.url, actor: byActor.url,
object: video.url object: video.url,
}
if (viewersCount === undefined) { expires: type === 'viewer'
return audiencify(base, audience) ? new Date(VideoViewsManager.Instance.buildViewerExpireTime()).toISOString()
}
return audiencify({
...base,
expires: new Date(VideoViewsManager.Instance.buildViewerExpireTime()).toISOString(),
result: isUsingViewersFederationV2()
? {
interactionType: 'WatchAction',
type: 'InteractionCounter',
userInteractionCount: viewersCount
}
: undefined : undefined
}, audience) },
audience
)
} }

View File

@ -89,8 +89,11 @@ async function getUser (usernameOrEmail?: string, password?: string, bypassLogin
let user = await UserModel.loadByEmail(bypassLogin.user.email) let user = await UserModel.loadByEmail(bypassLogin.user.email)
if (!user) user = await createUserFromExternal(bypassLogin.pluginName, bypassLogin.user) if (!user) {
else user = await updateUserFromExternal(user, bypassLogin.user, bypassLogin.userUpdater) user = await createUserFromExternal(bypassLogin.pluginName, bypassLogin.user)
} else if (user.pluginAuth === bypassLogin.pluginName) {
user = await updateUserFromExternal(user, bypassLogin.user, bypassLogin.userUpdater)
}
// Cannot create a user // Cannot create a user
if (!user) throw new AccessDeniedError('Cannot create such user: an actor with that name already exists.') if (!user) throw new AccessDeniedError('Cannot create such user: an actor with that name already exists.')

View File

@ -4,7 +4,7 @@ import express from 'express'
import { readFile } from 'fs/promises' import { readFile } from 'fs/promises'
import { join } from 'path' import { join } from 'path'
import { logger } from '../../../helpers/logger.js' import { logger } from '../../../helpers/logger.js'
import { CUSTOM_HTML_TAG_COMMENTS, FILES_CONTENT_HASH, PLUGIN_GLOBAL_CSS_PATH } from '../../../initializers/constants.js' import { CUSTOM_HTML_TAG_COMMENTS, FILES_CONTENT_HASH, PLUGIN_GLOBAL_CSS_PATH, WEBSERVER } from '../../../initializers/constants.js'
import { ServerConfigManager } from '../../server-config-manager.js' import { ServerConfigManager } from '../../server-config-manager.js'
import { TagsHtml } from './tags-html.js' import { TagsHtml } from './tags-html.js'
import { pathExists } from 'fs-extra/esm' import { pathExists } from 'fs-extra/esm'
@ -94,7 +94,7 @@ export class PageHtml {
// Save locale in cookies // Save locale in cookies
res.cookie('clientLanguage', lang, { res.cookie('clientLanguage', lang, {
secure: true, secure: WEBSERVER.SCHEME === 'https',
sameSite: 'none', sameSite: 'none',
maxAge: 1000 * 3600 * 24 * 90 // 3 months maxAge: 1000 * 3600 * 24 * 90 // 3 months
}) })

View File

@ -1,5 +1,4 @@
import { buildUUID, isTestOrDevInstance, isUsingViewersFederationV2, sha256 } from '@peertube/peertube-node-utils' import { buildUUID, isTestOrDevInstance, sha256 } from '@peertube/peertube-node-utils'
import { exists } from '@server/helpers/custom-validators/misc.js'
import { logger, loggerTagsFactory } from '@server/helpers/logger.js' import { logger, loggerTagsFactory } from '@server/helpers/logger.js'
import { VIEW_LIFETIME } from '@server/initializers/constants.js' import { VIEW_LIFETIME } from '@server/initializers/constants.js'
import { sendView } from '@server/lib/activitypub/send/send-view.js' import { sendView } from '@server/lib/activitypub/send/send-view.js'
@ -18,7 +17,6 @@ type Viewer = {
id: string id: string
viewerScope: ViewerScope viewerScope: ViewerScope
videoScope: VideoScope videoScope: VideoScope
viewerCount: number
lastFederation?: number lastFederation?: number
} }
@ -33,7 +31,7 @@ export class VideoViewerCounters {
private processingViewerCounters = false private processingViewerCounters = false
constructor () { constructor () {
setInterval(() => this.cleanViewerCounters(), VIEW_LIFETIME.VIEWER_COUNTER * 0.75) setInterval(() => this.cleanViewerCounters(), VIEW_LIFETIME.VIEWER_COUNTER)
} }
// --------------------------------------------------------------------------- // ---------------------------------------------------------------------------
@ -56,48 +54,22 @@ export class VideoViewerCounters {
return false return false
} }
const newViewer = this.addViewerToVideo({ viewerId, video, viewerScope: 'local', viewerCount: 1 }) const newViewer = await this.addViewerToVideo({ viewerId, video, viewerScope: 'local' })
await this.federateViewerIfNeeded(video, newViewer) await this.federateViewerIfNeeded(video, newViewer)
return true return true
} }
addRemoteViewerOnLocalVideo (options: { async addRemoteViewer (options: {
video: MVideo video: MVideo
viewerId: string viewerId: string
viewerExpires: Date viewerExpires: Date
}) { }) {
const { video, viewerExpires, viewerId } = options const { video, viewerExpires, viewerId } = options
logger.debug('Adding remote viewer to local video %s.', video.uuid, { viewerId, viewerExpires, ...lTags(video.uuid) }) logger.debug('Adding remote viewer to video %s.', video.uuid, { ...lTags(video.uuid) })
this.addViewerToVideo({ video, viewerExpires, viewerId, viewerScope: 'remote', viewerCount: 1 }) await this.addViewerToVideo({ video, viewerExpires, viewerId, viewerScope: 'remote' })
return true
}
addRemoteViewerOnRemoteVideo (options: {
video: MVideo
viewerId: string
viewerExpires: Date
viewerResultCounter?: number
}) {
const { video, viewerExpires, viewerId, viewerResultCounter } = options
logger.debug(
'Adding remote viewer to remote video %s.', video.uuid,
{ viewerId, viewerResultCounter, viewerExpires, ...lTags(video.uuid) }
)
this.addViewerToVideo({
video,
viewerExpires,
viewerId,
viewerScope: 'remote',
// The origin server sends a summary of all viewers, so we can replace our local copy
replaceCurrentViewers: exists(viewerResultCounter),
viewerCount: viewerResultCounter ?? 1
})
return true return true
} }
@ -111,17 +83,17 @@ export class VideoViewerCounters {
let total = 0 let total = 0
for (const viewers of this.viewersPerVideo.values()) { for (const viewers of this.viewersPerVideo.values()) {
total += viewers.filter(v => v.viewerScope === options.viewerScope && v.videoScope === options.videoScope) total += viewers.filter(v => v.viewerScope === options.viewerScope && v.videoScope === options.videoScope).length
.reduce((p, c) => p + c.viewerCount, 0)
} }
return total return total
} }
getTotalViewersOf (video: MVideoImmutable) { getViewers (video: MVideo) {
const viewers = this.viewersPerVideo.get(video.id) const viewers = this.viewersPerVideo.get(video.id)
if (!viewers) return 0
return viewers?.reduce((p, c) => p + c.viewerCount, 0) || 0 return viewers.length
} }
buildViewerExpireTime () { buildViewerExpireTime () {
@ -130,19 +102,17 @@ export class VideoViewerCounters {
// --------------------------------------------------------------------------- // ---------------------------------------------------------------------------
private addViewerToVideo (options: { private async addViewerToVideo (options: {
video: MVideoImmutable video: MVideoImmutable
viewerId: string viewerId: string
viewerScope: ViewerScope viewerScope: ViewerScope
viewerCount: number
replaceCurrentViewers?: boolean
viewerExpires?: Date viewerExpires?: Date
}) { }) {
const { video, viewerExpires, viewerId, viewerScope, viewerCount, replaceCurrentViewers } = options const { video, viewerExpires, viewerId, viewerScope } = options
let watchers = this.viewersPerVideo.get(video.id) let watchers = this.viewersPerVideo.get(video.id)
if (!watchers || replaceCurrentViewers) { if (!watchers) {
watchers = [] watchers = []
this.viewersPerVideo.set(video.id, watchers) this.viewersPerVideo.set(video.id, watchers)
} }
@ -155,12 +125,12 @@ export class VideoViewerCounters {
? 'remote' ? 'remote'
: 'local' : 'local'
const viewer = { id: viewerId, expires, videoScope, viewerScope, viewerCount } const viewer = { id: viewerId, expires, videoScope, viewerScope }
watchers.push(viewer) watchers.push(viewer)
this.idToViewer.set(viewerId, viewer) this.idToViewer.set(viewerId, viewer)
this.notifyClients(video) await this.notifyClients(video.id, watchers.length)
return viewer return viewer
} }
@ -192,16 +162,7 @@ export class VideoViewerCounters {
if (newViewers.length === 0) this.viewersPerVideo.delete(videoId) if (newViewers.length === 0) this.viewersPerVideo.delete(videoId)
else this.viewersPerVideo.set(videoId, newViewers) else this.viewersPerVideo.set(videoId, newViewers)
const video = await VideoModel.loadImmutableAttributes(videoId) await this.notifyClients(videoId, newViewers.length)
if (video) {
this.notifyClients(video)
// Let total viewers expire on remote instances if there are no more viewers
if (video.remote === false && newViewers.length !== 0) {
await this.federateTotalViewers(video)
}
}
} }
} catch (err) { } catch (err) {
logger.error('Error in video clean viewers scheduler.', { err, ...lTags() }) logger.error('Error in video clean viewers scheduler.', { err, ...lTags() })
@ -210,11 +171,13 @@ export class VideoViewerCounters {
this.processingViewerCounters = false this.processingViewerCounters = false
} }
private notifyClients (video: MVideoImmutable) { private async notifyClients (videoId: string | number, viewersLength: number) {
const totalViewers = this.getTotalViewersOf(video) const video = await VideoModel.loadImmutableAttributes(videoId)
PeerTubeSocket.Instance.sendVideoViewsUpdate(video, totalViewers) if (!video) return
logger.debug('Video viewers update for %s is %d.', video.url, totalViewers, lTags()) PeerTubeSocket.Instance.sendVideoViewsUpdate(video, viewersLength)
logger.debug('Video viewers update for %s is %d.', video.url, viewersLength, lTags())
} }
private generateViewerId (ip: string, videoUUID: string) { private generateViewerId (ip: string, videoUUID: string) {
@ -227,26 +190,8 @@ export class VideoViewerCounters {
const federationLimit = now - (VIEW_LIFETIME.VIEWER_COUNTER * 0.75) const federationLimit = now - (VIEW_LIFETIME.VIEWER_COUNTER * 0.75)
if (viewer.lastFederation && viewer.lastFederation > federationLimit) return if (viewer.lastFederation && viewer.lastFederation > federationLimit) return
if (video.remote === false && isUsingViewersFederationV2()) return
await sendView({
byActor: await getServerActor(),
video,
viewersCount: 1,
viewerIdentifier: viewer.id
})
await sendView({ byActor: await getServerActor(), video, type: 'viewer', viewerIdentifier: viewer.id })
viewer.lastFederation = now viewer.lastFederation = now
} }
private async federateTotalViewers (video: MVideoImmutable) {
if (!isUsingViewersFederationV2()) return
await sendView({
byActor: await getServerActor(),
video,
viewersCount: this.getTotalViewersOf(video),
viewerIdentifier: video.uuid
})
}
} }

View File

@ -3,7 +3,7 @@ import { VideoViewEvent } from '@peertube/peertube-models'
import { isTestOrDevInstance } from '@peertube/peertube-node-utils' import { isTestOrDevInstance } from '@peertube/peertube-node-utils'
import { GeoIP } from '@server/helpers/geo-ip.js' import { GeoIP } from '@server/helpers/geo-ip.js'
import { logger, loggerTagsFactory } from '@server/helpers/logger.js' import { logger, loggerTagsFactory } from '@server/helpers/logger.js'
import { MAX_LOCAL_VIEWER_WATCH_SECTIONS, VIEWER_SYNC_REDIS, VIEW_LIFETIME } from '@server/initializers/constants.js' import { MAX_LOCAL_VIEWER_WATCH_SECTIONS, VIEW_LIFETIME } from '@server/initializers/constants.js'
import { sequelizeTypescript } from '@server/initializers/database.js' import { sequelizeTypescript } from '@server/initializers/database.js'
import { sendCreateWatchAction } from '@server/lib/activitypub/send/index.js' import { sendCreateWatchAction } from '@server/lib/activitypub/send/index.js'
import { getLocalVideoViewerActivityPubUrl } from '@server/lib/activitypub/url.js' import { getLocalVideoViewerActivityPubUrl } from '@server/lib/activitypub/url.js'
@ -33,14 +33,11 @@ type LocalViewerStats = {
export class VideoViewerStats { export class VideoViewerStats {
private processingViewersStats = false private processingViewersStats = false
private processingRedisWrites = false
private readonly viewerCache = new Map<string, LocalViewerStats>() private readonly viewerCache = new Map<string, LocalViewerStats>()
private readonly redisPendingWrites = new Map<string, { ip: string, videoId: number, stats: LocalViewerStats }>()
constructor () { constructor () {
setInterval(() => this.processViewerStats(), VIEW_LIFETIME.VIEWER_STATS) setInterval(() => this.processViewerStats(), VIEW_LIFETIME.VIEWER_STATS)
setInterval(() => this.syncRedisWrites(), VIEWER_SYNC_REDIS)
} }
// --------------------------------------------------------------------------- // ---------------------------------------------------------------------------
@ -126,7 +123,7 @@ export class VideoViewerStats {
logger.debug('Set local video viewer stats for video %s.', video.uuid, { stats, ...lTags(video.uuid) }) logger.debug('Set local video viewer stats for video %s.', video.uuid, { stats, ...lTags(video.uuid) })
this.setLocalVideoViewer(ip, video.id, stats) await this.setLocalVideoViewer(ip, video.id, stats)
} }
async processViewerStats () { async processViewerStats () {
@ -138,8 +135,6 @@ export class VideoViewerStats {
const now = new Date().getTime() const now = new Date().getTime()
try { try {
await this.syncRedisWrites()
const allKeys = await Redis.Instance.listLocalVideoViewerKeys() const allKeys = await Redis.Instance.listLocalVideoViewerKeys()
for (const key of allKeys) { for (const key of allKeys) {
@ -227,7 +222,7 @@ export class VideoViewerStats {
const { viewerKey } = Redis.Instance.generateLocalVideoViewerKeys(ip, videoId) const { viewerKey } = Redis.Instance.generateLocalVideoViewerKeys(ip, videoId)
this.viewerCache.set(viewerKey, stats) this.viewerCache.set(viewerKey, stats)
this.redisPendingWrites.set(viewerKey, { ip, videoId, stats }) return Redis.Instance.setLocalVideoViewer(ip, videoId, stats)
} }
private deleteLocalVideoViewersKeys (key: string) { private deleteLocalVideoViewersKeys (key: string) {
@ -235,23 +230,4 @@ export class VideoViewerStats {
return Redis.Instance.deleteLocalVideoViewersKeys(key) return Redis.Instance.deleteLocalVideoViewersKeys(key)
} }
private async syncRedisWrites () {
if (this.processingRedisWrites) return
this.processingRedisWrites = true
for (const [ key, pendingWrite ] of this.redisPendingWrites) {
const { ip, videoId, stats } = pendingWrite
this.redisPendingWrites.delete(key)
try {
await Redis.Instance.setLocalVideoViewer(ip, videoId, stats)
} catch (err) {
logger.error('Cannot write viewer into redis', { ip, videoId, stats, err })
}
}
this.processingRedisWrites = false
}
} }

View File

@ -35,7 +35,7 @@ export class VideoViews {
await this.addView(video) await this.addView(video)
await sendView({ byActor: await getServerActor(), video, viewerIdentifier: buildUUID() }) await sendView({ byActor: await getServerActor(), video, type: 'view', viewerIdentifier: buildUUID() })
return true return true
} }

View File

@ -66,29 +66,17 @@ export class VideoViewsManager {
video: MVideo video: MVideo
viewerId: string | null viewerId: string | null
viewerExpires?: Date viewerExpires?: Date
viewerResultCounter?: number
}) { }) {
const { video, viewerId, viewerExpires, viewerResultCounter } = options const { video, viewerId, viewerExpires } = options
logger.debug('Processing remote view for %s.', video.url, { viewerExpires, viewerId, ...lTags() }) logger.debug('Processing remote view for %s.', video.url, { viewerExpires, viewerId, ...lTags() })
// Viewer if (viewerExpires) await this.videoViewerCounters.addRemoteViewer({ video, viewerId, viewerExpires })
if (viewerExpires) { else await this.videoViews.addRemoteView({ video })
if (video.remote === false) {
this.videoViewerCounters.addRemoteViewerOnLocalVideo({ video, viewerId, viewerExpires })
return
} }
this.videoViewerCounters.addRemoteViewerOnRemoteVideo({ video, viewerId, viewerExpires, viewerResultCounter }) getViewers (video: MVideo) {
return return this.videoViewerCounters.getViewers(video)
}
// Just a view
await this.videoViews.addRemoteView({ video })
}
getTotalViewersOf (video: MVideo) {
return this.videoViewerCounters.getTotalViewersOf(video)
} }
getTotalViewers (options: { getTotalViewers (options: {

View File

@ -1,3 +1,4 @@
import Bluebird from 'bluebird'
import { NextFunction, Request, RequestHandler, Response } from 'express' import { NextFunction, Request, RequestHandler, Response } from 'express'
import { ValidationChain } from 'express-validator' import { ValidationChain } from 'express-validator'
import { ExpressPromiseHandler } from '@server/types/express-handler.js' import { ExpressPromiseHandler } from '@server/types/express-handler.js'
@ -8,27 +9,22 @@ import { retryTransactionWrapper } from '../helpers/database-utils.js'
export type RequestPromiseHandler = ValidationChain | ExpressPromiseHandler export type RequestPromiseHandler = ValidationChain | ExpressPromiseHandler
function asyncMiddleware (fun: RequestPromiseHandler | RequestPromiseHandler[]) { function asyncMiddleware (fun: RequestPromiseHandler | RequestPromiseHandler[]) {
return async (req: Request, res: Response, next: NextFunction) => { return (req: Request, res: Response, next: NextFunction) => {
if (Array.isArray(fun) !== true) { if (Array.isArray(fun) === true) {
return Promise.resolve((fun as RequestHandler)(req, res, next)) return Bluebird.each(fun as RequestPromiseHandler[], f => {
.catch(err => next(err)) return new Promise<void>((resolve, reject) => {
}
try {
for (const f of (fun as RequestPromiseHandler[])) {
await new Promise<void>((resolve, reject) => {
return asyncMiddleware(f)(req, res, err => { return asyncMiddleware(f)(req, res, err => {
if (err) return reject(err) if (err) return reject(err)
return resolve() return resolve()
}) })
}) })
}).then(() => next())
.catch(err => next(err))
} }
next() return Promise.resolve((fun as RequestHandler)(req, res, next))
} catch (err) { .catch(err => next(err))
next(err)
}
} }
} }

View File

@ -873,6 +873,8 @@ export class UserModel extends Model<Partial<AttributesOnly<UserModel>>> {
} }
isPasswordMatch (password: string) { isPasswordMatch (password: string) {
if (!password || !this.password) return false
return comparePassword(password, this.password) return comparePassword(password, this.password)
} }

View File

@ -90,7 +90,7 @@ export function videoModelToFormattedJSON (video: MVideoFormattable, options: Vi
duration: video.duration, duration: video.duration,
views: video.views, views: video.views,
viewers: VideoViewsManager.Instance.getTotalViewersOf(video), viewers: VideoViewsManager.Instance.getViewers(video),
likes: video.likes, likes: video.likes,
dislikes: video.dislikes, dislikes: video.dislikes,

View File

@ -213,6 +213,9 @@ app.use(express.json({
} }
})) }))
// Cookies
app.use(cookieParser())
// W3C DNT Tracking Status // W3C DNT Tracking Status
app.use(advertiseDoNotTrack) app.use(advertiseDoNotTrack)
@ -227,6 +230,9 @@ app.use('/api/' + API_VERSION, apiRouter)
// Services (oembed...) // Services (oembed...)
app.use('/services', servicesRouter) app.use('/services', servicesRouter)
// Plugins & themes
app.use('/', pluginsRouter)
app.use('/', activityPubRouter) app.use('/', activityPubRouter)
app.use('/', feedsRouter) app.use('/', feedsRouter)
app.use('/', trackerRouter) app.use('/', trackerRouter)
@ -240,12 +246,6 @@ app.use('/', downloadRouter)
app.use('/', lazyStaticRouter) app.use('/', lazyStaticRouter)
app.use('/', objectStorageProxyRouter) app.use('/', objectStorageProxyRouter)
// Cookies for plugins and HTML
app.use(cookieParser())
// Plugins & themes
app.use('/', pluginsRouter)
// Client files, last valid routes! // Client files, last valid routes!
const cliOptions = cli.opts<{ client: boolean, plugins: boolean }>() const cliOptions = cli.opts<{ client: boolean, plugins: boolean }>()
if (cliOptions.client) app.use('/', clientsRouter) if (cliOptions.client) app.use('/', clientsRouter)

View File

@ -9766,10 +9766,10 @@ components:
description: P2P peers connected (doesn't include WebSeed peers) description: P2P peers connected (doesn't include WebSeed peers)
resolutionChanges: resolutionChanges:
type: number type: number
description: How many resolution changes occured since the last metric creation description: How many resolution changes occurred since the last metric creation
errors: errors:
type: number type: number
description: How many errors occured since the last metric creation description: How many errors occurred since the last metric creation
downloadedBytesP2P: downloadedBytesP2P:
type: number type: number
description: How many bytes were downloaded with P2P since the last metric creation description: How many bytes were downloaded with P2P since the last metric creation

View File

@ -870,7 +870,7 @@ function register ({ registerClientRoute }) {
} }
``` ```
You can then access the page on `/p/my-super/route` (please note the additionnal `/p/` in the path). You can then access the page on `/p/my-super/route` (please note the additional `/p/` in the path).
### Publishing ### Publishing

View File

@ -185,7 +185,7 @@ peertube-runner list-registered
## Server tools ## Server tools
Server tools are scripts that interect directly with the code of your PeerTube instance. Server tools are scripts that interact directly with the code of your PeerTube instance.
They must be run on the server, in `peertube-latest` directory. They must be run on the server, in `peertube-latest` directory.
### Parse logs ### Parse logs