Channel sync (#5135)
* Add external channel URL for channel update / creation (#754) * Disallow synchronisation if user has no video quota (#754) * More constraints serverside (#754) * Disable sync if server configuration does not allow HTTP import (#754) * Working version synchronizing videos with a job (#754) TODO: refactoring, too much code duplication * More logs and try/catch (#754) * Fix eslint error (#754) * WIP: support synchronization time change (#754) * New frontend #754 * WIP: Create sync front (#754) * Enhance UI, sync creation form (#754) * Warning message when HTTP upload is disallowed * More consistent names (#754) * Binding Front with API (#754) * Add a /me API (#754) * Improve list UI (#754) * Implement creation and deletion routes (#754) * Lint (#754) * Lint again (#754) * WIP: UI for triggering import existing videos (#754) * Implement jobs for syncing and importing channels * Don't sync videos before sync creation + avoid concurrency issue (#754) * Cleanup (#754) * Cleanup: OpenAPI + API rework (#754) * Remove dead code (#754) * Eslint (#754) * Revert the mess with whitespaces in constants.ts (#754) * Some fixes after rebase (#754) * Several fixes after PR remarks (#754) * Front + API: Rename video-channels-sync to video-channel-syncs (#754) * Allow enabling channel sync through UI (#754) * getChannelInfo (#754) * Minor fixes: openapi + model + sql (#754) * Simplified API validators (#754) * Rename MChannelSync to MChannelSyncChannel (#754) * Add command for VideoChannelSync (#754) * Use synchronization.enabled config (#754) * Check parameters test + some fixes (#754) * Fix conflict mistake (#754) * Restrict access to video channel sync list API (#754) * Start adding unit test for synchronization (#754) * Continue testing (#754) * Tests finished + convertion of job to scheduler (#754) * Add lastSyncAt field (#754) * Fix externalRemoteUrl sort + creation date not well formatted (#754) * Small fix (#754) * Factorize addYoutubeDLImport and buildVideo (#754) * Check duplicates on channel not on users (#754) * factorize thumbnail generation (#754) * Fetch error should return status 400 (#754) * Separate video-channel-import and video-channel-sync-latest (#754) * Bump DB migration version after rebase (#754) * Prettier states in UI table (#754) * Add DefaultScope in VideoChannelSyncModel (#754) * Fix audit logs (#754) * Ensure user can upload when importing channel + minor fixes (#754) * Mark synchronization as failed on exception + typos (#754) * Change REST API for importing videos into channel (#754) * Add option for fully synchronize a chnanel (#754) * Return a whole sync object on creation to avoid tricks in Front (#754) * Various remarks (#754) * Single quotes by default (#754) * Rename synchronization to video_channel_synchronization * Add check.latest_videos_count and max_per_user options (#754) * Better channel rendering in list #754 * Allow sorting with channel name and state (#754) * Add missing tests for channel imports (#754) * Prefer using a parent job for channel sync * Styling * Client styling Co-authored-by: Chocobozzz <me@florianbigard.com>
This commit is contained in:
parent
06ac128958
commit
2a491182e4
|
@ -285,6 +285,22 @@
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
</ng-container>
|
</ng-container>
|
||||||
|
|
||||||
|
<ng-container formGroupName="videoChannelSynchronization">
|
||||||
|
<div class="form-group">
|
||||||
|
<my-peertube-checkbox
|
||||||
|
inputName="importSynchronizationEnabled" formControlName="enabled"
|
||||||
|
i18n-labelText labelText="Allow channel synchronization with channel of other platforms like YouTube (requires allowing import with HTTP URL)"
|
||||||
|
>
|
||||||
|
<ng-container ngProjectAs="description">
|
||||||
|
<span i18n [hidden]="isImportVideosHttpEnabled()">
|
||||||
|
⛔ You need to allow import with HTTP URL to be able to activate this feature.
|
||||||
|
</span>
|
||||||
|
</ng-container>
|
||||||
|
</my-peertube-checkbox>
|
||||||
|
</div>
|
||||||
|
</ng-container>
|
||||||
|
|
||||||
</ng-container>
|
</ng-container>
|
||||||
|
|
||||||
<ng-container formGroupName="autoBlacklist">
|
<ng-container formGroupName="autoBlacklist">
|
||||||
|
|
|
@ -25,11 +25,12 @@ export class EditBasicConfigurationComponent implements OnInit, OnChanges {
|
||||||
private configService: ConfigService,
|
private configService: ConfigService,
|
||||||
private menuService: MenuService,
|
private menuService: MenuService,
|
||||||
private themeService: ThemeService
|
private themeService: ThemeService
|
||||||
) { }
|
) {}
|
||||||
|
|
||||||
ngOnInit () {
|
ngOnInit () {
|
||||||
this.buildLandingPageOptions()
|
this.buildLandingPageOptions()
|
||||||
this.checkSignupField()
|
this.checkSignupField()
|
||||||
|
this.checkImportSyncField()
|
||||||
|
|
||||||
this.availableThemes = this.themeService.buildAvailableThemes()
|
this.availableThemes = this.themeService.buildAvailableThemes()
|
||||||
}
|
}
|
||||||
|
@ -67,6 +68,14 @@ export class EditBasicConfigurationComponent implements OnInit, OnChanges {
|
||||||
return { 'disabled-checkbox-extra': !this.isSignupEnabled() }
|
return { 'disabled-checkbox-extra': !this.isSignupEnabled() }
|
||||||
}
|
}
|
||||||
|
|
||||||
|
isImportVideosHttpEnabled (): boolean {
|
||||||
|
return this.form.value['import']['videos']['http']['enabled'] === true
|
||||||
|
}
|
||||||
|
|
||||||
|
importSynchronizationChecked () {
|
||||||
|
return this.isImportVideosHttpEnabled() && this.form.value['import']['videoChannelSynchronization']['enabled']
|
||||||
|
}
|
||||||
|
|
||||||
hasUnlimitedSignup () {
|
hasUnlimitedSignup () {
|
||||||
return this.form.value['signup']['limit'] === -1
|
return this.form.value['signup']['limit'] === -1
|
||||||
}
|
}
|
||||||
|
@ -97,6 +106,21 @@ export class EditBasicConfigurationComponent implements OnInit, OnChanges {
|
||||||
return this.themeService.getDefaultThemeLabel()
|
return this.themeService.getDefaultThemeLabel()
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private checkImportSyncField () {
|
||||||
|
const importSyncControl = this.form.get('import.videoChannelSynchronization.enabled')
|
||||||
|
const importVideosHttpControl = this.form.get('import.videos.http.enabled')
|
||||||
|
|
||||||
|
importVideosHttpControl.valueChanges
|
||||||
|
.subscribe((httpImportEnabled) => {
|
||||||
|
importSyncControl.setValue(httpImportEnabled && importSyncControl.value)
|
||||||
|
if (httpImportEnabled) {
|
||||||
|
importSyncControl.enable()
|
||||||
|
} else {
|
||||||
|
importSyncControl.disable()
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
private checkSignupField () {
|
private checkSignupField () {
|
||||||
const signupControl = this.form.get('signup.enabled')
|
const signupControl = this.form.get('signup.enabled')
|
||||||
|
|
||||||
|
|
|
@ -144,6 +144,9 @@ export class EditCustomConfigComponent extends FormReactive implements OnInit {
|
||||||
torrent: {
|
torrent: {
|
||||||
enabled: null
|
enabled: null
|
||||||
}
|
}
|
||||||
|
},
|
||||||
|
videoChannelSynchronization: {
|
||||||
|
enabled: null
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
trending: {
|
trending: {
|
||||||
|
|
|
@ -38,7 +38,8 @@ export class JobsComponent extends RestTable implements OnInit {
|
||||||
'video-redundancy',
|
'video-redundancy',
|
||||||
'video-transcoding',
|
'video-transcoding',
|
||||||
'videos-views-stats',
|
'videos-views-stats',
|
||||||
'move-to-object-storage'
|
'move-to-object-storage',
|
||||||
|
'video-channel-import'
|
||||||
]
|
]
|
||||||
|
|
||||||
jobs: Job[] = []
|
jobs: Job[] = []
|
||||||
|
|
|
@ -61,7 +61,7 @@
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
<div class="form-group">
|
<div class="form-group">
|
||||||
<label for="support">Support</label>
|
<label i18n for="support">Support</label>
|
||||||
<my-help
|
<my-help
|
||||||
helpType="markdownEnhanced" i18n-preHtml preHtml="Short text to tell people how they can support the channel (membership platform...).<br /><br />
|
helpType="markdownEnhanced" i18n-preHtml preHtml="Short text to tell people how they can support the channel (membership platform...).<br /><br />
|
||||||
When a video is uploaded in this channel, the video support field will be automatically filled by this text."
|
When a video is uploaded in this channel, the video support field will be automatically filled by this text."
|
||||||
|
|
|
@ -1,7 +1,16 @@
|
||||||
<h1>
|
<h1>
|
||||||
|
<span>
|
||||||
<my-global-icon iconName="channel" aria-hidden="true"></my-global-icon>
|
<my-global-icon iconName="channel" aria-hidden="true"></my-global-icon>
|
||||||
<ng-container i18n>My channels</ng-container>
|
<ng-container i18n>My channels</ng-container>
|
||||||
<span *ngIf="totalItems" class="pt-badge badge-secondary">{{ totalItems }}</span>
|
<span *ngIf="totalItems" class="pt-badge badge-secondary">{{ totalItems }}</span>
|
||||||
|
</span>
|
||||||
|
|
||||||
|
<div>
|
||||||
|
<a routerLink="/my-library/video-channel-syncs" class="button-link">
|
||||||
|
<my-global-icon iconName="repeat" aria-hidden="true"></my-global-icon>
|
||||||
|
<ng-container i18n>My synchronizations</ng-container>
|
||||||
|
</a>
|
||||||
|
</div>
|
||||||
</h1>
|
</h1>
|
||||||
|
|
||||||
<my-channels-setup-message [hideLink]="true"></my-channels-setup-message>
|
<my-channels-setup-message [hideLink]="true"></my-channels-setup-message>
|
||||||
|
|
|
@ -1,9 +1,20 @@
|
||||||
@use '_variables' as *;
|
@use '_variables' as *;
|
||||||
@use '_mixins' as *;
|
@use '_mixins' as *;
|
||||||
|
|
||||||
h1 my-global-icon {
|
h1 {
|
||||||
|
display: flex;
|
||||||
|
justify-content: space-between;
|
||||||
|
|
||||||
|
my-global-icon {
|
||||||
position: relative;
|
position: relative;
|
||||||
top: -2px;
|
top: -2px;
|
||||||
|
}
|
||||||
|
|
||||||
|
.button-link {
|
||||||
|
@include peertube-button-link;
|
||||||
|
@include grey-button;
|
||||||
|
@include button-with-icon(18px, 3px, -1px);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
.create-button {
|
.create-button {
|
||||||
|
|
|
@ -6,6 +6,8 @@ import { MySubscriptionsComponent } from './my-follows/my-subscriptions.componen
|
||||||
import { MyHistoryComponent } from './my-history/my-history.component'
|
import { MyHistoryComponent } from './my-history/my-history.component'
|
||||||
import { MyLibraryComponent } from './my-library.component'
|
import { MyLibraryComponent } from './my-library.component'
|
||||||
import { MyOwnershipComponent } from './my-ownership/my-ownership.component'
|
import { MyOwnershipComponent } from './my-ownership/my-ownership.component'
|
||||||
|
import { MyVideoChannelSyncsComponent } from './my-video-channel-syncs/my-video-channel-syncs.component'
|
||||||
|
import { VideoChannelSyncEditComponent } from './my-video-channel-syncs/video-channel-sync-edit/video-channel-sync-edit.component'
|
||||||
import { MyVideoImportsComponent } from './my-video-imports/my-video-imports.component'
|
import { MyVideoImportsComponent } from './my-video-imports/my-video-imports.component'
|
||||||
import { MyVideoPlaylistCreateComponent } from './my-video-playlists/my-video-playlist-create.component'
|
import { MyVideoPlaylistCreateComponent } from './my-video-playlists/my-video-playlist-create.component'
|
||||||
import { MyVideoPlaylistElementsComponent } from './my-video-playlists/my-video-playlist-elements.component'
|
import { MyVideoPlaylistElementsComponent } from './my-video-playlists/my-video-playlist-elements.component'
|
||||||
|
@ -131,6 +133,26 @@ const myLibraryRoutes: Routes = [
|
||||||
key: 'my-videos-history-list'
|
key: 'my-videos-history-list'
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
},
|
||||||
|
|
||||||
|
{
|
||||||
|
path: 'video-channel-syncs',
|
||||||
|
component: MyVideoChannelSyncsComponent,
|
||||||
|
data: {
|
||||||
|
meta: {
|
||||||
|
title: $localize`My synchronizations`
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
|
||||||
|
{
|
||||||
|
path: 'video-channel-syncs/create',
|
||||||
|
component: VideoChannelSyncEditComponent,
|
||||||
|
data: {
|
||||||
|
meta: {
|
||||||
|
title: $localize`Create new synchronization`
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
]
|
]
|
||||||
}
|
}
|
||||||
|
|
|
@ -29,6 +29,8 @@ import { MyVideoPlaylistUpdateComponent } from './my-video-playlists/my-video-pl
|
||||||
import { MyVideoPlaylistsComponent } from './my-video-playlists/my-video-playlists.component'
|
import { MyVideoPlaylistsComponent } from './my-video-playlists/my-video-playlists.component'
|
||||||
import { VideoChangeOwnershipComponent } from './my-videos/modals/video-change-ownership.component'
|
import { VideoChangeOwnershipComponent } from './my-videos/modals/video-change-ownership.component'
|
||||||
import { MyVideosComponent } from './my-videos/my-videos.component'
|
import { MyVideosComponent } from './my-videos/my-videos.component'
|
||||||
|
import { MyVideoChannelSyncsComponent } from './my-video-channel-syncs/my-video-channel-syncs.component'
|
||||||
|
import { VideoChannelSyncEditComponent } from './my-video-channel-syncs/video-channel-sync-edit/video-channel-sync-edit.component'
|
||||||
|
|
||||||
@NgModule({
|
@NgModule({
|
||||||
imports: [
|
imports: [
|
||||||
|
@ -63,6 +65,8 @@ import { MyVideosComponent } from './my-videos/my-videos.component'
|
||||||
MyOwnershipComponent,
|
MyOwnershipComponent,
|
||||||
MyAcceptOwnershipComponent,
|
MyAcceptOwnershipComponent,
|
||||||
MyVideoImportsComponent,
|
MyVideoImportsComponent,
|
||||||
|
MyVideoChannelSyncsComponent,
|
||||||
|
VideoChannelSyncEditComponent,
|
||||||
MySubscriptionsComponent,
|
MySubscriptionsComponent,
|
||||||
MyFollowersComponent,
|
MyFollowersComponent,
|
||||||
MyHistoryComponent,
|
MyHistoryComponent,
|
||||||
|
|
|
@ -0,0 +1,83 @@
|
||||||
|
<div *ngIf="error" class="alert alert-danger">{{ error }}</div>
|
||||||
|
|
||||||
|
<h1>
|
||||||
|
<my-global-icon iconName="refresh" aria-hidden="true"></my-global-icon>
|
||||||
|
<ng-container i18n>My synchronizations</ng-container>
|
||||||
|
</h1>
|
||||||
|
|
||||||
|
<div *ngIf="!syncEnabled()">
|
||||||
|
<p class="muted" i18n>⚠️ The instance doesn't allow channel synchronization</p>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<p-table
|
||||||
|
*ngIf="syncEnabled()" [value]="channelSyncs" [lazy]="true"
|
||||||
|
[paginator]="totalRecords > 0" [totalRecords]="totalRecords" [rows]="rowsPerPage" [rowsPerPageOptions]="rowsPerPageOptions"
|
||||||
|
[sortField]="sort.field" [sortOrder]="sort.order" (onLazyLoad)="loadLazy($event)" dataKey="id"
|
||||||
|
[showCurrentPageReport]="true" i18n-currentPageReportTemplate
|
||||||
|
currentPageReportTemplate="Showing {{'{first}'}} to {{'{last}'}} of {{'{totalRecords}'}} synchronizations"
|
||||||
|
[expandedRowKeys]="expandedRows"
|
||||||
|
>
|
||||||
|
<ng-template pTemplate="caption">
|
||||||
|
<div class="caption">
|
||||||
|
<div class="left-buttons">
|
||||||
|
<a class="add-sync" routerLink="{{ getSyncCreateLink() }}">
|
||||||
|
<my-global-icon iconName="add" aria-hidden="true"></my-global-icon>
|
||||||
|
<ng-container i18n>Add synchronization</ng-container>
|
||||||
|
</a>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</ng-template>
|
||||||
|
|
||||||
|
<ng-template pTemplate="header">
|
||||||
|
<tr>
|
||||||
|
<th style="width: 10%"><my-global-icon iconName="columns"></my-global-icon></th>
|
||||||
|
<th style="width: 25%" i18n pSortableColumn="externalChannelUrl">External Channel <p-sortIcon field="externalChannelUrl"></p-sortIcon></th>
|
||||||
|
<th style="width: 25%" i18n pSortableColumn="videoChannel">Channel <p-sortIcon field="videoChannel"></p-sortIcon></th>
|
||||||
|
<th style="width: 10%" i18n pSortableColumn="state">State <p-sortIcon field="state"></p-sortIcon></th>
|
||||||
|
<th style="width: 10%" i18n pSortableColumn="createdAt">Created <p-sortIcon field="createdAt"></p-sortIcon></th>
|
||||||
|
<th style="width: 10%" i18n pSortableColumn="lastSyncAt">Last synchronization at <p-sortIcon field="lastSyncAt"></p-sortIcon></th>
|
||||||
|
</tr>
|
||||||
|
</ng-template>
|
||||||
|
|
||||||
|
<ng-template pTemplate="body" let-expanded="expanded" let-videoChannelSync>
|
||||||
|
<tr>
|
||||||
|
<td class="action-cell">
|
||||||
|
<my-action-dropdown
|
||||||
|
container="body"
|
||||||
|
[actions]="videoChannelSyncActions" [entry]="videoChannelSync"
|
||||||
|
></my-action-dropdown>
|
||||||
|
</td>
|
||||||
|
|
||||||
|
<td>
|
||||||
|
<a [href]="videoChannelSync.externalChannelUrl" target="_blank" rel="noopener noreferrer">{{ videoChannelSync.externalChannelUrl }}</a>
|
||||||
|
</td>
|
||||||
|
|
||||||
|
<td>
|
||||||
|
<div class="actor">
|
||||||
|
<my-actor-avatar
|
||||||
|
class="channel"
|
||||||
|
[actor]="videoChannelSync.channel" actorType="channel"
|
||||||
|
[internalHref]="[ '/c', videoChannelSync.channel.name ]"
|
||||||
|
size="25"
|
||||||
|
></my-actor-avatar>
|
||||||
|
|
||||||
|
<div class="actor-info">
|
||||||
|
<a [routerLink]="[ '/c', videoChannelSync.channel.name ]" class="actor-names" i18n-title title="Channel page">
|
||||||
|
<div class="actor-display-name">{{ videoChannelSync.channel.displayName }}</div>
|
||||||
|
<div class="actor-name">{{ videoChannelSync.channel.name }}</div>
|
||||||
|
</a>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</td>
|
||||||
|
|
||||||
|
<td>
|
||||||
|
<span [ngClass]="getSyncStateClass(videoChannelSync.state.id)">
|
||||||
|
{{ videoChannelSync.state.label }}
|
||||||
|
</span>
|
||||||
|
</td>
|
||||||
|
|
||||||
|
<td>{{ videoChannelSync.createdAt | date: 'short' }}</td>
|
||||||
|
<td>{{ videoChannelSync.lastSyncAt | date: 'short' }}</td>
|
||||||
|
</tr>
|
||||||
|
</ng-template>
|
||||||
|
</p-table>
|
|
@ -0,0 +1,14 @@
|
||||||
|
@use '_mixins' as *;
|
||||||
|
@use '_variables' as *;
|
||||||
|
@use '_actor' as *;
|
||||||
|
|
||||||
|
.add-sync {
|
||||||
|
@include create-button;
|
||||||
|
}
|
||||||
|
|
||||||
|
.actor {
|
||||||
|
@include actor-row($min-height: auto, $separator: true);
|
||||||
|
margin-bottom: 0;
|
||||||
|
padding-bottom: 0;
|
||||||
|
border: 0;
|
||||||
|
}
|
|
@ -0,0 +1,129 @@
|
||||||
|
import { Component, OnInit } from '@angular/core'
|
||||||
|
import { AuthService, Notifier, RestPagination, RestTable, ServerService } from '@app/core'
|
||||||
|
import { DropdownAction, VideoChannelService, VideoChannelSyncService } from '@app/shared/shared-main'
|
||||||
|
import { HTMLServerConfig } from '@shared/models/server'
|
||||||
|
import { VideoChannelSync, VideoChannelSyncState } from '@shared/models/videos'
|
||||||
|
import { SortMeta } from 'primeng/api'
|
||||||
|
import { mergeMap } from 'rxjs'
|
||||||
|
|
||||||
|
@Component({
|
||||||
|
templateUrl: './my-video-channel-syncs.component.html',
|
||||||
|
styleUrls: [ './my-video-channel-syncs.component.scss' ]
|
||||||
|
})
|
||||||
|
export class MyVideoChannelSyncsComponent extends RestTable implements OnInit {
|
||||||
|
error: string
|
||||||
|
|
||||||
|
channelSyncs: VideoChannelSync[] = []
|
||||||
|
totalRecords = 0
|
||||||
|
|
||||||
|
videoChannelSyncActions: DropdownAction<VideoChannelSync>[][] = []
|
||||||
|
sort: SortMeta = { field: 'createdAt', order: 1 }
|
||||||
|
pagination: RestPagination = { count: this.rowsPerPage, start: 0 }
|
||||||
|
|
||||||
|
private static STATE_CLASS_BY_ID = {
|
||||||
|
[VideoChannelSyncState.FAILED]: 'badge-red',
|
||||||
|
[VideoChannelSyncState.PROCESSING]: 'badge-blue',
|
||||||
|
[VideoChannelSyncState.SYNCED]: 'badge-green',
|
||||||
|
[VideoChannelSyncState.WAITING_FIRST_RUN]: 'badge-yellow'
|
||||||
|
}
|
||||||
|
|
||||||
|
private serverConfig: HTMLServerConfig
|
||||||
|
|
||||||
|
constructor (
|
||||||
|
private videoChannelsSyncService: VideoChannelSyncService,
|
||||||
|
private serverService: ServerService,
|
||||||
|
private notifier: Notifier,
|
||||||
|
private authService: AuthService,
|
||||||
|
private videoChannelService: VideoChannelService
|
||||||
|
) {
|
||||||
|
super()
|
||||||
|
}
|
||||||
|
|
||||||
|
ngOnInit () {
|
||||||
|
this.serverConfig = this.serverService.getHTMLConfig()
|
||||||
|
this.initialize()
|
||||||
|
|
||||||
|
this.videoChannelSyncActions = [
|
||||||
|
[
|
||||||
|
{
|
||||||
|
label: $localize`Delete`,
|
||||||
|
iconName: 'delete',
|
||||||
|
handler: videoChannelSync => this.deleteSync(videoChannelSync)
|
||||||
|
},
|
||||||
|
{
|
||||||
|
label: $localize`Fully synchronize the channel`,
|
||||||
|
description: $localize`This fetches any missing videos on the local channel`,
|
||||||
|
iconName: 'refresh',
|
||||||
|
handler: videoChannelSync => this.fullySynchronize(videoChannelSync)
|
||||||
|
}
|
||||||
|
]
|
||||||
|
]
|
||||||
|
}
|
||||||
|
|
||||||
|
protected reloadData () {
|
||||||
|
this.error = undefined
|
||||||
|
|
||||||
|
this.authService.userInformationLoaded
|
||||||
|
.pipe(mergeMap(() => {
|
||||||
|
const user = this.authService.getUser()
|
||||||
|
return this.videoChannelsSyncService.listAccountVideoChannelsSyncs({
|
||||||
|
sort: this.sort,
|
||||||
|
account: user.account,
|
||||||
|
pagination: this.pagination
|
||||||
|
})
|
||||||
|
}))
|
||||||
|
.subscribe({
|
||||||
|
next: res => {
|
||||||
|
this.channelSyncs = res.data
|
||||||
|
},
|
||||||
|
error: err => {
|
||||||
|
this.error = err.message
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
syncEnabled () {
|
||||||
|
return this.serverConfig.import.videoChannelSynchronization.enabled
|
||||||
|
}
|
||||||
|
|
||||||
|
deleteSync (videoChannelSync: VideoChannelSync) {
|
||||||
|
this.videoChannelsSyncService.deleteSync(videoChannelSync.id)
|
||||||
|
.subscribe({
|
||||||
|
next: () => {
|
||||||
|
this.notifier.success($localize`Synchronization removed successfully for ${videoChannelSync.channel.displayName}.`)
|
||||||
|
this.reloadData()
|
||||||
|
},
|
||||||
|
error: err => {
|
||||||
|
this.error = err.message
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
fullySynchronize (videoChannelSync: VideoChannelSync) {
|
||||||
|
this.videoChannelService.importVideos(videoChannelSync.channel.name, videoChannelSync.externalChannelUrl)
|
||||||
|
.subscribe({
|
||||||
|
next: () => {
|
||||||
|
this.notifier.success($localize`Full synchronization requested successfully for ${videoChannelSync.channel.displayName}.`)
|
||||||
|
},
|
||||||
|
error: err => {
|
||||||
|
this.error = err.message
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
getSyncCreateLink () {
|
||||||
|
return '/my-library/video-channel-syncs/create'
|
||||||
|
}
|
||||||
|
|
||||||
|
getSyncStateClass (stateId: number) {
|
||||||
|
return [ 'pt-badge', MyVideoChannelSyncsComponent.STATE_CLASS_BY_ID[stateId] ]
|
||||||
|
}
|
||||||
|
|
||||||
|
getIdentifier () {
|
||||||
|
return 'MyVideoChannelsSyncComponent'
|
||||||
|
}
|
||||||
|
|
||||||
|
getChannelUrl (name: string) {
|
||||||
|
return '/c/' + name
|
||||||
|
}
|
||||||
|
}
|
|
@ -0,0 +1,64 @@
|
||||||
|
<div *ngIf="error" class="alert alert-danger">{{ error }}</div>
|
||||||
|
|
||||||
|
<div class="margin-content">
|
||||||
|
<form role="form" (ngSubmit)="formValidated()" [formGroup]="form">
|
||||||
|
|
||||||
|
<div class="row">
|
||||||
|
<div class="col-12 col-lg-4 col-xl-3">
|
||||||
|
<div class="video-channel-sync-title" i18n>NEW SYNCHRONIZATION</div>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div class="col-12 col-lg-8 col-xl-9">
|
||||||
|
<div class="form-group">
|
||||||
|
<label i18n for="externalChannelUrl">Remote channel URL</label>
|
||||||
|
|
||||||
|
<div class="input-group">
|
||||||
|
<input
|
||||||
|
type="text"
|
||||||
|
id="externalChannelUrl"
|
||||||
|
i18n-placeholder
|
||||||
|
placeholder="Example: https://youtube.com/channel/UC_fancy_channel"
|
||||||
|
formControlName="externalChannelUrl"
|
||||||
|
[ngClass]="{ 'input-error': formErrors['externalChannelUrl'] }"
|
||||||
|
class="form-control"
|
||||||
|
>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div *ngIf="formErrors['externalChannelUrl']" class="form-error">
|
||||||
|
{{ formErrors['externalChannelUrl'] }}
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div class="form-group">
|
||||||
|
<label i18n for="videoChannel">Video Channel</label>
|
||||||
|
<my-select-channel required [items]="userVideoChannels" formControlName="videoChannel"></my-select-channel>
|
||||||
|
|
||||||
|
<div *ngIf="formErrors['videoChannel']" class="form-error">
|
||||||
|
{{ formErrors['videoChannel'] }}
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div class="form-group">
|
||||||
|
<label for="existingVideoStrategy" i18n>Options for existing videos on remote channel:</label>
|
||||||
|
|
||||||
|
<div class="peertube-radio-container">
|
||||||
|
<input type="radio" name="existingVideoStrategy" id="import" value="import" formControlName="existingVideoStrategy" required />
|
||||||
|
<label for="import" i18n>Import all and watch for new publications</label>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div class="peertube-radio-container">
|
||||||
|
<input type="radio" name="existingVideoStrategy" id="doNothing" value="nothing" formControlName="existingVideoStrategy" required />
|
||||||
|
<label for="doNothing" i18n>Only watch for new publications</label>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div class="row"> <!-- submit placement block -->
|
||||||
|
<div class="col-md-7 col-xl-5"></div>
|
||||||
|
<div class="col-md-5 col-xl-5 d-inline-flex">
|
||||||
|
<input type="submit" class="peertube-button orange-button ms-auto" value="{{ getFormButtonTitle() }}" [disabled]="!form.valid">
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</form>
|
||||||
|
</div>
|
|
@ -0,0 +1,17 @@
|
||||||
|
@use '_variables' as *;
|
||||||
|
@use '_mixins' as *;
|
||||||
|
|
||||||
|
$form-base-input-width: 480px;
|
||||||
|
|
||||||
|
input[type=text] {
|
||||||
|
@include peertube-input-text($form-base-input-width);
|
||||||
|
}
|
||||||
|
|
||||||
|
.video-channel-sync-title {
|
||||||
|
@include settings-big-title;
|
||||||
|
}
|
||||||
|
|
||||||
|
my-select-channel {
|
||||||
|
display: block;
|
||||||
|
max-width: $form-base-input-width;
|
||||||
|
}
|
|
@ -0,0 +1,76 @@
|
||||||
|
import { mergeMap } from 'rxjs'
|
||||||
|
import { SelectChannelItem } from 'src/types'
|
||||||
|
import { Component, OnInit } from '@angular/core'
|
||||||
|
import { Router } from '@angular/router'
|
||||||
|
import { AuthService, Notifier } from '@app/core'
|
||||||
|
import { listUserChannelsForSelect } from '@app/helpers'
|
||||||
|
import { VIDEO_CHANNEL_EXTERNAL_URL_VALIDATOR } from '@app/shared/form-validators/video-channel-validators'
|
||||||
|
import { FormReactive, FormValidatorService } from '@app/shared/shared-forms'
|
||||||
|
import { VideoChannelService, VideoChannelSyncService } from '@app/shared/shared-main'
|
||||||
|
import { VideoChannelSyncCreate } from '@shared/models/videos'
|
||||||
|
|
||||||
|
@Component({
|
||||||
|
selector: 'my-video-channel-sync-edit',
|
||||||
|
templateUrl: './video-channel-sync-edit.component.html',
|
||||||
|
styleUrls: [ './video-channel-sync-edit.component.scss' ]
|
||||||
|
})
|
||||||
|
export class VideoChannelSyncEditComponent extends FormReactive implements OnInit {
|
||||||
|
error: string
|
||||||
|
userVideoChannels: SelectChannelItem[] = []
|
||||||
|
existingVideosStrategy: string
|
||||||
|
|
||||||
|
constructor (
|
||||||
|
protected formValidatorService: FormValidatorService,
|
||||||
|
private authService: AuthService,
|
||||||
|
private router: Router,
|
||||||
|
private notifier: Notifier,
|
||||||
|
private videoChannelSyncService: VideoChannelSyncService,
|
||||||
|
private videoChannelService: VideoChannelService
|
||||||
|
) {
|
||||||
|
super()
|
||||||
|
}
|
||||||
|
|
||||||
|
ngOnInit () {
|
||||||
|
this.buildForm({
|
||||||
|
externalChannelUrl: VIDEO_CHANNEL_EXTERNAL_URL_VALIDATOR,
|
||||||
|
videoChannel: null,
|
||||||
|
existingVideoStrategy: null
|
||||||
|
})
|
||||||
|
|
||||||
|
listUserChannelsForSelect(this.authService)
|
||||||
|
.subscribe(channels => this.userVideoChannels = channels)
|
||||||
|
}
|
||||||
|
|
||||||
|
getFormButtonTitle () {
|
||||||
|
return $localize`Create`
|
||||||
|
}
|
||||||
|
|
||||||
|
formValidated () {
|
||||||
|
this.error = undefined
|
||||||
|
|
||||||
|
const body = this.form.value
|
||||||
|
const videoChannelSyncCreate: VideoChannelSyncCreate = {
|
||||||
|
externalChannelUrl: body.externalChannelUrl,
|
||||||
|
videoChannelId: body.videoChannel
|
||||||
|
}
|
||||||
|
|
||||||
|
const importExistingVideos = body['existingVideoStrategy'] === 'import'
|
||||||
|
|
||||||
|
this.videoChannelSyncService.createSync(videoChannelSyncCreate)
|
||||||
|
.pipe(mergeMap(({ videoChannelSync }) => {
|
||||||
|
return importExistingVideos
|
||||||
|
? this.videoChannelService.importVideos(videoChannelSync.channel.name, videoChannelSync.externalChannelUrl)
|
||||||
|
: Promise.resolve(null)
|
||||||
|
}))
|
||||||
|
.subscribe({
|
||||||
|
next: () => {
|
||||||
|
this.notifier.success($localize`Synchronization created successfully.`)
|
||||||
|
this.router.navigate([ '/my-library', 'video-channel-syncs' ])
|
||||||
|
},
|
||||||
|
|
||||||
|
error: err => {
|
||||||
|
this.error = err.message
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
|
@ -48,3 +48,16 @@ export const VIDEO_CHANNEL_SUPPORT_VALIDATOR: BuildFormValidator = {
|
||||||
maxlength: $localize`Support text cannot be more than 1000 characters long.`
|
maxlength: $localize`Support text cannot be more than 1000 characters long.`
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
export const VIDEO_CHANNEL_EXTERNAL_URL_VALIDATOR: BuildFormValidator = {
|
||||||
|
VALIDATORS: [
|
||||||
|
Validators.required,
|
||||||
|
Validators.pattern(/^https?:\/\//),
|
||||||
|
Validators.maxLength(1000)
|
||||||
|
],
|
||||||
|
MESSAGES: {
|
||||||
|
required: $localize`Remote channel url is required.`,
|
||||||
|
pattern: $localize`External channel URL must begin with "https://" or "http://"`,
|
||||||
|
maxlength: $localize`External channel URL cannot be more than 1000 characters long`
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
|
@ -106,6 +106,13 @@
|
||||||
</td>
|
</td>
|
||||||
</tr>
|
</tr>
|
||||||
|
|
||||||
|
<tr>
|
||||||
|
<th i18n class="sub-label" scope="row">Channel synchronization with other platforms (YouTube, Vimeo, ...)</th>
|
||||||
|
<td>
|
||||||
|
<my-feature-boolean [value]="serverConfig.import.videoChannelSynchronization.enabled"></my-feature-boolean>
|
||||||
|
</td>
|
||||||
|
</tr>
|
||||||
|
|
||||||
<tr>
|
<tr>
|
||||||
<th i18n class="label" colspan="2">Search</th>
|
<th i18n class="label" colspan="2">Search</th>
|
||||||
</tr>
|
</tr>
|
||||||
|
|
|
@ -13,3 +13,4 @@ export * from './video'
|
||||||
export * from './video-caption'
|
export * from './video-caption'
|
||||||
export * from './video-channel'
|
export * from './video-channel'
|
||||||
export * from './shared-main.module'
|
export * from './shared-main.module'
|
||||||
|
export * from './video-channel-sync'
|
||||||
|
|
|
@ -0,0 +1 @@
|
||||||
|
export * from './video-channel-sync.service'
|
|
@ -0,0 +1,50 @@
|
||||||
|
import { SortMeta } from 'primeng/api'
|
||||||
|
import { catchError, Observable } from 'rxjs'
|
||||||
|
import { environment } from 'src/environments/environment'
|
||||||
|
import { HttpClient, HttpParams } from '@angular/common/http'
|
||||||
|
import { Injectable } from '@angular/core'
|
||||||
|
import { RestExtractor, RestPagination, RestService } from '@app/core'
|
||||||
|
import { ResultList } from '@shared/models/common'
|
||||||
|
import { VideoChannelSync, VideoChannelSyncCreate } from '@shared/models/videos'
|
||||||
|
import { Account, AccountService } from '../account'
|
||||||
|
|
||||||
|
@Injectable({
|
||||||
|
providedIn: 'root'
|
||||||
|
})
|
||||||
|
export class VideoChannelSyncService {
|
||||||
|
static BASE_VIDEO_CHANNEL_URL = environment.apiUrl + '/api/v1/video-channel-syncs'
|
||||||
|
|
||||||
|
constructor (
|
||||||
|
private authHttp: HttpClient,
|
||||||
|
private restExtractor: RestExtractor,
|
||||||
|
private restService: RestService
|
||||||
|
) { }
|
||||||
|
|
||||||
|
listAccountVideoChannelsSyncs (parameters: {
|
||||||
|
sort: SortMeta
|
||||||
|
pagination: RestPagination
|
||||||
|
account: Account
|
||||||
|
}): Observable<ResultList<VideoChannelSync>> {
|
||||||
|
const { pagination, sort, account } = parameters
|
||||||
|
|
||||||
|
let params = new HttpParams()
|
||||||
|
params = this.restService.addRestGetParams(params, pagination, sort)
|
||||||
|
|
||||||
|
const url = AccountService.BASE_ACCOUNT_URL + account.nameWithHost + '/video-channel-syncs'
|
||||||
|
|
||||||
|
return this.authHttp.get<ResultList<VideoChannelSync>>(url, { params })
|
||||||
|
.pipe(catchError(err => this.restExtractor.handleError(err)))
|
||||||
|
}
|
||||||
|
|
||||||
|
createSync (body: VideoChannelSyncCreate) {
|
||||||
|
return this.authHttp.post<{ videoChannelSync: VideoChannelSync }>(VideoChannelSyncService.BASE_VIDEO_CHANNEL_URL, body)
|
||||||
|
.pipe(catchError(err => this.restExtractor.handleError(err)))
|
||||||
|
}
|
||||||
|
|
||||||
|
deleteSync (videoChannelsSyncId: number) {
|
||||||
|
const url = `${VideoChannelSyncService.BASE_VIDEO_CHANNEL_URL}/${videoChannelsSyncId}`
|
||||||
|
|
||||||
|
return this.authHttp.delete(url)
|
||||||
|
.pipe(catchError(err => this.restExtractor.handleError(err)))
|
||||||
|
}
|
||||||
|
}
|
|
@ -95,4 +95,10 @@ export class VideoChannelService {
|
||||||
return this.authHttp.delete(VideoChannelService.BASE_VIDEO_CHANNEL_URL + videoChannel.nameWithHost)
|
return this.authHttp.delete(VideoChannelService.BASE_VIDEO_CHANNEL_URL + videoChannel.nameWithHost)
|
||||||
.pipe(catchError(err => this.restExtractor.handleError(err)))
|
.pipe(catchError(err => this.restExtractor.handleError(err)))
|
||||||
}
|
}
|
||||||
|
|
||||||
|
importVideos (videoChannelName: string, externalChannelUrl: string) {
|
||||||
|
const path = VideoChannelService.BASE_VIDEO_CHANNEL_URL + videoChannelName + '/import-videos'
|
||||||
|
return this.authHttp.post(path, { externalChannelUrl })
|
||||||
|
.pipe(catchError(err => this.restExtractor.handleError(err)))
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -546,6 +546,17 @@ import:
|
||||||
# See https://docs.joinpeertube.org/maintain-configuration?id=security for more information
|
# See https://docs.joinpeertube.org/maintain-configuration?id=security for more information
|
||||||
enabled: false
|
enabled: false
|
||||||
|
|
||||||
|
# Add ability for your users to synchronize their channels with external channels, playlists, etc
|
||||||
|
video_channel_synchronization:
|
||||||
|
enabled: false
|
||||||
|
|
||||||
|
max_per_user: 10
|
||||||
|
|
||||||
|
check_interval: 1 hour
|
||||||
|
|
||||||
|
# Number of latest published videos to check and to potentially import when syncing a channel
|
||||||
|
videos_limit_per_synchronization: 10
|
||||||
|
|
||||||
auto_blacklist:
|
auto_blacklist:
|
||||||
# New videos automatically blacklisted so moderators can review before publishing
|
# New videos automatically blacklisted so moderators can review before publishing
|
||||||
videos:
|
videos:
|
||||||
|
|
|
@ -81,6 +81,11 @@ import:
|
||||||
enabled: true
|
enabled: true
|
||||||
torrent:
|
torrent:
|
||||||
enabled: true
|
enabled: true
|
||||||
|
video_channel_synchronization:
|
||||||
|
enabled: true
|
||||||
|
max_per_user: 10
|
||||||
|
check_interval: 5 minutes
|
||||||
|
videos_limit_per_synchronization: 3
|
||||||
|
|
||||||
instance:
|
instance:
|
||||||
default_nsfw_policy: 'display'
|
default_nsfw_policy: 'display'
|
||||||
|
|
|
@ -556,6 +556,17 @@ import:
|
||||||
# See https://docs.joinpeertube.org/maintain-configuration?id=security for more information
|
# See https://docs.joinpeertube.org/maintain-configuration?id=security for more information
|
||||||
enabled: false
|
enabled: false
|
||||||
|
|
||||||
|
# Add ability for your users to synchronize their channels with external channels, playlists, etc.
|
||||||
|
video_channel_synchronization:
|
||||||
|
enabled: false
|
||||||
|
|
||||||
|
max_per_user: 10
|
||||||
|
|
||||||
|
check_interval: 1 hour
|
||||||
|
|
||||||
|
# Number of latest published videos to check and to potentially import when syncing a channel
|
||||||
|
videos_limit_per_synchronization: 10
|
||||||
|
|
||||||
auto_blacklist:
|
auto_blacklist:
|
||||||
# New videos automatically blacklisted so moderators can review before publishing
|
# New videos automatically blacklisted so moderators can review before publishing
|
||||||
videos:
|
videos:
|
||||||
|
|
|
@ -139,6 +139,7 @@ import { VideoViewsManager } from '@server/lib/views/video-views-manager'
|
||||||
import { isTestOrDevInstance } from './server/helpers/core-utils'
|
import { isTestOrDevInstance } from './server/helpers/core-utils'
|
||||||
import { OpenTelemetryMetrics } from '@server/lib/opentelemetry/metrics'
|
import { OpenTelemetryMetrics } from '@server/lib/opentelemetry/metrics'
|
||||||
import { ApplicationModel } from '@server/models/application/application'
|
import { ApplicationModel } from '@server/models/application/application'
|
||||||
|
import { VideoChannelSyncLatestScheduler } from '@server/lib/schedulers/video-channel-sync-latest-scheduler'
|
||||||
|
|
||||||
// ----------- Command line -----------
|
// ----------- Command line -----------
|
||||||
|
|
||||||
|
@ -314,6 +315,7 @@ async function startApplication () {
|
||||||
PeerTubeVersionCheckScheduler.Instance.enable()
|
PeerTubeVersionCheckScheduler.Instance.enable()
|
||||||
AutoFollowIndexInstances.Instance.enable()
|
AutoFollowIndexInstances.Instance.enable()
|
||||||
RemoveDanglingResumableUploadsScheduler.Instance.enable()
|
RemoveDanglingResumableUploadsScheduler.Instance.enable()
|
||||||
|
VideoChannelSyncLatestScheduler.Instance.enable()
|
||||||
VideoViewsBufferScheduler.Instance.enable()
|
VideoViewsBufferScheduler.Instance.enable()
|
||||||
GeoIPUpdateScheduler.Instance.enable()
|
GeoIPUpdateScheduler.Instance.enable()
|
||||||
OpenTelemetryMetrics.Instance.registerMetrics()
|
OpenTelemetryMetrics.Instance.registerMetrics()
|
||||||
|
|
|
@ -25,8 +25,10 @@ import {
|
||||||
accountsFollowersSortValidator,
|
accountsFollowersSortValidator,
|
||||||
accountsSortValidator,
|
accountsSortValidator,
|
||||||
ensureAuthUserOwnsAccountValidator,
|
ensureAuthUserOwnsAccountValidator,
|
||||||
|
ensureCanManageUser,
|
||||||
videoChannelsSortValidator,
|
videoChannelsSortValidator,
|
||||||
videoChannelStatsValidator,
|
videoChannelStatsValidator,
|
||||||
|
videoChannelSyncsSortValidator,
|
||||||
videosSortValidator
|
videosSortValidator
|
||||||
} from '../../middlewares/validators'
|
} from '../../middlewares/validators'
|
||||||
import { commonVideoPlaylistFiltersValidator, videoPlaylistsSearchValidator } from '../../middlewares/validators/videos/video-playlists'
|
import { commonVideoPlaylistFiltersValidator, videoPlaylistsSearchValidator } from '../../middlewares/validators/videos/video-playlists'
|
||||||
|
@ -35,6 +37,7 @@ import { AccountVideoRateModel } from '../../models/account/account-video-rate'
|
||||||
import { VideoModel } from '../../models/video/video'
|
import { VideoModel } from '../../models/video/video'
|
||||||
import { VideoChannelModel } from '../../models/video/video-channel'
|
import { VideoChannelModel } from '../../models/video/video-channel'
|
||||||
import { VideoPlaylistModel } from '../../models/video/video-playlist'
|
import { VideoPlaylistModel } from '../../models/video/video-playlist'
|
||||||
|
import { VideoChannelSyncModel } from '@server/models/video/video-channel-sync'
|
||||||
|
|
||||||
const accountsRouter = express.Router()
|
const accountsRouter = express.Router()
|
||||||
|
|
||||||
|
@ -72,6 +75,17 @@ accountsRouter.get('/:accountName/video-channels',
|
||||||
asyncMiddleware(listAccountChannels)
|
asyncMiddleware(listAccountChannels)
|
||||||
)
|
)
|
||||||
|
|
||||||
|
accountsRouter.get('/:accountName/video-channel-syncs',
|
||||||
|
authenticate,
|
||||||
|
asyncMiddleware(accountNameWithHostGetValidator),
|
||||||
|
ensureCanManageUser,
|
||||||
|
paginationValidator,
|
||||||
|
videoChannelSyncsSortValidator,
|
||||||
|
setDefaultSort,
|
||||||
|
setDefaultPagination,
|
||||||
|
asyncMiddleware(listAccountChannelsSync)
|
||||||
|
)
|
||||||
|
|
||||||
accountsRouter.get('/:accountName/video-playlists',
|
accountsRouter.get('/:accountName/video-playlists',
|
||||||
optionalAuthenticate,
|
optionalAuthenticate,
|
||||||
asyncMiddleware(accountNameWithHostGetValidator),
|
asyncMiddleware(accountNameWithHostGetValidator),
|
||||||
|
@ -146,6 +160,20 @@ async function listAccountChannels (req: express.Request, res: express.Response)
|
||||||
return res.json(getFormattedObjects(resultList.data, resultList.total))
|
return res.json(getFormattedObjects(resultList.data, resultList.total))
|
||||||
}
|
}
|
||||||
|
|
||||||
|
async function listAccountChannelsSync (req: express.Request, res: express.Response) {
|
||||||
|
const options = {
|
||||||
|
accountId: res.locals.account.id,
|
||||||
|
start: req.query.start,
|
||||||
|
count: req.query.count,
|
||||||
|
sort: req.query.sort,
|
||||||
|
search: req.query.search
|
||||||
|
}
|
||||||
|
|
||||||
|
const resultList = await VideoChannelSyncModel.listByAccountForAPI(options)
|
||||||
|
|
||||||
|
return res.json(getFormattedObjects(resultList.data, resultList.total))
|
||||||
|
}
|
||||||
|
|
||||||
async function listAccountPlaylists (req: express.Request, res: express.Response) {
|
async function listAccountPlaylists (req: express.Request, res: express.Response) {
|
||||||
const serverActor = await getServerActor()
|
const serverActor = await getServerActor()
|
||||||
|
|
||||||
|
|
|
@ -273,6 +273,10 @@ function customConfig (): CustomConfig {
|
||||||
torrent: {
|
torrent: {
|
||||||
enabled: CONFIG.IMPORT.VIDEOS.TORRENT.ENABLED
|
enabled: CONFIG.IMPORT.VIDEOS.TORRENT.ENABLED
|
||||||
}
|
}
|
||||||
|
},
|
||||||
|
videoChannelSynchronization: {
|
||||||
|
enabled: CONFIG.IMPORT.VIDEO_CHANNEL_SYNCHRONIZATION.ENABLED,
|
||||||
|
maxPerUser: CONFIG.IMPORT.VIDEO_CHANNEL_SYNCHRONIZATION.MAX_PER_USER
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
trending: {
|
trending: {
|
||||||
|
|
|
@ -20,6 +20,7 @@ import { usersRouter } from './users'
|
||||||
import { videoChannelRouter } from './video-channel'
|
import { videoChannelRouter } from './video-channel'
|
||||||
import { videoPlaylistRouter } from './video-playlist'
|
import { videoPlaylistRouter } from './video-playlist'
|
||||||
import { videosRouter } from './videos'
|
import { videosRouter } from './videos'
|
||||||
|
import { videoChannelSyncRouter } from './video-channel-sync'
|
||||||
|
|
||||||
const apiRouter = express.Router()
|
const apiRouter = express.Router()
|
||||||
|
|
||||||
|
@ -43,6 +44,7 @@ apiRouter.use('/config', configRouter)
|
||||||
apiRouter.use('/users', usersRouter)
|
apiRouter.use('/users', usersRouter)
|
||||||
apiRouter.use('/accounts', accountsRouter)
|
apiRouter.use('/accounts', accountsRouter)
|
||||||
apiRouter.use('/video-channels', videoChannelRouter)
|
apiRouter.use('/video-channels', videoChannelRouter)
|
||||||
|
apiRouter.use('/video-channel-syncs', videoChannelSyncRouter)
|
||||||
apiRouter.use('/video-playlists', videoPlaylistRouter)
|
apiRouter.use('/video-playlists', videoPlaylistRouter)
|
||||||
apiRouter.use('/videos', videosRouter)
|
apiRouter.use('/videos', videosRouter)
|
||||||
apiRouter.use('/jobs', jobsRouter)
|
apiRouter.use('/jobs', jobsRouter)
|
||||||
|
|
|
@ -7,6 +7,7 @@ import { Debug, SendDebugCommand } from '@shared/models'
|
||||||
import { HttpStatusCode } from '../../../../shared/models/http/http-error-codes'
|
import { HttpStatusCode } from '../../../../shared/models/http/http-error-codes'
|
||||||
import { UserRight } from '../../../../shared/models/users'
|
import { UserRight } from '../../../../shared/models/users'
|
||||||
import { authenticate, ensureUserHasRight } from '../../../middlewares'
|
import { authenticate, ensureUserHasRight } from '../../../middlewares'
|
||||||
|
import { VideoChannelSyncLatestScheduler } from '@server/lib/schedulers/video-channel-sync-latest-scheduler'
|
||||||
|
|
||||||
const debugRouter = express.Router()
|
const debugRouter = express.Router()
|
||||||
|
|
||||||
|
@ -43,7 +44,8 @@ async function runCommand (req: express.Request, res: express.Response) {
|
||||||
const processors: { [id in SendDebugCommand['command']]: () => Promise<any> } = {
|
const processors: { [id in SendDebugCommand['command']]: () => Promise<any> } = {
|
||||||
'remove-dandling-resumable-uploads': () => RemoveDanglingResumableUploadsScheduler.Instance.execute(),
|
'remove-dandling-resumable-uploads': () => RemoveDanglingResumableUploadsScheduler.Instance.execute(),
|
||||||
'process-video-views-buffer': () => VideoViewsBufferScheduler.Instance.execute(),
|
'process-video-views-buffer': () => VideoViewsBufferScheduler.Instance.execute(),
|
||||||
'process-video-viewers': () => VideoViewsManager.Instance.processViewerStats()
|
'process-video-viewers': () => VideoViewsManager.Instance.processViewerStats(),
|
||||||
|
'process-video-channel-sync-latest': () => VideoChannelSyncLatestScheduler.Instance.execute()
|
||||||
}
|
}
|
||||||
|
|
||||||
await processors[body.command]()
|
await processors[body.command]()
|
||||||
|
|
76
server/controllers/api/video-channel-sync.ts
Normal file
76
server/controllers/api/video-channel-sync.ts
Normal file
|
@ -0,0 +1,76 @@
|
||||||
|
import express from 'express'
|
||||||
|
import { auditLoggerFactory, getAuditIdFromRes, VideoChannelSyncAuditView } from '@server/helpers/audit-logger'
|
||||||
|
import { logger } from '@server/helpers/logger'
|
||||||
|
import {
|
||||||
|
asyncMiddleware,
|
||||||
|
asyncRetryTransactionMiddleware,
|
||||||
|
authenticate,
|
||||||
|
ensureCanManageChannel as ensureCanManageSyncedChannel,
|
||||||
|
ensureSyncExists,
|
||||||
|
ensureSyncIsEnabled,
|
||||||
|
videoChannelSyncValidator
|
||||||
|
} from '@server/middlewares'
|
||||||
|
import { VideoChannelSyncModel } from '@server/models/video/video-channel-sync'
|
||||||
|
import { MChannelSyncFormattable } from '@server/types/models'
|
||||||
|
import { HttpStatusCode, VideoChannelSyncState } from '@shared/models'
|
||||||
|
|
||||||
|
const videoChannelSyncRouter = express.Router()
|
||||||
|
const auditLogger = auditLoggerFactory('channel-syncs')
|
||||||
|
|
||||||
|
videoChannelSyncRouter.post('/',
|
||||||
|
authenticate,
|
||||||
|
ensureSyncIsEnabled,
|
||||||
|
asyncMiddleware(videoChannelSyncValidator),
|
||||||
|
ensureCanManageSyncedChannel,
|
||||||
|
asyncRetryTransactionMiddleware(createVideoChannelSync)
|
||||||
|
)
|
||||||
|
|
||||||
|
videoChannelSyncRouter.delete('/:id',
|
||||||
|
authenticate,
|
||||||
|
asyncMiddleware(ensureSyncExists),
|
||||||
|
ensureCanManageSyncedChannel,
|
||||||
|
asyncRetryTransactionMiddleware(removeVideoChannelSync)
|
||||||
|
)
|
||||||
|
|
||||||
|
export { videoChannelSyncRouter }
|
||||||
|
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
async function createVideoChannelSync (req: express.Request, res: express.Response) {
|
||||||
|
const syncCreated: MChannelSyncFormattable = new VideoChannelSyncModel({
|
||||||
|
externalChannelUrl: req.body.externalChannelUrl,
|
||||||
|
videoChannelId: req.body.videoChannelId,
|
||||||
|
state: VideoChannelSyncState.WAITING_FIRST_RUN
|
||||||
|
})
|
||||||
|
|
||||||
|
await syncCreated.save()
|
||||||
|
syncCreated.VideoChannel = res.locals.videoChannel
|
||||||
|
|
||||||
|
auditLogger.create(getAuditIdFromRes(res), new VideoChannelSyncAuditView(syncCreated.toFormattedJSON()))
|
||||||
|
|
||||||
|
logger.info(
|
||||||
|
'Video synchronization for channel "%s" with external channel "%s" created.',
|
||||||
|
syncCreated.VideoChannel.name,
|
||||||
|
syncCreated.externalChannelUrl
|
||||||
|
)
|
||||||
|
|
||||||
|
return res.json({
|
||||||
|
videoChannelSync: syncCreated.toFormattedJSON()
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
async function removeVideoChannelSync (req: express.Request, res: express.Response) {
|
||||||
|
const syncInstance = res.locals.videoChannelSync
|
||||||
|
|
||||||
|
await syncInstance.destroy()
|
||||||
|
|
||||||
|
auditLogger.delete(getAuditIdFromRes(res), new VideoChannelSyncAuditView(syncInstance.toFormattedJSON()))
|
||||||
|
|
||||||
|
logger.info(
|
||||||
|
'Video synchronization for channel "%s" with external channel "%s" deleted.',
|
||||||
|
syncInstance.VideoChannel.name,
|
||||||
|
syncInstance.externalChannelUrl
|
||||||
|
)
|
||||||
|
|
||||||
|
return res.type('json').status(HttpStatusCode.NO_CONTENT_204).end()
|
||||||
|
}
|
|
@ -36,7 +36,9 @@ import {
|
||||||
videoPlaylistsSortValidator
|
videoPlaylistsSortValidator
|
||||||
} from '../../middlewares'
|
} from '../../middlewares'
|
||||||
import {
|
import {
|
||||||
|
ensureChannelOwnerCanUpload,
|
||||||
ensureIsLocalChannel,
|
ensureIsLocalChannel,
|
||||||
|
videoChannelImportVideosValidator,
|
||||||
videoChannelsFollowersSortValidator,
|
videoChannelsFollowersSortValidator,
|
||||||
videoChannelsListValidator,
|
videoChannelsListValidator,
|
||||||
videoChannelsNameWithHostValidator,
|
videoChannelsNameWithHostValidator,
|
||||||
|
@ -161,6 +163,16 @@ videoChannelRouter.get('/:nameWithHost/followers',
|
||||||
asyncMiddleware(listVideoChannelFollowers)
|
asyncMiddleware(listVideoChannelFollowers)
|
||||||
)
|
)
|
||||||
|
|
||||||
|
videoChannelRouter.post('/:nameWithHost/import-videos',
|
||||||
|
authenticate,
|
||||||
|
asyncMiddleware(videoChannelsNameWithHostValidator),
|
||||||
|
videoChannelImportVideosValidator,
|
||||||
|
ensureIsLocalChannel,
|
||||||
|
ensureCanManageChannel,
|
||||||
|
asyncMiddleware(ensureChannelOwnerCanUpload),
|
||||||
|
asyncMiddleware(importVideosInChannel)
|
||||||
|
)
|
||||||
|
|
||||||
// ---------------------------------------------------------------------------
|
// ---------------------------------------------------------------------------
|
||||||
|
|
||||||
export {
|
export {
|
||||||
|
@ -404,3 +416,19 @@ async function listVideoChannelFollowers (req: express.Request, res: express.Res
|
||||||
|
|
||||||
return res.json(getFormattedObjects(resultList.data, resultList.total))
|
return res.json(getFormattedObjects(resultList.data, resultList.total))
|
||||||
}
|
}
|
||||||
|
|
||||||
|
async function importVideosInChannel (req: express.Request, res: express.Response) {
|
||||||
|
const { externalChannelUrl } = req.body
|
||||||
|
|
||||||
|
await JobQueue.Instance.createJob({
|
||||||
|
type: 'video-channel-import',
|
||||||
|
payload: {
|
||||||
|
externalChannelUrl,
|
||||||
|
videoChannelId: res.locals.videoChannel.id
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
logger.info('Video import job for channel "%s" with url "%s" created.', res.locals.videoChannel.name, externalChannelUrl)
|
||||||
|
|
||||||
|
return res.type('json').status(HttpStatusCode.NO_CONTENT_204).end()
|
||||||
|
}
|
||||||
|
|
|
@ -1,49 +1,20 @@
|
||||||
import express from 'express'
|
import express from 'express'
|
||||||
import { move, readFile, remove } from 'fs-extra'
|
import { move, readFile } from 'fs-extra'
|
||||||
import { decode } from 'magnet-uri'
|
import { decode } from 'magnet-uri'
|
||||||
import parseTorrent, { Instance } from 'parse-torrent'
|
import parseTorrent, { Instance } from 'parse-torrent'
|
||||||
import { join } from 'path'
|
import { join } from 'path'
|
||||||
import { isVTTFileValid } from '@server/helpers/custom-validators/video-captions'
|
import { buildYoutubeDLImport, buildVideoFromImport, insertFromImportIntoDB, YoutubeDlImportError } from '@server/lib/video-import'
|
||||||
import { isVideoFileExtnameValid } from '@server/helpers/custom-validators/videos'
|
import { MThumbnail, MVideoThumbnail } from '@server/types/models'
|
||||||
import { isResolvingToUnicastOnly } from '@server/helpers/dns'
|
import { HttpStatusCode, ServerErrorCode, ThumbnailType, VideoImportCreate, VideoImportPayload, VideoImportState } from '@shared/models'
|
||||||
import { Hooks } from '@server/lib/plugins/hooks'
|
|
||||||
import { ServerConfigManager } from '@server/lib/server-config-manager'
|
|
||||||
import { setVideoTags } from '@server/lib/video'
|
|
||||||
import { FilteredModelAttributes } from '@server/types'
|
|
||||||
import {
|
|
||||||
MChannelAccountDefault,
|
|
||||||
MThumbnail,
|
|
||||||
MUser,
|
|
||||||
MVideoAccountDefault,
|
|
||||||
MVideoCaption,
|
|
||||||
MVideoTag,
|
|
||||||
MVideoThumbnail,
|
|
||||||
MVideoWithBlacklistLight
|
|
||||||
} from '@server/types/models'
|
|
||||||
import { MVideoImportFormattable } from '@server/types/models/video/video-import'
|
|
||||||
import {
|
|
||||||
HttpStatusCode,
|
|
||||||
ServerErrorCode,
|
|
||||||
ThumbnailType,
|
|
||||||
VideoImportCreate,
|
|
||||||
VideoImportState,
|
|
||||||
VideoPrivacy,
|
|
||||||
VideoState
|
|
||||||
} from '@shared/models'
|
|
||||||
import { auditLoggerFactory, getAuditIdFromRes, VideoImportAuditView } from '../../../helpers/audit-logger'
|
import { auditLoggerFactory, getAuditIdFromRes, VideoImportAuditView } from '../../../helpers/audit-logger'
|
||||||
import { moveAndProcessCaptionFile } from '../../../helpers/captions-utils'
|
|
||||||
import { isArray } from '../../../helpers/custom-validators/misc'
|
import { isArray } from '../../../helpers/custom-validators/misc'
|
||||||
import { cleanUpReqFiles, createReqFiles } from '../../../helpers/express-utils'
|
import { cleanUpReqFiles, createReqFiles } from '../../../helpers/express-utils'
|
||||||
import { logger } from '../../../helpers/logger'
|
import { logger } from '../../../helpers/logger'
|
||||||
import { getSecureTorrentName } from '../../../helpers/utils'
|
import { getSecureTorrentName } from '../../../helpers/utils'
|
||||||
import { YoutubeDLInfo, YoutubeDLWrapper } from '../../../helpers/youtube-dl'
|
|
||||||
import { CONFIG } from '../../../initializers/config'
|
import { CONFIG } from '../../../initializers/config'
|
||||||
import { MIMETYPES } from '../../../initializers/constants'
|
import { MIMETYPES } from '../../../initializers/constants'
|
||||||
import { sequelizeTypescript } from '../../../initializers/database'
|
|
||||||
import { getLocalVideoActivityPubUrl } from '../../../lib/activitypub/url'
|
|
||||||
import { JobQueue } from '../../../lib/job-queue/job-queue'
|
import { JobQueue } from '../../../lib/job-queue/job-queue'
|
||||||
import { updateVideoMiniatureFromExisting, updateVideoMiniatureFromUrl } from '../../../lib/thumbnail'
|
import { updateVideoMiniatureFromExisting } from '../../../lib/thumbnail'
|
||||||
import { autoBlacklistVideoIfNeeded } from '../../../lib/video-blacklist'
|
|
||||||
import {
|
import {
|
||||||
asyncMiddleware,
|
asyncMiddleware,
|
||||||
asyncRetryTransactionMiddleware,
|
asyncRetryTransactionMiddleware,
|
||||||
|
@ -52,9 +23,6 @@ import {
|
||||||
videoImportCancelValidator,
|
videoImportCancelValidator,
|
||||||
videoImportDeleteValidator
|
videoImportDeleteValidator
|
||||||
} from '../../../middlewares'
|
} from '../../../middlewares'
|
||||||
import { VideoModel } from '../../../models/video/video'
|
|
||||||
import { VideoCaptionModel } from '../../../models/video/video-caption'
|
|
||||||
import { VideoImportModel } from '../../../models/video/video-import'
|
|
||||||
|
|
||||||
const auditLogger = auditLoggerFactory('video-imports')
|
const auditLogger = auditLoggerFactory('video-imports')
|
||||||
const videoImportsRouter = express.Router()
|
const videoImportsRouter = express.Router()
|
||||||
|
@ -68,7 +36,7 @@ videoImportsRouter.post('/imports',
|
||||||
authenticate,
|
authenticate,
|
||||||
reqVideoFileImport,
|
reqVideoFileImport,
|
||||||
asyncMiddleware(videoImportAddValidator),
|
asyncMiddleware(videoImportAddValidator),
|
||||||
asyncRetryTransactionMiddleware(addVideoImport)
|
asyncRetryTransactionMiddleware(handleVideoImport)
|
||||||
)
|
)
|
||||||
|
|
||||||
videoImportsRouter.post('/imports/:id/cancel',
|
videoImportsRouter.post('/imports/:id/cancel',
|
||||||
|
@ -108,14 +76,14 @@ async function cancelVideoImport (req: express.Request, res: express.Response) {
|
||||||
return res.sendStatus(HttpStatusCode.NO_CONTENT_204)
|
return res.sendStatus(HttpStatusCode.NO_CONTENT_204)
|
||||||
}
|
}
|
||||||
|
|
||||||
function addVideoImport (req: express.Request, res: express.Response) {
|
function handleVideoImport (req: express.Request, res: express.Response) {
|
||||||
if (req.body.targetUrl) return addYoutubeDLImport(req, res)
|
if (req.body.targetUrl) return handleYoutubeDlImport(req, res)
|
||||||
|
|
||||||
const file = req.files?.['torrentfile']?.[0]
|
const file = req.files?.['torrentfile']?.[0]
|
||||||
if (req.body.magnetUri || file) return addTorrentImport(req, res, file)
|
if (req.body.magnetUri || file) return handleTorrentImport(req, res, file)
|
||||||
}
|
}
|
||||||
|
|
||||||
async function addTorrentImport (req: express.Request, res: express.Response, torrentfile: Express.Multer.File) {
|
async function handleTorrentImport (req: express.Request, res: express.Response, torrentfile: Express.Multer.File) {
|
||||||
const body: VideoImportCreate = req.body
|
const body: VideoImportCreate = req.body
|
||||||
const user = res.locals.oauth.token.User
|
const user = res.locals.oauth.token.User
|
||||||
|
|
||||||
|
@ -135,12 +103,17 @@ async function addTorrentImport (req: express.Request, res: express.Response, to
|
||||||
videoName = result.name
|
videoName = result.name
|
||||||
}
|
}
|
||||||
|
|
||||||
const video = await buildVideo(res.locals.videoChannel.id, body, { name: videoName })
|
const video = await buildVideoFromImport({
|
||||||
|
channelId: res.locals.videoChannel.id,
|
||||||
|
importData: { name: videoName },
|
||||||
|
importDataOverride: body,
|
||||||
|
importType: 'torrent'
|
||||||
|
})
|
||||||
|
|
||||||
const thumbnailModel = await processThumbnail(req, video)
|
const thumbnailModel = await processThumbnail(req, video)
|
||||||
const previewModel = await processPreview(req, video)
|
const previewModel = await processPreview(req, video)
|
||||||
|
|
||||||
const videoImport = await insertIntoDB({
|
const videoImport = await insertFromImportIntoDB({
|
||||||
video,
|
video,
|
||||||
thumbnailModel,
|
thumbnailModel,
|
||||||
previewModel,
|
previewModel,
|
||||||
|
@ -155,13 +128,12 @@ async function addTorrentImport (req: express.Request, res: express.Response, to
|
||||||
}
|
}
|
||||||
})
|
})
|
||||||
|
|
||||||
// Create job to import the video
|
const payload: VideoImportPayload = {
|
||||||
const payload = {
|
|
||||||
type: torrentfile
|
type: torrentfile
|
||||||
? 'torrent-file' as 'torrent-file'
|
? 'torrent-file'
|
||||||
: 'magnet-uri' as 'magnet-uri',
|
: 'magnet-uri',
|
||||||
videoImportId: videoImport.id,
|
videoImportId: videoImport.id,
|
||||||
magnetUri
|
preventException: false
|
||||||
}
|
}
|
||||||
await JobQueue.Instance.createJob({ type: 'video-import', payload })
|
await JobQueue.Instance.createJob({ type: 'video-import', payload })
|
||||||
|
|
||||||
|
@ -170,131 +142,49 @@ async function addTorrentImport (req: express.Request, res: express.Response, to
|
||||||
return res.json(videoImport.toFormattedJSON()).end()
|
return res.json(videoImport.toFormattedJSON()).end()
|
||||||
}
|
}
|
||||||
|
|
||||||
async function addYoutubeDLImport (req: express.Request, res: express.Response) {
|
function statusFromYtDlImportError (err: YoutubeDlImportError): number {
|
||||||
|
switch (err.code) {
|
||||||
|
case YoutubeDlImportError.CODE.NOT_ONLY_UNICAST_URL:
|
||||||
|
return HttpStatusCode.FORBIDDEN_403
|
||||||
|
|
||||||
|
case YoutubeDlImportError.CODE.FETCH_ERROR:
|
||||||
|
return HttpStatusCode.BAD_REQUEST_400
|
||||||
|
|
||||||
|
default:
|
||||||
|
return HttpStatusCode.INTERNAL_SERVER_ERROR_500
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async function handleYoutubeDlImport (req: express.Request, res: express.Response) {
|
||||||
const body: VideoImportCreate = req.body
|
const body: VideoImportCreate = req.body
|
||||||
const targetUrl = body.targetUrl
|
const targetUrl = body.targetUrl
|
||||||
const user = res.locals.oauth.token.User
|
const user = res.locals.oauth.token.User
|
||||||
|
|
||||||
const youtubeDL = new YoutubeDLWrapper(
|
|
||||||
targetUrl,
|
|
||||||
ServerConfigManager.Instance.getEnabledResolutions('vod'),
|
|
||||||
CONFIG.TRANSCODING.ALWAYS_TRANSCODE_ORIGINAL_RESOLUTION
|
|
||||||
)
|
|
||||||
|
|
||||||
// Get video infos
|
|
||||||
let youtubeDLInfo: YoutubeDLInfo
|
|
||||||
try {
|
try {
|
||||||
youtubeDLInfo = await youtubeDL.getInfoForDownload()
|
const { job, videoImport } = await buildYoutubeDLImport({
|
||||||
|
targetUrl,
|
||||||
|
channel: res.locals.videoChannel,
|
||||||
|
importDataOverride: body,
|
||||||
|
thumbnailFilePath: req.files?.['thumbnailfile']?.[0].path,
|
||||||
|
previewFilePath: req.files?.['previewfile']?.[0].path,
|
||||||
|
user
|
||||||
|
})
|
||||||
|
await JobQueue.Instance.createJob(job)
|
||||||
|
|
||||||
|
auditLogger.create(getAuditIdFromRes(res), new VideoImportAuditView(videoImport.toFormattedJSON()))
|
||||||
|
|
||||||
|
return res.json(videoImport.toFormattedJSON()).end()
|
||||||
} catch (err) {
|
} catch (err) {
|
||||||
logger.info('Cannot fetch information from import for URL %s.', targetUrl, { err })
|
logger.error('An error occurred while importing the video %s. ', targetUrl, { err })
|
||||||
|
|
||||||
return res.fail({
|
return res.fail({
|
||||||
message: 'Cannot fetch remote information of this URL.',
|
message: err.message,
|
||||||
|
status: statusFromYtDlImportError(err),
|
||||||
data: {
|
data: {
|
||||||
targetUrl
|
targetUrl
|
||||||
}
|
}
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
if (!await hasUnicastURLsOnly(youtubeDLInfo)) {
|
|
||||||
return res.fail({
|
|
||||||
status: HttpStatusCode.FORBIDDEN_403,
|
|
||||||
message: 'Cannot use non unicast IP as targetUrl.'
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
const video = await buildVideo(res.locals.videoChannel.id, body, youtubeDLInfo)
|
|
||||||
|
|
||||||
// Process video thumbnail from request.files
|
|
||||||
let thumbnailModel = await processThumbnail(req, video)
|
|
||||||
|
|
||||||
// Process video thumbnail from url if processing from request.files failed
|
|
||||||
if (!thumbnailModel && youtubeDLInfo.thumbnailUrl) {
|
|
||||||
try {
|
|
||||||
thumbnailModel = await processThumbnailFromUrl(youtubeDLInfo.thumbnailUrl, video)
|
|
||||||
} catch (err) {
|
|
||||||
logger.warn('Cannot process thumbnail %s from youtubedl.', youtubeDLInfo.thumbnailUrl, { err })
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Process video preview from request.files
|
|
||||||
let previewModel = await processPreview(req, video)
|
|
||||||
|
|
||||||
// Process video preview from url if processing from request.files failed
|
|
||||||
if (!previewModel && youtubeDLInfo.thumbnailUrl) {
|
|
||||||
try {
|
|
||||||
previewModel = await processPreviewFromUrl(youtubeDLInfo.thumbnailUrl, video)
|
|
||||||
} catch (err) {
|
|
||||||
logger.warn('Cannot process preview %s from youtubedl.', youtubeDLInfo.thumbnailUrl, { err })
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
const videoImport = await insertIntoDB({
|
|
||||||
video,
|
|
||||||
thumbnailModel,
|
|
||||||
previewModel,
|
|
||||||
videoChannel: res.locals.videoChannel,
|
|
||||||
tags: body.tags || youtubeDLInfo.tags,
|
|
||||||
user,
|
|
||||||
videoImportAttributes: {
|
|
||||||
targetUrl,
|
|
||||||
state: VideoImportState.PENDING,
|
|
||||||
userId: user.id
|
|
||||||
}
|
|
||||||
})
|
|
||||||
|
|
||||||
// Get video subtitles
|
|
||||||
await processYoutubeSubtitles(youtubeDL, targetUrl, video.id)
|
|
||||||
|
|
||||||
let fileExt = `.${youtubeDLInfo.ext}`
|
|
||||||
if (!isVideoFileExtnameValid(fileExt)) fileExt = '.mp4'
|
|
||||||
|
|
||||||
// Create job to import the video
|
|
||||||
const payload = {
|
|
||||||
type: 'youtube-dl' as 'youtube-dl',
|
|
||||||
videoImportId: videoImport.id,
|
|
||||||
fileExt
|
|
||||||
}
|
|
||||||
await JobQueue.Instance.createJob({ type: 'video-import', payload })
|
|
||||||
|
|
||||||
auditLogger.create(getAuditIdFromRes(res), new VideoImportAuditView(videoImport.toFormattedJSON()))
|
|
||||||
|
|
||||||
return res.json(videoImport.toFormattedJSON()).end()
|
|
||||||
}
|
|
||||||
|
|
||||||
async function buildVideo (channelId: number, body: VideoImportCreate, importData: YoutubeDLInfo): Promise<MVideoThumbnail> {
|
|
||||||
let videoData = {
|
|
||||||
name: body.name || importData.name || 'Unknown name',
|
|
||||||
remote: false,
|
|
||||||
category: body.category || importData.category,
|
|
||||||
licence: body.licence ?? importData.licence ?? CONFIG.DEFAULTS.PUBLISH.LICENCE,
|
|
||||||
language: body.language || importData.language,
|
|
||||||
commentsEnabled: body.commentsEnabled ?? CONFIG.DEFAULTS.PUBLISH.COMMENTS_ENABLED,
|
|
||||||
downloadEnabled: body.downloadEnabled ?? CONFIG.DEFAULTS.PUBLISH.DOWNLOAD_ENABLED,
|
|
||||||
waitTranscoding: body.waitTranscoding || false,
|
|
||||||
state: VideoState.TO_IMPORT,
|
|
||||||
nsfw: body.nsfw || importData.nsfw || false,
|
|
||||||
description: body.description || importData.description,
|
|
||||||
support: body.support || null,
|
|
||||||
privacy: body.privacy || VideoPrivacy.PRIVATE,
|
|
||||||
duration: 0, // duration will be set by the import job
|
|
||||||
channelId,
|
|
||||||
originallyPublishedAt: body.originallyPublishedAt
|
|
||||||
? new Date(body.originallyPublishedAt)
|
|
||||||
: importData.originallyPublishedAt
|
|
||||||
}
|
|
||||||
|
|
||||||
videoData = await Hooks.wrapObject(
|
|
||||||
videoData,
|
|
||||||
body.targetUrl
|
|
||||||
? 'filter:api.video.import-url.video-attribute.result'
|
|
||||||
: 'filter:api.video.import-torrent.video-attribute.result'
|
|
||||||
)
|
|
||||||
|
|
||||||
const video = new VideoModel(videoData)
|
|
||||||
video.url = getLocalVideoActivityPubUrl(video)
|
|
||||||
|
|
||||||
return video
|
|
||||||
}
|
}
|
||||||
|
|
||||||
async function processThumbnail (req: express.Request, video: MVideoThumbnail) {
|
async function processThumbnail (req: express.Request, video: MVideoThumbnail) {
|
||||||
|
@ -329,69 +219,6 @@ async function processPreview (req: express.Request, video: MVideoThumbnail): Pr
|
||||||
return undefined
|
return undefined
|
||||||
}
|
}
|
||||||
|
|
||||||
async function processThumbnailFromUrl (url: string, video: MVideoThumbnail) {
|
|
||||||
try {
|
|
||||||
return updateVideoMiniatureFromUrl({ downloadUrl: url, video, type: ThumbnailType.MINIATURE })
|
|
||||||
} catch (err) {
|
|
||||||
logger.warn('Cannot generate video thumbnail %s for %s.', url, video.url, { err })
|
|
||||||
return undefined
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
async function processPreviewFromUrl (url: string, video: MVideoThumbnail) {
|
|
||||||
try {
|
|
||||||
return updateVideoMiniatureFromUrl({ downloadUrl: url, video, type: ThumbnailType.PREVIEW })
|
|
||||||
} catch (err) {
|
|
||||||
logger.warn('Cannot generate video preview %s for %s.', url, video.url, { err })
|
|
||||||
return undefined
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
async function insertIntoDB (parameters: {
|
|
||||||
video: MVideoThumbnail
|
|
||||||
thumbnailModel: MThumbnail
|
|
||||||
previewModel: MThumbnail
|
|
||||||
videoChannel: MChannelAccountDefault
|
|
||||||
tags: string[]
|
|
||||||
videoImportAttributes: FilteredModelAttributes<VideoImportModel>
|
|
||||||
user: MUser
|
|
||||||
}): Promise<MVideoImportFormattable> {
|
|
||||||
const { video, thumbnailModel, previewModel, videoChannel, tags, videoImportAttributes, user } = parameters
|
|
||||||
|
|
||||||
const videoImport = await sequelizeTypescript.transaction(async t => {
|
|
||||||
const sequelizeOptions = { transaction: t }
|
|
||||||
|
|
||||||
// Save video object in database
|
|
||||||
const videoCreated = await video.save(sequelizeOptions) as (MVideoAccountDefault & MVideoWithBlacklistLight & MVideoTag)
|
|
||||||
videoCreated.VideoChannel = videoChannel
|
|
||||||
|
|
||||||
if (thumbnailModel) await videoCreated.addAndSaveThumbnail(thumbnailModel, t)
|
|
||||||
if (previewModel) await videoCreated.addAndSaveThumbnail(previewModel, t)
|
|
||||||
|
|
||||||
await autoBlacklistVideoIfNeeded({
|
|
||||||
video: videoCreated,
|
|
||||||
user,
|
|
||||||
notify: false,
|
|
||||||
isRemote: false,
|
|
||||||
isNew: true,
|
|
||||||
transaction: t
|
|
||||||
})
|
|
||||||
|
|
||||||
await setVideoTags({ video: videoCreated, tags, transaction: t })
|
|
||||||
|
|
||||||
// Create video import object in database
|
|
||||||
const videoImport = await VideoImportModel.create(
|
|
||||||
Object.assign({ videoId: videoCreated.id }, videoImportAttributes),
|
|
||||||
sequelizeOptions
|
|
||||||
) as MVideoImportFormattable
|
|
||||||
videoImport.Video = videoCreated
|
|
||||||
|
|
||||||
return videoImport
|
|
||||||
})
|
|
||||||
|
|
||||||
return videoImport
|
|
||||||
}
|
|
||||||
|
|
||||||
async function processTorrentOrAbortRequest (req: express.Request, res: express.Response, torrentfile: Express.Multer.File) {
|
async function processTorrentOrAbortRequest (req: express.Request, res: express.Response, torrentfile: Express.Multer.File) {
|
||||||
const torrentName = torrentfile.originalname
|
const torrentName = torrentfile.originalname
|
||||||
|
|
||||||
|
@ -432,46 +259,3 @@ function processMagnetURI (body: VideoImportCreate) {
|
||||||
function extractNameFromArray (name: string | string[]) {
|
function extractNameFromArray (name: string | string[]) {
|
||||||
return isArray(name) ? name[0] : name
|
return isArray(name) ? name[0] : name
|
||||||
}
|
}
|
||||||
|
|
||||||
async function processYoutubeSubtitles (youtubeDL: YoutubeDLWrapper, targetUrl: string, videoId: number) {
|
|
||||||
try {
|
|
||||||
const subtitles = await youtubeDL.getSubtitles()
|
|
||||||
|
|
||||||
logger.info('Will create %s subtitles from youtube import %s.', subtitles.length, targetUrl)
|
|
||||||
|
|
||||||
for (const subtitle of subtitles) {
|
|
||||||
if (!await isVTTFileValid(subtitle.path)) {
|
|
||||||
await remove(subtitle.path)
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
|
|
||||||
const videoCaption = new VideoCaptionModel({
|
|
||||||
videoId,
|
|
||||||
language: subtitle.language,
|
|
||||||
filename: VideoCaptionModel.generateCaptionName(subtitle.language)
|
|
||||||
}) as MVideoCaption
|
|
||||||
|
|
||||||
// Move physical file
|
|
||||||
await moveAndProcessCaptionFile(subtitle, videoCaption)
|
|
||||||
|
|
||||||
await sequelizeTypescript.transaction(async t => {
|
|
||||||
await VideoCaptionModel.insertOrReplaceLanguage(videoCaption, t)
|
|
||||||
})
|
|
||||||
}
|
|
||||||
} catch (err) {
|
|
||||||
logger.warn('Cannot get video subtitles.', { err })
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
async function hasUnicastURLsOnly (youtubeDLInfo: YoutubeDLInfo) {
|
|
||||||
const hosts = youtubeDLInfo.urls.map(u => new URL(u).hostname)
|
|
||||||
const uniqHosts = new Set(hosts)
|
|
||||||
|
|
||||||
for (const h of uniqHosts) {
|
|
||||||
if (await isResolvingToUnicastOnly(h) !== true) {
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return true
|
|
||||||
}
|
|
||||||
|
|
|
@ -5,7 +5,7 @@ import { chain } from 'lodash'
|
||||||
import { join } from 'path'
|
import { join } from 'path'
|
||||||
import { addColors, config, createLogger, format, transports } from 'winston'
|
import { addColors, config, createLogger, format, transports } from 'winston'
|
||||||
import { AUDIT_LOG_FILENAME } from '@server/initializers/constants'
|
import { AUDIT_LOG_FILENAME } from '@server/initializers/constants'
|
||||||
import { AdminAbuse, CustomConfig, User, VideoChannel, VideoComment, VideoDetails, VideoImport } from '@shared/models'
|
import { AdminAbuse, CustomConfig, User, VideoChannel, VideoChannelSync, VideoComment, VideoDetails, VideoImport } from '@shared/models'
|
||||||
import { CONFIG } from '../initializers/config'
|
import { CONFIG } from '../initializers/config'
|
||||||
import { jsonLoggerFormat, labelFormatter } from './logger'
|
import { jsonLoggerFormat, labelFormatter } from './logger'
|
||||||
|
|
||||||
|
@ -260,6 +260,18 @@ class CustomConfigAuditView extends EntityAuditView {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
const channelSyncKeysToKeep = [
|
||||||
|
'id',
|
||||||
|
'externalChannelUrl',
|
||||||
|
'channel-id',
|
||||||
|
'channel-name'
|
||||||
|
]
|
||||||
|
class VideoChannelSyncAuditView extends EntityAuditView {
|
||||||
|
constructor (channelSync: VideoChannelSync) {
|
||||||
|
super(channelSyncKeysToKeep, 'channelSync', channelSync)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
export {
|
export {
|
||||||
getAuditIdFromRes,
|
getAuditIdFromRes,
|
||||||
|
|
||||||
|
@ -270,5 +282,6 @@ export {
|
||||||
UserAuditView,
|
UserAuditView,
|
||||||
VideoAuditView,
|
VideoAuditView,
|
||||||
AbuseAuditView,
|
AbuseAuditView,
|
||||||
CustomConfigAuditView
|
CustomConfigAuditView,
|
||||||
|
VideoChannelSyncAuditView
|
||||||
}
|
}
|
||||||
|
|
6
server/helpers/custom-validators/video-channel-syncs.ts
Normal file
6
server/helpers/custom-validators/video-channel-syncs.ts
Normal file
|
@ -0,0 +1,6 @@
|
||||||
|
import { VIDEO_CHANNEL_SYNC_STATE } from '@server/initializers/constants'
|
||||||
|
import { exists } from './misc'
|
||||||
|
|
||||||
|
export function isVideoChannelSyncStateValid (value: any) {
|
||||||
|
return exists(value) && VIDEO_CHANNEL_SYNC_STATE[value] !== undefined
|
||||||
|
}
|
|
@ -87,6 +87,7 @@ export class YoutubeDLCLI {
|
||||||
return result.concat([
|
return result.concat([
|
||||||
'bestvideo[vcodec!*=av01][vcodec!*=vp9.2]+bestaudio',
|
'bestvideo[vcodec!*=av01][vcodec!*=vp9.2]+bestaudio',
|
||||||
'best[vcodec!*=av01][vcodec!*=vp9.2]', // case fallback for known formats
|
'best[vcodec!*=av01][vcodec!*=vp9.2]', // case fallback for known formats
|
||||||
|
'bestvideo[ext=mp4]+bestaudio[ext=m4a]',
|
||||||
'best' // Ultimate fallback
|
'best' // Ultimate fallback
|
||||||
]).join('/')
|
]).join('/')
|
||||||
}
|
}
|
||||||
|
@ -103,11 +104,14 @@ export class YoutubeDLCLI {
|
||||||
timeout?: number
|
timeout?: number
|
||||||
additionalYoutubeDLArgs?: string[]
|
additionalYoutubeDLArgs?: string[]
|
||||||
}) {
|
}) {
|
||||||
|
let args = options.additionalYoutubeDLArgs || []
|
||||||
|
args = args.concat([ '--merge-output-format', 'mp4', '-f', options.format, '-o', options.output ])
|
||||||
|
|
||||||
return this.run({
|
return this.run({
|
||||||
url: options.url,
|
url: options.url,
|
||||||
processOptions: options.processOptions,
|
processOptions: options.processOptions,
|
||||||
timeout: options.timeout,
|
timeout: options.timeout,
|
||||||
args: (options.additionalYoutubeDLArgs || []).concat([ '-f', options.format, '-o', options.output ])
|
args
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -129,6 +133,25 @@ export class YoutubeDLCLI {
|
||||||
: info
|
: info
|
||||||
}
|
}
|
||||||
|
|
||||||
|
getListInfo (options: {
|
||||||
|
url: string
|
||||||
|
latestVideosCount?: number
|
||||||
|
processOptions: execa.NodeOptions
|
||||||
|
}): Promise<{ upload_date: string, webpage_url: string }[]> {
|
||||||
|
const additionalYoutubeDLArgs = [ '--skip-download', '--playlist-reverse' ]
|
||||||
|
|
||||||
|
if (options.latestVideosCount !== undefined) {
|
||||||
|
additionalYoutubeDLArgs.push('--playlist-end', options.latestVideosCount.toString())
|
||||||
|
}
|
||||||
|
|
||||||
|
return this.getInfo({
|
||||||
|
url: options.url,
|
||||||
|
format: YoutubeDLCLI.getYoutubeDLVideoFormat([], false),
|
||||||
|
processOptions: options.processOptions,
|
||||||
|
additionalYoutubeDLArgs
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
async getSubs (options: {
|
async getSubs (options: {
|
||||||
url: string
|
url: string
|
||||||
format: 'vtt'
|
format: 'vtt'
|
||||||
|
@ -175,7 +198,7 @@ export class YoutubeDLCLI {
|
||||||
|
|
||||||
const output = await subProcess
|
const output = await subProcess
|
||||||
|
|
||||||
logger.debug('Runned youtube-dl command.', { command: output.command, ...lTags() })
|
logger.debug('Run youtube-dl command.', { command: output.command, ...lTags() })
|
||||||
|
|
||||||
return output.stdout
|
return output.stdout
|
||||||
? output.stdout.trim().split(/\r?\n/)
|
? output.stdout.trim().split(/\r?\n/)
|
||||||
|
|
|
@ -13,6 +13,7 @@ type YoutubeDLInfo = {
|
||||||
thumbnailUrl?: string
|
thumbnailUrl?: string
|
||||||
ext?: string
|
ext?: string
|
||||||
originallyPublishedAt?: Date
|
originallyPublishedAt?: Date
|
||||||
|
webpageUrl?: string
|
||||||
|
|
||||||
urls?: string[]
|
urls?: string[]
|
||||||
}
|
}
|
||||||
|
@ -81,7 +82,8 @@ class YoutubeDLInfoBuilder {
|
||||||
thumbnailUrl: obj.thumbnail || undefined,
|
thumbnailUrl: obj.thumbnail || undefined,
|
||||||
urls: this.buildAvailableUrl(obj),
|
urls: this.buildAvailableUrl(obj),
|
||||||
originallyPublishedAt: this.buildOriginallyPublishedAt(obj),
|
originallyPublishedAt: this.buildOriginallyPublishedAt(obj),
|
||||||
ext: obj.ext
|
ext: obj.ext,
|
||||||
|
webpageUrl: obj.webpage_url
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -46,6 +46,24 @@ class YoutubeDLWrapper {
|
||||||
return infoBuilder.getInfo()
|
return infoBuilder.getInfo()
|
||||||
}
|
}
|
||||||
|
|
||||||
|
async getInfoForListImport (options: {
|
||||||
|
latestVideosCount?: number
|
||||||
|
}) {
|
||||||
|
const youtubeDL = await YoutubeDLCLI.safeGet()
|
||||||
|
|
||||||
|
const list = await youtubeDL.getListInfo({
|
||||||
|
url: this.url,
|
||||||
|
latestVideosCount: options.latestVideosCount,
|
||||||
|
processOptions
|
||||||
|
})
|
||||||
|
|
||||||
|
return list.map(info => {
|
||||||
|
const infoBuilder = new YoutubeDLInfoBuilder(info)
|
||||||
|
|
||||||
|
return infoBuilder.getInfo()
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
async getSubtitles (): Promise<YoutubeDLSubs> {
|
async getSubtitles (): Promise<YoutubeDLSubs> {
|
||||||
const cwd = CONFIG.STORAGE.TMP_DIR
|
const cwd = CONFIG.STORAGE.TMP_DIR
|
||||||
|
|
||||||
|
@ -103,7 +121,7 @@ class YoutubeDLWrapper {
|
||||||
|
|
||||||
return remove(path)
|
return remove(path)
|
||||||
})
|
})
|
||||||
.catch(innerErr => logger.error('Cannot remove file in youtubeDL timeout.', { innerErr, ...lTags() }))
|
.catch(innerErr => logger.error('Cannot remove file in youtubeDL error.', { innerErr, ...lTags() }))
|
||||||
|
|
||||||
throw err
|
throw err
|
||||||
}
|
}
|
||||||
|
|
|
@ -48,6 +48,7 @@ function checkConfig () {
|
||||||
checkRemoteRedundancyConfig()
|
checkRemoteRedundancyConfig()
|
||||||
checkStorageConfig()
|
checkStorageConfig()
|
||||||
checkTranscodingConfig()
|
checkTranscodingConfig()
|
||||||
|
checkImportConfig()
|
||||||
checkBroadcastMessageConfig()
|
checkBroadcastMessageConfig()
|
||||||
checkSearchConfig()
|
checkSearchConfig()
|
||||||
checkLiveConfig()
|
checkLiveConfig()
|
||||||
|
@ -200,6 +201,12 @@ function checkTranscodingConfig () {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function checkImportConfig () {
|
||||||
|
if (CONFIG.IMPORT.VIDEO_CHANNEL_SYNCHRONIZATION.ENABLED && !CONFIG.IMPORT.VIDEOS.HTTP) {
|
||||||
|
throw new Error('You need to enable HTTP import to allow synchronization')
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
function checkBroadcastMessageConfig () {
|
function checkBroadcastMessageConfig () {
|
||||||
if (CONFIG.BROADCAST_MESSAGE.ENABLED) {
|
if (CONFIG.BROADCAST_MESSAGE.ENABLED) {
|
||||||
const currentLevel = CONFIG.BROADCAST_MESSAGE.LEVEL
|
const currentLevel = CONFIG.BROADCAST_MESSAGE.LEVEL
|
||||||
|
|
|
@ -32,6 +32,8 @@ function checkMissedConfig () {
|
||||||
'transcoding.resolutions.480p', 'transcoding.resolutions.720p', 'transcoding.resolutions.1080p', 'transcoding.resolutions.1440p',
|
'transcoding.resolutions.480p', 'transcoding.resolutions.720p', 'transcoding.resolutions.1080p', 'transcoding.resolutions.1440p',
|
||||||
'transcoding.resolutions.2160p', 'transcoding.always_transcode_original_resolution', 'video_studio.enabled',
|
'transcoding.resolutions.2160p', 'transcoding.always_transcode_original_resolution', 'video_studio.enabled',
|
||||||
'import.videos.http.enabled', 'import.videos.torrent.enabled', 'import.videos.concurrency', 'import.videos.timeout',
|
'import.videos.http.enabled', 'import.videos.torrent.enabled', 'import.videos.concurrency', 'import.videos.timeout',
|
||||||
|
'import.video_channel_synchronization.enabled', 'import.video_channel_synchronization.max_per_user',
|
||||||
|
'import.video_channel_synchronization.check_interval', 'import.video_channel_synchronization.videos_limit_per_synchronization',
|
||||||
'auto_blacklist.videos.of_users.enabled', 'trending.videos.interval_days',
|
'auto_blacklist.videos.of_users.enabled', 'trending.videos.interval_days',
|
||||||
'client.videos.miniature.display_author_avatar',
|
'client.videos.miniature.display_author_avatar',
|
||||||
'client.videos.miniature.prefer_author_display_name', 'client.menu.login.redirect_on_single_external_auth',
|
'client.videos.miniature.prefer_author_display_name', 'client.menu.login.redirect_on_single_external_auth',
|
||||||
|
|
|
@ -398,6 +398,14 @@ const CONFIG = {
|
||||||
TORRENT: {
|
TORRENT: {
|
||||||
get ENABLED () { return config.get<boolean>('import.videos.torrent.enabled') }
|
get ENABLED () { return config.get<boolean>('import.videos.torrent.enabled') }
|
||||||
}
|
}
|
||||||
|
},
|
||||||
|
VIDEO_CHANNEL_SYNCHRONIZATION: {
|
||||||
|
get ENABLED () { return config.get<boolean>('import.video_channel_synchronization.enabled') },
|
||||||
|
get MAX_PER_USER () { return config.get<number>('import.video_channel_synchronization.max_per_user') },
|
||||||
|
get CHECK_INTERVAL () { return parseDurationToMs(config.get<string>('import.video_channel_synchronization.check_interval')) },
|
||||||
|
get VIDEOS_LIMIT_PER_SYNCHRONIZATION () {
|
||||||
|
return config.get<number>('import.video_channel_synchronization.videos_limit_per_synchronization')
|
||||||
|
}
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
AUTO_BLACKLIST: {
|
AUTO_BLACKLIST: {
|
||||||
|
@ -499,6 +507,7 @@ const CONFIG = {
|
||||||
get IS_DEFAULT_SEARCH () { return config.get<boolean>('search.search_index.is_default_search') }
|
get IS_DEFAULT_SEARCH () { return config.get<boolean>('search.search_index.is_default_search') }
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
function registerConfigChangedHandler (fun: Function) {
|
function registerConfigChangedHandler (fun: Function) {
|
||||||
|
|
|
@ -6,6 +6,7 @@ import { randomInt, root } from '@shared/core-utils'
|
||||||
import {
|
import {
|
||||||
AbuseState,
|
AbuseState,
|
||||||
JobType,
|
JobType,
|
||||||
|
VideoChannelSyncState,
|
||||||
VideoImportState,
|
VideoImportState,
|
||||||
VideoPrivacy,
|
VideoPrivacy,
|
||||||
VideoRateType,
|
VideoRateType,
|
||||||
|
@ -24,7 +25,7 @@ import { CONFIG, registerConfigChangedHandler } from './config'
|
||||||
|
|
||||||
// ---------------------------------------------------------------------------
|
// ---------------------------------------------------------------------------
|
||||||
|
|
||||||
const LAST_MIGRATION_VERSION = 725
|
const LAST_MIGRATION_VERSION = 730
|
||||||
|
|
||||||
// ---------------------------------------------------------------------------
|
// ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
@ -64,6 +65,7 @@ const SORTABLE_COLUMNS = {
|
||||||
JOBS: [ 'createdAt' ],
|
JOBS: [ 'createdAt' ],
|
||||||
VIDEO_CHANNELS: [ 'id', 'name', 'updatedAt', 'createdAt' ],
|
VIDEO_CHANNELS: [ 'id', 'name', 'updatedAt', 'createdAt' ],
|
||||||
VIDEO_IMPORTS: [ 'createdAt' ],
|
VIDEO_IMPORTS: [ 'createdAt' ],
|
||||||
|
VIDEO_CHANNEL_SYNCS: [ 'externalChannelUrl', 'videoChannel', 'createdAt', 'lastSyncAt', 'state' ],
|
||||||
|
|
||||||
VIDEO_COMMENT_THREADS: [ 'createdAt', 'totalReplies' ],
|
VIDEO_COMMENT_THREADS: [ 'createdAt', 'totalReplies' ],
|
||||||
VIDEO_COMMENTS: [ 'createdAt' ],
|
VIDEO_COMMENTS: [ 'createdAt' ],
|
||||||
|
@ -156,6 +158,8 @@ const JOB_ATTEMPTS: { [id in JobType]: number } = {
|
||||||
'video-live-ending': 1,
|
'video-live-ending': 1,
|
||||||
'video-studio-edition': 1,
|
'video-studio-edition': 1,
|
||||||
'manage-video-torrent': 1,
|
'manage-video-torrent': 1,
|
||||||
|
'video-channel-import': 1,
|
||||||
|
'after-video-channel-import': 1,
|
||||||
'move-to-object-storage': 3,
|
'move-to-object-storage': 3,
|
||||||
'notify': 1,
|
'notify': 1,
|
||||||
'federate-video': 1
|
'federate-video': 1
|
||||||
|
@ -178,6 +182,8 @@ const JOB_CONCURRENCY: { [id in Exclude<JobType, 'video-transcoding' | 'video-im
|
||||||
'video-studio-edition': 1,
|
'video-studio-edition': 1,
|
||||||
'manage-video-torrent': 1,
|
'manage-video-torrent': 1,
|
||||||
'move-to-object-storage': 1,
|
'move-to-object-storage': 1,
|
||||||
|
'video-channel-import': 1,
|
||||||
|
'after-video-channel-import': 1,
|
||||||
'notify': 5,
|
'notify': 5,
|
||||||
'federate-video': 3
|
'federate-video': 3
|
||||||
}
|
}
|
||||||
|
@ -199,9 +205,11 @@ const JOB_TTL: { [id in JobType]: number } = {
|
||||||
'video-redundancy': 1000 * 3600 * 3, // 3 hours
|
'video-redundancy': 1000 * 3600 * 3, // 3 hours
|
||||||
'video-live-ending': 1000 * 60 * 10, // 10 minutes
|
'video-live-ending': 1000 * 60 * 10, // 10 minutes
|
||||||
'manage-video-torrent': 1000 * 3600 * 3, // 3 hours
|
'manage-video-torrent': 1000 * 3600 * 3, // 3 hours
|
||||||
|
'move-to-object-storage': 1000 * 60 * 60 * 3, // 3 hours
|
||||||
|
'video-channel-import': 1000 * 60 * 60 * 4, // 4 hours
|
||||||
|
'after-video-channel-import': 60000 * 5, // 5 minutes
|
||||||
'notify': 60000 * 5, // 5 minutes
|
'notify': 60000 * 5, // 5 minutes
|
||||||
'federate-video': 60000 * 5, // 5 minutes
|
'federate-video': 60000 * 5 // 5 minutes
|
||||||
'move-to-object-storage': 1000 * 60 * 60 * 3 // 3 hours
|
|
||||||
}
|
}
|
||||||
const REPEAT_JOBS: { [ id in JobType ]?: RepeatOptions } = {
|
const REPEAT_JOBS: { [ id in JobType ]?: RepeatOptions } = {
|
||||||
'videos-views-stats': {
|
'videos-views-stats': {
|
||||||
|
@ -246,7 +254,8 @@ const SCHEDULER_INTERVALS_MS = {
|
||||||
REMOVE_OLD_VIEWS: 60000 * 60 * 24, // 1 day
|
REMOVE_OLD_VIEWS: 60000 * 60 * 24, // 1 day
|
||||||
REMOVE_OLD_HISTORY: 60000 * 60 * 24, // 1 day
|
REMOVE_OLD_HISTORY: 60000 * 60 * 24, // 1 day
|
||||||
UPDATE_INBOX_STATS: 1000 * 60, // 1 minute
|
UPDATE_INBOX_STATS: 1000 * 60, // 1 minute
|
||||||
REMOVE_DANGLING_RESUMABLE_UPLOADS: 60000 * 60 // 1 hour
|
REMOVE_DANGLING_RESUMABLE_UPLOADS: 60000 * 60, // 1 hour
|
||||||
|
CHANNEL_SYNC_CHECK_INTERVAL: CONFIG.IMPORT.VIDEO_CHANNEL_SYNCHRONIZATION.CHECK_INTERVAL
|
||||||
}
|
}
|
||||||
|
|
||||||
// ---------------------------------------------------------------------------
|
// ---------------------------------------------------------------------------
|
||||||
|
@ -276,8 +285,12 @@ const CONSTRAINTS_FIELDS = {
|
||||||
NAME: { min: 1, max: 120 }, // Length
|
NAME: { min: 1, max: 120 }, // Length
|
||||||
DESCRIPTION: { min: 3, max: 1000 }, // Length
|
DESCRIPTION: { min: 3, max: 1000 }, // Length
|
||||||
SUPPORT: { min: 3, max: 1000 }, // Length
|
SUPPORT: { min: 3, max: 1000 }, // Length
|
||||||
|
EXTERNAL_CHANNEL_URL: { min: 3, max: 2000 }, // Length
|
||||||
URL: { min: 3, max: 2000 } // Length
|
URL: { min: 3, max: 2000 } // Length
|
||||||
},
|
},
|
||||||
|
VIDEO_CHANNEL_SYNCS: {
|
||||||
|
EXTERNAL_CHANNEL_URL: { min: 3, max: 2000 } // Length
|
||||||
|
},
|
||||||
VIDEO_CAPTIONS: {
|
VIDEO_CAPTIONS: {
|
||||||
CAPTION_FILE: {
|
CAPTION_FILE: {
|
||||||
EXTNAME: [ '.vtt', '.srt' ],
|
EXTNAME: [ '.vtt', '.srt' ],
|
||||||
|
@ -478,6 +491,13 @@ const VIDEO_IMPORT_STATES: { [ id in VideoImportState ]: string } = {
|
||||||
[VideoImportState.PROCESSING]: 'Processing'
|
[VideoImportState.PROCESSING]: 'Processing'
|
||||||
}
|
}
|
||||||
|
|
||||||
|
const VIDEO_CHANNEL_SYNC_STATE: { [ id in VideoChannelSyncState ]: string } = {
|
||||||
|
[VideoChannelSyncState.FAILED]: 'Failed',
|
||||||
|
[VideoChannelSyncState.SYNCED]: 'Synchronized',
|
||||||
|
[VideoChannelSyncState.PROCESSING]: 'Processing',
|
||||||
|
[VideoChannelSyncState.WAITING_FIRST_RUN]: 'Waiting first run'
|
||||||
|
}
|
||||||
|
|
||||||
const ABUSE_STATES: { [ id in AbuseState ]: string } = {
|
const ABUSE_STATES: { [ id in AbuseState ]: string } = {
|
||||||
[AbuseState.PENDING]: 'Pending',
|
[AbuseState.PENDING]: 'Pending',
|
||||||
[AbuseState.REJECTED]: 'Rejected',
|
[AbuseState.REJECTED]: 'Rejected',
|
||||||
|
@ -1005,6 +1025,7 @@ export {
|
||||||
JOB_COMPLETED_LIFETIME,
|
JOB_COMPLETED_LIFETIME,
|
||||||
HTTP_SIGNATURE,
|
HTTP_SIGNATURE,
|
||||||
VIDEO_IMPORT_STATES,
|
VIDEO_IMPORT_STATES,
|
||||||
|
VIDEO_CHANNEL_SYNC_STATE,
|
||||||
VIEW_LIFETIME,
|
VIEW_LIFETIME,
|
||||||
CONTACT_FORM_LIFETIME,
|
CONTACT_FORM_LIFETIME,
|
||||||
VIDEO_PLAYLIST_PRIVACIES,
|
VIDEO_PLAYLIST_PRIVACIES,
|
||||||
|
|
|
@ -50,6 +50,7 @@ import { VideoStreamingPlaylistModel } from '../models/video/video-streaming-pla
|
||||||
import { VideoTagModel } from '../models/video/video-tag'
|
import { VideoTagModel } from '../models/video/video-tag'
|
||||||
import { VideoViewModel } from '../models/view/video-view'
|
import { VideoViewModel } from '../models/view/video-view'
|
||||||
import { CONFIG } from './config'
|
import { CONFIG } from './config'
|
||||||
|
import { VideoChannelSyncModel } from '@server/models/video/video-channel-sync'
|
||||||
|
|
||||||
require('pg').defaults.parseInt8 = true // Avoid BIGINT to be converted to string
|
require('pg').defaults.parseInt8 = true // Avoid BIGINT to be converted to string
|
||||||
|
|
||||||
|
@ -153,7 +154,8 @@ async function initDatabaseModels (silent: boolean) {
|
||||||
VideoTrackerModel,
|
VideoTrackerModel,
|
||||||
PluginModel,
|
PluginModel,
|
||||||
ActorCustomPageModel,
|
ActorCustomPageModel,
|
||||||
VideoJobInfoModel
|
VideoJobInfoModel,
|
||||||
|
VideoChannelSyncModel
|
||||||
])
|
])
|
||||||
|
|
||||||
// Check extensions exist in the database
|
// Check extensions exist in the database
|
||||||
|
|
36
server/initializers/migrations/0730-video-channel-sync.ts
Normal file
36
server/initializers/migrations/0730-video-channel-sync.ts
Normal file
|
@ -0,0 +1,36 @@
|
||||||
|
import * as Sequelize from 'sequelize'
|
||||||
|
|
||||||
|
async function up (utils: {
|
||||||
|
transaction: Sequelize.Transaction
|
||||||
|
queryInterface: Sequelize.QueryInterface
|
||||||
|
sequelize: Sequelize.Sequelize
|
||||||
|
db: any
|
||||||
|
}): Promise<void> {
|
||||||
|
const query = `
|
||||||
|
CREATE TABLE IF NOT EXISTS "videoChannelSync" (
|
||||||
|
"id" SERIAL,
|
||||||
|
"externalChannelUrl" VARCHAR(2000) NOT NULL DEFAULT NULL,
|
||||||
|
"videoChannelId" INTEGER NOT NULL REFERENCES "videoChannel" ("id")
|
||||||
|
ON DELETE CASCADE
|
||||||
|
ON UPDATE CASCADE,
|
||||||
|
"state" INTEGER NOT NULL DEFAULT 1,
|
||||||
|
"createdAt" TIMESTAMP WITH TIME ZONE NOT NULL,
|
||||||
|
"updatedAt" TIMESTAMP WITH TIME ZONE NOT NULL,
|
||||||
|
"lastSyncAt" TIMESTAMP WITH TIME ZONE,
|
||||||
|
PRIMARY KEY ("id")
|
||||||
|
);
|
||||||
|
`
|
||||||
|
await utils.sequelize.query(query, { transaction: utils.transaction })
|
||||||
|
}
|
||||||
|
|
||||||
|
async function down (utils: {
|
||||||
|
queryInterface: Sequelize.QueryInterface
|
||||||
|
transaction: Sequelize.Transaction
|
||||||
|
}) {
|
||||||
|
await utils.queryInterface.dropTable('videoChannelSync', { transaction: utils.transaction })
|
||||||
|
}
|
||||||
|
|
||||||
|
export {
|
||||||
|
up,
|
||||||
|
down
|
||||||
|
}
|
37
server/lib/job-queue/handlers/after-video-channel-import.ts
Normal file
37
server/lib/job-queue/handlers/after-video-channel-import.ts
Normal file
|
@ -0,0 +1,37 @@
|
||||||
|
import { Job } from 'bullmq'
|
||||||
|
import { logger } from '@server/helpers/logger'
|
||||||
|
import { VideoChannelSyncModel } from '@server/models/video/video-channel-sync'
|
||||||
|
import { AfterVideoChannelImportPayload, VideoChannelSyncState, VideoImportPreventExceptionResult } from '@shared/models'
|
||||||
|
|
||||||
|
export async function processAfterVideoChannelImport (job: Job) {
|
||||||
|
const payload = job.data as AfterVideoChannelImportPayload
|
||||||
|
if (!payload.channelSyncId) return
|
||||||
|
|
||||||
|
logger.info('Processing after video channel import in job %s.', job.id)
|
||||||
|
|
||||||
|
const sync = await VideoChannelSyncModel.loadWithChannel(payload.channelSyncId)
|
||||||
|
if (!sync) {
|
||||||
|
logger.error('Unknown sync id %d.', payload.channelSyncId)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
const childrenValues = await job.getChildrenValues<VideoImportPreventExceptionResult>()
|
||||||
|
|
||||||
|
let errors = 0
|
||||||
|
let successes = 0
|
||||||
|
|
||||||
|
for (const value of Object.values(childrenValues)) {
|
||||||
|
if (value.resultType === 'success') successes++
|
||||||
|
else if (value.resultType === 'error') errors++
|
||||||
|
}
|
||||||
|
|
||||||
|
if (errors > 0) {
|
||||||
|
sync.state = VideoChannelSyncState.FAILED
|
||||||
|
logger.error(`Finished synchronizing "${sync.VideoChannel.Actor.preferredUsername}" with failures.`, { errors, successes })
|
||||||
|
} else {
|
||||||
|
sync.state = VideoChannelSyncState.SYNCED
|
||||||
|
logger.info(`Finished synchronizing "${sync.VideoChannel.Actor.preferredUsername}" successfully.`, { successes })
|
||||||
|
}
|
||||||
|
|
||||||
|
await sync.save()
|
||||||
|
}
|
36
server/lib/job-queue/handlers/video-channel-import.ts
Normal file
36
server/lib/job-queue/handlers/video-channel-import.ts
Normal file
|
@ -0,0 +1,36 @@
|
||||||
|
import { Job } from 'bullmq'
|
||||||
|
import { logger } from '@server/helpers/logger'
|
||||||
|
import { CONFIG } from '@server/initializers/config'
|
||||||
|
import { synchronizeChannel } from '@server/lib/sync-channel'
|
||||||
|
import { VideoChannelModel } from '@server/models/video/video-channel'
|
||||||
|
import { VideoChannelImportPayload } from '@shared/models'
|
||||||
|
|
||||||
|
export async function processVideoChannelImport (job: Job) {
|
||||||
|
const payload = job.data as VideoChannelImportPayload
|
||||||
|
|
||||||
|
logger.info('Processing video channel import in job %s.', job.id)
|
||||||
|
|
||||||
|
// Channel import requires only http upload to be allowed
|
||||||
|
if (!CONFIG.IMPORT.VIDEOS.HTTP.ENABLED) {
|
||||||
|
logger.error('Cannot import channel as the HTTP upload is disabled')
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
if (!CONFIG.IMPORT.VIDEO_CHANNEL_SYNCHRONIZATION.ENABLED) {
|
||||||
|
logger.error('Cannot import channel as the synchronization is disabled')
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
const videoChannel = await VideoChannelModel.loadAndPopulateAccount(payload.videoChannelId)
|
||||||
|
|
||||||
|
try {
|
||||||
|
logger.info(`Starting importing videos from external channel "${payload.externalChannelUrl}" to "${videoChannel.name}" `)
|
||||||
|
|
||||||
|
await synchronizeChannel({
|
||||||
|
channel: videoChannel,
|
||||||
|
externalChannelUrl: payload.externalChannelUrl
|
||||||
|
})
|
||||||
|
} catch (err) {
|
||||||
|
logger.error(`Failed to import channel ${videoChannel.name}`, { err })
|
||||||
|
}
|
||||||
|
}
|
|
@ -8,7 +8,7 @@ import { generateWebTorrentVideoFilename } from '@server/lib/paths'
|
||||||
import { Hooks } from '@server/lib/plugins/hooks'
|
import { Hooks } from '@server/lib/plugins/hooks'
|
||||||
import { ServerConfigManager } from '@server/lib/server-config-manager'
|
import { ServerConfigManager } from '@server/lib/server-config-manager'
|
||||||
import { isAbleToUploadVideo } from '@server/lib/user'
|
import { isAbleToUploadVideo } from '@server/lib/user'
|
||||||
import { buildOptimizeOrMergeAudioJob, buildMoveToObjectStorageJob } from '@server/lib/video'
|
import { buildMoveToObjectStorageJob, buildOptimizeOrMergeAudioJob } from '@server/lib/video'
|
||||||
import { VideoPathManager } from '@server/lib/video-path-manager'
|
import { VideoPathManager } from '@server/lib/video-path-manager'
|
||||||
import { buildNextVideoState } from '@server/lib/video-state'
|
import { buildNextVideoState } from '@server/lib/video-state'
|
||||||
import { ThumbnailModel } from '@server/models/video/thumbnail'
|
import { ThumbnailModel } from '@server/models/video/thumbnail'
|
||||||
|
@ -18,6 +18,7 @@ import { isAudioFile } from '@shared/extra-utils'
|
||||||
import {
|
import {
|
||||||
ThumbnailType,
|
ThumbnailType,
|
||||||
VideoImportPayload,
|
VideoImportPayload,
|
||||||
|
VideoImportPreventExceptionResult,
|
||||||
VideoImportState,
|
VideoImportState,
|
||||||
VideoImportTorrentPayload,
|
VideoImportTorrentPayload,
|
||||||
VideoImportTorrentPayloadType,
|
VideoImportTorrentPayloadType,
|
||||||
|
@ -41,20 +42,29 @@ import { Notifier } from '../../notifier'
|
||||||
import { generateVideoMiniature } from '../../thumbnail'
|
import { generateVideoMiniature } from '../../thumbnail'
|
||||||
import { JobQueue } from '../job-queue'
|
import { JobQueue } from '../job-queue'
|
||||||
|
|
||||||
async function processVideoImport (job: Job) {
|
async function processVideoImport (job: Job): Promise<VideoImportPreventExceptionResult> {
|
||||||
const payload = job.data as VideoImportPayload
|
const payload = job.data as VideoImportPayload
|
||||||
|
|
||||||
const videoImport = await getVideoImportOrDie(payload)
|
const videoImport = await getVideoImportOrDie(payload)
|
||||||
if (videoImport.state === VideoImportState.CANCELLED) {
|
if (videoImport.state === VideoImportState.CANCELLED) {
|
||||||
logger.info('Do not process import since it has been cancelled', { payload })
|
logger.info('Do not process import since it has been cancelled', { payload })
|
||||||
return
|
return { resultType: 'success' }
|
||||||
}
|
}
|
||||||
|
|
||||||
videoImport.state = VideoImportState.PROCESSING
|
videoImport.state = VideoImportState.PROCESSING
|
||||||
await videoImport.save()
|
await videoImport.save()
|
||||||
|
|
||||||
if (payload.type === 'youtube-dl') return processYoutubeDLImport(job, videoImport, payload)
|
try {
|
||||||
if (payload.type === 'magnet-uri' || payload.type === 'torrent-file') return processTorrentImport(job, videoImport, payload)
|
if (payload.type === 'youtube-dl') await processYoutubeDLImport(job, videoImport, payload)
|
||||||
|
if (payload.type === 'magnet-uri' || payload.type === 'torrent-file') await processTorrentImport(job, videoImport, payload)
|
||||||
|
|
||||||
|
return { resultType: 'success' }
|
||||||
|
} catch (err) {
|
||||||
|
if (!payload.preventException) throw err
|
||||||
|
|
||||||
|
logger.warn('Catch error in video import to send value to parent job.', { payload, err })
|
||||||
|
return { resultType: 'error' }
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// ---------------------------------------------------------------------------
|
// ---------------------------------------------------------------------------
|
||||||
|
|
|
@ -22,6 +22,7 @@ import {
|
||||||
ActivitypubHttpFetcherPayload,
|
ActivitypubHttpFetcherPayload,
|
||||||
ActivitypubHttpUnicastPayload,
|
ActivitypubHttpUnicastPayload,
|
||||||
ActorKeysPayload,
|
ActorKeysPayload,
|
||||||
|
AfterVideoChannelImportPayload,
|
||||||
DeleteResumableUploadMetaFilePayload,
|
DeleteResumableUploadMetaFilePayload,
|
||||||
EmailPayload,
|
EmailPayload,
|
||||||
FederateVideoPayload,
|
FederateVideoPayload,
|
||||||
|
@ -31,6 +32,7 @@ import {
|
||||||
MoveObjectStoragePayload,
|
MoveObjectStoragePayload,
|
||||||
NotifyPayload,
|
NotifyPayload,
|
||||||
RefreshPayload,
|
RefreshPayload,
|
||||||
|
VideoChannelImportPayload,
|
||||||
VideoFileImportPayload,
|
VideoFileImportPayload,
|
||||||
VideoImportPayload,
|
VideoImportPayload,
|
||||||
VideoLiveEndingPayload,
|
VideoLiveEndingPayload,
|
||||||
|
@ -53,12 +55,14 @@ import { processFederateVideo } from './handlers/federate-video'
|
||||||
import { processManageVideoTorrent } from './handlers/manage-video-torrent'
|
import { processManageVideoTorrent } from './handlers/manage-video-torrent'
|
||||||
import { onMoveToObjectStorageFailure, processMoveToObjectStorage } from './handlers/move-to-object-storage'
|
import { onMoveToObjectStorageFailure, processMoveToObjectStorage } from './handlers/move-to-object-storage'
|
||||||
import { processNotify } from './handlers/notify'
|
import { processNotify } from './handlers/notify'
|
||||||
|
import { processVideoChannelImport } from './handlers/video-channel-import'
|
||||||
import { processVideoFileImport } from './handlers/video-file-import'
|
import { processVideoFileImport } from './handlers/video-file-import'
|
||||||
import { processVideoImport } from './handlers/video-import'
|
import { processVideoImport } from './handlers/video-import'
|
||||||
import { processVideoLiveEnding } from './handlers/video-live-ending'
|
import { processVideoLiveEnding } from './handlers/video-live-ending'
|
||||||
import { processVideoStudioEdition } from './handlers/video-studio-edition'
|
import { processVideoStudioEdition } from './handlers/video-studio-edition'
|
||||||
import { processVideoTranscoding } from './handlers/video-transcoding'
|
import { processVideoTranscoding } from './handlers/video-transcoding'
|
||||||
import { processVideosViewsStats } from './handlers/video-views-stats'
|
import { processVideosViewsStats } from './handlers/video-views-stats'
|
||||||
|
import { processAfterVideoChannelImport } from './handlers/after-video-channel-import'
|
||||||
|
|
||||||
export type CreateJobArgument =
|
export type CreateJobArgument =
|
||||||
{ type: 'activitypub-http-broadcast', payload: ActivitypubHttpBroadcastPayload } |
|
{ type: 'activitypub-http-broadcast', payload: ActivitypubHttpBroadcastPayload } |
|
||||||
|
@ -79,6 +83,9 @@ export type CreateJobArgument =
|
||||||
{ type: 'delete-resumable-upload-meta-file', payload: DeleteResumableUploadMetaFilePayload } |
|
{ type: 'delete-resumable-upload-meta-file', payload: DeleteResumableUploadMetaFilePayload } |
|
||||||
{ type: 'video-studio-edition', payload: VideoStudioEditionPayload } |
|
{ type: 'video-studio-edition', payload: VideoStudioEditionPayload } |
|
||||||
{ type: 'manage-video-torrent', payload: ManageVideoTorrentPayload } |
|
{ type: 'manage-video-torrent', payload: ManageVideoTorrentPayload } |
|
||||||
|
{ type: 'move-to-object-storage', payload: MoveObjectStoragePayload } |
|
||||||
|
{ type: 'video-channel-import', payload: VideoChannelImportPayload } |
|
||||||
|
{ type: 'after-video-channel-import', payload: AfterVideoChannelImportPayload } |
|
||||||
{ type: 'notify', payload: NotifyPayload } |
|
{ type: 'notify', payload: NotifyPayload } |
|
||||||
{ type: 'move-to-object-storage', payload: MoveObjectStoragePayload } |
|
{ type: 'move-to-object-storage', payload: MoveObjectStoragePayload } |
|
||||||
{ type: 'federate-video', payload: FederateVideoPayload }
|
{ type: 'federate-video', payload: FederateVideoPayload }
|
||||||
|
@ -106,8 +113,10 @@ const handlers: { [id in JobType]: (job: Job) => Promise<any> } = {
|
||||||
'video-redundancy': processVideoRedundancy,
|
'video-redundancy': processVideoRedundancy,
|
||||||
'move-to-object-storage': processMoveToObjectStorage,
|
'move-to-object-storage': processMoveToObjectStorage,
|
||||||
'manage-video-torrent': processManageVideoTorrent,
|
'manage-video-torrent': processManageVideoTorrent,
|
||||||
'notify': processNotify,
|
|
||||||
'video-studio-edition': processVideoStudioEdition,
|
'video-studio-edition': processVideoStudioEdition,
|
||||||
|
'video-channel-import': processVideoChannelImport,
|
||||||
|
'after-video-channel-import': processAfterVideoChannelImport,
|
||||||
|
'notify': processNotify,
|
||||||
'federate-video': processFederateVideo
|
'federate-video': processFederateVideo
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -134,6 +143,8 @@ const jobTypes: JobType[] = [
|
||||||
'move-to-object-storage',
|
'move-to-object-storage',
|
||||||
'manage-video-torrent',
|
'manage-video-torrent',
|
||||||
'video-studio-edition',
|
'video-studio-edition',
|
||||||
|
'video-channel-import',
|
||||||
|
'after-video-channel-import',
|
||||||
'notify',
|
'notify',
|
||||||
'federate-video'
|
'federate-video'
|
||||||
]
|
]
|
||||||
|
@ -306,7 +317,7 @@ class JobQueue {
|
||||||
.catch(err => logger.error('Cannot create job.', { err, options }))
|
.catch(err => logger.error('Cannot create job.', { err, options }))
|
||||||
}
|
}
|
||||||
|
|
||||||
async createJob (options: CreateJobArgument & CreateJobOptions) {
|
createJob (options: CreateJobArgument & CreateJobOptions) {
|
||||||
const queue: Queue = this.queues[options.type]
|
const queue: Queue = this.queues[options.type]
|
||||||
if (queue === undefined) {
|
if (queue === undefined) {
|
||||||
logger.error('Unknown queue %s: cannot create job.', options.type)
|
logger.error('Unknown queue %s: cannot create job.', options.type)
|
||||||
|
@ -318,7 +329,7 @@ class JobQueue {
|
||||||
return queue.add('job', options.payload, jobOptions)
|
return queue.add('job', options.payload, jobOptions)
|
||||||
}
|
}
|
||||||
|
|
||||||
async createSequentialJobFlow (...jobs: ((CreateJobArgument & CreateJobOptions) | undefined)[]) {
|
createSequentialJobFlow (...jobs: ((CreateJobArgument & CreateJobOptions) | undefined)[]) {
|
||||||
let lastJob: FlowJob
|
let lastJob: FlowJob
|
||||||
|
|
||||||
for (const job of jobs) {
|
for (const job of jobs) {
|
||||||
|
@ -336,7 +347,7 @@ class JobQueue {
|
||||||
return this.flowProducer.add(lastJob)
|
return this.flowProducer.add(lastJob)
|
||||||
}
|
}
|
||||||
|
|
||||||
async createJobWithChildren (parent: CreateJobArgument & CreateJobOptions, children: (CreateJobArgument & CreateJobOptions)[]) {
|
createJobWithChildren (parent: CreateJobArgument & CreateJobOptions, children: (CreateJobArgument & CreateJobOptions)[]) {
|
||||||
return this.flowProducer.add({
|
return this.flowProducer.add({
|
||||||
...this.buildJobFlowOption(parent),
|
...this.buildJobFlowOption(parent),
|
||||||
|
|
||||||
|
|
61
server/lib/schedulers/video-channel-sync-latest-scheduler.ts
Normal file
61
server/lib/schedulers/video-channel-sync-latest-scheduler.ts
Normal file
|
@ -0,0 +1,61 @@
|
||||||
|
import { logger } from '@server/helpers/logger'
|
||||||
|
import { CONFIG } from '@server/initializers/config'
|
||||||
|
import { VideoChannelModel } from '@server/models/video/video-channel'
|
||||||
|
import { VideoChannelSyncModel } from '@server/models/video/video-channel-sync'
|
||||||
|
import { VideoChannelSyncState } from '@shared/models'
|
||||||
|
import { SCHEDULER_INTERVALS_MS } from '../../initializers/constants'
|
||||||
|
import { synchronizeChannel } from '../sync-channel'
|
||||||
|
import { AbstractScheduler } from './abstract-scheduler'
|
||||||
|
|
||||||
|
export class VideoChannelSyncLatestScheduler extends AbstractScheduler {
|
||||||
|
private static instance: AbstractScheduler
|
||||||
|
protected schedulerIntervalMs = SCHEDULER_INTERVALS_MS.CHANNEL_SYNC_CHECK_INTERVAL
|
||||||
|
|
||||||
|
private constructor () {
|
||||||
|
super()
|
||||||
|
}
|
||||||
|
|
||||||
|
protected async internalExecute () {
|
||||||
|
logger.debug('Running %s.%s', this.constructor.name, this.internalExecute.name)
|
||||||
|
|
||||||
|
if (!CONFIG.IMPORT.VIDEO_CHANNEL_SYNCHRONIZATION.ENABLED) {
|
||||||
|
logger.info('Discard channels synchronization as the feature is disabled')
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
const channelSyncs = await VideoChannelSyncModel.listSyncs()
|
||||||
|
|
||||||
|
for (const sync of channelSyncs) {
|
||||||
|
const channel = await VideoChannelModel.loadAndPopulateAccount(sync.videoChannelId)
|
||||||
|
|
||||||
|
try {
|
||||||
|
logger.info(
|
||||||
|
'Creating video import jobs for "%s" sync with external channel "%s"',
|
||||||
|
channel.Actor.preferredUsername, sync.externalChannelUrl
|
||||||
|
)
|
||||||
|
|
||||||
|
const onlyAfter = sync.lastSyncAt || sync.createdAt
|
||||||
|
|
||||||
|
sync.state = VideoChannelSyncState.PROCESSING
|
||||||
|
sync.lastSyncAt = new Date()
|
||||||
|
await sync.save()
|
||||||
|
|
||||||
|
await synchronizeChannel({
|
||||||
|
channel,
|
||||||
|
externalChannelUrl: sync.externalChannelUrl,
|
||||||
|
videosCountLimit: CONFIG.IMPORT.VIDEO_CHANNEL_SYNCHRONIZATION.VIDEOS_LIMIT_PER_SYNCHRONIZATION,
|
||||||
|
channelSync: sync,
|
||||||
|
onlyAfter
|
||||||
|
})
|
||||||
|
} catch (err) {
|
||||||
|
logger.error(`Failed to synchronize channel ${channel.Actor.preferredUsername}`, { err })
|
||||||
|
sync.state = VideoChannelSyncState.FAILED
|
||||||
|
await sync.save()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
static get Instance () {
|
||||||
|
return this.instance || (this.instance = new this())
|
||||||
|
}
|
||||||
|
}
|
|
@ -170,6 +170,9 @@ class ServerConfigManager {
|
||||||
torrent: {
|
torrent: {
|
||||||
enabled: CONFIG.IMPORT.VIDEOS.TORRENT.ENABLED
|
enabled: CONFIG.IMPORT.VIDEOS.TORRENT.ENABLED
|
||||||
}
|
}
|
||||||
|
},
|
||||||
|
videoChannelSynchronization: {
|
||||||
|
enabled: CONFIG.IMPORT.VIDEO_CHANNEL_SYNCHRONIZATION.ENABLED
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
autoBlacklist: {
|
autoBlacklist: {
|
||||||
|
|
81
server/lib/sync-channel.ts
Normal file
81
server/lib/sync-channel.ts
Normal file
|
@ -0,0 +1,81 @@
|
||||||
|
import { logger } from '@server/helpers/logger'
|
||||||
|
import { YoutubeDLWrapper } from '@server/helpers/youtube-dl'
|
||||||
|
import { CONFIG } from '@server/initializers/config'
|
||||||
|
import { buildYoutubeDLImport } from '@server/lib/video-import'
|
||||||
|
import { UserModel } from '@server/models/user/user'
|
||||||
|
import { VideoImportModel } from '@server/models/video/video-import'
|
||||||
|
import { MChannelAccountDefault, MChannelSync } from '@server/types/models'
|
||||||
|
import { VideoChannelSyncState, VideoPrivacy } from '@shared/models'
|
||||||
|
import { CreateJobArgument, JobQueue } from './job-queue'
|
||||||
|
import { ServerConfigManager } from './server-config-manager'
|
||||||
|
|
||||||
|
export async function synchronizeChannel (options: {
|
||||||
|
channel: MChannelAccountDefault
|
||||||
|
externalChannelUrl: string
|
||||||
|
channelSync?: MChannelSync
|
||||||
|
videosCountLimit?: number
|
||||||
|
onlyAfter?: Date
|
||||||
|
}) {
|
||||||
|
const { channel, externalChannelUrl, videosCountLimit, onlyAfter, channelSync } = options
|
||||||
|
|
||||||
|
const user = await UserModel.loadByChannelActorId(channel.actorId)
|
||||||
|
const youtubeDL = new YoutubeDLWrapper(
|
||||||
|
externalChannelUrl,
|
||||||
|
ServerConfigManager.Instance.getEnabledResolutions('vod'),
|
||||||
|
CONFIG.TRANSCODING.ALWAYS_TRANSCODE_ORIGINAL_RESOLUTION
|
||||||
|
)
|
||||||
|
|
||||||
|
const infoList = await youtubeDL.getInfoForListImport({ latestVideosCount: videosCountLimit })
|
||||||
|
|
||||||
|
const targetUrls = infoList
|
||||||
|
.filter(videoInfo => {
|
||||||
|
if (!onlyAfter) return true
|
||||||
|
|
||||||
|
return videoInfo.originallyPublishedAt.getTime() >= onlyAfter.getTime()
|
||||||
|
})
|
||||||
|
.map(videoInfo => videoInfo.webpageUrl)
|
||||||
|
|
||||||
|
logger.info(
|
||||||
|
'Fetched %d candidate URLs for sync channel %s.',
|
||||||
|
targetUrls.length, channel.Actor.preferredUsername, { targetUrls }
|
||||||
|
)
|
||||||
|
|
||||||
|
if (targetUrls.length === 0) {
|
||||||
|
if (channelSync) {
|
||||||
|
channelSync.state = VideoChannelSyncState.SYNCED
|
||||||
|
await channelSync.save()
|
||||||
|
}
|
||||||
|
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
const children: CreateJobArgument[] = []
|
||||||
|
|
||||||
|
for (const targetUrl of targetUrls) {
|
||||||
|
if (await VideoImportModel.urlAlreadyImported(channel.id, targetUrl)) {
|
||||||
|
logger.debug('%s is already imported for channel %s, skipping video channel synchronization.', channel.name, targetUrl)
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
const { job } = await buildYoutubeDLImport({
|
||||||
|
user,
|
||||||
|
channel,
|
||||||
|
targetUrl,
|
||||||
|
channelSync,
|
||||||
|
importDataOverride: {
|
||||||
|
privacy: VideoPrivacy.PUBLIC
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
children.push(job)
|
||||||
|
}
|
||||||
|
|
||||||
|
const parent: CreateJobArgument = {
|
||||||
|
type: 'after-video-channel-import',
|
||||||
|
payload: {
|
||||||
|
channelSyncId: channelSync?.id
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
await JobQueue.Instance.createJobWithChildren(parent, children)
|
||||||
|
}
|
308
server/lib/video-import.ts
Normal file
308
server/lib/video-import.ts
Normal file
|
@ -0,0 +1,308 @@
|
||||||
|
import { remove } from 'fs-extra'
|
||||||
|
import { moveAndProcessCaptionFile } from '@server/helpers/captions-utils'
|
||||||
|
import { isVTTFileValid } from '@server/helpers/custom-validators/video-captions'
|
||||||
|
import { isVideoFileExtnameValid } from '@server/helpers/custom-validators/videos'
|
||||||
|
import { isResolvingToUnicastOnly } from '@server/helpers/dns'
|
||||||
|
import { logger } from '@server/helpers/logger'
|
||||||
|
import { YoutubeDLInfo, YoutubeDLWrapper } from '@server/helpers/youtube-dl'
|
||||||
|
import { CONFIG } from '@server/initializers/config'
|
||||||
|
import { sequelizeTypescript } from '@server/initializers/database'
|
||||||
|
import { Hooks } from '@server/lib/plugins/hooks'
|
||||||
|
import { ServerConfigManager } from '@server/lib/server-config-manager'
|
||||||
|
import { setVideoTags } from '@server/lib/video'
|
||||||
|
import { autoBlacklistVideoIfNeeded } from '@server/lib/video-blacklist'
|
||||||
|
import { VideoModel } from '@server/models/video/video'
|
||||||
|
import { VideoCaptionModel } from '@server/models/video/video-caption'
|
||||||
|
import { VideoImportModel } from '@server/models/video/video-import'
|
||||||
|
import { FilteredModelAttributes } from '@server/types'
|
||||||
|
import {
|
||||||
|
MChannelAccountDefault,
|
||||||
|
MChannelSync,
|
||||||
|
MThumbnail,
|
||||||
|
MUser,
|
||||||
|
MVideoAccountDefault,
|
||||||
|
MVideoCaption,
|
||||||
|
MVideoImportFormattable,
|
||||||
|
MVideoTag,
|
||||||
|
MVideoThumbnail,
|
||||||
|
MVideoWithBlacklistLight
|
||||||
|
} from '@server/types/models'
|
||||||
|
import { ThumbnailType, VideoImportCreate, VideoImportPayload, VideoImportState, VideoPrivacy, VideoState } from '@shared/models'
|
||||||
|
import { getLocalVideoActivityPubUrl } from './activitypub/url'
|
||||||
|
import { updateVideoMiniatureFromExisting, updateVideoMiniatureFromUrl } from './thumbnail'
|
||||||
|
|
||||||
|
class YoutubeDlImportError extends Error {
|
||||||
|
code: YoutubeDlImportError.CODE
|
||||||
|
cause?: Error // Property to remove once ES2022 is used
|
||||||
|
constructor ({ message, code }) {
|
||||||
|
super(message)
|
||||||
|
this.code = code
|
||||||
|
}
|
||||||
|
|
||||||
|
static fromError (err: Error, code: YoutubeDlImportError.CODE, message?: string) {
|
||||||
|
const ytDlErr = new this({ message: message ?? err.message, code })
|
||||||
|
ytDlErr.cause = err
|
||||||
|
ytDlErr.stack = err.stack // Useless once ES2022 is used
|
||||||
|
return ytDlErr
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
namespace YoutubeDlImportError {
|
||||||
|
export enum CODE {
|
||||||
|
FETCH_ERROR,
|
||||||
|
NOT_ONLY_UNICAST_URL
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
async function insertFromImportIntoDB (parameters: {
|
||||||
|
video: MVideoThumbnail
|
||||||
|
thumbnailModel: MThumbnail
|
||||||
|
previewModel: MThumbnail
|
||||||
|
videoChannel: MChannelAccountDefault
|
||||||
|
tags: string[]
|
||||||
|
videoImportAttributes: FilteredModelAttributes<VideoImportModel>
|
||||||
|
user: MUser
|
||||||
|
}): Promise<MVideoImportFormattable> {
|
||||||
|
const { video, thumbnailModel, previewModel, videoChannel, tags, videoImportAttributes, user } = parameters
|
||||||
|
|
||||||
|
const videoImport = await sequelizeTypescript.transaction(async t => {
|
||||||
|
const sequelizeOptions = { transaction: t }
|
||||||
|
|
||||||
|
// Save video object in database
|
||||||
|
const videoCreated = await video.save(sequelizeOptions) as (MVideoAccountDefault & MVideoWithBlacklistLight & MVideoTag)
|
||||||
|
videoCreated.VideoChannel = videoChannel
|
||||||
|
|
||||||
|
if (thumbnailModel) await videoCreated.addAndSaveThumbnail(thumbnailModel, t)
|
||||||
|
if (previewModel) await videoCreated.addAndSaveThumbnail(previewModel, t)
|
||||||
|
|
||||||
|
await autoBlacklistVideoIfNeeded({
|
||||||
|
video: videoCreated,
|
||||||
|
user,
|
||||||
|
notify: false,
|
||||||
|
isRemote: false,
|
||||||
|
isNew: true,
|
||||||
|
transaction: t
|
||||||
|
})
|
||||||
|
|
||||||
|
await setVideoTags({ video: videoCreated, tags, transaction: t })
|
||||||
|
|
||||||
|
// Create video import object in database
|
||||||
|
const videoImport = await VideoImportModel.create(
|
||||||
|
Object.assign({ videoId: videoCreated.id }, videoImportAttributes),
|
||||||
|
sequelizeOptions
|
||||||
|
) as MVideoImportFormattable
|
||||||
|
videoImport.Video = videoCreated
|
||||||
|
|
||||||
|
return videoImport
|
||||||
|
})
|
||||||
|
|
||||||
|
return videoImport
|
||||||
|
}
|
||||||
|
|
||||||
|
async function buildVideoFromImport ({ channelId, importData, importDataOverride, importType }: {
|
||||||
|
channelId: number
|
||||||
|
importData: YoutubeDLInfo
|
||||||
|
importDataOverride?: Partial<VideoImportCreate>
|
||||||
|
importType: 'url' | 'torrent'
|
||||||
|
}): Promise<MVideoThumbnail> {
|
||||||
|
let videoData = {
|
||||||
|
name: importDataOverride?.name || importData.name || 'Unknown name',
|
||||||
|
remote: false,
|
||||||
|
category: importDataOverride?.category || importData.category,
|
||||||
|
licence: importDataOverride?.licence ?? importData.licence ?? CONFIG.DEFAULTS.PUBLISH.LICENCE,
|
||||||
|
language: importDataOverride?.language || importData.language,
|
||||||
|
commentsEnabled: importDataOverride?.commentsEnabled ?? CONFIG.DEFAULTS.PUBLISH.COMMENTS_ENABLED,
|
||||||
|
downloadEnabled: importDataOverride?.downloadEnabled ?? CONFIG.DEFAULTS.PUBLISH.DOWNLOAD_ENABLED,
|
||||||
|
waitTranscoding: importDataOverride?.waitTranscoding || false,
|
||||||
|
state: VideoState.TO_IMPORT,
|
||||||
|
nsfw: importDataOverride?.nsfw || importData.nsfw || false,
|
||||||
|
description: importDataOverride?.description || importData.description,
|
||||||
|
support: importDataOverride?.support || null,
|
||||||
|
privacy: importDataOverride?.privacy || VideoPrivacy.PRIVATE,
|
||||||
|
duration: 0, // duration will be set by the import job
|
||||||
|
channelId,
|
||||||
|
originallyPublishedAt: importDataOverride?.originallyPublishedAt
|
||||||
|
? new Date(importDataOverride?.originallyPublishedAt)
|
||||||
|
: importData.originallyPublishedAt
|
||||||
|
}
|
||||||
|
|
||||||
|
videoData = await Hooks.wrapObject(
|
||||||
|
videoData,
|
||||||
|
importType === 'url'
|
||||||
|
? 'filter:api.video.import-url.video-attribute.result'
|
||||||
|
: 'filter:api.video.import-torrent.video-attribute.result'
|
||||||
|
)
|
||||||
|
|
||||||
|
const video = new VideoModel(videoData)
|
||||||
|
video.url = getLocalVideoActivityPubUrl(video)
|
||||||
|
|
||||||
|
return video
|
||||||
|
}
|
||||||
|
|
||||||
|
async function buildYoutubeDLImport (options: {
|
||||||
|
targetUrl: string
|
||||||
|
channel: MChannelAccountDefault
|
||||||
|
user: MUser
|
||||||
|
channelSync?: MChannelSync
|
||||||
|
importDataOverride?: Partial<VideoImportCreate>
|
||||||
|
thumbnailFilePath?: string
|
||||||
|
previewFilePath?: string
|
||||||
|
}) {
|
||||||
|
const { targetUrl, channel, channelSync, importDataOverride, thumbnailFilePath, previewFilePath, user } = options
|
||||||
|
|
||||||
|
const youtubeDL = new YoutubeDLWrapper(
|
||||||
|
targetUrl,
|
||||||
|
ServerConfigManager.Instance.getEnabledResolutions('vod'),
|
||||||
|
CONFIG.TRANSCODING.ALWAYS_TRANSCODE_ORIGINAL_RESOLUTION
|
||||||
|
)
|
||||||
|
|
||||||
|
// Get video infos
|
||||||
|
let youtubeDLInfo: YoutubeDLInfo
|
||||||
|
try {
|
||||||
|
youtubeDLInfo = await youtubeDL.getInfoForDownload()
|
||||||
|
} catch (err) {
|
||||||
|
throw YoutubeDlImportError.fromError(
|
||||||
|
err, YoutubeDlImportError.CODE.FETCH_ERROR, `Cannot fetch information from import for URL ${targetUrl}`
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
if (!await hasUnicastURLsOnly(youtubeDLInfo)) {
|
||||||
|
throw new YoutubeDlImportError({
|
||||||
|
message: 'Cannot use non unicast IP as targetUrl.',
|
||||||
|
code: YoutubeDlImportError.CODE.NOT_ONLY_UNICAST_URL
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
const video = await buildVideoFromImport({
|
||||||
|
channelId: channel.id,
|
||||||
|
importData: youtubeDLInfo,
|
||||||
|
importDataOverride,
|
||||||
|
importType: 'url'
|
||||||
|
})
|
||||||
|
|
||||||
|
const thumbnailModel = await forgeThumbnail({
|
||||||
|
inputPath: thumbnailFilePath,
|
||||||
|
downloadUrl: youtubeDLInfo.thumbnailUrl,
|
||||||
|
video,
|
||||||
|
type: ThumbnailType.MINIATURE
|
||||||
|
})
|
||||||
|
|
||||||
|
const previewModel = await forgeThumbnail({
|
||||||
|
inputPath: previewFilePath,
|
||||||
|
downloadUrl: youtubeDLInfo.thumbnailUrl,
|
||||||
|
video,
|
||||||
|
type: ThumbnailType.PREVIEW
|
||||||
|
})
|
||||||
|
|
||||||
|
const videoImport = await insertFromImportIntoDB({
|
||||||
|
video,
|
||||||
|
thumbnailModel,
|
||||||
|
previewModel,
|
||||||
|
videoChannel: channel,
|
||||||
|
tags: importDataOverride?.tags || youtubeDLInfo.tags,
|
||||||
|
user,
|
||||||
|
videoImportAttributes: {
|
||||||
|
targetUrl,
|
||||||
|
state: VideoImportState.PENDING,
|
||||||
|
userId: user.id
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
// Get video subtitles
|
||||||
|
await processYoutubeSubtitles(youtubeDL, targetUrl, video.id)
|
||||||
|
|
||||||
|
let fileExt = `.${youtubeDLInfo.ext}`
|
||||||
|
if (!isVideoFileExtnameValid(fileExt)) fileExt = '.mp4'
|
||||||
|
|
||||||
|
const payload: VideoImportPayload = {
|
||||||
|
type: 'youtube-dl' as 'youtube-dl',
|
||||||
|
videoImportId: videoImport.id,
|
||||||
|
fileExt,
|
||||||
|
// If part of a sync process, there is a parent job that will aggregate children results
|
||||||
|
preventException: !!channelSync
|
||||||
|
}
|
||||||
|
|
||||||
|
return {
|
||||||
|
videoImport,
|
||||||
|
job: { type: 'video-import' as 'video-import', payload }
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
export {
|
||||||
|
buildYoutubeDLImport,
|
||||||
|
YoutubeDlImportError,
|
||||||
|
insertFromImportIntoDB,
|
||||||
|
buildVideoFromImport
|
||||||
|
}
|
||||||
|
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
async function forgeThumbnail ({ inputPath, video, downloadUrl, type }: {
|
||||||
|
inputPath?: string
|
||||||
|
downloadUrl?: string
|
||||||
|
video: MVideoThumbnail
|
||||||
|
type: ThumbnailType
|
||||||
|
}): Promise<MThumbnail> {
|
||||||
|
if (inputPath) {
|
||||||
|
return updateVideoMiniatureFromExisting({
|
||||||
|
inputPath,
|
||||||
|
video,
|
||||||
|
type,
|
||||||
|
automaticallyGenerated: false
|
||||||
|
})
|
||||||
|
} else if (downloadUrl) {
|
||||||
|
try {
|
||||||
|
return await updateVideoMiniatureFromUrl({ downloadUrl, video, type })
|
||||||
|
} catch (err) {
|
||||||
|
logger.warn('Cannot process thumbnail %s from youtubedl.', downloadUrl, { err })
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return null
|
||||||
|
}
|
||||||
|
|
||||||
|
async function processYoutubeSubtitles (youtubeDL: YoutubeDLWrapper, targetUrl: string, videoId: number) {
|
||||||
|
try {
|
||||||
|
const subtitles = await youtubeDL.getSubtitles()
|
||||||
|
|
||||||
|
logger.info('Will create %s subtitles from youtube import %s.', subtitles.length, targetUrl)
|
||||||
|
|
||||||
|
for (const subtitle of subtitles) {
|
||||||
|
if (!await isVTTFileValid(subtitle.path)) {
|
||||||
|
await remove(subtitle.path)
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
const videoCaption = new VideoCaptionModel({
|
||||||
|
videoId,
|
||||||
|
language: subtitle.language,
|
||||||
|
filename: VideoCaptionModel.generateCaptionName(subtitle.language)
|
||||||
|
}) as MVideoCaption
|
||||||
|
|
||||||
|
// Move physical file
|
||||||
|
await moveAndProcessCaptionFile(subtitle, videoCaption)
|
||||||
|
|
||||||
|
await sequelizeTypescript.transaction(async t => {
|
||||||
|
await VideoCaptionModel.insertOrReplaceLanguage(videoCaption, t)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
} catch (err) {
|
||||||
|
logger.warn('Cannot get video subtitles.', { err })
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async function hasUnicastURLsOnly (youtubeDLInfo: YoutubeDLInfo) {
|
||||||
|
const hosts = youtubeDLInfo.urls.map(u => new URL(u).hostname)
|
||||||
|
const uniqHosts = new Set(hosts)
|
||||||
|
|
||||||
|
for (const h of uniqHosts) {
|
||||||
|
if (await isResolvingToUnicastOnly(h) !== true) {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return true
|
||||||
|
}
|
|
@ -66,6 +66,8 @@ const customConfigUpdateValidator = [
|
||||||
body('import.videos.http.enabled').isBoolean().withMessage('Should have a valid import video http enabled boolean'),
|
body('import.videos.http.enabled').isBoolean().withMessage('Should have a valid import video http enabled boolean'),
|
||||||
body('import.videos.torrent.enabled').isBoolean().withMessage('Should have a valid import video torrent enabled boolean'),
|
body('import.videos.torrent.enabled').isBoolean().withMessage('Should have a valid import video torrent enabled boolean'),
|
||||||
|
|
||||||
|
body('import.videoChannelSynchronization.enabled').isBoolean().withMessage('Should have a valid synchronization enabled boolean'),
|
||||||
|
|
||||||
body('trending.videos.algorithms.default').exists().withMessage('Should have a valid default trending algorithm'),
|
body('trending.videos.algorithms.default').exists().withMessage('Should have a valid default trending algorithm'),
|
||||||
body('trending.videos.algorithms.enabled').exists().withMessage('Should have a valid array of enabled trending algorithms'),
|
body('trending.videos.algorithms.enabled').exists().withMessage('Should have a valid array of enabled trending algorithms'),
|
||||||
|
|
||||||
|
@ -110,6 +112,7 @@ const customConfigUpdateValidator = [
|
||||||
if (areValidationErrors(req, res)) return
|
if (areValidationErrors(req, res)) return
|
||||||
if (!checkInvalidConfigIfEmailDisabled(req.body, res)) return
|
if (!checkInvalidConfigIfEmailDisabled(req.body, res)) return
|
||||||
if (!checkInvalidTranscodingConfig(req.body, res)) return
|
if (!checkInvalidTranscodingConfig(req.body, res)) return
|
||||||
|
if (!checkInvalidSynchronizationConfig(req.body, res)) return
|
||||||
if (!checkInvalidLiveConfig(req.body, res)) return
|
if (!checkInvalidLiveConfig(req.body, res)) return
|
||||||
if (!checkInvalidVideoStudioConfig(req.body, res)) return
|
if (!checkInvalidVideoStudioConfig(req.body, res)) return
|
||||||
|
|
||||||
|
@ -157,6 +160,14 @@ function checkInvalidTranscodingConfig (customConfig: CustomConfig, res: express
|
||||||
return true
|
return true
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function checkInvalidSynchronizationConfig (customConfig: CustomConfig, res: express.Response) {
|
||||||
|
if (customConfig.import.videoChannelSynchronization.enabled && !customConfig.import.videos.http.enabled) {
|
||||||
|
res.fail({ message: 'You need to enable HTTP video import in order to enable channel synchronization' })
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
|
||||||
function checkInvalidLiveConfig (customConfig: CustomConfig, res: express.Response) {
|
function checkInvalidLiveConfig (customConfig: CustomConfig, res: express.Response) {
|
||||||
if (customConfig.live.enabled === false) return true
|
if (customConfig.live.enabled === false) return true
|
||||||
|
|
||||||
|
|
|
@ -52,6 +52,7 @@ const videoPlaylistsSortValidator = checkSortFactory(SORTABLE_COLUMNS.VIDEO_PLAY
|
||||||
const pluginsSortValidator = checkSortFactory(SORTABLE_COLUMNS.PLUGINS)
|
const pluginsSortValidator = checkSortFactory(SORTABLE_COLUMNS.PLUGINS)
|
||||||
const availablePluginsSortValidator = checkSortFactory(SORTABLE_COLUMNS.AVAILABLE_PLUGINS)
|
const availablePluginsSortValidator = checkSortFactory(SORTABLE_COLUMNS.AVAILABLE_PLUGINS)
|
||||||
const videoRedundanciesSortValidator = checkSortFactory(SORTABLE_COLUMNS.VIDEO_REDUNDANCIES)
|
const videoRedundanciesSortValidator = checkSortFactory(SORTABLE_COLUMNS.VIDEO_REDUNDANCIES)
|
||||||
|
const videoChannelSyncsSortValidator = checkSortFactory(SORTABLE_COLUMNS.VIDEO_CHANNEL_SYNCS)
|
||||||
|
|
||||||
const accountsFollowersSortValidator = checkSortFactory(SORTABLE_COLUMNS.ACCOUNT_FOLLOWERS)
|
const accountsFollowersSortValidator = checkSortFactory(SORTABLE_COLUMNS.ACCOUNT_FOLLOWERS)
|
||||||
const videoChannelsFollowersSortValidator = checkSortFactory(SORTABLE_COLUMNS.CHANNEL_FOLLOWERS)
|
const videoChannelsFollowersSortValidator = checkSortFactory(SORTABLE_COLUMNS.CHANNEL_FOLLOWERS)
|
||||||
|
@ -84,5 +85,6 @@ export {
|
||||||
videoPlaylistsSearchSortValidator,
|
videoPlaylistsSearchSortValidator,
|
||||||
accountsFollowersSortValidator,
|
accountsFollowersSortValidator,
|
||||||
videoChannelsFollowersSortValidator,
|
videoChannelsFollowersSortValidator,
|
||||||
|
videoChannelSyncsSortValidator,
|
||||||
pluginsSortValidator
|
pluginsSortValidator
|
||||||
}
|
}
|
||||||
|
|
|
@ -14,3 +14,4 @@ export * from './video-stats'
|
||||||
export * from './video-studio'
|
export * from './video-studio'
|
||||||
export * from './video-transcoding'
|
export * from './video-transcoding'
|
||||||
export * from './videos'
|
export * from './videos'
|
||||||
|
export * from './video-channel-sync'
|
||||||
|
|
66
server/middlewares/validators/videos/video-channel-sync.ts
Normal file
66
server/middlewares/validators/videos/video-channel-sync.ts
Normal file
|
@ -0,0 +1,66 @@
|
||||||
|
import * as express from 'express'
|
||||||
|
import { body, param } from 'express-validator'
|
||||||
|
import { isUrlValid } from '@server/helpers/custom-validators/activitypub/misc'
|
||||||
|
import { logger } from '@server/helpers/logger'
|
||||||
|
import { CONFIG } from '@server/initializers/config'
|
||||||
|
import { VideoChannelModel } from '@server/models/video/video-channel'
|
||||||
|
import { VideoChannelSyncModel } from '@server/models/video/video-channel-sync'
|
||||||
|
import { HttpStatusCode, VideoChannelSyncCreate } from '@shared/models'
|
||||||
|
import { areValidationErrors, doesVideoChannelIdExist } from '../shared'
|
||||||
|
|
||||||
|
export const ensureSyncIsEnabled = (req: express.Request, res: express.Response, next: express.NextFunction) => {
|
||||||
|
if (!CONFIG.IMPORT.VIDEO_CHANNEL_SYNCHRONIZATION.ENABLED) {
|
||||||
|
return res.fail({
|
||||||
|
status: HttpStatusCode.FORBIDDEN_403,
|
||||||
|
message: 'Synchronization is impossible as video channel synchronization is not enabled on the server'
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
return next()
|
||||||
|
}
|
||||||
|
|
||||||
|
export const videoChannelSyncValidator = [
|
||||||
|
body('externalChannelUrl').custom(isUrlValid).withMessage('Should have a valid channel url'),
|
||||||
|
body('videoChannelId').isInt().withMessage('Should have a valid video channel id'),
|
||||||
|
|
||||||
|
async (req: express.Request, res: express.Response, next: express.NextFunction) => {
|
||||||
|
logger.debug('Checking videoChannelSync parameters', { parameters: req.body })
|
||||||
|
|
||||||
|
if (areValidationErrors(req, res)) return
|
||||||
|
|
||||||
|
const body: VideoChannelSyncCreate = req.body
|
||||||
|
if (!await doesVideoChannelIdExist(body.videoChannelId, res)) return
|
||||||
|
|
||||||
|
const count = await VideoChannelSyncModel.countByAccount(res.locals.videoChannel.accountId)
|
||||||
|
if (count >= CONFIG.IMPORT.VIDEO_CHANNEL_SYNCHRONIZATION.MAX_PER_USER) {
|
||||||
|
return res.fail({
|
||||||
|
message: `You cannot create more than ${CONFIG.IMPORT.VIDEO_CHANNEL_SYNCHRONIZATION.MAX_PER_USER} channel synchronizations`
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
return next()
|
||||||
|
}
|
||||||
|
]
|
||||||
|
|
||||||
|
export const ensureSyncExists = [
|
||||||
|
param('id').exists().isInt().withMessage('Should have an sync id'),
|
||||||
|
|
||||||
|
async (req: express.Request, res: express.Response, next: express.NextFunction) => {
|
||||||
|
if (areValidationErrors(req, res)) return
|
||||||
|
|
||||||
|
const syncId = parseInt(req.params.id, 10)
|
||||||
|
const sync = await VideoChannelSyncModel.loadWithChannel(syncId)
|
||||||
|
|
||||||
|
if (!sync) {
|
||||||
|
return res.fail({
|
||||||
|
status: HttpStatusCode.NOT_FOUND_404,
|
||||||
|
message: 'Synchronization not found'
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
res.locals.videoChannelSync = sync
|
||||||
|
res.locals.videoChannel = await VideoChannelModel.loadAndPopulateAccount(sync.videoChannelId)
|
||||||
|
|
||||||
|
return next()
|
||||||
|
}
|
||||||
|
]
|
|
@ -1,5 +1,6 @@
|
||||||
import express from 'express'
|
import express from 'express'
|
||||||
import { body, param, query } from 'express-validator'
|
import { body, param, query } from 'express-validator'
|
||||||
|
import { isUrlValid } from '@server/helpers/custom-validators/activitypub/misc'
|
||||||
import { CONFIG } from '@server/initializers/config'
|
import { CONFIG } from '@server/initializers/config'
|
||||||
import { MChannelAccountDefault } from '@server/types/models'
|
import { MChannelAccountDefault } from '@server/types/models'
|
||||||
import { HttpStatusCode } from '../../../../shared/models/http/http-error-codes'
|
import { HttpStatusCode } from '../../../../shared/models/http/http-error-codes'
|
||||||
|
@ -13,9 +14,9 @@ import {
|
||||||
import { logger } from '../../../helpers/logger'
|
import { logger } from '../../../helpers/logger'
|
||||||
import { ActorModel } from '../../../models/actor/actor'
|
import { ActorModel } from '../../../models/actor/actor'
|
||||||
import { VideoChannelModel } from '../../../models/video/video-channel'
|
import { VideoChannelModel } from '../../../models/video/video-channel'
|
||||||
import { areValidationErrors, doesVideoChannelNameWithHostExist } from '../shared'
|
import { areValidationErrors, checkUserQuota, doesVideoChannelNameWithHostExist } from '../shared'
|
||||||
|
|
||||||
const videoChannelsAddValidator = [
|
export const videoChannelsAddValidator = [
|
||||||
body('name').custom(isVideoChannelUsernameValid).withMessage('Should have a valid channel name'),
|
body('name').custom(isVideoChannelUsernameValid).withMessage('Should have a valid channel name'),
|
||||||
body('displayName').custom(isVideoChannelDisplayNameValid).withMessage('Should have a valid display name'),
|
body('displayName').custom(isVideoChannelDisplayNameValid).withMessage('Should have a valid display name'),
|
||||||
body('description').optional().custom(isVideoChannelDescriptionValid).withMessage('Should have a valid description'),
|
body('description').optional().custom(isVideoChannelDescriptionValid).withMessage('Should have a valid description'),
|
||||||
|
@ -45,7 +46,7 @@ const videoChannelsAddValidator = [
|
||||||
}
|
}
|
||||||
]
|
]
|
||||||
|
|
||||||
const videoChannelsUpdateValidator = [
|
export const videoChannelsUpdateValidator = [
|
||||||
param('nameWithHost').exists().withMessage('Should have an video channel name with host'),
|
param('nameWithHost').exists().withMessage('Should have an video channel name with host'),
|
||||||
body('displayName')
|
body('displayName')
|
||||||
.optional()
|
.optional()
|
||||||
|
@ -69,7 +70,7 @@ const videoChannelsUpdateValidator = [
|
||||||
}
|
}
|
||||||
]
|
]
|
||||||
|
|
||||||
const videoChannelsRemoveValidator = [
|
export const videoChannelsRemoveValidator = [
|
||||||
async (req: express.Request, res: express.Response, next: express.NextFunction) => {
|
async (req: express.Request, res: express.Response, next: express.NextFunction) => {
|
||||||
logger.debug('Checking videoChannelsRemove parameters', { parameters: req.params })
|
logger.debug('Checking videoChannelsRemove parameters', { parameters: req.params })
|
||||||
|
|
||||||
|
@ -79,7 +80,7 @@ const videoChannelsRemoveValidator = [
|
||||||
}
|
}
|
||||||
]
|
]
|
||||||
|
|
||||||
const videoChannelsNameWithHostValidator = [
|
export const videoChannelsNameWithHostValidator = [
|
||||||
param('nameWithHost').exists().withMessage('Should have an video channel name with host'),
|
param('nameWithHost').exists().withMessage('Should have an video channel name with host'),
|
||||||
|
|
||||||
async (req: express.Request, res: express.Response, next: express.NextFunction) => {
|
async (req: express.Request, res: express.Response, next: express.NextFunction) => {
|
||||||
|
@ -93,7 +94,7 @@ const videoChannelsNameWithHostValidator = [
|
||||||
}
|
}
|
||||||
]
|
]
|
||||||
|
|
||||||
const ensureIsLocalChannel = [
|
export const ensureIsLocalChannel = [
|
||||||
(req: express.Request, res: express.Response, next: express.NextFunction) => {
|
(req: express.Request, res: express.Response, next: express.NextFunction) => {
|
||||||
if (res.locals.videoChannel.Actor.isOwned() === false) {
|
if (res.locals.videoChannel.Actor.isOwned() === false) {
|
||||||
return res.fail({
|
return res.fail({
|
||||||
|
@ -106,7 +107,18 @@ const ensureIsLocalChannel = [
|
||||||
}
|
}
|
||||||
]
|
]
|
||||||
|
|
||||||
const videoChannelStatsValidator = [
|
export const ensureChannelOwnerCanUpload = [
|
||||||
|
async (req: express.Request, res: express.Response, next: express.NextFunction) => {
|
||||||
|
const channel = res.locals.videoChannel
|
||||||
|
const user = { id: channel.Account.userId }
|
||||||
|
|
||||||
|
if (!await checkUserQuota(user, 1, res)) return
|
||||||
|
|
||||||
|
next()
|
||||||
|
}
|
||||||
|
]
|
||||||
|
|
||||||
|
export const videoChannelStatsValidator = [
|
||||||
query('withStats')
|
query('withStats')
|
||||||
.optional()
|
.optional()
|
||||||
.customSanitizer(toBooleanOrNull)
|
.customSanitizer(toBooleanOrNull)
|
||||||
|
@ -118,7 +130,7 @@ const videoChannelStatsValidator = [
|
||||||
}
|
}
|
||||||
]
|
]
|
||||||
|
|
||||||
const videoChannelsListValidator = [
|
export const videoChannelsListValidator = [
|
||||||
query('search').optional().not().isEmpty().withMessage('Should have a valid search'),
|
query('search').optional().not().isEmpty().withMessage('Should have a valid search'),
|
||||||
|
|
||||||
(req: express.Request, res: express.Response, next: express.NextFunction) => {
|
(req: express.Request, res: express.Response, next: express.NextFunction) => {
|
||||||
|
@ -130,17 +142,24 @@ const videoChannelsListValidator = [
|
||||||
}
|
}
|
||||||
]
|
]
|
||||||
|
|
||||||
// ---------------------------------------------------------------------------
|
export const videoChannelImportVideosValidator = [
|
||||||
|
body('externalChannelUrl').custom(isUrlValid).withMessage('Should have a valid channel url'),
|
||||||
|
|
||||||
export {
|
(req: express.Request, res: express.Response, next: express.NextFunction) => {
|
||||||
videoChannelsAddValidator,
|
logger.debug('Checking videoChannelImport parameters', { parameters: req.body })
|
||||||
videoChannelsUpdateValidator,
|
|
||||||
videoChannelsRemoveValidator,
|
if (areValidationErrors(req, res)) return
|
||||||
videoChannelsNameWithHostValidator,
|
|
||||||
ensureIsLocalChannel,
|
if (!CONFIG.IMPORT.VIDEOS.HTTP.ENABLED) {
|
||||||
videoChannelsListValidator,
|
return res.fail({
|
||||||
videoChannelStatsValidator
|
status: HttpStatusCode.FORBIDDEN_403,
|
||||||
}
|
message: 'Channel import is impossible as video upload via HTTP is not enabled on the server'
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
return next()
|
||||||
|
}
|
||||||
|
]
|
||||||
|
|
||||||
// ---------------------------------------------------------------------------
|
// ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
|
|
@ -117,6 +117,16 @@ function getInstanceFollowsSort (value: string, lastSort: OrderItem = [ 'id', 'A
|
||||||
return getSort(value, lastSort)
|
return getSort(value, lastSort)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function getChannelSyncSort (value: string): OrderItem[] {
|
||||||
|
const { direction, field } = buildDirectionAndField(value)
|
||||||
|
if (field.toLowerCase() === 'videochannel') {
|
||||||
|
return [
|
||||||
|
[ literal('"VideoChannel.name"'), direction ]
|
||||||
|
]
|
||||||
|
}
|
||||||
|
return [ [ field, direction ] ]
|
||||||
|
}
|
||||||
|
|
||||||
function isOutdated (model: { createdAt: Date, updatedAt: Date }, refreshInterval: number) {
|
function isOutdated (model: { createdAt: Date, updatedAt: Date }, refreshInterval: number) {
|
||||||
if (!model.createdAt || !model.updatedAt) {
|
if (!model.createdAt || !model.updatedAt) {
|
||||||
throw new Error('Miss createdAt & updatedAt attributes to model')
|
throw new Error('Miss createdAt & updatedAt attributes to model')
|
||||||
|
@ -280,6 +290,7 @@ export {
|
||||||
getAdminUsersSort,
|
getAdminUsersSort,
|
||||||
getVideoSort,
|
getVideoSort,
|
||||||
getBlacklistSort,
|
getBlacklistSort,
|
||||||
|
getChannelSyncSort,
|
||||||
createSimilarityAttribute,
|
createSimilarityAttribute,
|
||||||
throwIfNotValid,
|
throwIfNotValid,
|
||||||
buildServerIdsFollowedBy,
|
buildServerIdsFollowedBy,
|
||||||
|
|
176
server/models/video/video-channel-sync.ts
Normal file
176
server/models/video/video-channel-sync.ts
Normal file
|
@ -0,0 +1,176 @@
|
||||||
|
import { Op } from 'sequelize'
|
||||||
|
import {
|
||||||
|
AllowNull,
|
||||||
|
BelongsTo,
|
||||||
|
Column,
|
||||||
|
CreatedAt,
|
||||||
|
DataType,
|
||||||
|
Default,
|
||||||
|
DefaultScope,
|
||||||
|
ForeignKey,
|
||||||
|
Is,
|
||||||
|
Model,
|
||||||
|
Table,
|
||||||
|
UpdatedAt
|
||||||
|
} from 'sequelize-typescript'
|
||||||
|
import { isUrlValid } from '@server/helpers/custom-validators/activitypub/misc'
|
||||||
|
import { isVideoChannelSyncStateValid } from '@server/helpers/custom-validators/video-channel-syncs'
|
||||||
|
import { CONSTRAINTS_FIELDS, VIDEO_CHANNEL_SYNC_STATE } from '@server/initializers/constants'
|
||||||
|
import { MChannelSync, MChannelSyncChannel, MChannelSyncFormattable } from '@server/types/models'
|
||||||
|
import { VideoChannelSync, VideoChannelSyncState } from '@shared/models'
|
||||||
|
import { AttributesOnly } from '@shared/typescript-utils'
|
||||||
|
import { AccountModel } from '../account/account'
|
||||||
|
import { UserModel } from '../user/user'
|
||||||
|
import { getChannelSyncSort, throwIfNotValid } from '../utils'
|
||||||
|
import { VideoChannelModel } from './video-channel'
|
||||||
|
|
||||||
|
@DefaultScope(() => ({
|
||||||
|
include: [
|
||||||
|
{
|
||||||
|
model: VideoChannelModel, // Default scope includes avatar and server
|
||||||
|
required: true
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}))
|
||||||
|
@Table({
|
||||||
|
tableName: 'videoChannelSync',
|
||||||
|
indexes: [
|
||||||
|
{
|
||||||
|
fields: [ 'videoChannelId' ]
|
||||||
|
}
|
||||||
|
]
|
||||||
|
})
|
||||||
|
export class VideoChannelSyncModel extends Model<Partial<AttributesOnly<VideoChannelSyncModel>>> {
|
||||||
|
|
||||||
|
@AllowNull(false)
|
||||||
|
@Default(null)
|
||||||
|
@Is('VideoChannelExternalChannelUrl', value => throwIfNotValid(value, isUrlValid, 'externalChannelUrl', true))
|
||||||
|
@Column(DataType.STRING(CONSTRAINTS_FIELDS.VIDEO_CHANNEL_SYNCS.EXTERNAL_CHANNEL_URL.max))
|
||||||
|
externalChannelUrl: string
|
||||||
|
|
||||||
|
@CreatedAt
|
||||||
|
createdAt: Date
|
||||||
|
|
||||||
|
@UpdatedAt
|
||||||
|
updatedAt: Date
|
||||||
|
|
||||||
|
@ForeignKey(() => VideoChannelModel)
|
||||||
|
@Column
|
||||||
|
videoChannelId: number
|
||||||
|
|
||||||
|
@BelongsTo(() => VideoChannelModel, {
|
||||||
|
foreignKey: {
|
||||||
|
allowNull: false
|
||||||
|
},
|
||||||
|
onDelete: 'cascade'
|
||||||
|
})
|
||||||
|
VideoChannel: VideoChannelModel
|
||||||
|
|
||||||
|
@AllowNull(false)
|
||||||
|
@Default(VideoChannelSyncState.WAITING_FIRST_RUN)
|
||||||
|
@Is('VideoChannelSyncState', value => throwIfNotValid(value, isVideoChannelSyncStateValid, 'state'))
|
||||||
|
@Column
|
||||||
|
state: VideoChannelSyncState
|
||||||
|
|
||||||
|
@AllowNull(true)
|
||||||
|
@Column(DataType.DATE)
|
||||||
|
lastSyncAt: Date
|
||||||
|
|
||||||
|
static listByAccountForAPI (options: {
|
||||||
|
accountId: number
|
||||||
|
start: number
|
||||||
|
count: number
|
||||||
|
sort: string
|
||||||
|
}) {
|
||||||
|
const getQuery = (forCount: boolean) => {
|
||||||
|
const videoChannelModel = forCount
|
||||||
|
? VideoChannelModel.unscoped()
|
||||||
|
: VideoChannelModel
|
||||||
|
|
||||||
|
return {
|
||||||
|
offset: options.start,
|
||||||
|
limit: options.count,
|
||||||
|
order: getChannelSyncSort(options.sort),
|
||||||
|
include: [
|
||||||
|
{
|
||||||
|
model: videoChannelModel,
|
||||||
|
required: true,
|
||||||
|
where: {
|
||||||
|
accountId: options.accountId
|
||||||
|
}
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return Promise.all([
|
||||||
|
VideoChannelSyncModel.unscoped().count(getQuery(true)),
|
||||||
|
VideoChannelSyncModel.unscoped().findAll(getQuery(false))
|
||||||
|
]).then(([ total, data ]) => ({ total, data }))
|
||||||
|
}
|
||||||
|
|
||||||
|
static countByAccount (accountId: number) {
|
||||||
|
const query = {
|
||||||
|
include: [
|
||||||
|
{
|
||||||
|
model: VideoChannelModel.unscoped(),
|
||||||
|
required: true,
|
||||||
|
where: {
|
||||||
|
accountId
|
||||||
|
}
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
||||||
|
|
||||||
|
return VideoChannelSyncModel.unscoped().count(query)
|
||||||
|
}
|
||||||
|
|
||||||
|
static loadWithChannel (id: number): Promise<MChannelSyncChannel> {
|
||||||
|
return VideoChannelSyncModel.findByPk(id)
|
||||||
|
}
|
||||||
|
|
||||||
|
static async listSyncs (): Promise<MChannelSync[]> {
|
||||||
|
const query = {
|
||||||
|
include: [
|
||||||
|
{
|
||||||
|
model: VideoChannelModel.unscoped(),
|
||||||
|
required: true,
|
||||||
|
include: [
|
||||||
|
{
|
||||||
|
model: AccountModel.unscoped(),
|
||||||
|
required: true,
|
||||||
|
include: [ {
|
||||||
|
attributes: [],
|
||||||
|
model: UserModel.unscoped(),
|
||||||
|
required: true,
|
||||||
|
where: {
|
||||||
|
videoQuota: {
|
||||||
|
[Op.ne]: 0
|
||||||
|
},
|
||||||
|
videoQuotaDaily: {
|
||||||
|
[Op.ne]: 0
|
||||||
|
}
|
||||||
|
}
|
||||||
|
} ]
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
||||||
|
return VideoChannelSyncModel.unscoped().findAll(query)
|
||||||
|
}
|
||||||
|
|
||||||
|
toFormattedJSON (this: MChannelSyncFormattable): VideoChannelSync {
|
||||||
|
return {
|
||||||
|
id: this.id,
|
||||||
|
state: {
|
||||||
|
id: this.state,
|
||||||
|
label: VIDEO_CHANNEL_SYNC_STATE[this.state]
|
||||||
|
},
|
||||||
|
externalChannelUrl: this.externalChannelUrl,
|
||||||
|
createdAt: this.createdAt.toISOString(),
|
||||||
|
channel: this.VideoChannel.toFormattedSummaryJSON(),
|
||||||
|
lastSyncAt: this.lastSyncAt?.toISOString()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
|
@ -1,4 +1,4 @@
|
||||||
import { WhereOptions } from 'sequelize'
|
import { Op, WhereOptions } from 'sequelize'
|
||||||
import {
|
import {
|
||||||
AfterUpdate,
|
AfterUpdate,
|
||||||
AllowNull,
|
AllowNull,
|
||||||
|
@ -161,6 +161,28 @@ export class VideoImportModel extends Model<Partial<AttributesOnly<VideoImportMo
|
||||||
]).then(([ total, data ]) => ({ total, data }))
|
]).then(([ total, data ]) => ({ total, data }))
|
||||||
}
|
}
|
||||||
|
|
||||||
|
static async urlAlreadyImported (channelId: number, targetUrl: string): Promise<boolean> {
|
||||||
|
const element = await VideoImportModel.unscoped().findOne({
|
||||||
|
where: {
|
||||||
|
targetUrl,
|
||||||
|
state: {
|
||||||
|
[Op.in]: [ VideoImportState.PENDING, VideoImportState.PROCESSING, VideoImportState.SUCCESS ]
|
||||||
|
}
|
||||||
|
},
|
||||||
|
include: [
|
||||||
|
{
|
||||||
|
model: VideoModel,
|
||||||
|
required: true,
|
||||||
|
where: {
|
||||||
|
channelId
|
||||||
|
}
|
||||||
|
}
|
||||||
|
]
|
||||||
|
})
|
||||||
|
|
||||||
|
return !!element
|
||||||
|
}
|
||||||
|
|
||||||
getTargetIdentifier () {
|
getTargetIdentifier () {
|
||||||
return this.targetUrl || this.magnetUri || this.torrentName
|
return this.targetUrl || this.magnetUri || this.torrentName
|
||||||
}
|
}
|
||||||
|
|
|
@ -1,7 +1,8 @@
|
||||||
/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */
|
/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */
|
||||||
|
|
||||||
import 'mocha'
|
import 'mocha'
|
||||||
import { omit } from 'lodash'
|
import { merge, omit } from 'lodash'
|
||||||
|
import { CustomConfig, HttpStatusCode } from '@shared/models'
|
||||||
import {
|
import {
|
||||||
cleanupTests,
|
cleanupTests,
|
||||||
createSingleServer,
|
createSingleServer,
|
||||||
|
@ -11,7 +12,6 @@ import {
|
||||||
PeerTubeServer,
|
PeerTubeServer,
|
||||||
setAccessTokensToServers
|
setAccessTokensToServers
|
||||||
} from '@shared/server-commands'
|
} from '@shared/server-commands'
|
||||||
import { CustomConfig, HttpStatusCode } from '@shared/models'
|
|
||||||
|
|
||||||
describe('Test config API validators', function () {
|
describe('Test config API validators', function () {
|
||||||
const path = '/api/v1/config/custom'
|
const path = '/api/v1/config/custom'
|
||||||
|
@ -162,6 +162,10 @@ describe('Test config API validators', function () {
|
||||||
torrent: {
|
torrent: {
|
||||||
enabled: false
|
enabled: false
|
||||||
}
|
}
|
||||||
|
},
|
||||||
|
videoChannelSynchronization: {
|
||||||
|
enabled: false,
|
||||||
|
maxPerUser: 10
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
trending: {
|
trending: {
|
||||||
|
@ -346,7 +350,26 @@ describe('Test config API validators', function () {
|
||||||
})
|
})
|
||||||
})
|
})
|
||||||
|
|
||||||
it('Should success with the correct parameters', async function () {
|
it('Should fail with a disabled http upload & enabled sync', async function () {
|
||||||
|
const newUpdateParams: CustomConfig = merge({}, updateParams, {
|
||||||
|
import: {
|
||||||
|
videos: {
|
||||||
|
http: { enabled: false }
|
||||||
|
},
|
||||||
|
videoChannelSynchronization: { enabled: true }
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
await makePutBodyRequest({
|
||||||
|
url: server.url,
|
||||||
|
path,
|
||||||
|
fields: newUpdateParams,
|
||||||
|
token: server.accessToken,
|
||||||
|
expectedStatus: HttpStatusCode.BAD_REQUEST_400
|
||||||
|
})
|
||||||
|
})
|
||||||
|
|
||||||
|
it('Should succeed with the correct parameters', async function () {
|
||||||
await makePutBodyRequest({
|
await makePutBodyRequest({
|
||||||
url: server.url,
|
url: server.url,
|
||||||
path,
|
path,
|
||||||
|
|
|
@ -27,6 +27,7 @@ import './video-channels'
|
||||||
import './video-comments'
|
import './video-comments'
|
||||||
import './video-files'
|
import './video-files'
|
||||||
import './video-imports'
|
import './video-imports'
|
||||||
|
import './video-channel-syncs'
|
||||||
import './video-playlists'
|
import './video-playlists'
|
||||||
import './video-source'
|
import './video-source'
|
||||||
import './video-studio'
|
import './video-studio'
|
||||||
|
|
|
@ -70,7 +70,7 @@ describe('Test upload quota', function () {
|
||||||
})
|
})
|
||||||
|
|
||||||
it('Should fail to import with HTTP/Torrent/magnet', async function () {
|
it('Should fail to import with HTTP/Torrent/magnet', async function () {
|
||||||
this.timeout(120000)
|
this.timeout(120_000)
|
||||||
|
|
||||||
const baseAttributes = {
|
const baseAttributes = {
|
||||||
channelId: server.store.channel.id,
|
channelId: server.store.channel.id,
|
||||||
|
|
318
server/tests/api/check-params/video-channel-syncs.ts
Normal file
318
server/tests/api/check-params/video-channel-syncs.ts
Normal file
|
@ -0,0 +1,318 @@
|
||||||
|
import { checkBadCountPagination, checkBadSortPagination, checkBadStartPagination, FIXTURE_URLS } from '@server/tests/shared'
|
||||||
|
import { HttpStatusCode, VideoChannelSyncCreate } from '@shared/models'
|
||||||
|
import {
|
||||||
|
ChannelSyncsCommand,
|
||||||
|
createSingleServer,
|
||||||
|
makePostBodyRequest,
|
||||||
|
PeerTubeServer,
|
||||||
|
setAccessTokensToServers,
|
||||||
|
setDefaultVideoChannel
|
||||||
|
} from '@shared/server-commands'
|
||||||
|
|
||||||
|
describe('Test video channel sync API validator', () => {
|
||||||
|
const path = '/api/v1/video-channel-syncs'
|
||||||
|
let server: PeerTubeServer
|
||||||
|
let command: ChannelSyncsCommand
|
||||||
|
let rootChannelId: number
|
||||||
|
let rootChannelSyncId: number
|
||||||
|
const userInfo = {
|
||||||
|
accessToken: '',
|
||||||
|
username: 'user1',
|
||||||
|
id: -1,
|
||||||
|
channelId: -1,
|
||||||
|
syncId: -1
|
||||||
|
}
|
||||||
|
|
||||||
|
async function withChannelSyncDisabled<T> (callback: () => Promise<T>): Promise<void> {
|
||||||
|
try {
|
||||||
|
await server.config.disableChannelSync()
|
||||||
|
await callback()
|
||||||
|
} finally {
|
||||||
|
await server.config.enableChannelSync()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async function withMaxSyncsPerUser<T> (maxSync: number, callback: () => Promise<T>): Promise<void> {
|
||||||
|
const origConfig = await server.config.getCustomConfig()
|
||||||
|
|
||||||
|
await server.config.updateExistingSubConfig({
|
||||||
|
newConfig: {
|
||||||
|
import: {
|
||||||
|
videoChannelSynchronization: {
|
||||||
|
maxPerUser: maxSync
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
try {
|
||||||
|
await callback()
|
||||||
|
} finally {
|
||||||
|
await server.config.updateCustomConfig({ newCustomConfig: origConfig })
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
before(async function () {
|
||||||
|
this.timeout(30_000)
|
||||||
|
|
||||||
|
server = await createSingleServer(1)
|
||||||
|
|
||||||
|
await setAccessTokensToServers([ server ])
|
||||||
|
await setDefaultVideoChannel([ server ])
|
||||||
|
|
||||||
|
command = server.channelSyncs
|
||||||
|
|
||||||
|
rootChannelId = server.store.channel.id
|
||||||
|
|
||||||
|
{
|
||||||
|
userInfo.accessToken = await server.users.generateUserAndToken(userInfo.username)
|
||||||
|
|
||||||
|
const { videoChannels, id: userId } = await server.users.getMyInfo({ token: userInfo.accessToken })
|
||||||
|
userInfo.id = userId
|
||||||
|
userInfo.channelId = videoChannels[0].id
|
||||||
|
}
|
||||||
|
|
||||||
|
await server.config.enableChannelSync()
|
||||||
|
})
|
||||||
|
|
||||||
|
describe('When creating a sync', function () {
|
||||||
|
let baseCorrectParams: VideoChannelSyncCreate
|
||||||
|
|
||||||
|
before(function () {
|
||||||
|
baseCorrectParams = {
|
||||||
|
externalChannelUrl: FIXTURE_URLS.youtubeChannel,
|
||||||
|
videoChannelId: rootChannelId
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
it('Should fail when sync is disabled', async function () {
|
||||||
|
await withChannelSyncDisabled(async () => {
|
||||||
|
await command.create({
|
||||||
|
token: server.accessToken,
|
||||||
|
attributes: baseCorrectParams,
|
||||||
|
expectedStatus: HttpStatusCode.FORBIDDEN_403
|
||||||
|
})
|
||||||
|
})
|
||||||
|
})
|
||||||
|
|
||||||
|
it('Should fail with nothing', async function () {
|
||||||
|
const fields = {}
|
||||||
|
await makePostBodyRequest({
|
||||||
|
url: server.url,
|
||||||
|
path,
|
||||||
|
token: server.accessToken,
|
||||||
|
fields,
|
||||||
|
expectedStatus: HttpStatusCode.BAD_REQUEST_400
|
||||||
|
})
|
||||||
|
})
|
||||||
|
|
||||||
|
it('Should fail with no authentication', async function () {
|
||||||
|
await command.create({
|
||||||
|
token: null,
|
||||||
|
attributes: baseCorrectParams,
|
||||||
|
expectedStatus: HttpStatusCode.UNAUTHORIZED_401
|
||||||
|
})
|
||||||
|
})
|
||||||
|
|
||||||
|
it('Should fail without a target url', async function () {
|
||||||
|
const attributes: VideoChannelSyncCreate = {
|
||||||
|
...baseCorrectParams,
|
||||||
|
externalChannelUrl: null
|
||||||
|
}
|
||||||
|
await command.create({
|
||||||
|
token: server.accessToken,
|
||||||
|
attributes,
|
||||||
|
expectedStatus: HttpStatusCode.BAD_REQUEST_400
|
||||||
|
})
|
||||||
|
})
|
||||||
|
|
||||||
|
it('Should fail without a channelId', async function () {
|
||||||
|
const attributes: VideoChannelSyncCreate = {
|
||||||
|
...baseCorrectParams,
|
||||||
|
videoChannelId: null
|
||||||
|
}
|
||||||
|
await command.create({
|
||||||
|
token: server.accessToken,
|
||||||
|
attributes,
|
||||||
|
expectedStatus: HttpStatusCode.BAD_REQUEST_400
|
||||||
|
})
|
||||||
|
})
|
||||||
|
|
||||||
|
it('Should fail with a channelId refering nothing', async function () {
|
||||||
|
const attributes: VideoChannelSyncCreate = {
|
||||||
|
...baseCorrectParams,
|
||||||
|
videoChannelId: 42
|
||||||
|
}
|
||||||
|
await command.create({
|
||||||
|
token: server.accessToken,
|
||||||
|
attributes,
|
||||||
|
expectedStatus: HttpStatusCode.NOT_FOUND_404
|
||||||
|
})
|
||||||
|
})
|
||||||
|
|
||||||
|
it('Should fail to create a sync when the user does not own the channel', async function () {
|
||||||
|
await command.create({
|
||||||
|
token: userInfo.accessToken,
|
||||||
|
attributes: baseCorrectParams,
|
||||||
|
expectedStatus: HttpStatusCode.FORBIDDEN_403
|
||||||
|
})
|
||||||
|
})
|
||||||
|
|
||||||
|
it('Should succeed to create a sync with root and for another user\'s channel', async function () {
|
||||||
|
const { videoChannelSync } = await command.create({
|
||||||
|
token: server.accessToken,
|
||||||
|
attributes: {
|
||||||
|
...baseCorrectParams,
|
||||||
|
videoChannelId: userInfo.channelId
|
||||||
|
},
|
||||||
|
expectedStatus: HttpStatusCode.OK_200
|
||||||
|
})
|
||||||
|
userInfo.syncId = videoChannelSync.id
|
||||||
|
})
|
||||||
|
|
||||||
|
it('Should succeed with the correct parameters', async function () {
|
||||||
|
const { videoChannelSync } = await command.create({
|
||||||
|
token: server.accessToken,
|
||||||
|
attributes: baseCorrectParams,
|
||||||
|
expectedStatus: HttpStatusCode.OK_200
|
||||||
|
})
|
||||||
|
rootChannelSyncId = videoChannelSync.id
|
||||||
|
})
|
||||||
|
|
||||||
|
it('Should fail when the user exceeds allowed number of synchronizations', async function () {
|
||||||
|
await withMaxSyncsPerUser(1, async () => {
|
||||||
|
await command.create({
|
||||||
|
token: server.accessToken,
|
||||||
|
attributes: {
|
||||||
|
...baseCorrectParams,
|
||||||
|
videoChannelId: userInfo.channelId
|
||||||
|
},
|
||||||
|
expectedStatus: HttpStatusCode.BAD_REQUEST_400
|
||||||
|
})
|
||||||
|
})
|
||||||
|
})
|
||||||
|
})
|
||||||
|
|
||||||
|
describe('When listing my channel syncs', function () {
|
||||||
|
const myPath = '/api/v1/accounts/root/video-channel-syncs'
|
||||||
|
|
||||||
|
it('Should fail with a bad start pagination', async function () {
|
||||||
|
await checkBadStartPagination(server.url, myPath, server.accessToken)
|
||||||
|
})
|
||||||
|
|
||||||
|
it('Should fail with a bad count pagination', async function () {
|
||||||
|
await checkBadCountPagination(server.url, myPath, server.accessToken)
|
||||||
|
})
|
||||||
|
|
||||||
|
it('Should fail with an incorrect sort', async function () {
|
||||||
|
await checkBadSortPagination(server.url, myPath, server.accessToken)
|
||||||
|
})
|
||||||
|
|
||||||
|
it('Should succeed with the correct parameters', async function () {
|
||||||
|
await command.listByAccount({
|
||||||
|
accountName: 'root',
|
||||||
|
token: server.accessToken,
|
||||||
|
expectedStatus: HttpStatusCode.OK_200
|
||||||
|
})
|
||||||
|
})
|
||||||
|
|
||||||
|
it('Should fail with no authentication', async function () {
|
||||||
|
await command.listByAccount({
|
||||||
|
accountName: 'root',
|
||||||
|
token: null,
|
||||||
|
expectedStatus: HttpStatusCode.UNAUTHORIZED_401
|
||||||
|
})
|
||||||
|
})
|
||||||
|
|
||||||
|
it('Should fail when a simple user lists another user\'s synchronizations', async function () {
|
||||||
|
await command.listByAccount({
|
||||||
|
accountName: 'root',
|
||||||
|
token: userInfo.accessToken,
|
||||||
|
expectedStatus: HttpStatusCode.FORBIDDEN_403
|
||||||
|
})
|
||||||
|
})
|
||||||
|
|
||||||
|
it('Should succeed when root lists another user\'s synchronizations', async function () {
|
||||||
|
await command.listByAccount({
|
||||||
|
accountName: userInfo.username,
|
||||||
|
token: server.accessToken,
|
||||||
|
expectedStatus: HttpStatusCode.OK_200
|
||||||
|
})
|
||||||
|
})
|
||||||
|
|
||||||
|
it('Should succeed even with synchronization disabled', async function () {
|
||||||
|
await withChannelSyncDisabled(async function () {
|
||||||
|
await command.listByAccount({
|
||||||
|
accountName: 'root',
|
||||||
|
token: server.accessToken,
|
||||||
|
expectedStatus: HttpStatusCode.OK_200
|
||||||
|
})
|
||||||
|
})
|
||||||
|
})
|
||||||
|
})
|
||||||
|
|
||||||
|
describe('When triggering deletion', function () {
|
||||||
|
it('should fail with no authentication', async function () {
|
||||||
|
await command.delete({
|
||||||
|
channelSyncId: userInfo.syncId,
|
||||||
|
token: null,
|
||||||
|
expectedStatus: HttpStatusCode.UNAUTHORIZED_401
|
||||||
|
})
|
||||||
|
})
|
||||||
|
|
||||||
|
it('Should fail when channelSyncId does not refer to any sync', async function () {
|
||||||
|
await command.delete({
|
||||||
|
channelSyncId: 42,
|
||||||
|
token: server.accessToken,
|
||||||
|
expectedStatus: HttpStatusCode.NOT_FOUND_404
|
||||||
|
})
|
||||||
|
})
|
||||||
|
|
||||||
|
it('Should fail when sync is not owned by the user', async function () {
|
||||||
|
await command.delete({
|
||||||
|
channelSyncId: rootChannelSyncId,
|
||||||
|
token: userInfo.accessToken,
|
||||||
|
expectedStatus: HttpStatusCode.FORBIDDEN_403
|
||||||
|
})
|
||||||
|
})
|
||||||
|
|
||||||
|
it('Should succeed when root delete a sync they do not own', async function () {
|
||||||
|
await command.delete({
|
||||||
|
channelSyncId: userInfo.syncId,
|
||||||
|
token: server.accessToken,
|
||||||
|
expectedStatus: HttpStatusCode.NO_CONTENT_204
|
||||||
|
})
|
||||||
|
})
|
||||||
|
|
||||||
|
it('should succeed when user delete a sync they own', async function () {
|
||||||
|
const { videoChannelSync } = await command.create({
|
||||||
|
attributes: {
|
||||||
|
externalChannelUrl: FIXTURE_URLS.youtubeChannel,
|
||||||
|
videoChannelId: userInfo.channelId
|
||||||
|
},
|
||||||
|
token: server.accessToken,
|
||||||
|
expectedStatus: HttpStatusCode.OK_200
|
||||||
|
})
|
||||||
|
|
||||||
|
await command.delete({
|
||||||
|
channelSyncId: videoChannelSync.id,
|
||||||
|
token: server.accessToken,
|
||||||
|
expectedStatus: HttpStatusCode.NO_CONTENT_204
|
||||||
|
})
|
||||||
|
})
|
||||||
|
|
||||||
|
it('Should succeed even when synchronization is disabled', async function () {
|
||||||
|
await withChannelSyncDisabled(async function () {
|
||||||
|
await command.delete({
|
||||||
|
channelSyncId: rootChannelSyncId,
|
||||||
|
token: server.accessToken,
|
||||||
|
expectedStatus: HttpStatusCode.NO_CONTENT_204
|
||||||
|
})
|
||||||
|
})
|
||||||
|
})
|
||||||
|
})
|
||||||
|
|
||||||
|
after(async function () {
|
||||||
|
await server?.kill()
|
||||||
|
})
|
||||||
|
})
|
|
@ -3,8 +3,8 @@
|
||||||
import 'mocha'
|
import 'mocha'
|
||||||
import * as chai from 'chai'
|
import * as chai from 'chai'
|
||||||
import { omit } from 'lodash'
|
import { omit } from 'lodash'
|
||||||
import { checkBadCountPagination, checkBadSortPagination, checkBadStartPagination } from '@server/tests/shared'
|
import { checkBadCountPagination, checkBadSortPagination, checkBadStartPagination, FIXTURE_URLS } from '@server/tests/shared'
|
||||||
import { buildAbsoluteFixturePath } from '@shared/core-utils'
|
import { areHttpImportTestsDisabled, buildAbsoluteFixturePath } from '@shared/core-utils'
|
||||||
import { HttpStatusCode, VideoChannelUpdate } from '@shared/models'
|
import { HttpStatusCode, VideoChannelUpdate } from '@shared/models'
|
||||||
import {
|
import {
|
||||||
ChannelsCommand,
|
ChannelsCommand,
|
||||||
|
@ -23,7 +23,13 @@ const expect = chai.expect
|
||||||
describe('Test video channels API validator', function () {
|
describe('Test video channels API validator', function () {
|
||||||
const videoChannelPath = '/api/v1/video-channels'
|
const videoChannelPath = '/api/v1/video-channels'
|
||||||
let server: PeerTubeServer
|
let server: PeerTubeServer
|
||||||
let accessTokenUser: string
|
const userInfo = {
|
||||||
|
accessToken: '',
|
||||||
|
channelName: 'fake_channel',
|
||||||
|
id: -1,
|
||||||
|
videoQuota: -1,
|
||||||
|
videoQuotaDaily: -1
|
||||||
|
}
|
||||||
let command: ChannelsCommand
|
let command: ChannelsCommand
|
||||||
|
|
||||||
// ---------------------------------------------------------------
|
// ---------------------------------------------------------------
|
||||||
|
@ -35,14 +41,15 @@ describe('Test video channels API validator', function () {
|
||||||
|
|
||||||
await setAccessTokensToServers([ server ])
|
await setAccessTokensToServers([ server ])
|
||||||
|
|
||||||
const user = {
|
const userCreds = {
|
||||||
username: 'fake',
|
username: 'fake',
|
||||||
password: 'fake_password'
|
password: 'fake_password'
|
||||||
}
|
}
|
||||||
|
|
||||||
{
|
{
|
||||||
await server.users.create({ username: user.username, password: user.password })
|
const user = await server.users.create({ username: userCreds.username, password: userCreds.password })
|
||||||
accessTokenUser = await server.login.getAccessToken(user)
|
userInfo.id = user.id
|
||||||
|
userInfo.accessToken = await server.login.getAccessToken(userCreds)
|
||||||
}
|
}
|
||||||
|
|
||||||
command = server.channels
|
command = server.channels
|
||||||
|
@ -191,7 +198,7 @@ describe('Test video channels API validator', function () {
|
||||||
await makePutBodyRequest({
|
await makePutBodyRequest({
|
||||||
url: server.url,
|
url: server.url,
|
||||||
path,
|
path,
|
||||||
token: accessTokenUser,
|
token: userInfo.accessToken,
|
||||||
fields: baseCorrectParams,
|
fields: baseCorrectParams,
|
||||||
expectedStatus: HttpStatusCode.FORBIDDEN_403
|
expectedStatus: HttpStatusCode.FORBIDDEN_403
|
||||||
})
|
})
|
||||||
|
@ -339,7 +346,7 @@ describe('Test video channels API validator', function () {
|
||||||
})
|
})
|
||||||
|
|
||||||
it('Should fail with a another user', async function () {
|
it('Should fail with a another user', async function () {
|
||||||
await makeGetRequest({ url: server.url, path, token: accessTokenUser, expectedStatus: HttpStatusCode.FORBIDDEN_403 })
|
await makeGetRequest({ url: server.url, path, token: userInfo.accessToken, expectedStatus: HttpStatusCode.FORBIDDEN_403 })
|
||||||
})
|
})
|
||||||
|
|
||||||
it('Should succeed with the correct params', async function () {
|
it('Should succeed with the correct params', async function () {
|
||||||
|
@ -347,13 +354,122 @@ describe('Test video channels API validator', function () {
|
||||||
})
|
})
|
||||||
})
|
})
|
||||||
|
|
||||||
|
describe('When triggering full synchronization', function () {
|
||||||
|
|
||||||
|
it('Should fail when HTTP upload is disabled', async function () {
|
||||||
|
await server.config.disableImports()
|
||||||
|
|
||||||
|
await command.importVideos({
|
||||||
|
channelName: 'super_channel',
|
||||||
|
externalChannelUrl: FIXTURE_URLS.youtubeChannel,
|
||||||
|
token: server.accessToken,
|
||||||
|
expectedStatus: HttpStatusCode.FORBIDDEN_403
|
||||||
|
})
|
||||||
|
|
||||||
|
await server.config.enableImports()
|
||||||
|
})
|
||||||
|
|
||||||
|
it('Should fail when externalChannelUrl is not provided', async function () {
|
||||||
|
await command.importVideos({
|
||||||
|
channelName: 'super_channel',
|
||||||
|
externalChannelUrl: null,
|
||||||
|
token: server.accessToken,
|
||||||
|
expectedStatus: HttpStatusCode.BAD_REQUEST_400
|
||||||
|
})
|
||||||
|
})
|
||||||
|
|
||||||
|
it('Should fail when externalChannelUrl is malformed', async function () {
|
||||||
|
await command.importVideos({
|
||||||
|
channelName: 'super_channel',
|
||||||
|
externalChannelUrl: 'not-a-url',
|
||||||
|
token: server.accessToken,
|
||||||
|
expectedStatus: HttpStatusCode.BAD_REQUEST_400
|
||||||
|
})
|
||||||
|
})
|
||||||
|
|
||||||
|
it('Should fail with no authentication', async function () {
|
||||||
|
await command.importVideos({
|
||||||
|
channelName: 'super_channel',
|
||||||
|
externalChannelUrl: FIXTURE_URLS.youtubeChannel,
|
||||||
|
token: null,
|
||||||
|
expectedStatus: HttpStatusCode.UNAUTHORIZED_401
|
||||||
|
})
|
||||||
|
})
|
||||||
|
|
||||||
|
it('Should fail when sync is not owned by the user', async function () {
|
||||||
|
await command.importVideos({
|
||||||
|
channelName: 'super_channel',
|
||||||
|
externalChannelUrl: FIXTURE_URLS.youtubeChannel,
|
||||||
|
token: userInfo.accessToken,
|
||||||
|
expectedStatus: HttpStatusCode.FORBIDDEN_403
|
||||||
|
})
|
||||||
|
})
|
||||||
|
|
||||||
|
it('Should fail when the user has no quota', async function () {
|
||||||
|
await server.users.update({
|
||||||
|
userId: userInfo.id,
|
||||||
|
videoQuota: 0
|
||||||
|
})
|
||||||
|
|
||||||
|
await command.importVideos({
|
||||||
|
channelName: 'fake_channel',
|
||||||
|
externalChannelUrl: FIXTURE_URLS.youtubeChannel,
|
||||||
|
token: userInfo.accessToken,
|
||||||
|
expectedStatus: HttpStatusCode.PAYLOAD_TOO_LARGE_413
|
||||||
|
})
|
||||||
|
|
||||||
|
await server.users.update({
|
||||||
|
userId: userInfo.id,
|
||||||
|
videoQuota: userInfo.videoQuota
|
||||||
|
})
|
||||||
|
})
|
||||||
|
|
||||||
|
it('Should fail when the user has no daily quota', async function () {
|
||||||
|
await server.users.update({
|
||||||
|
userId: userInfo.id,
|
||||||
|
videoQuotaDaily: 0
|
||||||
|
})
|
||||||
|
|
||||||
|
await command.importVideos({
|
||||||
|
channelName: 'fake_channel',
|
||||||
|
externalChannelUrl: FIXTURE_URLS.youtubeChannel,
|
||||||
|
token: userInfo.accessToken,
|
||||||
|
expectedStatus: HttpStatusCode.PAYLOAD_TOO_LARGE_413
|
||||||
|
})
|
||||||
|
|
||||||
|
await server.users.update({
|
||||||
|
userId: userInfo.id,
|
||||||
|
videoQuotaDaily: userInfo.videoQuotaDaily
|
||||||
|
})
|
||||||
|
})
|
||||||
|
|
||||||
|
it('Should succeed when sync is run by its owner', async function () {
|
||||||
|
if (!areHttpImportTestsDisabled()) return
|
||||||
|
|
||||||
|
await command.importVideos({
|
||||||
|
channelName: 'fake_channel',
|
||||||
|
externalChannelUrl: FIXTURE_URLS.youtubeChannel,
|
||||||
|
token: userInfo.accessToken
|
||||||
|
})
|
||||||
|
})
|
||||||
|
|
||||||
|
it('Should succeed when sync is run with root and for another user\'s channel', async function () {
|
||||||
|
if (!areHttpImportTestsDisabled()) return
|
||||||
|
|
||||||
|
await command.importVideos({
|
||||||
|
channelName: 'fake_channel',
|
||||||
|
externalChannelUrl: FIXTURE_URLS.youtubeChannel
|
||||||
|
})
|
||||||
|
})
|
||||||
|
})
|
||||||
|
|
||||||
describe('When deleting a video channel', function () {
|
describe('When deleting a video channel', function () {
|
||||||
it('Should fail with a non authenticated user', async function () {
|
it('Should fail with a non authenticated user', async function () {
|
||||||
await command.delete({ token: 'coucou', channelName: 'super_channel', expectedStatus: HttpStatusCode.UNAUTHORIZED_401 })
|
await command.delete({ token: 'coucou', channelName: 'super_channel', expectedStatus: HttpStatusCode.UNAUTHORIZED_401 })
|
||||||
})
|
})
|
||||||
|
|
||||||
it('Should fail with another authenticated user', async function () {
|
it('Should fail with another authenticated user', async function () {
|
||||||
await command.delete({ token: accessTokenUser, channelName: 'super_channel', expectedStatus: HttpStatusCode.FORBIDDEN_403 })
|
await command.delete({ token: userInfo.accessToken, channelName: 'super_channel', expectedStatus: HttpStatusCode.FORBIDDEN_403 })
|
||||||
})
|
})
|
||||||
|
|
||||||
it('Should fail with an unknown video channel id', async function () {
|
it('Should fail with an unknown video channel id', async function () {
|
||||||
|
|
|
@ -88,7 +88,13 @@ describe('Test video imports API validator', function () {
|
||||||
|
|
||||||
it('Should fail with nothing', async function () {
|
it('Should fail with nothing', async function () {
|
||||||
const fields = {}
|
const fields = {}
|
||||||
await makePostBodyRequest({ url: server.url, path, token: server.accessToken, fields })
|
await makePostBodyRequest({
|
||||||
|
url: server.url,
|
||||||
|
path,
|
||||||
|
token: server.accessToken,
|
||||||
|
fields,
|
||||||
|
expectedStatus: HttpStatusCode.BAD_REQUEST_400
|
||||||
|
})
|
||||||
})
|
})
|
||||||
|
|
||||||
it('Should fail without a target url', async function () {
|
it('Should fail without a target url', async function () {
|
||||||
|
|
|
@ -368,6 +368,10 @@ const newCustomConfig: CustomConfig = {
|
||||||
torrent: {
|
torrent: {
|
||||||
enabled: false
|
enabled: false
|
||||||
}
|
}
|
||||||
|
},
|
||||||
|
videoChannelSynchronization: {
|
||||||
|
enabled: false,
|
||||||
|
maxPerUser: 10
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
trending: {
|
trending: {
|
||||||
|
|
50
server/tests/api/videos/channel-import-videos.ts
Normal file
50
server/tests/api/videos/channel-import-videos.ts
Normal file
|
@ -0,0 +1,50 @@
|
||||||
|
import { expect } from 'chai'
|
||||||
|
import { FIXTURE_URLS } from '@server/tests/shared'
|
||||||
|
import { areHttpImportTestsDisabled } from '@shared/core-utils'
|
||||||
|
import {
|
||||||
|
createSingleServer,
|
||||||
|
getServerImportConfig,
|
||||||
|
PeerTubeServer,
|
||||||
|
setAccessTokensToServers,
|
||||||
|
setDefaultVideoChannel,
|
||||||
|
waitJobs
|
||||||
|
} from '@shared/server-commands'
|
||||||
|
|
||||||
|
describe('Test videos import in a channel', function () {
|
||||||
|
if (areHttpImportTestsDisabled()) return
|
||||||
|
|
||||||
|
function runSuite (mode: 'youtube-dl' | 'yt-dlp') {
|
||||||
|
|
||||||
|
describe('Import using ' + mode, function () {
|
||||||
|
let server: PeerTubeServer
|
||||||
|
|
||||||
|
before(async function () {
|
||||||
|
this.timeout(120_000)
|
||||||
|
|
||||||
|
server = await createSingleServer(1, getServerImportConfig(mode))
|
||||||
|
|
||||||
|
await setAccessTokensToServers([ server ])
|
||||||
|
await setDefaultVideoChannel([ server ])
|
||||||
|
|
||||||
|
await server.config.enableChannelSync()
|
||||||
|
})
|
||||||
|
|
||||||
|
it('Should import a whole channel', async function () {
|
||||||
|
this.timeout(240_000)
|
||||||
|
|
||||||
|
await server.channels.importVideos({ channelName: server.store.channel.name, externalChannelUrl: FIXTURE_URLS.youtubeChannel })
|
||||||
|
await waitJobs(server)
|
||||||
|
|
||||||
|
const videos = await server.videos.listByChannel({ handle: server.store.channel.name })
|
||||||
|
expect(videos.total).to.equal(2)
|
||||||
|
})
|
||||||
|
|
||||||
|
after(async function () {
|
||||||
|
await server?.kill()
|
||||||
|
})
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
runSuite('yt-dlp')
|
||||||
|
runSuite('youtube-dl')
|
||||||
|
})
|
|
@ -4,6 +4,8 @@ import './single-server'
|
||||||
import './video-captions'
|
import './video-captions'
|
||||||
import './video-change-ownership'
|
import './video-change-ownership'
|
||||||
import './video-channels'
|
import './video-channels'
|
||||||
|
import './channel-import-videos'
|
||||||
|
import './video-channel-syncs'
|
||||||
import './video-comments'
|
import './video-comments'
|
||||||
import './video-description'
|
import './video-description'
|
||||||
import './video-files'
|
import './video-files'
|
||||||
|
|
226
server/tests/api/videos/video-channel-syncs.ts
Normal file
226
server/tests/api/videos/video-channel-syncs.ts
Normal file
|
@ -0,0 +1,226 @@
|
||||||
|
import 'mocha'
|
||||||
|
import { expect } from 'chai'
|
||||||
|
import { FIXTURE_URLS } from '@server/tests/shared'
|
||||||
|
import { areHttpImportTestsDisabled } from '@shared/core-utils'
|
||||||
|
import { HttpStatusCode, VideoChannelSyncState, VideoInclude, VideoPrivacy } from '@shared/models'
|
||||||
|
import {
|
||||||
|
ChannelSyncsCommand,
|
||||||
|
createSingleServer,
|
||||||
|
getServerImportConfig,
|
||||||
|
PeerTubeServer,
|
||||||
|
setAccessTokensToServers,
|
||||||
|
setDefaultAccountAvatar,
|
||||||
|
setDefaultChannelAvatar,
|
||||||
|
setDefaultVideoChannel,
|
||||||
|
waitJobs
|
||||||
|
} from '@shared/server-commands'
|
||||||
|
|
||||||
|
describe('Test channel synchronizations', function () {
|
||||||
|
if (areHttpImportTestsDisabled()) return
|
||||||
|
|
||||||
|
function runSuite (mode: 'youtube-dl' | 'yt-dlp') {
|
||||||
|
|
||||||
|
describe('Sync using ' + mode, function () {
|
||||||
|
let server: PeerTubeServer
|
||||||
|
let command: ChannelSyncsCommand
|
||||||
|
let startTestDate: Date
|
||||||
|
const userInfo = {
|
||||||
|
accessToken: '',
|
||||||
|
username: 'user1',
|
||||||
|
channelName: 'user1_channel',
|
||||||
|
channelId: -1,
|
||||||
|
syncId: -1
|
||||||
|
}
|
||||||
|
|
||||||
|
async function changeDateForSync (channelSyncId: number, newDate: string) {
|
||||||
|
await server.sql.updateQuery(
|
||||||
|
`UPDATE "videoChannelSync" ` +
|
||||||
|
`SET "createdAt"='${newDate}', "lastSyncAt"='${newDate}' ` +
|
||||||
|
`WHERE id=${channelSyncId}`
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
before(async function () {
|
||||||
|
this.timeout(120_000)
|
||||||
|
|
||||||
|
startTestDate = new Date()
|
||||||
|
|
||||||
|
server = await createSingleServer(1, getServerImportConfig(mode))
|
||||||
|
|
||||||
|
await setAccessTokensToServers([ server ])
|
||||||
|
await setDefaultVideoChannel([ server ])
|
||||||
|
await setDefaultChannelAvatar([ server ])
|
||||||
|
await setDefaultAccountAvatar([ server ])
|
||||||
|
|
||||||
|
await server.config.enableChannelSync()
|
||||||
|
|
||||||
|
command = server.channelSyncs
|
||||||
|
|
||||||
|
{
|
||||||
|
userInfo.accessToken = await server.users.generateUserAndToken(userInfo.username)
|
||||||
|
|
||||||
|
const { videoChannels } = await server.users.getMyInfo({ token: userInfo.accessToken })
|
||||||
|
userInfo.channelId = videoChannels[0].id
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
it('Should fetch the latest channel videos of a remote channel', async function () {
|
||||||
|
this.timeout(120_000)
|
||||||
|
|
||||||
|
{
|
||||||
|
const { video } = await server.imports.importVideo({
|
||||||
|
attributes: {
|
||||||
|
channelId: server.store.channel.id,
|
||||||
|
privacy: VideoPrivacy.PUBLIC,
|
||||||
|
targetUrl: FIXTURE_URLS.youtube
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
expect(video.name).to.equal('small video - youtube')
|
||||||
|
|
||||||
|
const { total } = await server.videos.listByChannel({ handle: 'root_channel', include: VideoInclude.NOT_PUBLISHED_STATE })
|
||||||
|
expect(total).to.equal(1)
|
||||||
|
}
|
||||||
|
|
||||||
|
const { videoChannelSync } = await command.create({
|
||||||
|
attributes: {
|
||||||
|
externalChannelUrl: FIXTURE_URLS.youtubeChannel,
|
||||||
|
videoChannelId: server.store.channel.id
|
||||||
|
},
|
||||||
|
token: server.accessToken,
|
||||||
|
expectedStatus: HttpStatusCode.OK_200
|
||||||
|
})
|
||||||
|
|
||||||
|
// Ensure any missing video not already fetched will be considered as new
|
||||||
|
await changeDateForSync(videoChannelSync.id, '1970-01-01')
|
||||||
|
|
||||||
|
await server.debug.sendCommand({
|
||||||
|
body: {
|
||||||
|
command: 'process-video-channel-sync-latest'
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
{
|
||||||
|
await waitJobs(server)
|
||||||
|
|
||||||
|
const { total, data } = await server.videos.listByChannel({ handle: 'root_channel', include: VideoInclude.NOT_PUBLISHED_STATE })
|
||||||
|
expect(total).to.equal(2)
|
||||||
|
expect(data[0].name).to.equal('test')
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
it('Should add another synchronization', async function () {
|
||||||
|
const externalChannelUrl = FIXTURE_URLS.youtubeChannel + '?foo=bar'
|
||||||
|
|
||||||
|
const { videoChannelSync } = await command.create({
|
||||||
|
attributes: {
|
||||||
|
externalChannelUrl,
|
||||||
|
videoChannelId: server.store.channel.id
|
||||||
|
},
|
||||||
|
token: server.accessToken,
|
||||||
|
expectedStatus: HttpStatusCode.OK_200
|
||||||
|
})
|
||||||
|
|
||||||
|
expect(videoChannelSync.externalChannelUrl).to.equal(externalChannelUrl)
|
||||||
|
expect(videoChannelSync.channel).to.include({
|
||||||
|
id: server.store.channel.id,
|
||||||
|
name: 'root_channel'
|
||||||
|
})
|
||||||
|
expect(videoChannelSync.state.id).to.equal(VideoChannelSyncState.WAITING_FIRST_RUN)
|
||||||
|
expect(new Date(videoChannelSync.createdAt)).to.be.above(startTestDate).and.to.be.at.most(new Date())
|
||||||
|
})
|
||||||
|
|
||||||
|
it('Should add a synchronization for another user', async function () {
|
||||||
|
const { videoChannelSync } = await command.create({
|
||||||
|
attributes: {
|
||||||
|
externalChannelUrl: FIXTURE_URLS.youtubeChannel + '?baz=qux',
|
||||||
|
videoChannelId: userInfo.channelId
|
||||||
|
},
|
||||||
|
token: userInfo.accessToken
|
||||||
|
})
|
||||||
|
userInfo.syncId = videoChannelSync.id
|
||||||
|
})
|
||||||
|
|
||||||
|
it('Should not import a channel if not asked', async function () {
|
||||||
|
await waitJobs(server)
|
||||||
|
|
||||||
|
const { data } = await command.listByAccount({ accountName: userInfo.username })
|
||||||
|
|
||||||
|
expect(data[0].state).to.contain({
|
||||||
|
id: VideoChannelSyncState.WAITING_FIRST_RUN,
|
||||||
|
label: 'Waiting first run'
|
||||||
|
})
|
||||||
|
})
|
||||||
|
|
||||||
|
it('Should only fetch the videos newer than the creation date', async function () {
|
||||||
|
this.timeout(120_000)
|
||||||
|
|
||||||
|
await changeDateForSync(userInfo.syncId, '2019-03-01')
|
||||||
|
|
||||||
|
await server.debug.sendCommand({
|
||||||
|
body: {
|
||||||
|
command: 'process-video-channel-sync-latest'
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
await waitJobs(server)
|
||||||
|
|
||||||
|
const { data, total } = await server.videos.listByChannel({
|
||||||
|
handle: userInfo.channelName,
|
||||||
|
include: VideoInclude.NOT_PUBLISHED_STATE
|
||||||
|
})
|
||||||
|
|
||||||
|
expect(total).to.equal(1)
|
||||||
|
expect(data[0].name).to.equal('test')
|
||||||
|
})
|
||||||
|
|
||||||
|
it('Should list channel synchronizations', async function () {
|
||||||
|
// Root
|
||||||
|
{
|
||||||
|
const { total, data } = await command.listByAccount({ accountName: 'root' })
|
||||||
|
expect(total).to.equal(2)
|
||||||
|
|
||||||
|
expect(data[0]).to.deep.contain({
|
||||||
|
externalChannelUrl: FIXTURE_URLS.youtubeChannel,
|
||||||
|
state: {
|
||||||
|
id: VideoChannelSyncState.SYNCED,
|
||||||
|
label: 'Synchronized'
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
expect(new Date(data[0].lastSyncAt)).to.be.greaterThan(startTestDate)
|
||||||
|
|
||||||
|
expect(data[0].channel).to.contain({ id: server.store.channel.id })
|
||||||
|
expect(data[1]).to.contain({ externalChannelUrl: FIXTURE_URLS.youtubeChannel + '?foo=bar' })
|
||||||
|
}
|
||||||
|
|
||||||
|
// User
|
||||||
|
{
|
||||||
|
const { total, data } = await command.listByAccount({ accountName: userInfo.username })
|
||||||
|
expect(total).to.equal(1)
|
||||||
|
expect(data[0]).to.deep.contain({
|
||||||
|
externalChannelUrl: FIXTURE_URLS.youtubeChannel + '?baz=qux',
|
||||||
|
state: {
|
||||||
|
id: VideoChannelSyncState.SYNCED,
|
||||||
|
label: 'Synchronized'
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
it('Should remove user\'s channel synchronizations', async function () {
|
||||||
|
await command.delete({ channelSyncId: userInfo.syncId })
|
||||||
|
|
||||||
|
const { total } = await command.listByAccount({ accountName: userInfo.username })
|
||||||
|
expect(total).to.equal(0)
|
||||||
|
})
|
||||||
|
|
||||||
|
after(async function () {
|
||||||
|
await server?.kill()
|
||||||
|
})
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
runSuite('youtube-dl')
|
||||||
|
runSuite('yt-dlp')
|
||||||
|
})
|
|
@ -12,6 +12,7 @@ import {
|
||||||
createMultipleServers,
|
createMultipleServers,
|
||||||
createSingleServer,
|
createSingleServer,
|
||||||
doubleFollow,
|
doubleFollow,
|
||||||
|
getServerImportConfig,
|
||||||
PeerTubeServer,
|
PeerTubeServer,
|
||||||
setAccessTokensToServers,
|
setAccessTokensToServers,
|
||||||
setDefaultVideoChannel,
|
setDefaultVideoChannel,
|
||||||
|
@ -84,24 +85,9 @@ describe('Test video imports', function () {
|
||||||
let servers: PeerTubeServer[] = []
|
let servers: PeerTubeServer[] = []
|
||||||
|
|
||||||
before(async function () {
|
before(async function () {
|
||||||
this.timeout(30_000)
|
this.timeout(60_000)
|
||||||
|
|
||||||
// Run servers
|
servers = await createMultipleServers(2, getServerImportConfig(mode))
|
||||||
servers = await createMultipleServers(2, {
|
|
||||||
import: {
|
|
||||||
videos: {
|
|
||||||
http: {
|
|
||||||
youtube_dl_release: {
|
|
||||||
url: mode === 'youtube-dl'
|
|
||||||
? 'https://yt-dl.org/downloads/latest/youtube-dl'
|
|
||||||
: 'https://api.github.com/repos/yt-dlp/yt-dlp/releases',
|
|
||||||
|
|
||||||
name: mode
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
})
|
|
||||||
|
|
||||||
await setAccessTokensToServers(servers)
|
await setAccessTokensToServers(servers)
|
||||||
await setDefaultVideoChannel(servers)
|
await setDefaultVideoChannel(servers)
|
||||||
|
|
|
@ -16,6 +16,8 @@ const FIXTURE_URLS = {
|
||||||
*/
|
*/
|
||||||
youtubeHDR: 'https://www.youtube.com/watch?v=RQgnBB9z_N4',
|
youtubeHDR: 'https://www.youtube.com/watch?v=RQgnBB9z_N4',
|
||||||
|
|
||||||
|
youtubeChannel: 'https://youtube.com/channel/UCtnlZdXv3-xQzxiqfn6cjIA',
|
||||||
|
|
||||||
// eslint-disable-next-line max-len
|
// eslint-disable-next-line max-len
|
||||||
magnet: 'magnet:?xs=https%3A%2F%2Fpeertube2.cpy.re%2Flazy-static%2Ftorrents%2Fb209ca00-c8bb-4b2b-b421-1ede169f3dbc-720.torrent&xt=urn:btih:0f498834733e8057ed5c6f2ee2b4efd8d84a76ee&dn=super+peertube2+video&tr=https%3A%2F%2Fpeertube2.cpy.re%2Ftracker%2Fannounce&tr=wss%3A%2F%2Fpeertube2.cpy.re%3A443%2Ftracker%2Fsocket&ws=https%3A%2F%2Fpeertube2.cpy.re%2Fstatic%2Fwebseed%2Fb209ca00-c8bb-4b2b-b421-1ede169f3dbc-720.mp4',
|
magnet: 'magnet:?xs=https%3A%2F%2Fpeertube2.cpy.re%2Flazy-static%2Ftorrents%2Fb209ca00-c8bb-4b2b-b421-1ede169f3dbc-720.torrent&xt=urn:btih:0f498834733e8057ed5c6f2ee2b4efd8d84a76ee&dn=super+peertube2+video&tr=https%3A%2F%2Fpeertube2.cpy.re%2Ftracker%2Fannounce&tr=wss%3A%2F%2Fpeertube2.cpy.re%3A443%2Ftracker%2Fsocket&ws=https%3A%2F%2Fpeertube2.cpy.re%2Fstatic%2Fwebseed%2Fb209ca00-c8bb-4b2b-b421-1ede169f3dbc-720.mp4',
|
||||||
|
|
||||||
|
|
3
server/types/express.d.ts
vendored
3
server/types/express.d.ts
vendored
|
@ -8,6 +8,7 @@ import {
|
||||||
MActorFollowActorsDefault,
|
MActorFollowActorsDefault,
|
||||||
MActorUrl,
|
MActorUrl,
|
||||||
MChannelBannerAccountDefault,
|
MChannelBannerAccountDefault,
|
||||||
|
MChannelSyncChannel,
|
||||||
MStreamingPlaylist,
|
MStreamingPlaylist,
|
||||||
MVideoChangeOwnershipFull,
|
MVideoChangeOwnershipFull,
|
||||||
MVideoFile,
|
MVideoFile,
|
||||||
|
@ -145,6 +146,7 @@ declare module 'express' {
|
||||||
videoStreamingPlaylist?: MStreamingPlaylist
|
videoStreamingPlaylist?: MStreamingPlaylist
|
||||||
|
|
||||||
videoChannel?: MChannelBannerAccountDefault
|
videoChannel?: MChannelBannerAccountDefault
|
||||||
|
videoChannelSync?: MChannelSyncChannel
|
||||||
|
|
||||||
videoPlaylistFull?: MVideoPlaylistFull
|
videoPlaylistFull?: MVideoPlaylistFull
|
||||||
videoPlaylistSummary?: MVideoPlaylistFullSummary
|
videoPlaylistSummary?: MVideoPlaylistFullSummary
|
||||||
|
@ -194,6 +196,7 @@ declare module 'express' {
|
||||||
plugin?: MPlugin
|
plugin?: MPlugin
|
||||||
|
|
||||||
localViewerFull?: MLocalVideoViewerWithWatchSections
|
localViewerFull?: MLocalVideoViewerWithWatchSections
|
||||||
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -8,6 +8,7 @@ export * from './video'
|
||||||
export * from './video-blacklist'
|
export * from './video-blacklist'
|
||||||
export * from './video-caption'
|
export * from './video-caption'
|
||||||
export * from './video-change-ownership'
|
export * from './video-change-ownership'
|
||||||
|
export * from './video-channel-sync'
|
||||||
export * from './video-channels'
|
export * from './video-channels'
|
||||||
export * from './video-comment'
|
export * from './video-comment'
|
||||||
export * from './video-file'
|
export * from './video-file'
|
||||||
|
|
17
server/types/models/video/video-channel-sync.ts
Normal file
17
server/types/models/video/video-channel-sync.ts
Normal file
|
@ -0,0 +1,17 @@
|
||||||
|
import { VideoChannelSyncModel } from '@server/models/video/video-channel-sync'
|
||||||
|
import { FunctionProperties, PickWith } from '@shared/typescript-utils'
|
||||||
|
import { MChannelAccountDefault, MChannelFormattable } from './video-channels'
|
||||||
|
|
||||||
|
type Use<K extends keyof VideoChannelSyncModel, M> = PickWith<VideoChannelSyncModel, K, M>
|
||||||
|
|
||||||
|
export type MChannelSync = Omit<VideoChannelSyncModel, 'VideoChannel'>
|
||||||
|
|
||||||
|
export type MChannelSyncChannel =
|
||||||
|
MChannelSync &
|
||||||
|
Use<'VideoChannel', MChannelAccountDefault> &
|
||||||
|
FunctionProperties<VideoChannelSyncModel>
|
||||||
|
|
||||||
|
export type MChannelSyncFormattable =
|
||||||
|
FunctionProperties<MChannelSyncChannel> &
|
||||||
|
Use<'VideoChannel', MChannelFormattable> &
|
||||||
|
MChannelSync
|
|
@ -165,6 +165,10 @@ export interface CustomConfig {
|
||||||
enabled: boolean
|
enabled: boolean
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
videoChannelSynchronization: {
|
||||||
|
enabled: boolean
|
||||||
|
maxPerUser: number
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
trending: {
|
trending: {
|
||||||
|
|
|
@ -4,5 +4,8 @@ export interface Debug {
|
||||||
}
|
}
|
||||||
|
|
||||||
export interface SendDebugCommand {
|
export interface SendDebugCommand {
|
||||||
command: 'remove-dandling-resumable-uploads' | 'process-video-views-buffer' | 'process-video-viewers'
|
command: 'remove-dandling-resumable-uploads'
|
||||||
|
| 'process-video-views-buffer'
|
||||||
|
| 'process-video-viewers'
|
||||||
|
| 'process-video-channel-sync-latest'
|
||||||
}
|
}
|
||||||
|
|
|
@ -25,6 +25,8 @@ export type JobType =
|
||||||
| 'manage-video-torrent'
|
| 'manage-video-torrent'
|
||||||
| 'move-to-object-storage'
|
| 'move-to-object-storage'
|
||||||
| 'video-studio-edition'
|
| 'video-studio-edition'
|
||||||
|
| 'video-channel-import'
|
||||||
|
| 'after-video-channel-import'
|
||||||
| 'notify'
|
| 'notify'
|
||||||
| 'federate-video'
|
| 'federate-video'
|
||||||
|
|
||||||
|
@ -82,20 +84,32 @@ export type VideoFileImportPayload = {
|
||||||
filePath: string
|
filePath: string
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
|
||||||
export type VideoImportTorrentPayloadType = 'magnet-uri' | 'torrent-file'
|
export type VideoImportTorrentPayloadType = 'magnet-uri' | 'torrent-file'
|
||||||
export type VideoImportYoutubeDLPayloadType = 'youtube-dl'
|
export type VideoImportYoutubeDLPayloadType = 'youtube-dl'
|
||||||
|
|
||||||
export type VideoImportYoutubeDLPayload = {
|
export interface VideoImportYoutubeDLPayload {
|
||||||
type: VideoImportYoutubeDLPayloadType
|
type: VideoImportYoutubeDLPayloadType
|
||||||
videoImportId: number
|
videoImportId: number
|
||||||
|
|
||||||
fileExt?: string
|
fileExt?: string
|
||||||
}
|
}
|
||||||
export type VideoImportTorrentPayload = {
|
|
||||||
|
export interface VideoImportTorrentPayload {
|
||||||
type: VideoImportTorrentPayloadType
|
type: VideoImportTorrentPayloadType
|
||||||
videoImportId: number
|
videoImportId: number
|
||||||
}
|
}
|
||||||
export type VideoImportPayload = VideoImportYoutubeDLPayload | VideoImportTorrentPayload
|
|
||||||
|
export type VideoImportPayload = (VideoImportYoutubeDLPayload | VideoImportTorrentPayload) & {
|
||||||
|
preventException: boolean
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface VideoImportPreventExceptionResult {
|
||||||
|
resultType: 'success' | 'error'
|
||||||
|
}
|
||||||
|
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
|
||||||
export type VideoRedundancyPayload = {
|
export type VideoRedundancyPayload = {
|
||||||
videoId: number
|
videoId: number
|
||||||
|
@ -219,6 +233,17 @@ export interface VideoStudioEditionPayload {
|
||||||
|
|
||||||
// ---------------------------------------------------------------------------
|
// ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
export interface VideoChannelImportPayload {
|
||||||
|
externalChannelUrl: string
|
||||||
|
videoChannelId: number
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface AfterVideoChannelImportPayload {
|
||||||
|
channelSyncId: number
|
||||||
|
}
|
||||||
|
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
|
||||||
export type NotifyPayload =
|
export type NotifyPayload =
|
||||||
{
|
{
|
||||||
action: 'new-video'
|
action: 'new-video'
|
||||||
|
|
|
@ -188,6 +188,9 @@ export interface ServerConfig {
|
||||||
enabled: boolean
|
enabled: boolean
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
videoChannelSynchronization: {
|
||||||
|
enabled: boolean
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
autoBlacklist: {
|
autoBlacklist: {
|
||||||
|
|
3
shared/models/videos/channel-sync/index.ts
Normal file
3
shared/models/videos/channel-sync/index.ts
Normal file
|
@ -0,0 +1,3 @@
|
||||||
|
export * from './video-channel-sync-state.enum'
|
||||||
|
export * from './video-channel-sync.model'
|
||||||
|
export * from './video-channel-sync-create.model'
|
|
@ -0,0 +1,4 @@
|
||||||
|
export interface VideoChannelSyncCreate {
|
||||||
|
externalChannelUrl: string
|
||||||
|
videoChannelId: number
|
||||||
|
}
|
|
@ -0,0 +1,6 @@
|
||||||
|
export const enum VideoChannelSyncState {
|
||||||
|
WAITING_FIRST_RUN = 1,
|
||||||
|
PROCESSING = 2,
|
||||||
|
SYNCED = 3,
|
||||||
|
FAILED = 4
|
||||||
|
}
|
|
@ -0,0 +1,14 @@
|
||||||
|
import { VideoChannelSummary } from '../channel/video-channel.model'
|
||||||
|
import { VideoConstant } from '../video-constant.model'
|
||||||
|
import { VideoChannelSyncState } from './video-channel-sync-state.enum'
|
||||||
|
|
||||||
|
export interface VideoChannelSync {
|
||||||
|
id: number
|
||||||
|
|
||||||
|
externalChannelUrl: string
|
||||||
|
|
||||||
|
createdAt: string
|
||||||
|
channel: VideoChannelSummary
|
||||||
|
state: VideoConstant<VideoChannelSyncState>
|
||||||
|
lastSyncAt: string
|
||||||
|
}
|
|
@ -11,6 +11,7 @@ export * from './playlist'
|
||||||
export * from './rate'
|
export * from './rate'
|
||||||
export * from './stats'
|
export * from './stats'
|
||||||
export * from './transcoding'
|
export * from './transcoding'
|
||||||
|
export * from './channel-sync'
|
||||||
|
|
||||||
export * from './nsfw-policy.type'
|
export * from './nsfw-policy.type'
|
||||||
|
|
||||||
|
|
|
@ -18,17 +18,25 @@ export class ConfigCommand extends AbstractCommand {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
disableImports () {
|
||||||
|
return this.setImportsEnabled(false)
|
||||||
|
}
|
||||||
|
|
||||||
enableImports () {
|
enableImports () {
|
||||||
|
return this.setImportsEnabled(true)
|
||||||
|
}
|
||||||
|
|
||||||
|
private setImportsEnabled (enabled: boolean) {
|
||||||
return this.updateExistingSubConfig({
|
return this.updateExistingSubConfig({
|
||||||
newConfig: {
|
newConfig: {
|
||||||
import: {
|
import: {
|
||||||
videos: {
|
videos: {
|
||||||
http: {
|
http: {
|
||||||
enabled: true
|
enabled
|
||||||
},
|
},
|
||||||
|
|
||||||
torrent: {
|
torrent: {
|
||||||
enabled: true
|
enabled
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -36,6 +44,26 @@ export class ConfigCommand extends AbstractCommand {
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private setChannelSyncEnabled (enabled: boolean) {
|
||||||
|
return this.updateExistingSubConfig({
|
||||||
|
newConfig: {
|
||||||
|
import: {
|
||||||
|
videoChannelSynchronization: {
|
||||||
|
enabled
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
enableChannelSync () {
|
||||||
|
return this.setChannelSyncEnabled(true)
|
||||||
|
}
|
||||||
|
|
||||||
|
disableChannelSync () {
|
||||||
|
return this.setChannelSyncEnabled(false)
|
||||||
|
}
|
||||||
|
|
||||||
enableLive (options: {
|
enableLive (options: {
|
||||||
allowReplay?: boolean
|
allowReplay?: boolean
|
||||||
transcoding?: boolean
|
transcoding?: boolean
|
||||||
|
@ -356,6 +384,10 @@ export class ConfigCommand extends AbstractCommand {
|
||||||
torrent: {
|
torrent: {
|
||||||
enabled: false
|
enabled: false
|
||||||
}
|
}
|
||||||
|
},
|
||||||
|
videoChannelSynchronization: {
|
||||||
|
enabled: false,
|
||||||
|
maxPerUser: 10
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
trending: {
|
trending: {
|
||||||
|
|
|
@ -19,6 +19,7 @@ import {
|
||||||
CaptionsCommand,
|
CaptionsCommand,
|
||||||
ChangeOwnershipCommand,
|
ChangeOwnershipCommand,
|
||||||
ChannelsCommand,
|
ChannelsCommand,
|
||||||
|
ChannelSyncsCommand,
|
||||||
HistoryCommand,
|
HistoryCommand,
|
||||||
ImportsCommand,
|
ImportsCommand,
|
||||||
LiveCommand,
|
LiveCommand,
|
||||||
|
@ -118,6 +119,7 @@ export class PeerTubeServer {
|
||||||
playlists?: PlaylistsCommand
|
playlists?: PlaylistsCommand
|
||||||
history?: HistoryCommand
|
history?: HistoryCommand
|
||||||
imports?: ImportsCommand
|
imports?: ImportsCommand
|
||||||
|
channelSyncs?: ChannelSyncsCommand
|
||||||
streamingPlaylists?: StreamingPlaylistsCommand
|
streamingPlaylists?: StreamingPlaylistsCommand
|
||||||
channels?: ChannelsCommand
|
channels?: ChannelsCommand
|
||||||
comments?: CommentsCommand
|
comments?: CommentsCommand
|
||||||
|
@ -390,6 +392,7 @@ export class PeerTubeServer {
|
||||||
this.playlists = new PlaylistsCommand(this)
|
this.playlists = new PlaylistsCommand(this)
|
||||||
this.history = new HistoryCommand(this)
|
this.history = new HistoryCommand(this)
|
||||||
this.imports = new ImportsCommand(this)
|
this.imports = new ImportsCommand(this)
|
||||||
|
this.channelSyncs = new ChannelSyncsCommand(this)
|
||||||
this.streamingPlaylists = new StreamingPlaylistsCommand(this)
|
this.streamingPlaylists = new StreamingPlaylistsCommand(this)
|
||||||
this.channels = new ChannelsCommand(this)
|
this.channels = new ChannelsCommand(this)
|
||||||
this.comments = new CommentsCommand(this)
|
this.comments = new CommentsCommand(this)
|
||||||
|
|
|
@ -39,11 +39,30 @@ async function cleanupTests (servers: PeerTubeServer[]) {
|
||||||
return Promise.all(p)
|
return Promise.all(p)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function getServerImportConfig (mode: 'youtube-dl' | 'yt-dlp') {
|
||||||
|
return {
|
||||||
|
import: {
|
||||||
|
videos: {
|
||||||
|
http: {
|
||||||
|
youtube_dl_release: {
|
||||||
|
url: mode === 'youtube-dl'
|
||||||
|
? 'https://yt-dl.org/downloads/latest/youtube-dl'
|
||||||
|
: 'https://api.github.com/repos/yt-dlp/yt-dlp/releases',
|
||||||
|
|
||||||
|
name: mode
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// ---------------------------------------------------------------------------
|
// ---------------------------------------------------------------------------
|
||||||
|
|
||||||
export {
|
export {
|
||||||
createSingleServer,
|
createSingleServer,
|
||||||
createMultipleServers,
|
createMultipleServers,
|
||||||
cleanupTests,
|
cleanupTests,
|
||||||
killallServers
|
killallServers,
|
||||||
|
getServerImportConfig
|
||||||
}
|
}
|
||||||
|
|
55
shared/server-commands/videos/channel-syncs-command.ts
Normal file
55
shared/server-commands/videos/channel-syncs-command.ts
Normal file
|
@ -0,0 +1,55 @@
|
||||||
|
import { HttpStatusCode, ResultList, VideoChannelSync, VideoChannelSyncCreate } from '@shared/models'
|
||||||
|
import { pick } from '@shared/core-utils'
|
||||||
|
import { unwrapBody } from '../requests'
|
||||||
|
import { AbstractCommand, OverrideCommandOptions } from '../shared'
|
||||||
|
|
||||||
|
export class ChannelSyncsCommand extends AbstractCommand {
|
||||||
|
private static readonly API_PATH = '/api/v1/video-channel-syncs'
|
||||||
|
|
||||||
|
listByAccount (options: OverrideCommandOptions & {
|
||||||
|
accountName: string
|
||||||
|
start?: number
|
||||||
|
count?: number
|
||||||
|
sort?: string
|
||||||
|
}) {
|
||||||
|
const { accountName, sort = 'createdAt' } = options
|
||||||
|
|
||||||
|
const path = `/api/v1/accounts/${accountName}/video-channel-syncs`
|
||||||
|
|
||||||
|
return this.getRequestBody<ResultList<VideoChannelSync>>({
|
||||||
|
...options,
|
||||||
|
|
||||||
|
path,
|
||||||
|
query: { sort, ...pick(options, [ 'start', 'count' ]) },
|
||||||
|
implicitToken: true,
|
||||||
|
defaultExpectedStatus: HttpStatusCode.OK_200
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
async create (options: OverrideCommandOptions & {
|
||||||
|
attributes: VideoChannelSyncCreate
|
||||||
|
}) {
|
||||||
|
return unwrapBody<{ videoChannelSync: VideoChannelSync }>(this.postBodyRequest({
|
||||||
|
...options,
|
||||||
|
|
||||||
|
path: ChannelSyncsCommand.API_PATH,
|
||||||
|
fields: options.attributes,
|
||||||
|
implicitToken: true,
|
||||||
|
defaultExpectedStatus: HttpStatusCode.OK_200
|
||||||
|
}))
|
||||||
|
}
|
||||||
|
|
||||||
|
delete (options: OverrideCommandOptions & {
|
||||||
|
channelSyncId: number
|
||||||
|
}) {
|
||||||
|
const path = `${ChannelSyncsCommand.API_PATH}/${options.channelSyncId}`
|
||||||
|
|
||||||
|
return this.deleteRequest({
|
||||||
|
...options,
|
||||||
|
|
||||||
|
path,
|
||||||
|
implicitToken: true,
|
||||||
|
defaultExpectedStatus: HttpStatusCode.NO_CONTENT_204
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
|
@ -181,4 +181,22 @@ export class ChannelsCommand extends AbstractCommand {
|
||||||
defaultExpectedStatus: HttpStatusCode.OK_200
|
defaultExpectedStatus: HttpStatusCode.OK_200
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
importVideos (options: OverrideCommandOptions & {
|
||||||
|
channelName: string
|
||||||
|
externalChannelUrl: string
|
||||||
|
}) {
|
||||||
|
const { channelName, externalChannelUrl } = options
|
||||||
|
|
||||||
|
const path = `/api/v1/video-channels/${channelName}/import-videos`
|
||||||
|
|
||||||
|
return this.postBodyRequest({
|
||||||
|
...options,
|
||||||
|
|
||||||
|
path,
|
||||||
|
fields: { externalChannelUrl },
|
||||||
|
implicitToken: true,
|
||||||
|
defaultExpectedStatus: HttpStatusCode.NO_CONTENT_204
|
||||||
|
})
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -3,6 +3,7 @@ export * from './captions-command'
|
||||||
export * from './change-ownership-command'
|
export * from './change-ownership-command'
|
||||||
export * from './channels'
|
export * from './channels'
|
||||||
export * from './channels-command'
|
export * from './channels-command'
|
||||||
|
export * from './channel-syncs-command'
|
||||||
export * from './comments-command'
|
export * from './comments-command'
|
||||||
export * from './history-command'
|
export * from './history-command'
|
||||||
export * from './imports-command'
|
export * from './imports-command'
|
||||||
|
|
|
@ -254,6 +254,8 @@ tags:
|
||||||
download it for you, saving you as much bandwidth and avoiding any instability or limitation your network might have.
|
download it for you, saving you as much bandwidth and avoiding any instability or limitation your network might have.
|
||||||
- name: Video Imports
|
- name: Video Imports
|
||||||
description: Operations dealing with listing, adding and removing video imports.
|
description: Operations dealing with listing, adding and removing video imports.
|
||||||
|
- name: Channels Sync
|
||||||
|
description: Operations dealing with synchronizing PeerTube user's channel with channels of other platforms
|
||||||
- name: Video Captions
|
- name: Video Captions
|
||||||
description: Operations dealing with listing, adding and removing closed captions of a video.
|
description: Operations dealing with listing, adding and removing closed captions of a video.
|
||||||
- name: Video Channels
|
- name: Video Channels
|
||||||
|
@ -327,6 +329,7 @@ x-tagGroups:
|
||||||
- Video Transcoding
|
- Video Transcoding
|
||||||
- Live Videos
|
- Live Videos
|
||||||
- Feeds
|
- Feeds
|
||||||
|
- Channels Sync
|
||||||
- name: Search
|
- name: Search
|
||||||
tags:
|
tags:
|
||||||
- Search
|
- Search
|
||||||
|
@ -3050,7 +3053,7 @@ paths:
|
||||||
tags:
|
tags:
|
||||||
- Video Channels
|
- Video Channels
|
||||||
responses:
|
responses:
|
||||||
'204':
|
'200':
|
||||||
description: successful operation
|
description: successful operation
|
||||||
content:
|
content:
|
||||||
application/json:
|
application/json:
|
||||||
|
@ -3288,6 +3291,59 @@ paths:
|
||||||
'204':
|
'204':
|
||||||
description: successful operation
|
description: successful operation
|
||||||
|
|
||||||
|
'/video-channel-syncs':
|
||||||
|
post:
|
||||||
|
summary: Create a synchronization for a video channel
|
||||||
|
operationId: addVideoChannelSync
|
||||||
|
security:
|
||||||
|
- OAuth2: []
|
||||||
|
tags:
|
||||||
|
- Channels Sync
|
||||||
|
requestBody:
|
||||||
|
content:
|
||||||
|
application/json:
|
||||||
|
schema:
|
||||||
|
$ref: '#/components/schemas/VideoChannelSyncCreate'
|
||||||
|
responses:
|
||||||
|
'200':
|
||||||
|
description: successful operation
|
||||||
|
content:
|
||||||
|
application/json:
|
||||||
|
schema:
|
||||||
|
type: object
|
||||||
|
properties:
|
||||||
|
videoChannelSync:
|
||||||
|
$ref: "#/components/schemas/VideoChannelSync"
|
||||||
|
|
||||||
|
'/video-channel-syncs/{channelSyncId}':
|
||||||
|
delete:
|
||||||
|
summary: Delete a video channel synchronization
|
||||||
|
operationId: delVideoChannelSync
|
||||||
|
security:
|
||||||
|
- OAuth2: []
|
||||||
|
tags:
|
||||||
|
- Channels Sync
|
||||||
|
parameters:
|
||||||
|
- $ref: '#/components/parameters/channelSyncId'
|
||||||
|
responses:
|
||||||
|
'204':
|
||||||
|
description: successful operation
|
||||||
|
|
||||||
|
'/video-channel-syncs/{channelSyncId}/sync':
|
||||||
|
post:
|
||||||
|
summary: Triggers the channel synchronization job, fetching all the videos from the remote channel
|
||||||
|
operationId: triggerVideoChannelSync
|
||||||
|
security:
|
||||||
|
- OAuth2: []
|
||||||
|
tags:
|
||||||
|
- Channels Sync
|
||||||
|
parameters:
|
||||||
|
- $ref: '#/components/parameters/channelSyncId'
|
||||||
|
responses:
|
||||||
|
'204':
|
||||||
|
description: successful operation
|
||||||
|
|
||||||
|
|
||||||
/video-playlists/privacies:
|
/video-playlists/privacies:
|
||||||
get:
|
get:
|
||||||
summary: List available playlist privacy policies
|
summary: List available playlist privacy policies
|
||||||
|
@ -3659,6 +3715,26 @@ paths:
|
||||||
schema:
|
schema:
|
||||||
$ref: '#/components/schemas/VideoChannelList'
|
$ref: '#/components/schemas/VideoChannelList'
|
||||||
|
|
||||||
|
'/accounts/{name}/video-channel-syncs':
|
||||||
|
get:
|
||||||
|
summary: List the synchronizations of video channels of an account
|
||||||
|
tags:
|
||||||
|
- Video Channels
|
||||||
|
- Channels Sync
|
||||||
|
- Accounts
|
||||||
|
parameters:
|
||||||
|
- $ref: '#/components/parameters/name'
|
||||||
|
- $ref: '#/components/parameters/start'
|
||||||
|
- $ref: '#/components/parameters/count'
|
||||||
|
- $ref: '#/components/parameters/sort'
|
||||||
|
responses:
|
||||||
|
'200':
|
||||||
|
description: successful operation
|
||||||
|
content:
|
||||||
|
application/json:
|
||||||
|
schema:
|
||||||
|
$ref: '#/components/schemas/VideoChannelSyncList'
|
||||||
|
|
||||||
'/accounts/{name}/ratings':
|
'/accounts/{name}/ratings':
|
||||||
get:
|
get:
|
||||||
summary: List ratings of an account
|
summary: List ratings of an account
|
||||||
|
@ -5141,6 +5217,13 @@ components:
|
||||||
schema:
|
schema:
|
||||||
type: string
|
type: string
|
||||||
example: my_username | my_username@example.com
|
example: my_username | my_username@example.com
|
||||||
|
channelSyncId:
|
||||||
|
name: channelSyncId
|
||||||
|
in: path
|
||||||
|
required: true
|
||||||
|
description: Channel Sync id
|
||||||
|
schema:
|
||||||
|
$ref: '#/components/schemas/Abuse/properties/id'
|
||||||
subscriptionHandle:
|
subscriptionHandle:
|
||||||
name: subscriptionHandle
|
name: subscriptionHandle
|
||||||
in: path
|
in: path
|
||||||
|
@ -5347,6 +5430,7 @@ components:
|
||||||
- activitypub-refresher
|
- activitypub-refresher
|
||||||
- video-redundancy
|
- video-redundancy
|
||||||
- video-live-ending
|
- video-live-ending
|
||||||
|
- video-channel-import
|
||||||
followState:
|
followState:
|
||||||
name: state
|
name: state
|
||||||
in: query
|
in: query
|
||||||
|
@ -6497,6 +6581,11 @@ components:
|
||||||
properties:
|
properties:
|
||||||
enabled:
|
enabled:
|
||||||
type: boolean
|
type: boolean
|
||||||
|
videoChannelSynchronization:
|
||||||
|
type: object
|
||||||
|
properties:
|
||||||
|
enabled:
|
||||||
|
type: boolean
|
||||||
autoBlacklist:
|
autoBlacklist:
|
||||||
type: object
|
type: object
|
||||||
properties:
|
properties:
|
||||||
|
@ -6861,6 +6950,11 @@ components:
|
||||||
properties:
|
properties:
|
||||||
enabled:
|
enabled:
|
||||||
type: boolean
|
type: boolean
|
||||||
|
video_channel_synchronization:
|
||||||
|
type: object
|
||||||
|
properties:
|
||||||
|
enabled:
|
||||||
|
type: boolean
|
||||||
autoBlacklist:
|
autoBlacklist:
|
||||||
type: object
|
type: object
|
||||||
properties:
|
properties:
|
||||||
|
@ -6953,6 +7047,7 @@ components:
|
||||||
- videos-views-stats
|
- videos-views-stats
|
||||||
- activitypub-refresher
|
- activitypub-refresher
|
||||||
- video-redundancy
|
- video-redundancy
|
||||||
|
- video-channel-import
|
||||||
data:
|
data:
|
||||||
type: object
|
type: object
|
||||||
additionalProperties: true
|
additionalProperties: true
|
||||||
|
@ -7473,6 +7568,7 @@ components:
|
||||||
type: integer
|
type: integer
|
||||||
uuid:
|
uuid:
|
||||||
$ref: '#/components/schemas/UUIDv4'
|
$ref: '#/components/schemas/UUIDv4'
|
||||||
|
|
||||||
VideoChannelCreate:
|
VideoChannelCreate:
|
||||||
allOf:
|
allOf:
|
||||||
- $ref: '#/components/schemas/VideoChannel'
|
- $ref: '#/components/schemas/VideoChannel'
|
||||||
|
@ -7503,6 +7599,51 @@ components:
|
||||||
- $ref: '#/components/schemas/VideoChannel'
|
- $ref: '#/components/schemas/VideoChannel'
|
||||||
- $ref: '#/components/schemas/Actor'
|
- $ref: '#/components/schemas/Actor'
|
||||||
|
|
||||||
|
VideoChannelSync:
|
||||||
|
type: object
|
||||||
|
properties:
|
||||||
|
id:
|
||||||
|
$ref: '#/components/schemas/id'
|
||||||
|
state:
|
||||||
|
type: object
|
||||||
|
properties:
|
||||||
|
id:
|
||||||
|
type: integer
|
||||||
|
example: 2
|
||||||
|
label:
|
||||||
|
type: string
|
||||||
|
example: PROCESSING
|
||||||
|
externalChannelUrl:
|
||||||
|
type: string
|
||||||
|
example: 'https://youtube.com/c/UC_myfancychannel'
|
||||||
|
createdAt:
|
||||||
|
type: string
|
||||||
|
format: date-time
|
||||||
|
lastSyncAt:
|
||||||
|
type: string
|
||||||
|
format: date-time
|
||||||
|
nullable: true
|
||||||
|
channel:
|
||||||
|
$ref: '#/components/schemas/VideoChannel'
|
||||||
|
VideoChannelSyncList:
|
||||||
|
type: object
|
||||||
|
properties:
|
||||||
|
total:
|
||||||
|
type: integer
|
||||||
|
example: 1
|
||||||
|
data:
|
||||||
|
type: array
|
||||||
|
items:
|
||||||
|
allOf:
|
||||||
|
- $ref: '#/components/schemas/VideoChannelSync'
|
||||||
|
VideoChannelSyncCreate:
|
||||||
|
type: object
|
||||||
|
properties:
|
||||||
|
externalChannelUrl:
|
||||||
|
type: string
|
||||||
|
example: https://youtube.com/c/UC_myfancychannel
|
||||||
|
videoChannelId:
|
||||||
|
$ref: '#/components/schemas/id'
|
||||||
MRSSPeerLink:
|
MRSSPeerLink:
|
||||||
type: object
|
type: object
|
||||||
xml:
|
xml:
|
||||||
|
|
Loading…
Reference in New Issue
Block a user