Channel sync (#5135)

* Add external channel URL for channel update / creation (#754)

* Disallow synchronisation if user has no video quota (#754)

* More constraints serverside (#754)

* Disable sync if server configuration does not allow HTTP import (#754)

* Working version synchronizing videos with a job (#754)

TODO: refactoring, too much code duplication

* More logs and try/catch (#754)

* Fix eslint error (#754)

* WIP: support synchronization time change (#754)

* New frontend #754

* WIP: Create sync front (#754)

* Enhance UI, sync creation form (#754)

* Warning message when HTTP upload is disallowed

* More consistent names (#754)

* Binding Front with API (#754)

* Add a /me API (#754)

* Improve list UI (#754)

* Implement creation and deletion routes (#754)

* Lint (#754)

* Lint again (#754)

* WIP: UI for triggering import existing videos (#754)

* Implement jobs for syncing and importing channels

* Don't sync videos before sync creation + avoid concurrency issue (#754)

* Cleanup (#754)

* Cleanup: OpenAPI + API rework (#754)

* Remove dead code (#754)

* Eslint (#754)

* Revert the mess with whitespaces in constants.ts (#754)

* Some fixes after rebase (#754)

* Several fixes after PR remarks (#754)

* Front + API: Rename video-channels-sync to video-channel-syncs (#754)

* Allow enabling channel sync through UI (#754)

* getChannelInfo (#754)

* Minor fixes: openapi + model + sql (#754)

* Simplified API validators (#754)

* Rename MChannelSync to MChannelSyncChannel (#754)

* Add command for VideoChannelSync (#754)

* Use synchronization.enabled config (#754)

* Check parameters test + some fixes (#754)

* Fix conflict mistake (#754)

* Restrict access to video channel sync list API (#754)

* Start adding unit test for synchronization (#754)

* Continue testing (#754)

* Tests finished + convertion of job to scheduler (#754)

* Add lastSyncAt field (#754)

* Fix externalRemoteUrl sort + creation date not well formatted (#754)

* Small fix (#754)

* Factorize addYoutubeDLImport and buildVideo (#754)

* Check duplicates on channel not on users (#754)

* factorize thumbnail generation (#754)

* Fetch error should return status 400 (#754)

* Separate video-channel-import and video-channel-sync-latest (#754)

* Bump DB migration version after rebase (#754)

* Prettier states in UI table (#754)

* Add DefaultScope in VideoChannelSyncModel (#754)

* Fix audit logs (#754)

* Ensure user can upload when importing channel + minor fixes (#754)

* Mark synchronization as failed on exception + typos (#754)

* Change REST API for importing videos into channel (#754)

* Add option for fully synchronize a chnanel (#754)

* Return a whole sync object on creation to avoid tricks in Front (#754)

* Various remarks (#754)

* Single quotes by default (#754)

* Rename synchronization to video_channel_synchronization

* Add check.latest_videos_count and max_per_user options (#754)

* Better channel rendering in list #754

* Allow sorting with channel name and state (#754)

* Add missing tests for channel imports (#754)

* Prefer using a parent job for channel sync

* Styling

* Client styling

Co-authored-by: Chocobozzz <me@florianbigard.com>

client/src/app/+admin/config/edit-custom-config/edit-basic-configuration.component.html

@@ -285,6 +285,22 @@
           </div>
 
         </ng-container>
+
+        <ng-container formGroupName="videoChannelSynchronization">
+          <div class="form-group">
+            <my-peertube-checkbox
+              inputName="importSynchronizationEnabled" formControlName="enabled"
+              i18n-labelText labelText="Allow channel synchronization with channel of other platforms like YouTube (requires allowing import with HTTP URL)"
+            >
+            <ng-container ngProjectAs="description">
+              <span i18n [hidden]="isImportVideosHttpEnabled()">
+                  ⛔ You need to allow  import with HTTP URL to be able to activate this feature.
+              </span>
+            </ng-container>
+            </my-peertube-checkbox>
+          </div>
+        </ng-container>
+
       </ng-container>
 
       <ng-container formGroupName="autoBlacklist">

client/src/app/+admin/config/edit-custom-config/edit-basic-configuration.component.ts

@@ -25,11 +25,12 @@ export class EditBasicConfigurationComponent implements OnInit, OnChanges {
     private configService: ConfigService,
     private menuService: MenuService,
     private themeService: ThemeService
-  ) { }
+  ) {}
 
   ngOnInit () {
     this.buildLandingPageOptions()
     this.checkSignupField()
+    this.checkImportSyncField()
 
     this.availableThemes = this.themeService.buildAvailableThemes()
   }
@@ -67,6 +68,14 @@ export class EditBasicConfigurationComponent implements OnInit, OnChanges {
     return { 'disabled-checkbox-extra': !this.isSignupEnabled() }
   }
 
+  isImportVideosHttpEnabled (): boolean {
+    return this.form.value['import']['videos']['http']['enabled'] === true
+  }
+
+  importSynchronizationChecked () {
+    return this.isImportVideosHttpEnabled() && this.form.value['import']['videoChannelSynchronization']['enabled']
+  }
+
   hasUnlimitedSignup () {
     return this.form.value['signup']['limit'] === -1
   }
@@ -97,6 +106,21 @@ export class EditBasicConfigurationComponent implements OnInit, OnChanges {
     return this.themeService.getDefaultThemeLabel()
   }
 
+  private checkImportSyncField () {
+    const importSyncControl = this.form.get('import.videoChannelSynchronization.enabled')
+    const importVideosHttpControl = this.form.get('import.videos.http.enabled')
+
+    importVideosHttpControl.valueChanges
+      .subscribe((httpImportEnabled) => {
+        importSyncControl.setValue(httpImportEnabled && importSyncControl.value)
+        if (httpImportEnabled) {
+          importSyncControl.enable()
+        } else {
+          importSyncControl.disable()
+        }
+      })
+  }
+
   private checkSignupField () {
     const signupControl = this.form.get('signup.enabled')
 

client/src/app/+admin/config/edit-custom-config/edit-custom-config.component.ts

@@ -144,6 +144,9 @@ export class EditCustomConfigComponent extends FormReactive implements OnInit {
           torrent: {
             enabled: null
           }
+        },
+        videoChannelSynchronization: {
+          enabled: null
         }
       },
       trending: {

client/src/app/+admin/system/jobs/jobs.component.ts

@@ -38,7 +38,8 @@ export class JobsComponent extends RestTable implements OnInit {
     'video-redundancy',
     'video-transcoding',
     'videos-views-stats',
-    'move-to-object-storage'
+    'move-to-object-storage',
+    'video-channel-import'
   ]
 
   jobs: Job[] = []

client/src/app/+manage/video-channel-edit/video-channel-edit.component.html

@@ -61,7 +61,7 @@
         </div>
 
         <div class="form-group">
-          <label for="support">Support</label>
+          <label i18n for="support">Support</label>
           <my-help
             helpType="markdownEnhanced" i18n-preHtml preHtml="Short text to tell people how they can support the channel (membership platform...).<br /><br />
       When a video is uploaded in this channel, the video support field will be automatically filled by this text."

client/src/app/+my-library/+my-video-channels/my-video-channels.component.html

@@ -1,7 +1,16 @@
 <h1>
+  <span>
     <my-global-icon iconName="channel" aria-hidden="true"></my-global-icon>
     <ng-container i18n>My channels</ng-container>
     <span *ngIf="totalItems" class="pt-badge badge-secondary">{{ totalItems }}</span>
+  </span>
+
+  <div>
+    <a routerLink="/my-library/video-channel-syncs" class="button-link">
+      <my-global-icon iconName="repeat" aria-hidden="true"></my-global-icon>
+      <ng-container i18n>My synchronizations</ng-container>
+    </a>
+  </div>
 </h1>
 
 <my-channels-setup-message [hideLink]="true"></my-channels-setup-message>

client/src/app/+my-library/+my-video-channels/my-video-channels.component.scss

@@ -1,9 +1,20 @@
 @use '_variables' as *;
 @use '_mixins' as *;
 
-h1 my-global-icon {
+h1 {
+  display: flex;
+  justify-content: space-between;
+
+  my-global-icon {
     position: relative;
     top: -2px;
+  }
+
+  .button-link {
+    @include peertube-button-link;
+    @include grey-button;
+    @include button-with-icon(18px, 3px, -1px);
+  }
 }
 
 .create-button {

client/src/app/+my-library/my-library-routing.module.ts

@@ -6,6 +6,8 @@ import { MySubscriptionsComponent } from './my-follows/my-subscriptions.componen
 import { MyHistoryComponent } from './my-history/my-history.component'
 import { MyLibraryComponent } from './my-library.component'
 import { MyOwnershipComponent } from './my-ownership/my-ownership.component'
+import { MyVideoChannelSyncsComponent } from './my-video-channel-syncs/my-video-channel-syncs.component'
+import { VideoChannelSyncEditComponent } from './my-video-channel-syncs/video-channel-sync-edit/video-channel-sync-edit.component'
 import { MyVideoImportsComponent } from './my-video-imports/my-video-imports.component'
 import { MyVideoPlaylistCreateComponent } from './my-video-playlists/my-video-playlist-create.component'
 import { MyVideoPlaylistElementsComponent } from './my-video-playlists/my-video-playlist-elements.component'
@@ -131,6 +133,26 @@ const myLibraryRoutes: Routes = [
             key: 'my-videos-history-list'
           }
         }
+      },
+
+      {
+        path: 'video-channel-syncs',
+        component: MyVideoChannelSyncsComponent,
+        data: {
+          meta: {
+            title: $localize`My synchronizations`
+          }
+        }
+      },
+
+      {
+        path: 'video-channel-syncs/create',
+        component: VideoChannelSyncEditComponent,
+        data: {
+          meta: {
+            title: $localize`Create new synchronization`
+          }
+        }
       }
     ]
   }

client/src/app/+my-library/my-library.module.ts

@@ -29,6 +29,8 @@ import { MyVideoPlaylistUpdateComponent } from './my-video-playlists/my-video-pl
 import { MyVideoPlaylistsComponent } from './my-video-playlists/my-video-playlists.component'
 import { VideoChangeOwnershipComponent } from './my-videos/modals/video-change-ownership.component'
 import { MyVideosComponent } from './my-videos/my-videos.component'
+import { MyVideoChannelSyncsComponent } from './my-video-channel-syncs/my-video-channel-syncs.component'
+import { VideoChannelSyncEditComponent } from './my-video-channel-syncs/video-channel-sync-edit/video-channel-sync-edit.component'
 
 @NgModule({
   imports: [
@@ -63,6 +65,8 @@ import { MyVideosComponent } from './my-videos/my-videos.component'
     MyOwnershipComponent,
     MyAcceptOwnershipComponent,
     MyVideoImportsComponent,
+    MyVideoChannelSyncsComponent,
+    VideoChannelSyncEditComponent,
     MySubscriptionsComponent,
     MyFollowersComponent,
     MyHistoryComponent,

client/src/app/+my-library/my-video-channel-syncs/my-video-channel-syncs.component.html

@@ -0,0 +1,83 @@
+<div *ngIf="error" class="alert alert-danger">{{ error }}</div>
+
+<h1>
+  <my-global-icon iconName="refresh" aria-hidden="true"></my-global-icon>
+  <ng-container i18n>My synchronizations</ng-container>
+</h1>
+
+<div *ngIf="!syncEnabled()">
+  <p class="muted" i18n>⚠️ The instance doesn't allow channel synchronization</p>
+</div>
+
+<p-table
+  *ngIf="syncEnabled()" [value]="channelSyncs" [lazy]="true"
+  [paginator]="totalRecords > 0" [totalRecords]="totalRecords" [rows]="rowsPerPage" [rowsPerPageOptions]="rowsPerPageOptions"
+  [sortField]="sort.field" [sortOrder]="sort.order" (onLazyLoad)="loadLazy($event)" dataKey="id"
+  [showCurrentPageReport]="true" i18n-currentPageReportTemplate
+  currentPageReportTemplate="Showing {{'{first}'}} to {{'{last}'}} of {{'{totalRecords}'}} synchronizations"
+  [expandedRowKeys]="expandedRows"
+>
+  <ng-template pTemplate="caption">
+    <div class="caption">
+      <div class="left-buttons">
+        <a class="add-sync" routerLink="{{ getSyncCreateLink() }}">
+          <my-global-icon iconName="add" aria-hidden="true"></my-global-icon>
+          <ng-container i18n>Add synchronization</ng-container>
+        </a>
+      </div>
+    </div>
+  </ng-template>
+
+  <ng-template pTemplate="header">
+    <tr>
+      <th style="width: 10%"><my-global-icon iconName="columns"></my-global-icon></th>
+      <th style="width: 25%" i18n pSortableColumn="externalChannelUrl">External Channel <p-sortIcon field="externalChannelUrl"></p-sortIcon></th>
+      <th style="width: 25%" i18n pSortableColumn="videoChannel">Channel <p-sortIcon field="videoChannel"></p-sortIcon></th>
+      <th style="width: 10%" i18n pSortableColumn="state">State <p-sortIcon field="state"></p-sortIcon></th>
+      <th style="width: 10%" i18n pSortableColumn="createdAt">Created <p-sortIcon field="createdAt"></p-sortIcon></th>
+      <th style="width: 10%" i18n pSortableColumn="lastSyncAt">Last synchronization at <p-sortIcon field="lastSyncAt"></p-sortIcon></th>
+    </tr>
+  </ng-template>
+
+  <ng-template pTemplate="body" let-expanded="expanded" let-videoChannelSync>
+    <tr>
+      <td class="action-cell">
+        <my-action-dropdown
+           container="body"
+          [actions]="videoChannelSyncActions" [entry]="videoChannelSync"
+        ></my-action-dropdown>
+      </td>
+
+      <td>
+        <a [href]="videoChannelSync.externalChannelUrl" target="_blank" rel="noopener noreferrer">{{ videoChannelSync.externalChannelUrl }}</a>
+      </td>
+
+      <td>
+        <div class="actor">
+          <my-actor-avatar
+            class="channel"
+            [actor]="videoChannelSync.channel" actorType="channel"
+            [internalHref]="[ '/c', videoChannelSync.channel.name ]"
+            size="25"
+          ></my-actor-avatar>
+
+          <div class="actor-info">
+            <a [routerLink]="[ '/c', videoChannelSync.channel.name ]" class="actor-names" i18n-title title="Channel page">
+              <div class="actor-display-name">{{ videoChannelSync.channel.displayName }}</div>
+              <div class="actor-name">{{ videoChannelSync.channel.name }}</div>
+            </a>
+          </div>
+        </div>
+      </td>
+
+      <td>
+        <span [ngClass]="getSyncStateClass(videoChannelSync.state.id)">
+          {{ videoChannelSync.state.label }}
+        </span>
+      </td>
+
+      <td>{{ videoChannelSync.createdAt | date: 'short' }}</td>
+      <td>{{ videoChannelSync.lastSyncAt | date: 'short' }}</td>
+    </tr>
+  </ng-template>
+</p-table>

client/src/app/+my-library/my-video-channel-syncs/my-video-channel-syncs.component.scss

@@ -0,0 +1,14 @@
+@use '_mixins' as *;
+@use '_variables' as *;
+@use '_actor' as *;
+
+.add-sync {
+  @include create-button;
+}
+
+.actor {
+  @include actor-row($min-height: auto, $separator: true);
+  margin-bottom: 0;
+  padding-bottom: 0;
+  border: 0;
+}

client/src/app/+my-library/my-video-channel-syncs/my-video-channel-syncs.component.ts

@@ -0,0 +1,129 @@
+import { Component, OnInit } from '@angular/core'
+import { AuthService, Notifier, RestPagination, RestTable, ServerService } from '@app/core'
+import { DropdownAction, VideoChannelService, VideoChannelSyncService } from '@app/shared/shared-main'
+import { HTMLServerConfig } from '@shared/models/server'
+import { VideoChannelSync, VideoChannelSyncState } from '@shared/models/videos'
+import { SortMeta } from 'primeng/api'
+import { mergeMap } from 'rxjs'
+
+@Component({
+  templateUrl: './my-video-channel-syncs.component.html',
+  styleUrls: [ './my-video-channel-syncs.component.scss' ]
+})
+export class MyVideoChannelSyncsComponent extends RestTable implements OnInit {
+  error: string
+
+  channelSyncs: VideoChannelSync[] = []
+  totalRecords = 0
+
+  videoChannelSyncActions: DropdownAction<VideoChannelSync>[][] = []
+  sort: SortMeta = { field: 'createdAt', order: 1 }
+  pagination: RestPagination = { count: this.rowsPerPage, start: 0 }
+
+  private static STATE_CLASS_BY_ID = {
+    [VideoChannelSyncState.FAILED]: 'badge-red',
+    [VideoChannelSyncState.PROCESSING]: 'badge-blue',
+    [VideoChannelSyncState.SYNCED]: 'badge-green',
+    [VideoChannelSyncState.WAITING_FIRST_RUN]: 'badge-yellow'
+  }
+
+  private serverConfig: HTMLServerConfig
+
+  constructor (
+    private videoChannelsSyncService: VideoChannelSyncService,
+    private serverService: ServerService,
+    private notifier: Notifier,
+    private authService: AuthService,
+    private videoChannelService: VideoChannelService
+  ) {
+    super()
+  }
+
+  ngOnInit () {
+    this.serverConfig = this.serverService.getHTMLConfig()
+    this.initialize()
+
+    this.videoChannelSyncActions = [
+      [
+        {
+          label: $localize`Delete`,
+          iconName: 'delete',
+          handler: videoChannelSync => this.deleteSync(videoChannelSync)
+        },
+        {
+          label: $localize`Fully synchronize the channel`,
+          description: $localize`This fetches any missing videos on the local channel`,
+          iconName: 'refresh',
+          handler: videoChannelSync => this.fullySynchronize(videoChannelSync)
+        }
+      ]
+    ]
+  }
+
+  protected reloadData () {
+    this.error = undefined
+
+    this.authService.userInformationLoaded
+      .pipe(mergeMap(() => {
+        const user = this.authService.getUser()
+        return this.videoChannelsSyncService.listAccountVideoChannelsSyncs({
+          sort: this.sort,
+          account: user.account,
+          pagination: this.pagination
+        })
+      }))
+      .subscribe({
+        next: res => {
+          this.channelSyncs = res.data
+        },
+        error: err => {
+          this.error = err.message
+        }
+      })
+  }
+
+  syncEnabled () {
+    return this.serverConfig.import.videoChannelSynchronization.enabled
+  }
+
+  deleteSync (videoChannelSync: VideoChannelSync) {
+    this.videoChannelsSyncService.deleteSync(videoChannelSync.id)
+      .subscribe({
+        next: () => {
+          this.notifier.success($localize`Synchronization removed successfully for ${videoChannelSync.channel.displayName}.`)
+          this.reloadData()
+        },
+        error: err => {
+          this.error = err.message
+        }
+      })
+  }
+
+  fullySynchronize (videoChannelSync: VideoChannelSync) {
+    this.videoChannelService.importVideos(videoChannelSync.channel.name, videoChannelSync.externalChannelUrl)
+      .subscribe({
+        next: () => {
+          this.notifier.success($localize`Full synchronization requested successfully for ${videoChannelSync.channel.displayName}.`)
+        },
+        error: err => {
+          this.error = err.message
+        }
+      })
+  }
+
+  getSyncCreateLink () {
+    return '/my-library/video-channel-syncs/create'
+  }
+
+  getSyncStateClass (stateId: number) {
+    return [ 'pt-badge', MyVideoChannelSyncsComponent.STATE_CLASS_BY_ID[stateId] ]
+  }
+
+  getIdentifier () {
+    return 'MyVideoChannelsSyncComponent'
+  }
+
+  getChannelUrl (name: string) {
+    return '/c/' + name
+  }
+}

client/src/app/+my-library/my-video-channel-syncs/video-channel-sync-edit/video-channel-sync-edit.component.html

@@ -0,0 +1,64 @@
+<div *ngIf="error" class="alert alert-danger">{{ error }}</div>
+
+<div class="margin-content">
+  <form role="form" (ngSubmit)="formValidated()" [formGroup]="form">
+
+    <div class="row">
+      <div class="col-12 col-lg-4 col-xl-3">
+        <div class="video-channel-sync-title" i18n>NEW SYNCHRONIZATION</div>
+      </div>
+
+      <div class="col-12 col-lg-8 col-xl-9">
+        <div class="form-group">
+          <label i18n for="externalChannelUrl">Remote channel URL</label>
+
+          <div class="input-group">
+            <input
+              type="text"
+              id="externalChannelUrl"
+              i18n-placeholder
+              placeholder="Example: https://youtube.com/channel/UC_fancy_channel"
+              formControlName="externalChannelUrl"
+              [ngClass]="{ 'input-error': formErrors['externalChannelUrl'] }"
+              class="form-control"
+            >
+          </div>
+
+          <div *ngIf="formErrors['externalChannelUrl']" class="form-error">
+            {{ formErrors['externalChannelUrl'] }}
+          </div>
+        </div>
+
+        <div class="form-group">
+          <label i18n for="videoChannel">Video Channel</label>
+          <my-select-channel required [items]="userVideoChannels" formControlName="videoChannel"></my-select-channel>
+
+          <div *ngIf="formErrors['videoChannel']" class="form-error">
+            {{ formErrors['videoChannel'] }}
+          </div>
+        </div>
+
+        <div class="form-group">
+          <label for="existingVideoStrategy" i18n>Options for existing videos on remote channel:</label>
+
+          <div class="peertube-radio-container">
+            <input type="radio" name="existingVideoStrategy" id="import" value="import" formControlName="existingVideoStrategy" required />
+            <label for="import" i18n>Import all and watch for new publications</label>
+          </div>
+
+          <div class="peertube-radio-container">
+            <input type="radio" name="existingVideoStrategy" id="doNothing" value="nothing" formControlName="existingVideoStrategy" required />
+            <label for="doNothing" i18n>Only watch for new publications</label>
+          </div>
+        </div>
+      </div>
+    </div>
+
+    <div class="row"> <!-- submit placement block -->
+      <div class="col-md-7 col-xl-5"></div>
+      <div class="col-md-5 col-xl-5 d-inline-flex">
+        <input type="submit" class="peertube-button orange-button ms-auto" value="{{ getFormButtonTitle() }}" [disabled]="!form.valid">
+      </div>
+    </div>
+  </form>
+</div>

client/src/app/+my-library/my-video-channel-syncs/video-channel-sync-edit/video-channel-sync-edit.component.scss

@@ -0,0 +1,17 @@
+@use '_variables' as *;
+@use '_mixins' as *;
+
+$form-base-input-width: 480px;
+
+input[type=text] {
+  @include peertube-input-text($form-base-input-width);
+}
+
+.video-channel-sync-title {
+  @include settings-big-title;
+}
+
+my-select-channel {
+  display: block;
+  max-width: $form-base-input-width;
+}

client/src/app/+my-library/my-video-channel-syncs/video-channel-sync-edit/video-channel-sync-edit.component.ts

@@ -0,0 +1,76 @@
+import { mergeMap } from 'rxjs'
+import { SelectChannelItem } from 'src/types'
+import { Component, OnInit } from '@angular/core'
+import { Router } from '@angular/router'
+import { AuthService, Notifier } from '@app/core'
+import { listUserChannelsForSelect } from '@app/helpers'
+import { VIDEO_CHANNEL_EXTERNAL_URL_VALIDATOR } from '@app/shared/form-validators/video-channel-validators'
+import { FormReactive, FormValidatorService } from '@app/shared/shared-forms'
+import { VideoChannelService, VideoChannelSyncService } from '@app/shared/shared-main'
+import { VideoChannelSyncCreate } from '@shared/models/videos'
+
+@Component({
+  selector: 'my-video-channel-sync-edit',
+  templateUrl: './video-channel-sync-edit.component.html',
+  styleUrls: [ './video-channel-sync-edit.component.scss' ]
+})
+export class VideoChannelSyncEditComponent extends FormReactive implements OnInit {
+  error: string
+  userVideoChannels: SelectChannelItem[] = []
+  existingVideosStrategy: string
+
+  constructor (
+    protected formValidatorService: FormValidatorService,
+    private authService: AuthService,
+    private router: Router,
+    private notifier: Notifier,
+    private videoChannelSyncService: VideoChannelSyncService,
+    private videoChannelService: VideoChannelService
+  ) {
+    super()
+  }
+
+  ngOnInit () {
+    this.buildForm({
+      externalChannelUrl: VIDEO_CHANNEL_EXTERNAL_URL_VALIDATOR,
+      videoChannel: null,
+      existingVideoStrategy: null
+    })
+
+    listUserChannelsForSelect(this.authService)
+      .subscribe(channels => this.userVideoChannels = channels)
+  }
+
+  getFormButtonTitle () {
+    return $localize`Create`
+  }
+
+  formValidated () {
+    this.error = undefined
+
+    const body = this.form.value
+    const videoChannelSyncCreate: VideoChannelSyncCreate = {
+      externalChannelUrl: body.externalChannelUrl,
+      videoChannelId: body.videoChannel
+    }
+
+    const importExistingVideos = body['existingVideoStrategy'] === 'import'
+
+    this.videoChannelSyncService.createSync(videoChannelSyncCreate)
+      .pipe(mergeMap(({ videoChannelSync }) => {
+        return importExistingVideos
+          ? this.videoChannelService.importVideos(videoChannelSync.channel.name, videoChannelSync.externalChannelUrl)
+          : Promise.resolve(null)
+      }))
+      .subscribe({
+        next: () => {
+          this.notifier.success($localize`Synchronization created successfully.`)
+          this.router.navigate([ '/my-library', 'video-channel-syncs' ])
+        },
+
+        error: err => {
+          this.error = err.message
+        }
+      })
+  }
+}

client/src/app/shared/form-validators/video-channel-validators.ts

@@ -48,3 +48,16 @@ export const VIDEO_CHANNEL_SUPPORT_VALIDATOR: BuildFormValidator = {
     maxlength: $localize`Support text cannot be more than 1000 characters long.`
   }
 }
+
+export const VIDEO_CHANNEL_EXTERNAL_URL_VALIDATOR: BuildFormValidator = {
+  VALIDATORS: [
+    Validators.required,
+    Validators.pattern(/^https?:\/\//),
+    Validators.maxLength(1000)
+  ],
+  MESSAGES: {
+    required: $localize`Remote channel url is required.`,
+    pattern: $localize`External channel URL must begin with "https://" or "http://"`,
+    maxlength: $localize`External channel URL cannot be more than 1000 characters long`
+  }
+}

client/src/app/shared/shared-instance/instance-features-table.component.html

@@ -106,6 +106,13 @@
       </td>
     </tr>
 
+    <tr>
+      <th i18n class="sub-label" scope="row">Channel synchronization with other platforms (YouTube, Vimeo, ...)</th>
+      <td>
+        <my-feature-boolean [value]="serverConfig.import.videoChannelSynchronization.enabled"></my-feature-boolean>
+      </td>
+    </tr>
+
     <tr>
       <th i18n class="label" colspan="2">Search</th>
     </tr>

client/src/app/shared/shared-main/index.ts

@@ -13,3 +13,4 @@ export * from './video'
 export * from './video-caption'
 export * from './video-channel'
 export * from './shared-main.module'
+export * from './video-channel-sync'

client/src/app/shared/shared-main/video-channel-sync/index.ts

@@ -0,0 +1 @@
+export * from './video-channel-sync.service'

client/src/app/shared/shared-main/video-channel-sync/video-channel-sync.service.ts

@@ -0,0 +1,50 @@
+import { SortMeta } from 'primeng/api'
+import { catchError, Observable } from 'rxjs'
+import { environment } from 'src/environments/environment'
+import { HttpClient, HttpParams } from '@angular/common/http'
+import { Injectable } from '@angular/core'
+import { RestExtractor, RestPagination, RestService } from '@app/core'
+import { ResultList } from '@shared/models/common'
+import { VideoChannelSync, VideoChannelSyncCreate } from '@shared/models/videos'
+import { Account, AccountService } from '../account'
+
+@Injectable({
+  providedIn: 'root'
+})
+export class VideoChannelSyncService {
+  static BASE_VIDEO_CHANNEL_URL = environment.apiUrl + '/api/v1/video-channel-syncs'
+
+  constructor (
+    private authHttp: HttpClient,
+    private restExtractor: RestExtractor,
+    private restService: RestService
+  ) { }
+
+  listAccountVideoChannelsSyncs (parameters: {
+    sort: SortMeta
+    pagination: RestPagination
+    account: Account
+  }): Observable<ResultList<VideoChannelSync>> {
+    const { pagination, sort, account } = parameters
+
+    let params = new HttpParams()
+    params = this.restService.addRestGetParams(params, pagination, sort)
+
+    const url = AccountService.BASE_ACCOUNT_URL + account.nameWithHost + '/video-channel-syncs'
+
+    return this.authHttp.get<ResultList<VideoChannelSync>>(url, { params })
+               .pipe(catchError(err => this.restExtractor.handleError(err)))
+  }
+
+  createSync (body: VideoChannelSyncCreate) {
+    return this.authHttp.post<{ videoChannelSync: VideoChannelSync }>(VideoChannelSyncService.BASE_VIDEO_CHANNEL_URL, body)
+               .pipe(catchError(err => this.restExtractor.handleError(err)))
+  }
+
+  deleteSync (videoChannelsSyncId: number) {
+    const url = `${VideoChannelSyncService.BASE_VIDEO_CHANNEL_URL}/${videoChannelsSyncId}`
+
+    return this.authHttp.delete(url)
+               .pipe(catchError(err => this.restExtractor.handleError(err)))
+  }
+}

client/src/app/shared/shared-main/video-channel/video-channel.service.ts

@@ -95,4 +95,10 @@ export class VideoChannelService {
     return this.authHttp.delete(VideoChannelService.BASE_VIDEO_CHANNEL_URL + videoChannel.nameWithHost)
                .pipe(catchError(err => this.restExtractor.handleError(err)))
   }
+
+  importVideos (videoChannelName: string, externalChannelUrl: string) {
+    const path = VideoChannelService.BASE_VIDEO_CHANNEL_URL + videoChannelName + '/import-videos'
+    return this.authHttp.post(path, { externalChannelUrl })
+               .pipe(catchError(err => this.restExtractor.handleError(err)))
+  }
 }

config/default.yaml

@@ -546,6 +546,17 @@ import:
       # See https://docs.joinpeertube.org/maintain-configuration?id=security for more information
       enabled: false
 
+  # Add ability for your users to synchronize their channels with external channels, playlists, etc
+  video_channel_synchronization:
+    enabled: false
+
+    max_per_user: 10
+
+    check_interval: 1 hour
+
+    # Number of latest published videos to check and to potentially import when syncing a channel
+    videos_limit_per_synchronization: 10
+
 auto_blacklist:
   # New videos automatically blacklisted so moderators can review before publishing
   videos:

config/dev.yaml

@@ -81,6 +81,11 @@ import:
       enabled: true
     torrent:
       enabled: true
+  video_channel_synchronization:
+    enabled: true
+    max_per_user: 10
+    check_interval: 5 minutes
+    videos_limit_per_synchronization: 3
 
 instance:
   default_nsfw_policy: 'display'

config/production.yaml.example

@@ -556,6 +556,17 @@ import:
       # See https://docs.joinpeertube.org/maintain-configuration?id=security for more information
       enabled: false
 
+  # Add ability for your users to synchronize their channels with external channels, playlists, etc.
+  video_channel_synchronization:
+    enabled: false
+
+    max_per_user: 10
+
+    check_interval: 1 hour
+
+    # Number of latest published videos to check and to potentially import when syncing a channel
+    videos_limit_per_synchronization: 10
+
 auto_blacklist:
   # New videos automatically blacklisted so moderators can review before publishing
   videos:

server.ts

@@ -139,6 +139,7 @@ import { VideoViewsManager } from '@server/lib/views/video-views-manager'
 import { isTestOrDevInstance } from './server/helpers/core-utils'
 import { OpenTelemetryMetrics } from '@server/lib/opentelemetry/metrics'
 import { ApplicationModel } from '@server/models/application/application'
+import { VideoChannelSyncLatestScheduler } from '@server/lib/schedulers/video-channel-sync-latest-scheduler'
 
 // ----------- Command line -----------
 
@@ -314,6 +315,7 @@ async function startApplication () {
   PeerTubeVersionCheckScheduler.Instance.enable()
   AutoFollowIndexInstances.Instance.enable()
   RemoveDanglingResumableUploadsScheduler.Instance.enable()
+  VideoChannelSyncLatestScheduler.Instance.enable()
   VideoViewsBufferScheduler.Instance.enable()
   GeoIPUpdateScheduler.Instance.enable()
   OpenTelemetryMetrics.Instance.registerMetrics()

server/controllers/api/accounts.ts

@@ -25,8 +25,10 @@ import {
   accountsFollowersSortValidator,
   accountsSortValidator,
   ensureAuthUserOwnsAccountValidator,
+  ensureCanManageUser,
   videoChannelsSortValidator,
   videoChannelStatsValidator,
+  videoChannelSyncsSortValidator,
   videosSortValidator
 } from '../../middlewares/validators'
 import { commonVideoPlaylistFiltersValidator, videoPlaylistsSearchValidator } from '../../middlewares/validators/videos/video-playlists'
@@ -35,6 +37,7 @@ import { AccountVideoRateModel } from '../../models/account/account-video-rate'
 import { VideoModel } from '../../models/video/video'
 import { VideoChannelModel } from '../../models/video/video-channel'
 import { VideoPlaylistModel } from '../../models/video/video-playlist'
+import { VideoChannelSyncModel } from '@server/models/video/video-channel-sync'
 
 const accountsRouter = express.Router()
 
@@ -72,6 +75,17 @@ accountsRouter.get('/:accountName/video-channels',
   asyncMiddleware(listAccountChannels)
 )
 
+accountsRouter.get('/:accountName/video-channel-syncs',
+  authenticate,
+  asyncMiddleware(accountNameWithHostGetValidator),
+  ensureCanManageUser,
+  paginationValidator,
+  videoChannelSyncsSortValidator,
+  setDefaultSort,
+  setDefaultPagination,
+  asyncMiddleware(listAccountChannelsSync)
+)
+
 accountsRouter.get('/:accountName/video-playlists',
   optionalAuthenticate,
   asyncMiddleware(accountNameWithHostGetValidator),
@@ -146,6 +160,20 @@ async function listAccountChannels (req: express.Request, res: express.Response)
   return res.json(getFormattedObjects(resultList.data, resultList.total))
 }
 
+async function listAccountChannelsSync (req: express.Request, res: express.Response) {
+  const options = {
+    accountId: res.locals.account.id,
+    start: req.query.start,
+    count: req.query.count,
+    sort: req.query.sort,
+    search: req.query.search
+  }
+
+  const resultList = await VideoChannelSyncModel.listByAccountForAPI(options)
+
+  return res.json(getFormattedObjects(resultList.data, resultList.total))
+}
+
 async function listAccountPlaylists (req: express.Request, res: express.Response) {
   const serverActor = await getServerActor()
 

server/controllers/api/config.ts

@@ -273,6 +273,10 @@ function customConfig (): CustomConfig {
         torrent: {
           enabled: CONFIG.IMPORT.VIDEOS.TORRENT.ENABLED
         }
+      },
+      videoChannelSynchronization: {
+        enabled: CONFIG.IMPORT.VIDEO_CHANNEL_SYNCHRONIZATION.ENABLED,
+        maxPerUser: CONFIG.IMPORT.VIDEO_CHANNEL_SYNCHRONIZATION.MAX_PER_USER
       }
     },
     trending: {

server/controllers/api/index.ts

@@ -20,6 +20,7 @@ import { usersRouter } from './users'
 import { videoChannelRouter } from './video-channel'
 import { videoPlaylistRouter } from './video-playlist'
 import { videosRouter } from './videos'
+import { videoChannelSyncRouter } from './video-channel-sync'
 
 const apiRouter = express.Router()
 
@@ -43,6 +44,7 @@ apiRouter.use('/config', configRouter)
 apiRouter.use('/users', usersRouter)
 apiRouter.use('/accounts', accountsRouter)
 apiRouter.use('/video-channels', videoChannelRouter)
+apiRouter.use('/video-channel-syncs', videoChannelSyncRouter)
 apiRouter.use('/video-playlists', videoPlaylistRouter)
 apiRouter.use('/videos', videosRouter)
 apiRouter.use('/jobs', jobsRouter)

server/controllers/api/server/debug.ts

@@ -7,6 +7,7 @@ import { Debug, SendDebugCommand } from '@shared/models'
 import { HttpStatusCode } from '../../../../shared/models/http/http-error-codes'
 import { UserRight } from '../../../../shared/models/users'
 import { authenticate, ensureUserHasRight } from '../../../middlewares'
+import { VideoChannelSyncLatestScheduler } from '@server/lib/schedulers/video-channel-sync-latest-scheduler'
 
 const debugRouter = express.Router()
 
@@ -43,7 +44,8 @@ async function runCommand (req: express.Request, res: express.Response) {
   const processors: { [id in SendDebugCommand['command']]: () => Promise<any> } = {
     'remove-dandling-resumable-uploads': () => RemoveDanglingResumableUploadsScheduler.Instance.execute(),
     'process-video-views-buffer': () => VideoViewsBufferScheduler.Instance.execute(),
-    'process-video-viewers': () => VideoViewsManager.Instance.processViewerStats()
+    'process-video-viewers': () => VideoViewsManager.Instance.processViewerStats(),
+    'process-video-channel-sync-latest': () => VideoChannelSyncLatestScheduler.Instance.execute()
   }
 
   await processors[body.command]()

server/controllers/api/video-channel-sync.ts

@@ -0,0 +1,76 @@
+import express from 'express'
+import { auditLoggerFactory, getAuditIdFromRes, VideoChannelSyncAuditView } from '@server/helpers/audit-logger'
+import { logger } from '@server/helpers/logger'
+import {
+  asyncMiddleware,
+  asyncRetryTransactionMiddleware,
+  authenticate,
+  ensureCanManageChannel as ensureCanManageSyncedChannel,
+  ensureSyncExists,
+  ensureSyncIsEnabled,
+  videoChannelSyncValidator
+} from '@server/middlewares'
+import { VideoChannelSyncModel } from '@server/models/video/video-channel-sync'
+import { MChannelSyncFormattable } from '@server/types/models'
+import { HttpStatusCode, VideoChannelSyncState } from '@shared/models'
+
+const videoChannelSyncRouter = express.Router()
+const auditLogger = auditLoggerFactory('channel-syncs')
+
+videoChannelSyncRouter.post('/',
+  authenticate,
+  ensureSyncIsEnabled,
+  asyncMiddleware(videoChannelSyncValidator),
+  ensureCanManageSyncedChannel,
+  asyncRetryTransactionMiddleware(createVideoChannelSync)
+)
+
+videoChannelSyncRouter.delete('/:id',
+  authenticate,
+  asyncMiddleware(ensureSyncExists),
+  ensureCanManageSyncedChannel,
+  asyncRetryTransactionMiddleware(removeVideoChannelSync)
+)
+
+export { videoChannelSyncRouter }
+
+// ---------------------------------------------------------------------------
+
+async function createVideoChannelSync (req: express.Request, res: express.Response) {
+  const syncCreated: MChannelSyncFormattable = new VideoChannelSyncModel({
+    externalChannelUrl: req.body.externalChannelUrl,
+    videoChannelId: req.body.videoChannelId,
+    state: VideoChannelSyncState.WAITING_FIRST_RUN
+  })
+
+  await syncCreated.save()
+  syncCreated.VideoChannel = res.locals.videoChannel
+
+  auditLogger.create(getAuditIdFromRes(res), new VideoChannelSyncAuditView(syncCreated.toFormattedJSON()))
+
+  logger.info(
+    'Video synchronization for channel "%s" with external channel "%s" created.',
+    syncCreated.VideoChannel.name,
+    syncCreated.externalChannelUrl
+  )
+
+  return res.json({
+    videoChannelSync: syncCreated.toFormattedJSON()
+  })
+}
+
+async function removeVideoChannelSync (req: express.Request, res: express.Response) {
+  const syncInstance = res.locals.videoChannelSync
+
+  await syncInstance.destroy()
+
+  auditLogger.delete(getAuditIdFromRes(res), new VideoChannelSyncAuditView(syncInstance.toFormattedJSON()))
+
+  logger.info(
+    'Video synchronization for channel "%s" with external channel "%s" deleted.',
+    syncInstance.VideoChannel.name,
+    syncInstance.externalChannelUrl
+  )
+
+  return res.type('json').status(HttpStatusCode.NO_CONTENT_204).end()
+}

server/controllers/api/video-channel.ts

@@ -36,7 +36,9 @@ import {
   videoPlaylistsSortValidator
 } from '../../middlewares'
 import {
+  ensureChannelOwnerCanUpload,
   ensureIsLocalChannel,
+  videoChannelImportVideosValidator,
   videoChannelsFollowersSortValidator,
   videoChannelsListValidator,
   videoChannelsNameWithHostValidator,
@@ -161,6 +163,16 @@ videoChannelRouter.get('/:nameWithHost/followers',
   asyncMiddleware(listVideoChannelFollowers)
 )
 
+videoChannelRouter.post('/:nameWithHost/import-videos',
+  authenticate,
+  asyncMiddleware(videoChannelsNameWithHostValidator),
+  videoChannelImportVideosValidator,
+  ensureIsLocalChannel,
+  ensureCanManageChannel,
+  asyncMiddleware(ensureChannelOwnerCanUpload),
+  asyncMiddleware(importVideosInChannel)
+)
+
 // ---------------------------------------------------------------------------
 
 export {
@@ -404,3 +416,19 @@ async function listVideoChannelFollowers (req: express.Request, res: express.Res
 
   return res.json(getFormattedObjects(resultList.data, resultList.total))
 }
+
+async function importVideosInChannel (req: express.Request, res: express.Response) {
+  const { externalChannelUrl } = req.body
+
+  await JobQueue.Instance.createJob({
+    type: 'video-channel-import',
+    payload: {
+      externalChannelUrl,
+      videoChannelId: res.locals.videoChannel.id
+    }
+  })
+
+  logger.info('Video import job for channel "%s" with url "%s" created.', res.locals.videoChannel.name, externalChannelUrl)
+
+  return res.type('json').status(HttpStatusCode.NO_CONTENT_204).end()
+}

server/controllers/api/videos/import.ts

@@ -1,49 +1,20 @@
 import express from 'express'
-import { move, readFile, remove } from 'fs-extra'
+import { move, readFile } from 'fs-extra'
 import { decode } from 'magnet-uri'
 import parseTorrent, { Instance } from 'parse-torrent'
 import { join } from 'path'
-import { isVTTFileValid } from '@server/helpers/custom-validators/video-captions'
+import { buildYoutubeDLImport, buildVideoFromImport, insertFromImportIntoDB, YoutubeDlImportError } from '@server/lib/video-import'
-import { isVideoFileExtnameValid } from '@server/helpers/custom-validators/videos'
+import { MThumbnail, MVideoThumbnail } from '@server/types/models'
-import { isResolvingToUnicastOnly } from '@server/helpers/dns'
+import { HttpStatusCode, ServerErrorCode, ThumbnailType, VideoImportCreate, VideoImportPayload, VideoImportState } from '@shared/models'
-import { Hooks } from '@server/lib/plugins/hooks'
-import { ServerConfigManager } from '@server/lib/server-config-manager'
-import { setVideoTags } from '@server/lib/video'
-import { FilteredModelAttributes } from '@server/types'
-import {
-  MChannelAccountDefault,
-  MThumbnail,
-  MUser,
-  MVideoAccountDefault,
-  MVideoCaption,
-  MVideoTag,
-  MVideoThumbnail,
-  MVideoWithBlacklistLight
-} from '@server/types/models'
-import { MVideoImportFormattable } from '@server/types/models/video/video-import'
-import {
-  HttpStatusCode,
-  ServerErrorCode,
-  ThumbnailType,
-  VideoImportCreate,
-  VideoImportState,
-  VideoPrivacy,
-  VideoState
-} from '@shared/models'
 import { auditLoggerFactory, getAuditIdFromRes, VideoImportAuditView } from '../../../helpers/audit-logger'
-import { moveAndProcessCaptionFile } from '../../../helpers/captions-utils'
 import { isArray } from '../../../helpers/custom-validators/misc'
 import { cleanUpReqFiles, createReqFiles } from '../../../helpers/express-utils'
 import { logger } from '../../../helpers/logger'
 import { getSecureTorrentName } from '../../../helpers/utils'
-import { YoutubeDLInfo, YoutubeDLWrapper } from '../../../helpers/youtube-dl'
 import { CONFIG } from '../../../initializers/config'
 import { MIMETYPES } from '../../../initializers/constants'
-import { sequelizeTypescript } from '../../../initializers/database'
-import { getLocalVideoActivityPubUrl } from '../../../lib/activitypub/url'
 import { JobQueue } from '../../../lib/job-queue/job-queue'
-import { updateVideoMiniatureFromExisting, updateVideoMiniatureFromUrl } from '../../../lib/thumbnail'
+import { updateVideoMiniatureFromExisting } from '../../../lib/thumbnail'
-import { autoBlacklistVideoIfNeeded } from '../../../lib/video-blacklist'
 import {
   asyncMiddleware,
   asyncRetryTransactionMiddleware,
@@ -52,9 +23,6 @@ import {
   videoImportCancelValidator,
   videoImportDeleteValidator
 } from '../../../middlewares'
-import { VideoModel } from '../../../models/video/video'
-import { VideoCaptionModel } from '../../../models/video/video-caption'
-import { VideoImportModel } from '../../../models/video/video-import'
 
 const auditLogger = auditLoggerFactory('video-imports')
 const videoImportsRouter = express.Router()
@@ -68,7 +36,7 @@ videoImportsRouter.post('/imports',
   authenticate,
   reqVideoFileImport,
   asyncMiddleware(videoImportAddValidator),
-  asyncRetryTransactionMiddleware(addVideoImport)
+  asyncRetryTransactionMiddleware(handleVideoImport)
 )
 
 videoImportsRouter.post('/imports/:id/cancel',
@@ -108,14 +76,14 @@ async function cancelVideoImport (req: express.Request, res: express.Response) {
   return res.sendStatus(HttpStatusCode.NO_CONTENT_204)
 }
 
-function addVideoImport (req: express.Request, res: express.Response) {
+function handleVideoImport (req: express.Request, res: express.Response) {
-  if (req.body.targetUrl) return addYoutubeDLImport(req, res)
+  if (req.body.targetUrl) return handleYoutubeDlImport(req, res)
 
   const file = req.files?.['torrentfile']?.[0]
-  if (req.body.magnetUri || file) return addTorrentImport(req, res, file)
+  if (req.body.magnetUri || file) return handleTorrentImport(req, res, file)
 }
 
-async function addTorrentImport (req: express.Request, res: express.Response, torrentfile: Express.Multer.File) {
+async function handleTorrentImport (req: express.Request, res: express.Response, torrentfile: Express.Multer.File) {
   const body: VideoImportCreate = req.body
   const user = res.locals.oauth.token.User
 
@@ -135,12 +103,17 @@ async function addTorrentImport (req: express.Request, res: express.Response, to
     videoName = result.name
   }
 
-  const video = await buildVideo(res.locals.videoChannel.id, body, { name: videoName })
+  const video = await buildVideoFromImport({
+    channelId: res.locals.videoChannel.id,
+    importData: { name: videoName },
+    importDataOverride: body,
+    importType: 'torrent'
+  })
 
   const thumbnailModel = await processThumbnail(req, video)
   const previewModel = await processPreview(req, video)
 
-  const videoImport = await insertIntoDB({
+  const videoImport = await insertFromImportIntoDB({
     video,
     thumbnailModel,
     previewModel,
@@ -155,13 +128,12 @@ async function addTorrentImport (req: express.Request, res: express.Response, to
     }
   })
 
-  // Create job to import the video
+  const payload: VideoImportPayload = {
-  const payload = {
     type: torrentfile
-      ? 'torrent-file' as 'torrent-file'
+      ? 'torrent-file'
-      : 'magnet-uri' as 'magnet-uri',
+      : 'magnet-uri',
     videoImportId: videoImport.id,
-    magnetUri
+    preventException: false
   }
   await JobQueue.Instance.createJob({ type: 'video-import', payload })
 
@@ -170,131 +142,49 @@ async function addTorrentImport (req: express.Request, res: express.Response, to
   return res.json(videoImport.toFormattedJSON()).end()
 }
 
-async function addYoutubeDLImport (req: express.Request, res: express.Response) {
+function statusFromYtDlImportError (err: YoutubeDlImportError): number {
+  switch (err.code) {
+    case YoutubeDlImportError.CODE.NOT_ONLY_UNICAST_URL:
+      return HttpStatusCode.FORBIDDEN_403
+
+    case YoutubeDlImportError.CODE.FETCH_ERROR:
+      return HttpStatusCode.BAD_REQUEST_400
+
+    default:
+      return HttpStatusCode.INTERNAL_SERVER_ERROR_500
+  }
+}
+
+async function handleYoutubeDlImport (req: express.Request, res: express.Response) {
   const body: VideoImportCreate = req.body
   const targetUrl = body.targetUrl
   const user = res.locals.oauth.token.User
 
-  const youtubeDL = new YoutubeDLWrapper(
-    targetUrl,
-    ServerConfigManager.Instance.getEnabledResolutions('vod'),
-    CONFIG.TRANSCODING.ALWAYS_TRANSCODE_ORIGINAL_RESOLUTION
-  )
-
-  // Get video infos
-  let youtubeDLInfo: YoutubeDLInfo
   try {
-    youtubeDLInfo = await youtubeDL.getInfoForDownload()
+    const { job, videoImport } = await buildYoutubeDLImport({
+      targetUrl,
+      channel: res.locals.videoChannel,
+      importDataOverride: body,
+      thumbnailFilePath: req.files?.['thumbnailfile']?.[0].path,
+      previewFilePath: req.files?.['previewfile']?.[0].path,
+      user
+    })
+    await JobQueue.Instance.createJob(job)
+
+    auditLogger.create(getAuditIdFromRes(res), new VideoImportAuditView(videoImport.toFormattedJSON()))
+
+    return res.json(videoImport.toFormattedJSON()).end()
   } catch (err) {
-    logger.info('Cannot fetch information from import for URL %s.', targetUrl, { err })
+    logger.error('An error occurred while importing the video %s. ', targetUrl, { err })
 
     return res.fail({
-      message: 'Cannot fetch remote information of this URL.',
+      message: err.message,
+      status: statusFromYtDlImportError(err),
       data: {
         targetUrl
       }
     })
   }
-
-  if (!await hasUnicastURLsOnly(youtubeDLInfo)) {
-    return res.fail({
-      status: HttpStatusCode.FORBIDDEN_403,
-      message: 'Cannot use non unicast IP as targetUrl.'
-    })
-  }
-
-  const video = await buildVideo(res.locals.videoChannel.id, body, youtubeDLInfo)
-
-  // Process video thumbnail from request.files
-  let thumbnailModel = await processThumbnail(req, video)
-
-  // Process video thumbnail from url if processing from request.files failed
-  if (!thumbnailModel && youtubeDLInfo.thumbnailUrl) {
-    try {
-      thumbnailModel = await processThumbnailFromUrl(youtubeDLInfo.thumbnailUrl, video)
-    } catch (err) {
-      logger.warn('Cannot process thumbnail %s from youtubedl.', youtubeDLInfo.thumbnailUrl, { err })
-    }
-  }
-
-  // Process video preview from request.files
-  let previewModel = await processPreview(req, video)
-
-  // Process video preview from url if processing from request.files failed
-  if (!previewModel && youtubeDLInfo.thumbnailUrl) {
-    try {
-      previewModel = await processPreviewFromUrl(youtubeDLInfo.thumbnailUrl, video)
-    } catch (err) {
-      logger.warn('Cannot process preview %s from youtubedl.', youtubeDLInfo.thumbnailUrl, { err })
-    }
-  }
-
-  const videoImport = await insertIntoDB({
-    video,
-    thumbnailModel,
-    previewModel,
-    videoChannel: res.locals.videoChannel,
-    tags: body.tags || youtubeDLInfo.tags,
-    user,
-    videoImportAttributes: {
-      targetUrl,
-      state: VideoImportState.PENDING,
-      userId: user.id
-    }
-  })
-
-  // Get video subtitles
-  await processYoutubeSubtitles(youtubeDL, targetUrl, video.id)
-
-  let fileExt = `.${youtubeDLInfo.ext}`
-  if (!isVideoFileExtnameValid(fileExt)) fileExt = '.mp4'
-
-  // Create job to import the video
-  const payload = {
-    type: 'youtube-dl' as 'youtube-dl',
-    videoImportId: videoImport.id,
-    fileExt
-  }
-  await JobQueue.Instance.createJob({ type: 'video-import', payload })
-
-  auditLogger.create(getAuditIdFromRes(res), new VideoImportAuditView(videoImport.toFormattedJSON()))
-
-  return res.json(videoImport.toFormattedJSON()).end()
-}
-
-async function buildVideo (channelId: number, body: VideoImportCreate, importData: YoutubeDLInfo): Promise<MVideoThumbnail> {
-  let videoData = {
-    name: body.name || importData.name || 'Unknown name',
-    remote: false,
-    category: body.category || importData.category,
-    licence: body.licence ?? importData.licence ?? CONFIG.DEFAULTS.PUBLISH.LICENCE,
-    language: body.language || importData.language,
-    commentsEnabled: body.commentsEnabled ?? CONFIG.DEFAULTS.PUBLISH.COMMENTS_ENABLED,
-    downloadEnabled: body.downloadEnabled ?? CONFIG.DEFAULTS.PUBLISH.DOWNLOAD_ENABLED,
-    waitTranscoding: body.waitTranscoding || false,
-    state: VideoState.TO_IMPORT,
-    nsfw: body.nsfw || importData.nsfw || false,
-    description: body.description || importData.description,
-    support: body.support || null,
-    privacy: body.privacy || VideoPrivacy.PRIVATE,
-    duration: 0, // duration will be set by the import job
-    channelId,
-    originallyPublishedAt: body.originallyPublishedAt
-      ? new Date(body.originallyPublishedAt)
-      : importData.originallyPublishedAt
-  }
-
-  videoData = await Hooks.wrapObject(
-    videoData,
-    body.targetUrl
-      ? 'filter:api.video.import-url.video-attribute.result'
-      : 'filter:api.video.import-torrent.video-attribute.result'
-  )
-
-  const video = new VideoModel(videoData)
-  video.url = getLocalVideoActivityPubUrl(video)
-
-  return video
 }
 
 async function processThumbnail (req: express.Request, video: MVideoThumbnail) {
@@ -329,69 +219,6 @@ async function processPreview (req: express.Request, video: MVideoThumbnail): Pr
   return undefined
 }
 
-async function processThumbnailFromUrl (url: string, video: MVideoThumbnail) {
-  try {
-    return updateVideoMiniatureFromUrl({ downloadUrl: url, video, type: ThumbnailType.MINIATURE })
-  } catch (err) {
-    logger.warn('Cannot generate video thumbnail %s for %s.', url, video.url, { err })
-    return undefined
-  }
-}
-
-async function processPreviewFromUrl (url: string, video: MVideoThumbnail) {
-  try {
-    return updateVideoMiniatureFromUrl({ downloadUrl: url, video, type: ThumbnailType.PREVIEW })
-  } catch (err) {
-    logger.warn('Cannot generate video preview %s for %s.', url, video.url, { err })
-    return undefined
-  }
-}
-
-async function insertIntoDB (parameters: {
-  video: MVideoThumbnail
-  thumbnailModel: MThumbnail
-  previewModel: MThumbnail
-  videoChannel: MChannelAccountDefault
-  tags: string[]
-  videoImportAttributes: FilteredModelAttributes<VideoImportModel>
-  user: MUser
-}): Promise<MVideoImportFormattable> {
-  const { video, thumbnailModel, previewModel, videoChannel, tags, videoImportAttributes, user } = parameters
-
-  const videoImport = await sequelizeTypescript.transaction(async t => {
-    const sequelizeOptions = { transaction: t }
-
-    // Save video object in database
-    const videoCreated = await video.save(sequelizeOptions) as (MVideoAccountDefault & MVideoWithBlacklistLight & MVideoTag)
-    videoCreated.VideoChannel = videoChannel
-
-    if (thumbnailModel) await videoCreated.addAndSaveThumbnail(thumbnailModel, t)
-    if (previewModel) await videoCreated.addAndSaveThumbnail(previewModel, t)
-
-    await autoBlacklistVideoIfNeeded({
-      video: videoCreated,
-      user,
-      notify: false,
-      isRemote: false,
-      isNew: true,
-      transaction: t
-    })
-
-    await setVideoTags({ video: videoCreated, tags, transaction: t })
-
-    // Create video import object in database
-    const videoImport = await VideoImportModel.create(
-      Object.assign({ videoId: videoCreated.id }, videoImportAttributes),
-      sequelizeOptions
-    ) as MVideoImportFormattable
-    videoImport.Video = videoCreated
-
-    return videoImport
-  })
-
-  return videoImport
-}
-
 async function processTorrentOrAbortRequest (req: express.Request, res: express.Response, torrentfile: Express.Multer.File) {
   const torrentName = torrentfile.originalname
 
@@ -432,46 +259,3 @@ function processMagnetURI (body: VideoImportCreate) {
 function extractNameFromArray (name: string | string[]) {
   return isArray(name) ? name[0] : name
 }
-
-async function processYoutubeSubtitles (youtubeDL: YoutubeDLWrapper, targetUrl: string, videoId: number) {
-  try {
-    const subtitles = await youtubeDL.getSubtitles()
-
-    logger.info('Will create %s subtitles from youtube import %s.', subtitles.length, targetUrl)
-
-    for (const subtitle of subtitles) {
-      if (!await isVTTFileValid(subtitle.path)) {
-        await remove(subtitle.path)
-        continue
-      }
-
-      const videoCaption = new VideoCaptionModel({
-        videoId,
-        language: subtitle.language,
-        filename: VideoCaptionModel.generateCaptionName(subtitle.language)
-      }) as MVideoCaption
-
-      // Move physical file
-      await moveAndProcessCaptionFile(subtitle, videoCaption)
-
-      await sequelizeTypescript.transaction(async t => {
-        await VideoCaptionModel.insertOrReplaceLanguage(videoCaption, t)
-      })
-    }
-  } catch (err) {
-    logger.warn('Cannot get video subtitles.', { err })
-  }
-}
-
-async function hasUnicastURLsOnly (youtubeDLInfo: YoutubeDLInfo) {
-  const hosts = youtubeDLInfo.urls.map(u => new URL(u).hostname)
-  const uniqHosts = new Set(hosts)
-
-  for (const h of uniqHosts) {
-    if (await isResolvingToUnicastOnly(h) !== true) {
-      return false
-    }
-  }
-
-  return true
-}

server/helpers/audit-logger.ts

@@ -5,7 +5,7 @@ import { chain } from 'lodash'
 import { join } from 'path'
 import { addColors, config, createLogger, format, transports } from 'winston'
 import { AUDIT_LOG_FILENAME } from '@server/initializers/constants'
-import { AdminAbuse, CustomConfig, User, VideoChannel, VideoComment, VideoDetails, VideoImport } from '@shared/models'
+import { AdminAbuse, CustomConfig, User, VideoChannel, VideoChannelSync, VideoComment, VideoDetails, VideoImport } from '@shared/models'
 import { CONFIG } from '../initializers/config'
 import { jsonLoggerFormat, labelFormatter } from './logger'
 
@@ -260,6 +260,18 @@ class CustomConfigAuditView extends EntityAuditView {
   }
 }
 
+const channelSyncKeysToKeep = [
+  'id',
+  'externalChannelUrl',
+  'channel-id',
+  'channel-name'
+]
+class VideoChannelSyncAuditView extends EntityAuditView {
+  constructor (channelSync: VideoChannelSync) {
+    super(channelSyncKeysToKeep, 'channelSync', channelSync)
+  }
+}
+
 export {
   getAuditIdFromRes,
 
@@ -270,5 +282,6 @@ export {
   UserAuditView,
   VideoAuditView,
   AbuseAuditView,
-  CustomConfigAuditView
+  CustomConfigAuditView,
+  VideoChannelSyncAuditView
 }

server/helpers/custom-validators/video-channel-syncs.ts

@@ -0,0 +1,6 @@
+import { VIDEO_CHANNEL_SYNC_STATE } from '@server/initializers/constants'
+import { exists } from './misc'
+
+export function isVideoChannelSyncStateValid (value: any) {
+  return exists(value) && VIDEO_CHANNEL_SYNC_STATE[value] !== undefined
+}

server/helpers/youtube-dl/youtube-dl-cli.ts

@@ -87,6 +87,7 @@ export class YoutubeDLCLI {
     return result.concat([
       'bestvideo[vcodec!*=av01][vcodec!*=vp9.2]+bestaudio',
       'best[vcodec!*=av01][vcodec!*=vp9.2]', // case fallback for known formats
+      'bestvideo[ext=mp4]+bestaudio[ext=m4a]',
       'best' // Ultimate fallback
     ]).join('/')
   }
@@ -103,11 +104,14 @@ export class YoutubeDLCLI {
     timeout?: number
     additionalYoutubeDLArgs?: string[]
   }) {
+    let args = options.additionalYoutubeDLArgs || []
+    args = args.concat([ '--merge-output-format', 'mp4', '-f', options.format, '-o', options.output ])
+
     return this.run({
       url: options.url,
       processOptions: options.processOptions,
       timeout: options.timeout,
-      args: (options.additionalYoutubeDLArgs || []).concat([ '-f', options.format, '-o', options.output ])
+      args
     })
   }
 
@@ -129,6 +133,25 @@ export class YoutubeDLCLI {
       : info
   }
 
+  getListInfo (options: {
+    url: string
+    latestVideosCount?: number
+    processOptions: execa.NodeOptions
+  }): Promise<{ upload_date: string, webpage_url: string }[]> {
+    const additionalYoutubeDLArgs = [ '--skip-download', '--playlist-reverse' ]
+
+    if (options.latestVideosCount !== undefined) {
+      additionalYoutubeDLArgs.push('--playlist-end', options.latestVideosCount.toString())
+    }
+
+    return this.getInfo({
+      url: options.url,
+      format: YoutubeDLCLI.getYoutubeDLVideoFormat([], false),
+      processOptions: options.processOptions,
+      additionalYoutubeDLArgs
+    })
+  }
+
   async getSubs (options: {
     url: string
     format: 'vtt'
@@ -175,7 +198,7 @@ export class YoutubeDLCLI {
 
     const output = await subProcess
 
-    logger.debug('Runned youtube-dl command.', { command: output.command, ...lTags() })
+    logger.debug('Run youtube-dl command.', { command: output.command, ...lTags() })
 
     return output.stdout
       ? output.stdout.trim().split(/\r?\n/)

server/helpers/youtube-dl/youtube-dl-info-builder.ts

@@ -13,6 +13,7 @@ type YoutubeDLInfo = {
   thumbnailUrl?: string
   ext?: string
   originallyPublishedAt?: Date
+  webpageUrl?: string
 
   urls?: string[]
 }
@@ -81,7 +82,8 @@ class YoutubeDLInfoBuilder {
       thumbnailUrl: obj.thumbnail || undefined,
       urls: this.buildAvailableUrl(obj),
       originallyPublishedAt: this.buildOriginallyPublishedAt(obj),
-      ext: obj.ext
+      ext: obj.ext,
+      webpageUrl: obj.webpage_url
     }
   }
 

server/helpers/youtube-dl/youtube-dl-wrapper.ts

@@ -46,6 +46,24 @@ class YoutubeDLWrapper {
     return infoBuilder.getInfo()
   }
 
+  async getInfoForListImport (options: {
+    latestVideosCount?: number
+  }) {
+    const youtubeDL = await YoutubeDLCLI.safeGet()
+
+    const list = await youtubeDL.getListInfo({
+      url: this.url,
+      latestVideosCount: options.latestVideosCount,
+      processOptions
+    })
+
+    return list.map(info => {
+      const infoBuilder = new YoutubeDLInfoBuilder(info)
+
+      return infoBuilder.getInfo()
+    })
+  }
+
   async getSubtitles (): Promise<YoutubeDLSubs> {
     const cwd = CONFIG.STORAGE.TMP_DIR
 
@@ -103,7 +121,7 @@ class YoutubeDLWrapper {
 
           return remove(path)
         })
-        .catch(innerErr => logger.error('Cannot remove file in youtubeDL timeout.', { innerErr, ...lTags() }))
+        .catch(innerErr => logger.error('Cannot remove file in youtubeDL error.', { innerErr, ...lTags() }))
 
       throw err
     }

server/initializers/checker-after-init.ts

@@ -48,6 +48,7 @@ function checkConfig () {
   checkRemoteRedundancyConfig()
   checkStorageConfig()
   checkTranscodingConfig()
+  checkImportConfig()
   checkBroadcastMessageConfig()
   checkSearchConfig()
   checkLiveConfig()
@@ -200,6 +201,12 @@ function checkTranscodingConfig () {
   }
 }
 
+function checkImportConfig () {
+  if (CONFIG.IMPORT.VIDEO_CHANNEL_SYNCHRONIZATION.ENABLED && !CONFIG.IMPORT.VIDEOS.HTTP) {
+    throw new Error('You need to enable HTTP import to allow synchronization')
+  }
+}
+
 function checkBroadcastMessageConfig () {
   if (CONFIG.BROADCAST_MESSAGE.ENABLED) {
     const currentLevel = CONFIG.BROADCAST_MESSAGE.LEVEL

server/initializers/checker-before-init.ts

@@ -32,6 +32,8 @@ function checkMissedConfig () {
     'transcoding.resolutions.480p', 'transcoding.resolutions.720p', 'transcoding.resolutions.1080p', 'transcoding.resolutions.1440p',
     'transcoding.resolutions.2160p', 'transcoding.always_transcode_original_resolution', 'video_studio.enabled',
     'import.videos.http.enabled', 'import.videos.torrent.enabled', 'import.videos.concurrency', 'import.videos.timeout',
+    'import.video_channel_synchronization.enabled', 'import.video_channel_synchronization.max_per_user',
+    'import.video_channel_synchronization.check_interval', 'import.video_channel_synchronization.videos_limit_per_synchronization',
     'auto_blacklist.videos.of_users.enabled', 'trending.videos.interval_days',
     'client.videos.miniature.display_author_avatar',
     'client.videos.miniature.prefer_author_display_name', 'client.menu.login.redirect_on_single_external_auth',

server/initializers/config.ts

@@ -398,6 +398,14 @@ const CONFIG = {
       TORRENT: {
         get ENABLED () { return config.get<boolean>('import.videos.torrent.enabled') }
       }
+    },
+    VIDEO_CHANNEL_SYNCHRONIZATION: {
+      get ENABLED () { return config.get<boolean>('import.video_channel_synchronization.enabled') },
+      get MAX_PER_USER () { return config.get<number>('import.video_channel_synchronization.max_per_user') },
+      get CHECK_INTERVAL () { return parseDurationToMs(config.get<string>('import.video_channel_synchronization.check_interval')) },
+      get VIDEOS_LIMIT_PER_SYNCHRONIZATION () {
+        return config.get<number>('import.video_channel_synchronization.videos_limit_per_synchronization')
+      }
     }
   },
   AUTO_BLACKLIST: {
@@ -499,6 +507,7 @@ const CONFIG = {
       get IS_DEFAULT_SEARCH () { return config.get<boolean>('search.search_index.is_default_search') }
     }
   }
+
 }
 
 function registerConfigChangedHandler (fun: Function) {

server/initializers/constants.ts

@@ -6,6 +6,7 @@ import { randomInt, root } from '@shared/core-utils'
 import {
   AbuseState,
   JobType,
+  VideoChannelSyncState,
   VideoImportState,
   VideoPrivacy,
   VideoRateType,
@@ -24,7 +25,7 @@ import { CONFIG, registerConfigChangedHandler } from './config'
 
 // ---------------------------------------------------------------------------
 
-const LAST_MIGRATION_VERSION = 725
+const LAST_MIGRATION_VERSION = 730
 
 // ---------------------------------------------------------------------------
 
@@ -64,6 +65,7 @@ const SORTABLE_COLUMNS = {
   JOBS: [ 'createdAt' ],
   VIDEO_CHANNELS: [ 'id', 'name', 'updatedAt', 'createdAt' ],
   VIDEO_IMPORTS: [ 'createdAt' ],
+  VIDEO_CHANNEL_SYNCS: [ 'externalChannelUrl', 'videoChannel', 'createdAt', 'lastSyncAt', 'state' ],
 
   VIDEO_COMMENT_THREADS: [ 'createdAt', 'totalReplies' ],
   VIDEO_COMMENTS: [ 'createdAt' ],
@@ -156,6 +158,8 @@ const JOB_ATTEMPTS: { [id in JobType]: number } = {
   'video-live-ending': 1,
   'video-studio-edition': 1,
   'manage-video-torrent': 1,
+  'video-channel-import': 1,
+  'after-video-channel-import': 1,
   'move-to-object-storage': 3,
   'notify': 1,
   'federate-video': 1
@@ -178,6 +182,8 @@ const JOB_CONCURRENCY: { [id in Exclude<JobType, 'video-transcoding' | 'video-im
   'video-studio-edition': 1,
   'manage-video-torrent': 1,
   'move-to-object-storage': 1,
+  'video-channel-import': 1,
+  'after-video-channel-import': 1,
   'notify': 5,
   'federate-video': 3
 }
@@ -199,9 +205,11 @@ const JOB_TTL: { [id in JobType]: number } = {
   'video-redundancy': 1000 * 3600 * 3, // 3 hours
   'video-live-ending': 1000 * 60 * 10, // 10 minutes
   'manage-video-torrent': 1000 * 3600 * 3, // 3 hours
+  'move-to-object-storage': 1000 * 60 * 60 * 3, // 3 hours
+  'video-channel-import': 1000 * 60 * 60 * 4, // 4 hours
+  'after-video-channel-import': 60000 * 5, // 5 minutes
   'notify': 60000 * 5, // 5 minutes
-  'federate-video': 60000 * 5, // 5 minutes
+  'federate-video': 60000 * 5 // 5 minutes
-  'move-to-object-storage': 1000 * 60 * 60 * 3 // 3 hours
 }
 const REPEAT_JOBS: { [ id in JobType ]?: RepeatOptions } = {
   'videos-views-stats': {
@@ -246,7 +254,8 @@ const SCHEDULER_INTERVALS_MS = {
   REMOVE_OLD_VIEWS: 60000 * 60 * 24, // 1 day
   REMOVE_OLD_HISTORY: 60000 * 60 * 24, // 1 day
   UPDATE_INBOX_STATS: 1000 * 60, // 1 minute
-  REMOVE_DANGLING_RESUMABLE_UPLOADS: 60000 * 60 // 1 hour
+  REMOVE_DANGLING_RESUMABLE_UPLOADS: 60000 * 60, // 1 hour
+  CHANNEL_SYNC_CHECK_INTERVAL: CONFIG.IMPORT.VIDEO_CHANNEL_SYNCHRONIZATION.CHECK_INTERVAL
 }
 
 // ---------------------------------------------------------------------------
@@ -276,8 +285,12 @@ const CONSTRAINTS_FIELDS = {
     NAME: { min: 1, max: 120 }, // Length
     DESCRIPTION: { min: 3, max: 1000 }, // Length
     SUPPORT: { min: 3, max: 1000 }, // Length
+    EXTERNAL_CHANNEL_URL: { min: 3, max: 2000 }, // Length
     URL: { min: 3, max: 2000 } // Length
   },
+  VIDEO_CHANNEL_SYNCS: {
+    EXTERNAL_CHANNEL_URL: { min: 3, max: 2000 } // Length
+  },
   VIDEO_CAPTIONS: {
     CAPTION_FILE: {
       EXTNAME: [ '.vtt', '.srt' ],
@@ -478,6 +491,13 @@ const VIDEO_IMPORT_STATES: { [ id in VideoImportState ]: string } = {
   [VideoImportState.PROCESSING]: 'Processing'
 }
 
+const VIDEO_CHANNEL_SYNC_STATE: { [ id in VideoChannelSyncState ]: string } = {
+  [VideoChannelSyncState.FAILED]: 'Failed',
+  [VideoChannelSyncState.SYNCED]: 'Synchronized',
+  [VideoChannelSyncState.PROCESSING]: 'Processing',
+  [VideoChannelSyncState.WAITING_FIRST_RUN]: 'Waiting first run'
+}
+
 const ABUSE_STATES: { [ id in AbuseState ]: string } = {
   [AbuseState.PENDING]: 'Pending',
   [AbuseState.REJECTED]: 'Rejected',
@@ -1005,6 +1025,7 @@ export {
   JOB_COMPLETED_LIFETIME,
   HTTP_SIGNATURE,
   VIDEO_IMPORT_STATES,
+  VIDEO_CHANNEL_SYNC_STATE,
   VIEW_LIFETIME,
   CONTACT_FORM_LIFETIME,
   VIDEO_PLAYLIST_PRIVACIES,

server/initializers/database.ts

@@ -50,6 +50,7 @@ import { VideoStreamingPlaylistModel } from '../models/video/video-streaming-pla
 import { VideoTagModel } from '../models/video/video-tag'
 import { VideoViewModel } from '../models/view/video-view'
 import { CONFIG } from './config'
+import { VideoChannelSyncModel } from '@server/models/video/video-channel-sync'
 
 require('pg').defaults.parseInt8 = true // Avoid BIGINT to be converted to string
 
@@ -153,7 +154,8 @@ async function initDatabaseModels (silent: boolean) {
     VideoTrackerModel,
     PluginModel,
     ActorCustomPageModel,
-    VideoJobInfoModel
+    VideoJobInfoModel,
+    VideoChannelSyncModel
   ])
 
   // Check extensions exist in the database

server/initializers/migrations/0730-video-channel-sync.ts

@@ -0,0 +1,36 @@
+import * as Sequelize from 'sequelize'
+
+async function up (utils: {
+  transaction: Sequelize.Transaction
+  queryInterface: Sequelize.QueryInterface
+  sequelize: Sequelize.Sequelize
+  db: any
+}): Promise<void> {
+  const query = `
+    CREATE TABLE IF NOT EXISTS "videoChannelSync" (
+      "id"   SERIAL,
+      "externalChannelUrl" VARCHAR(2000) NOT NULL DEFAULT NULL,
+      "videoChannelId" INTEGER NOT NULL REFERENCES "videoChannel" ("id")
+        ON DELETE CASCADE
+        ON UPDATE CASCADE,
+      "state" INTEGER NOT NULL DEFAULT 1,
+      "createdAt" TIMESTAMP WITH TIME ZONE NOT NULL,
+      "updatedAt" TIMESTAMP WITH TIME ZONE NOT NULL,
+      "lastSyncAt" TIMESTAMP WITH TIME ZONE,
+      PRIMARY KEY ("id")
+    );
+  `
+  await utils.sequelize.query(query, { transaction: utils.transaction })
+}
+
+async function down (utils: {
+  queryInterface: Sequelize.QueryInterface
+  transaction: Sequelize.Transaction
+}) {
+  await utils.queryInterface.dropTable('videoChannelSync', { transaction: utils.transaction })
+}
+
+export {
+  up,
+  down
+}

server/lib/job-queue/handlers/after-video-channel-import.ts

@@ -0,0 +1,37 @@
+import { Job } from 'bullmq'
+import { logger } from '@server/helpers/logger'
+import { VideoChannelSyncModel } from '@server/models/video/video-channel-sync'
+import { AfterVideoChannelImportPayload, VideoChannelSyncState, VideoImportPreventExceptionResult } from '@shared/models'
+
+export async function processAfterVideoChannelImport (job: Job) {
+  const payload = job.data as AfterVideoChannelImportPayload
+  if (!payload.channelSyncId) return
+
+  logger.info('Processing after video channel import in job %s.', job.id)
+
+  const sync = await VideoChannelSyncModel.loadWithChannel(payload.channelSyncId)
+  if (!sync) {
+    logger.error('Unknown sync id %d.', payload.channelSyncId)
+    return
+  }
+
+  const childrenValues = await job.getChildrenValues<VideoImportPreventExceptionResult>()
+
+  let errors = 0
+  let successes = 0
+
+  for (const value of Object.values(childrenValues)) {
+    if (value.resultType === 'success') successes++
+    else if (value.resultType === 'error') errors++
+  }
+
+  if (errors > 0) {
+    sync.state = VideoChannelSyncState.FAILED
+    logger.error(`Finished synchronizing "${sync.VideoChannel.Actor.preferredUsername}" with failures.`, { errors, successes })
+  } else {
+    sync.state = VideoChannelSyncState.SYNCED
+    logger.info(`Finished synchronizing "${sync.VideoChannel.Actor.preferredUsername}" successfully.`, { successes })
+  }
+
+  await sync.save()
+}

server/lib/job-queue/handlers/video-channel-import.ts

@@ -0,0 +1,36 @@
+import { Job } from 'bullmq'
+import { logger } from '@server/helpers/logger'
+import { CONFIG } from '@server/initializers/config'
+import { synchronizeChannel } from '@server/lib/sync-channel'
+import { VideoChannelModel } from '@server/models/video/video-channel'
+import { VideoChannelImportPayload } from '@shared/models'
+
+export async function processVideoChannelImport (job: Job) {
+  const payload = job.data as VideoChannelImportPayload
+
+  logger.info('Processing video channel import in job %s.', job.id)
+
+  // Channel import requires only http upload to be allowed
+  if (!CONFIG.IMPORT.VIDEOS.HTTP.ENABLED) {
+    logger.error('Cannot import channel as the HTTP upload is disabled')
+    return
+  }
+
+  if (!CONFIG.IMPORT.VIDEO_CHANNEL_SYNCHRONIZATION.ENABLED) {
+    logger.error('Cannot import channel as the synchronization is disabled')
+    return
+  }
+
+  const videoChannel = await VideoChannelModel.loadAndPopulateAccount(payload.videoChannelId)
+
+  try {
+    logger.info(`Starting importing videos from external channel "${payload.externalChannelUrl}" to "${videoChannel.name}" `)
+
+    await synchronizeChannel({
+      channel: videoChannel,
+      externalChannelUrl: payload.externalChannelUrl
+    })
+  } catch (err) {
+    logger.error(`Failed to import channel ${videoChannel.name}`, { err })
+  }
+}

server/lib/job-queue/handlers/video-import.ts

@@ -8,7 +8,7 @@ import { generateWebTorrentVideoFilename } from '@server/lib/paths'
 import { Hooks } from '@server/lib/plugins/hooks'
 import { ServerConfigManager } from '@server/lib/server-config-manager'
 import { isAbleToUploadVideo } from '@server/lib/user'
-import { buildOptimizeOrMergeAudioJob, buildMoveToObjectStorageJob } from '@server/lib/video'
+import { buildMoveToObjectStorageJob, buildOptimizeOrMergeAudioJob } from '@server/lib/video'
 import { VideoPathManager } from '@server/lib/video-path-manager'
 import { buildNextVideoState } from '@server/lib/video-state'
 import { ThumbnailModel } from '@server/models/video/thumbnail'
@@ -18,6 +18,7 @@ import { isAudioFile } from '@shared/extra-utils'
 import {
   ThumbnailType,
   VideoImportPayload,
+  VideoImportPreventExceptionResult,
   VideoImportState,
   VideoImportTorrentPayload,
   VideoImportTorrentPayloadType,
@@ -41,20 +42,29 @@ import { Notifier } from '../../notifier'
 import { generateVideoMiniature } from '../../thumbnail'
 import { JobQueue } from '../job-queue'
 
-async function processVideoImport (job: Job) {
+async function processVideoImport (job: Job): Promise<VideoImportPreventExceptionResult> {
   const payload = job.data as VideoImportPayload
 
   const videoImport = await getVideoImportOrDie(payload)
   if (videoImport.state === VideoImportState.CANCELLED) {
     logger.info('Do not process import since it has been cancelled', { payload })
-    return
+    return { resultType: 'success' }
   }
 
   videoImport.state = VideoImportState.PROCESSING
   await videoImport.save()
 
-  if (payload.type === 'youtube-dl') return processYoutubeDLImport(job, videoImport, payload)
+  try {
-  if (payload.type === 'magnet-uri' || payload.type === 'torrent-file') return processTorrentImport(job, videoImport, payload)
+    if (payload.type === 'youtube-dl') await processYoutubeDLImport(job, videoImport, payload)
+    if (payload.type === 'magnet-uri' || payload.type === 'torrent-file') await processTorrentImport(job, videoImport, payload)
+
+    return { resultType: 'success' }
+  } catch (err) {
+    if (!payload.preventException) throw err
+
+    logger.warn('Catch error in video import to send value to parent job.', { payload, err })
+    return { resultType: 'error' }
+  }
 }
 
 // ---------------------------------------------------------------------------

server/lib/job-queue/job-queue.ts

@@ -22,6 +22,7 @@ import {
   ActivitypubHttpFetcherPayload,
   ActivitypubHttpUnicastPayload,
   ActorKeysPayload,
+  AfterVideoChannelImportPayload,
   DeleteResumableUploadMetaFilePayload,
   EmailPayload,
   FederateVideoPayload,
@@ -31,6 +32,7 @@ import {
   MoveObjectStoragePayload,
   NotifyPayload,
   RefreshPayload,
+  VideoChannelImportPayload,
   VideoFileImportPayload,
   VideoImportPayload,
   VideoLiveEndingPayload,
@@ -53,12 +55,14 @@ import { processFederateVideo } from './handlers/federate-video'
 import { processManageVideoTorrent } from './handlers/manage-video-torrent'
 import { onMoveToObjectStorageFailure, processMoveToObjectStorage } from './handlers/move-to-object-storage'
 import { processNotify } from './handlers/notify'
+import { processVideoChannelImport } from './handlers/video-channel-import'
 import { processVideoFileImport } from './handlers/video-file-import'
 import { processVideoImport } from './handlers/video-import'
 import { processVideoLiveEnding } from './handlers/video-live-ending'
 import { processVideoStudioEdition } from './handlers/video-studio-edition'
 import { processVideoTranscoding } from './handlers/video-transcoding'
 import { processVideosViewsStats } from './handlers/video-views-stats'
+import { processAfterVideoChannelImport } from './handlers/after-video-channel-import'
 
 export type CreateJobArgument =
   { type: 'activitypub-http-broadcast', payload: ActivitypubHttpBroadcastPayload } |
@@ -79,6 +83,9 @@ export type CreateJobArgument =
   { type: 'delete-resumable-upload-meta-file', payload: DeleteResumableUploadMetaFilePayload } |
   { type: 'video-studio-edition', payload: VideoStudioEditionPayload } |
   { type: 'manage-video-torrent', payload: ManageVideoTorrentPayload } |
+  { type: 'move-to-object-storage', payload: MoveObjectStoragePayload } |
+  { type: 'video-channel-import', payload: VideoChannelImportPayload } |
+  { type: 'after-video-channel-import', payload: AfterVideoChannelImportPayload } |
   { type: 'notify', payload: NotifyPayload } |
   { type: 'move-to-object-storage', payload: MoveObjectStoragePayload } |
   { type: 'federate-video', payload: FederateVideoPayload }
@@ -106,8 +113,10 @@ const handlers: { [id in JobType]: (job: Job) => Promise<any> } = {
   'video-redundancy': processVideoRedundancy,
   'move-to-object-storage': processMoveToObjectStorage,
   'manage-video-torrent': processManageVideoTorrent,
-  'notify': processNotify,
   'video-studio-edition': processVideoStudioEdition,
+  'video-channel-import': processVideoChannelImport,
+  'after-video-channel-import': processAfterVideoChannelImport,
+  'notify': processNotify,
   'federate-video': processFederateVideo
 }
 
@@ -134,6 +143,8 @@ const jobTypes: JobType[] = [
   'move-to-object-storage',
   'manage-video-torrent',
   'video-studio-edition',
+  'video-channel-import',
+  'after-video-channel-import',
   'notify',
   'federate-video'
 ]
@@ -306,7 +317,7 @@ class JobQueue {
         .catch(err => logger.error('Cannot create job.', { err, options }))
   }
 
-  async createJob (options: CreateJobArgument & CreateJobOptions) {
+  createJob (options: CreateJobArgument & CreateJobOptions) {
     const queue: Queue = this.queues[options.type]
     if (queue === undefined) {
       logger.error('Unknown queue %s: cannot create job.', options.type)
@@ -318,7 +329,7 @@ class JobQueue {
     return queue.add('job', options.payload, jobOptions)
   }
 
-  async createSequentialJobFlow (...jobs: ((CreateJobArgument & CreateJobOptions) | undefined)[]) {
+  createSequentialJobFlow (...jobs: ((CreateJobArgument & CreateJobOptions) | undefined)[]) {
     let lastJob: FlowJob
 
     for (const job of jobs) {
@@ -336,7 +347,7 @@ class JobQueue {
     return this.flowProducer.add(lastJob)
   }
 
-  async createJobWithChildren (parent: CreateJobArgument & CreateJobOptions, children: (CreateJobArgument & CreateJobOptions)[]) {
+  createJobWithChildren (parent: CreateJobArgument & CreateJobOptions, children: (CreateJobArgument & CreateJobOptions)[]) {
     return this.flowProducer.add({
       ...this.buildJobFlowOption(parent),
 

server/lib/schedulers/video-channel-sync-latest-scheduler.ts

@@ -0,0 +1,61 @@
+import { logger } from '@server/helpers/logger'
+import { CONFIG } from '@server/initializers/config'
+import { VideoChannelModel } from '@server/models/video/video-channel'
+import { VideoChannelSyncModel } from '@server/models/video/video-channel-sync'
+import { VideoChannelSyncState } from '@shared/models'
+import { SCHEDULER_INTERVALS_MS } from '../../initializers/constants'
+import { synchronizeChannel } from '../sync-channel'
+import { AbstractScheduler } from './abstract-scheduler'
+
+export class VideoChannelSyncLatestScheduler extends AbstractScheduler {
+  private static instance: AbstractScheduler
+  protected schedulerIntervalMs = SCHEDULER_INTERVALS_MS.CHANNEL_SYNC_CHECK_INTERVAL
+
+  private constructor () {
+    super()
+  }
+
+  protected async internalExecute () {
+    logger.debug('Running %s.%s', this.constructor.name, this.internalExecute.name)
+
+    if (!CONFIG.IMPORT.VIDEO_CHANNEL_SYNCHRONIZATION.ENABLED) {
+      logger.info('Discard channels synchronization as the feature is disabled')
+      return
+    }
+
+    const channelSyncs = await VideoChannelSyncModel.listSyncs()
+
+    for (const sync of channelSyncs) {
+      const channel = await VideoChannelModel.loadAndPopulateAccount(sync.videoChannelId)
+
+      try {
+        logger.info(
+          'Creating video import jobs for "%s" sync with external channel "%s"',
+          channel.Actor.preferredUsername, sync.externalChannelUrl
+        )
+
+        const onlyAfter = sync.lastSyncAt || sync.createdAt
+
+        sync.state = VideoChannelSyncState.PROCESSING
+        sync.lastSyncAt = new Date()
+        await sync.save()
+
+        await synchronizeChannel({
+          channel,
+          externalChannelUrl: sync.externalChannelUrl,
+          videosCountLimit: CONFIG.IMPORT.VIDEO_CHANNEL_SYNCHRONIZATION.VIDEOS_LIMIT_PER_SYNCHRONIZATION,
+          channelSync: sync,
+          onlyAfter
+        })
+      } catch (err) {
+        logger.error(`Failed to synchronize channel ${channel.Actor.preferredUsername}`, { err })
+        sync.state = VideoChannelSyncState.FAILED
+        await sync.save()
+      }
+    }
+  }
+
+  static get Instance () {
+    return this.instance || (this.instance = new this())
+  }
+}

server/lib/server-config-manager.ts

@@ -170,6 +170,9 @@ class ServerConfigManager {
           torrent: {
             enabled: CONFIG.IMPORT.VIDEOS.TORRENT.ENABLED
           }
+        },
+        videoChannelSynchronization: {
+          enabled: CONFIG.IMPORT.VIDEO_CHANNEL_SYNCHRONIZATION.ENABLED
         }
       },
       autoBlacklist: {

server/lib/sync-channel.ts

@@ -0,0 +1,81 @@
+import { logger } from '@server/helpers/logger'
+import { YoutubeDLWrapper } from '@server/helpers/youtube-dl'
+import { CONFIG } from '@server/initializers/config'
+import { buildYoutubeDLImport } from '@server/lib/video-import'
+import { UserModel } from '@server/models/user/user'
+import { VideoImportModel } from '@server/models/video/video-import'
+import { MChannelAccountDefault, MChannelSync } from '@server/types/models'
+import { VideoChannelSyncState, VideoPrivacy } from '@shared/models'
+import { CreateJobArgument, JobQueue } from './job-queue'
+import { ServerConfigManager } from './server-config-manager'
+
+export async function synchronizeChannel (options: {
+  channel: MChannelAccountDefault
+  externalChannelUrl: string
+  channelSync?: MChannelSync
+  videosCountLimit?: number
+  onlyAfter?: Date
+}) {
+  const { channel, externalChannelUrl, videosCountLimit, onlyAfter, channelSync } = options
+
+  const user = await UserModel.loadByChannelActorId(channel.actorId)
+  const youtubeDL = new YoutubeDLWrapper(
+    externalChannelUrl,
+    ServerConfigManager.Instance.getEnabledResolutions('vod'),
+    CONFIG.TRANSCODING.ALWAYS_TRANSCODE_ORIGINAL_RESOLUTION
+  )
+
+  const infoList = await youtubeDL.getInfoForListImport({ latestVideosCount: videosCountLimit })
+
+  const targetUrls = infoList
+    .filter(videoInfo => {
+      if (!onlyAfter) return true
+
+      return videoInfo.originallyPublishedAt.getTime() >= onlyAfter.getTime()
+    })
+    .map(videoInfo => videoInfo.webpageUrl)
+
+  logger.info(
+    'Fetched %d candidate URLs for sync channel %s.',
+    targetUrls.length, channel.Actor.preferredUsername, { targetUrls }
+  )
+
+  if (targetUrls.length === 0) {
+    if (channelSync) {
+      channelSync.state = VideoChannelSyncState.SYNCED
+      await channelSync.save()
+    }
+
+    return
+  }
+
+  const children: CreateJobArgument[] = []
+
+  for (const targetUrl of targetUrls) {
+    if (await VideoImportModel.urlAlreadyImported(channel.id, targetUrl)) {
+      logger.debug('%s is already imported for channel %s, skipping video channel synchronization.', channel.name, targetUrl)
+      continue
+    }
+
+    const { job } = await buildYoutubeDLImport({
+      user,
+      channel,
+      targetUrl,
+      channelSync,
+      importDataOverride: {
+        privacy: VideoPrivacy.PUBLIC
+      }
+    })
+
+    children.push(job)
+  }
+
+  const parent: CreateJobArgument = {
+    type: 'after-video-channel-import',
+    payload: {
+      channelSyncId: channelSync?.id
+    }
+  }
+
+  await JobQueue.Instance.createJobWithChildren(parent, children)
+}

server/lib/video-import.ts

@@ -0,0 +1,308 @@
+import { remove } from 'fs-extra'
+import { moveAndProcessCaptionFile } from '@server/helpers/captions-utils'
+import { isVTTFileValid } from '@server/helpers/custom-validators/video-captions'
+import { isVideoFileExtnameValid } from '@server/helpers/custom-validators/videos'
+import { isResolvingToUnicastOnly } from '@server/helpers/dns'
+import { logger } from '@server/helpers/logger'
+import { YoutubeDLInfo, YoutubeDLWrapper } from '@server/helpers/youtube-dl'
+import { CONFIG } from '@server/initializers/config'
+import { sequelizeTypescript } from '@server/initializers/database'
+import { Hooks } from '@server/lib/plugins/hooks'
+import { ServerConfigManager } from '@server/lib/server-config-manager'
+import { setVideoTags } from '@server/lib/video'
+import { autoBlacklistVideoIfNeeded } from '@server/lib/video-blacklist'
+import { VideoModel } from '@server/models/video/video'
+import { VideoCaptionModel } from '@server/models/video/video-caption'
+import { VideoImportModel } from '@server/models/video/video-import'
+import { FilteredModelAttributes } from '@server/types'
+import {
+  MChannelAccountDefault,
+  MChannelSync,
+  MThumbnail,
+  MUser,
+  MVideoAccountDefault,
+  MVideoCaption,
+  MVideoImportFormattable,
+  MVideoTag,
+  MVideoThumbnail,
+  MVideoWithBlacklistLight
+} from '@server/types/models'
+import { ThumbnailType, VideoImportCreate, VideoImportPayload, VideoImportState, VideoPrivacy, VideoState } from '@shared/models'
+import { getLocalVideoActivityPubUrl } from './activitypub/url'
+import { updateVideoMiniatureFromExisting, updateVideoMiniatureFromUrl } from './thumbnail'
+
+class YoutubeDlImportError extends Error {
+  code: YoutubeDlImportError.CODE
+  cause?: Error // Property to remove once ES2022 is used
+  constructor ({ message, code }) {
+    super(message)
+    this.code = code
+  }
+
+  static fromError (err: Error, code: YoutubeDlImportError.CODE, message?: string) {
+    const ytDlErr = new this({ message: message ?? err.message, code })
+    ytDlErr.cause = err
+    ytDlErr.stack = err.stack // Useless once ES2022 is used
+    return ytDlErr
+  }
+}
+
+namespace YoutubeDlImportError {
+  export enum CODE {
+    FETCH_ERROR,
+    NOT_ONLY_UNICAST_URL
+  }
+}
+
+// ---------------------------------------------------------------------------
+
+async function insertFromImportIntoDB (parameters: {
+  video: MVideoThumbnail
+  thumbnailModel: MThumbnail
+  previewModel: MThumbnail
+  videoChannel: MChannelAccountDefault
+  tags: string[]
+  videoImportAttributes: FilteredModelAttributes<VideoImportModel>
+  user: MUser
+}): Promise<MVideoImportFormattable> {
+  const { video, thumbnailModel, previewModel, videoChannel, tags, videoImportAttributes, user } = parameters
+
+  const videoImport = await sequelizeTypescript.transaction(async t => {
+    const sequelizeOptions = { transaction: t }
+
+    // Save video object in database
+    const videoCreated = await video.save(sequelizeOptions) as (MVideoAccountDefault & MVideoWithBlacklistLight & MVideoTag)
+    videoCreated.VideoChannel = videoChannel
+
+    if (thumbnailModel) await videoCreated.addAndSaveThumbnail(thumbnailModel, t)
+    if (previewModel) await videoCreated.addAndSaveThumbnail(previewModel, t)
+
+    await autoBlacklistVideoIfNeeded({
+      video: videoCreated,
+      user,
+      notify: false,
+      isRemote: false,
+      isNew: true,
+      transaction: t
+    })
+
+    await setVideoTags({ video: videoCreated, tags, transaction: t })
+
+    // Create video import object in database
+    const videoImport = await VideoImportModel.create(
+      Object.assign({ videoId: videoCreated.id }, videoImportAttributes),
+      sequelizeOptions
+    ) as MVideoImportFormattable
+    videoImport.Video = videoCreated
+
+    return videoImport
+  })
+
+  return videoImport
+}
+
+async function buildVideoFromImport ({ channelId, importData, importDataOverride, importType }: {
+  channelId: number
+  importData: YoutubeDLInfo
+  importDataOverride?: Partial<VideoImportCreate>
+  importType: 'url' | 'torrent'
+}): Promise<MVideoThumbnail> {
+  let videoData = {
+    name: importDataOverride?.name || importData.name || 'Unknown name',
+    remote: false,
+    category: importDataOverride?.category || importData.category,
+    licence: importDataOverride?.licence ?? importData.licence ?? CONFIG.DEFAULTS.PUBLISH.LICENCE,
+    language: importDataOverride?.language || importData.language,
+    commentsEnabled: importDataOverride?.commentsEnabled ?? CONFIG.DEFAULTS.PUBLISH.COMMENTS_ENABLED,
+    downloadEnabled: importDataOverride?.downloadEnabled ?? CONFIG.DEFAULTS.PUBLISH.DOWNLOAD_ENABLED,
+    waitTranscoding: importDataOverride?.waitTranscoding || false,
+    state: VideoState.TO_IMPORT,
+    nsfw: importDataOverride?.nsfw || importData.nsfw || false,
+    description: importDataOverride?.description || importData.description,
+    support: importDataOverride?.support || null,
+    privacy: importDataOverride?.privacy || VideoPrivacy.PRIVATE,
+    duration: 0, // duration will be set by the import job
+    channelId,
+    originallyPublishedAt: importDataOverride?.originallyPublishedAt
+      ? new Date(importDataOverride?.originallyPublishedAt)
+      : importData.originallyPublishedAt
+  }
+
+  videoData = await Hooks.wrapObject(
+    videoData,
+    importType === 'url'
+      ? 'filter:api.video.import-url.video-attribute.result'
+      : 'filter:api.video.import-torrent.video-attribute.result'
+  )
+
+  const video = new VideoModel(videoData)
+  video.url = getLocalVideoActivityPubUrl(video)
+
+  return video
+}
+
+async function buildYoutubeDLImport (options: {
+  targetUrl: string
+  channel: MChannelAccountDefault
+  user: MUser
+  channelSync?: MChannelSync
+  importDataOverride?: Partial<VideoImportCreate>
+  thumbnailFilePath?: string
+  previewFilePath?: string
+}) {
+  const { targetUrl, channel, channelSync, importDataOverride, thumbnailFilePath, previewFilePath, user } = options
+
+  const youtubeDL = new YoutubeDLWrapper(
+    targetUrl,
+    ServerConfigManager.Instance.getEnabledResolutions('vod'),
+    CONFIG.TRANSCODING.ALWAYS_TRANSCODE_ORIGINAL_RESOLUTION
+  )
+
+  // Get video infos
+  let youtubeDLInfo: YoutubeDLInfo
+  try {
+    youtubeDLInfo = await youtubeDL.getInfoForDownload()
+  } catch (err) {
+    throw YoutubeDlImportError.fromError(
+      err, YoutubeDlImportError.CODE.FETCH_ERROR, `Cannot fetch information from import for URL ${targetUrl}`
+    )
+  }
+
+  if (!await hasUnicastURLsOnly(youtubeDLInfo)) {
+    throw new YoutubeDlImportError({
+      message: 'Cannot use non unicast IP as targetUrl.',
+      code: YoutubeDlImportError.CODE.NOT_ONLY_UNICAST_URL
+    })
+  }
+
+  const video = await buildVideoFromImport({
+    channelId: channel.id,
+    importData: youtubeDLInfo,
+    importDataOverride,
+    importType: 'url'
+  })
+
+  const thumbnailModel = await forgeThumbnail({
+    inputPath: thumbnailFilePath,
+    downloadUrl: youtubeDLInfo.thumbnailUrl,
+    video,
+    type: ThumbnailType.MINIATURE
+  })
+
+  const previewModel = await forgeThumbnail({
+    inputPath: previewFilePath,
+    downloadUrl: youtubeDLInfo.thumbnailUrl,
+    video,
+    type: ThumbnailType.PREVIEW
+  })
+
+  const videoImport = await insertFromImportIntoDB({
+    video,
+    thumbnailModel,
+    previewModel,
+    videoChannel: channel,
+    tags: importDataOverride?.tags || youtubeDLInfo.tags,
+    user,
+    videoImportAttributes: {
+      targetUrl,
+      state: VideoImportState.PENDING,
+      userId: user.id
+    }
+  })
+
+  // Get video subtitles
+  await processYoutubeSubtitles(youtubeDL, targetUrl, video.id)
+
+  let fileExt = `.${youtubeDLInfo.ext}`
+  if (!isVideoFileExtnameValid(fileExt)) fileExt = '.mp4'
+
+  const payload: VideoImportPayload = {
+    type: 'youtube-dl' as 'youtube-dl',
+    videoImportId: videoImport.id,
+    fileExt,
+    // If part of a sync process, there is a parent job that will aggregate children results
+    preventException: !!channelSync
+  }
+
+  return {
+    videoImport,
+    job: { type: 'video-import' as 'video-import', payload }
+  }
+}
+
+// ---------------------------------------------------------------------------
+
+export {
+  buildYoutubeDLImport,
+  YoutubeDlImportError,
+  insertFromImportIntoDB,
+  buildVideoFromImport
+}
+
+// ---------------------------------------------------------------------------
+
+async function forgeThumbnail ({ inputPath, video, downloadUrl, type }: {
+  inputPath?: string
+  downloadUrl?: string
+  video: MVideoThumbnail
+  type: ThumbnailType
+}): Promise<MThumbnail> {
+  if (inputPath) {
+    return updateVideoMiniatureFromExisting({
+      inputPath,
+      video,
+      type,
+      automaticallyGenerated: false
+    })
+  } else if (downloadUrl) {
+    try {
+      return await updateVideoMiniatureFromUrl({ downloadUrl, video, type })
+    } catch (err) {
+      logger.warn('Cannot process thumbnail %s from youtubedl.', downloadUrl, { err })
+    }
+  }
+  return null
+}
+
+async function processYoutubeSubtitles (youtubeDL: YoutubeDLWrapper, targetUrl: string, videoId: number) {
+  try {
+    const subtitles = await youtubeDL.getSubtitles()
+
+    logger.info('Will create %s subtitles from youtube import %s.', subtitles.length, targetUrl)
+
+    for (const subtitle of subtitles) {
+      if (!await isVTTFileValid(subtitle.path)) {
+        await remove(subtitle.path)
+        continue
+      }
+
+      const videoCaption = new VideoCaptionModel({
+        videoId,
+        language: subtitle.language,
+        filename: VideoCaptionModel.generateCaptionName(subtitle.language)
+      }) as MVideoCaption
+
+      // Move physical file
+      await moveAndProcessCaptionFile(subtitle, videoCaption)
+
+      await sequelizeTypescript.transaction(async t => {
+        await VideoCaptionModel.insertOrReplaceLanguage(videoCaption, t)
+      })
+    }
+  } catch (err) {
+    logger.warn('Cannot get video subtitles.', { err })
+  }
+}
+
+async function hasUnicastURLsOnly (youtubeDLInfo: YoutubeDLInfo) {
+  const hosts = youtubeDLInfo.urls.map(u => new URL(u).hostname)
+  const uniqHosts = new Set(hosts)
+
+  for (const h of uniqHosts) {
+    if (await isResolvingToUnicastOnly(h) !== true) {
+      return false
+    }
+  }
+
+  return true
+}

server/middlewares/validators/config.ts

@@ -66,6 +66,8 @@ const customConfigUpdateValidator = [
   body('import.videos.http.enabled').isBoolean().withMessage('Should have a valid import video http enabled boolean'),
   body('import.videos.torrent.enabled').isBoolean().withMessage('Should have a valid import video torrent enabled boolean'),
 
+  body('import.videoChannelSynchronization.enabled').isBoolean().withMessage('Should have a valid synchronization enabled boolean'),
+
   body('trending.videos.algorithms.default').exists().withMessage('Should have a valid default trending algorithm'),
   body('trending.videos.algorithms.enabled').exists().withMessage('Should have a valid array of enabled trending algorithms'),
 
@@ -110,6 +112,7 @@ const customConfigUpdateValidator = [
     if (areValidationErrors(req, res)) return
     if (!checkInvalidConfigIfEmailDisabled(req.body, res)) return
     if (!checkInvalidTranscodingConfig(req.body, res)) return
+    if (!checkInvalidSynchronizationConfig(req.body, res)) return
     if (!checkInvalidLiveConfig(req.body, res)) return
     if (!checkInvalidVideoStudioConfig(req.body, res)) return
 
@@ -157,6 +160,14 @@ function checkInvalidTranscodingConfig (customConfig: CustomConfig, res: express
   return true
 }
 
+function checkInvalidSynchronizationConfig (customConfig: CustomConfig, res: express.Response) {
+  if (customConfig.import.videoChannelSynchronization.enabled && !customConfig.import.videos.http.enabled) {
+    res.fail({ message: 'You need to enable HTTP video import in order to enable channel synchronization' })
+    return false
+  }
+  return true
+}
+
 function checkInvalidLiveConfig (customConfig: CustomConfig, res: express.Response) {
   if (customConfig.live.enabled === false) return true
 

server/middlewares/validators/sort.ts

@@ -52,6 +52,7 @@ const videoPlaylistsSortValidator = checkSortFactory(SORTABLE_COLUMNS.VIDEO_PLAY
 const pluginsSortValidator = checkSortFactory(SORTABLE_COLUMNS.PLUGINS)
 const availablePluginsSortValidator = checkSortFactory(SORTABLE_COLUMNS.AVAILABLE_PLUGINS)
 const videoRedundanciesSortValidator = checkSortFactory(SORTABLE_COLUMNS.VIDEO_REDUNDANCIES)
+const videoChannelSyncsSortValidator = checkSortFactory(SORTABLE_COLUMNS.VIDEO_CHANNEL_SYNCS)
 
 const accountsFollowersSortValidator = checkSortFactory(SORTABLE_COLUMNS.ACCOUNT_FOLLOWERS)
 const videoChannelsFollowersSortValidator = checkSortFactory(SORTABLE_COLUMNS.CHANNEL_FOLLOWERS)
@@ -84,5 +85,6 @@ export {
   videoPlaylistsSearchSortValidator,
   accountsFollowersSortValidator,
   videoChannelsFollowersSortValidator,
+  videoChannelSyncsSortValidator,
   pluginsSortValidator
 }

server/middlewares/validators/videos/index.ts

@@ -14,3 +14,4 @@ export * from './video-stats'
 export * from './video-studio'
 export * from './video-transcoding'
 export * from './videos'
+export * from './video-channel-sync'

server/middlewares/validators/videos/video-channel-sync.ts

@@ -0,0 +1,66 @@
+import * as express from 'express'
+import { body, param } from 'express-validator'
+import { isUrlValid } from '@server/helpers/custom-validators/activitypub/misc'
+import { logger } from '@server/helpers/logger'
+import { CONFIG } from '@server/initializers/config'
+import { VideoChannelModel } from '@server/models/video/video-channel'
+import { VideoChannelSyncModel } from '@server/models/video/video-channel-sync'
+import { HttpStatusCode, VideoChannelSyncCreate } from '@shared/models'
+import { areValidationErrors, doesVideoChannelIdExist } from '../shared'
+
+export const ensureSyncIsEnabled = (req: express.Request, res: express.Response, next: express.NextFunction) => {
+  if (!CONFIG.IMPORT.VIDEO_CHANNEL_SYNCHRONIZATION.ENABLED) {
+    return res.fail({
+      status: HttpStatusCode.FORBIDDEN_403,
+      message: 'Synchronization is impossible as video channel synchronization is not enabled on the server'
+    })
+  }
+
+  return next()
+}
+
+export const videoChannelSyncValidator = [
+  body('externalChannelUrl').custom(isUrlValid).withMessage('Should have a valid channel url'),
+  body('videoChannelId').isInt().withMessage('Should have a valid video channel id'),
+
+  async (req: express.Request, res: express.Response, next: express.NextFunction) => {
+    logger.debug('Checking videoChannelSync parameters', { parameters: req.body })
+
+    if (areValidationErrors(req, res)) return
+
+    const body: VideoChannelSyncCreate = req.body
+    if (!await doesVideoChannelIdExist(body.videoChannelId, res)) return
+
+    const count = await VideoChannelSyncModel.countByAccount(res.locals.videoChannel.accountId)
+    if (count >= CONFIG.IMPORT.VIDEO_CHANNEL_SYNCHRONIZATION.MAX_PER_USER) {
+      return res.fail({
+        message: `You cannot create more than ${CONFIG.IMPORT.VIDEO_CHANNEL_SYNCHRONIZATION.MAX_PER_USER} channel synchronizations`
+      })
+    }
+
+    return next()
+  }
+]
+
+export const ensureSyncExists = [
+  param('id').exists().isInt().withMessage('Should have an sync id'),
+
+  async (req: express.Request, res: express.Response, next: express.NextFunction) => {
+    if (areValidationErrors(req, res)) return
+
+    const syncId = parseInt(req.params.id, 10)
+    const sync = await VideoChannelSyncModel.loadWithChannel(syncId)
+
+    if (!sync) {
+      return res.fail({
+        status: HttpStatusCode.NOT_FOUND_404,
+        message: 'Synchronization not found'
+      })
+    }
+
+    res.locals.videoChannelSync = sync
+    res.locals.videoChannel = await VideoChannelModel.loadAndPopulateAccount(sync.videoChannelId)
+
+    return next()
+  }
+]

server/middlewares/validators/videos/video-channels.ts

@@ -1,5 +1,6 @@
 import express from 'express'
 import { body, param, query } from 'express-validator'
+import { isUrlValid } from '@server/helpers/custom-validators/activitypub/misc'
 import { CONFIG } from '@server/initializers/config'
 import { MChannelAccountDefault } from '@server/types/models'
 import { HttpStatusCode } from '../../../../shared/models/http/http-error-codes'
@@ -13,9 +14,9 @@ import {
 import { logger } from '../../../helpers/logger'
 import { ActorModel } from '../../../models/actor/actor'
 import { VideoChannelModel } from '../../../models/video/video-channel'
-import { areValidationErrors, doesVideoChannelNameWithHostExist } from '../shared'
+import { areValidationErrors, checkUserQuota, doesVideoChannelNameWithHostExist } from '../shared'
 
-const videoChannelsAddValidator = [
+export const videoChannelsAddValidator = [
   body('name').custom(isVideoChannelUsernameValid).withMessage('Should have a valid channel name'),
   body('displayName').custom(isVideoChannelDisplayNameValid).withMessage('Should have a valid display name'),
   body('description').optional().custom(isVideoChannelDescriptionValid).withMessage('Should have a valid description'),
@@ -45,7 +46,7 @@ const videoChannelsAddValidator = [
   }
 ]
 
-const videoChannelsUpdateValidator = [
+export const videoChannelsUpdateValidator = [
   param('nameWithHost').exists().withMessage('Should have an video channel name with host'),
   body('displayName')
     .optional()
@@ -69,7 +70,7 @@ const videoChannelsUpdateValidator = [
   }
 ]
 
-const videoChannelsRemoveValidator = [
+export const videoChannelsRemoveValidator = [
   async (req: express.Request, res: express.Response, next: express.NextFunction) => {
     logger.debug('Checking videoChannelsRemove parameters', { parameters: req.params })
 
@@ -79,7 +80,7 @@ const videoChannelsRemoveValidator = [
   }
 ]
 
-const videoChannelsNameWithHostValidator = [
+export const videoChannelsNameWithHostValidator = [
   param('nameWithHost').exists().withMessage('Should have an video channel name with host'),
 
   async (req: express.Request, res: express.Response, next: express.NextFunction) => {
@@ -93,7 +94,7 @@ const videoChannelsNameWithHostValidator = [
   }
 ]
 
-const ensureIsLocalChannel = [
+export const ensureIsLocalChannel = [
   (req: express.Request, res: express.Response, next: express.NextFunction) => {
     if (res.locals.videoChannel.Actor.isOwned() === false) {
       return res.fail({
@@ -106,7 +107,18 @@ const ensureIsLocalChannel = [
   }
 ]
 
-const videoChannelStatsValidator = [
+export const ensureChannelOwnerCanUpload = [
+  async (req: express.Request, res: express.Response, next: express.NextFunction) => {
+    const channel = res.locals.videoChannel
+    const user = { id: channel.Account.userId }
+
+    if (!await checkUserQuota(user, 1, res)) return
+
+    next()
+  }
+]
+
+export const videoChannelStatsValidator = [
   query('withStats')
     .optional()
     .customSanitizer(toBooleanOrNull)
@@ -118,7 +130,7 @@ const videoChannelStatsValidator = [
   }
 ]
 
-const videoChannelsListValidator = [
+export const videoChannelsListValidator = [
   query('search').optional().not().isEmpty().withMessage('Should have a valid search'),
 
   (req: express.Request, res: express.Response, next: express.NextFunction) => {
@@ -130,17 +142,24 @@ const videoChannelsListValidator = [
   }
 ]
 
-// ---------------------------------------------------------------------------
+export const videoChannelImportVideosValidator = [
+  body('externalChannelUrl').custom(isUrlValid).withMessage('Should have a valid channel url'),
 
-export {
+  (req: express.Request, res: express.Response, next: express.NextFunction) => {
-  videoChannelsAddValidator,
+    logger.debug('Checking videoChannelImport parameters', { parameters: req.body })
-  videoChannelsUpdateValidator,
+
-  videoChannelsRemoveValidator,
+    if (areValidationErrors(req, res)) return
-  videoChannelsNameWithHostValidator,
+
-  ensureIsLocalChannel,
+    if (!CONFIG.IMPORT.VIDEOS.HTTP.ENABLED) {
-  videoChannelsListValidator,
+      return res.fail({
-  videoChannelStatsValidator
+        status: HttpStatusCode.FORBIDDEN_403,
-}
+        message: 'Channel import is impossible as video upload via HTTP is not enabled on the server'
+      })
+    }
+
+    return next()
+  }
+]
 
 // ---------------------------------------------------------------------------
 

server/models/utils.ts

@@ -117,6 +117,16 @@ function getInstanceFollowsSort (value: string, lastSort: OrderItem = [ 'id', 'A
   return getSort(value, lastSort)
 }
 
+function getChannelSyncSort (value: string): OrderItem[] {
+  const { direction, field } = buildDirectionAndField(value)
+  if (field.toLowerCase() === 'videochannel') {
+    return [
+      [ literal('"VideoChannel.name"'), direction ]
+    ]
+  }
+  return [ [ field, direction ] ]
+}
+
 function isOutdated (model: { createdAt: Date, updatedAt: Date }, refreshInterval: number) {
   if (!model.createdAt || !model.updatedAt) {
     throw new Error('Miss createdAt & updatedAt attributes to model')
@@ -280,6 +290,7 @@ export {
   getAdminUsersSort,
   getVideoSort,
   getBlacklistSort,
+  getChannelSyncSort,
   createSimilarityAttribute,
   throwIfNotValid,
   buildServerIdsFollowedBy,

server/models/video/video-channel-sync.ts

@@ -0,0 +1,176 @@
+import { Op } from 'sequelize'
+import {
+  AllowNull,
+  BelongsTo,
+  Column,
+  CreatedAt,
+  DataType,
+  Default,
+  DefaultScope,
+  ForeignKey,
+  Is,
+  Model,
+  Table,
+  UpdatedAt
+} from 'sequelize-typescript'
+import { isUrlValid } from '@server/helpers/custom-validators/activitypub/misc'
+import { isVideoChannelSyncStateValid } from '@server/helpers/custom-validators/video-channel-syncs'
+import { CONSTRAINTS_FIELDS, VIDEO_CHANNEL_SYNC_STATE } from '@server/initializers/constants'
+import { MChannelSync, MChannelSyncChannel, MChannelSyncFormattable } from '@server/types/models'
+import { VideoChannelSync, VideoChannelSyncState } from '@shared/models'
+import { AttributesOnly } from '@shared/typescript-utils'
+import { AccountModel } from '../account/account'
+import { UserModel } from '../user/user'
+import { getChannelSyncSort, throwIfNotValid } from '../utils'
+import { VideoChannelModel } from './video-channel'
+
+@DefaultScope(() => ({
+  include: [
+    {
+      model: VideoChannelModel, // Default scope includes avatar and server
+      required: true
+    }
+  ]
+}))
+@Table({
+  tableName: 'videoChannelSync',
+  indexes: [
+    {
+      fields: [ 'videoChannelId' ]
+    }
+  ]
+})
+export class VideoChannelSyncModel extends Model<Partial<AttributesOnly<VideoChannelSyncModel>>> {
+
+  @AllowNull(false)
+  @Default(null)
+  @Is('VideoChannelExternalChannelUrl', value => throwIfNotValid(value, isUrlValid, 'externalChannelUrl', true))
+  @Column(DataType.STRING(CONSTRAINTS_FIELDS.VIDEO_CHANNEL_SYNCS.EXTERNAL_CHANNEL_URL.max))
+  externalChannelUrl: string
+
+  @CreatedAt
+  createdAt: Date
+
+  @UpdatedAt
+  updatedAt: Date
+
+  @ForeignKey(() => VideoChannelModel)
+  @Column
+  videoChannelId: number
+
+  @BelongsTo(() => VideoChannelModel, {
+    foreignKey: {
+      allowNull: false
+    },
+    onDelete: 'cascade'
+  })
+  VideoChannel: VideoChannelModel
+
+  @AllowNull(false)
+  @Default(VideoChannelSyncState.WAITING_FIRST_RUN)
+  @Is('VideoChannelSyncState', value => throwIfNotValid(value, isVideoChannelSyncStateValid, 'state'))
+  @Column
+  state: VideoChannelSyncState
+
+  @AllowNull(true)
+  @Column(DataType.DATE)
+  lastSyncAt: Date
+
+  static listByAccountForAPI (options: {
+    accountId: number
+    start: number
+    count: number
+    sort: string
+  }) {
+    const getQuery = (forCount: boolean) => {
+      const videoChannelModel = forCount
+        ? VideoChannelModel.unscoped()
+        : VideoChannelModel
+
+      return {
+        offset: options.start,
+        limit: options.count,
+        order: getChannelSyncSort(options.sort),
+        include: [
+          {
+            model: videoChannelModel,
+            required: true,
+            where: {
+              accountId: options.accountId
+            }
+          }
+        ]
+      }
+    }
+
+    return Promise.all([
+      VideoChannelSyncModel.unscoped().count(getQuery(true)),
+      VideoChannelSyncModel.unscoped().findAll(getQuery(false))
+    ]).then(([ total, data ]) => ({ total, data }))
+  }
+
+  static countByAccount (accountId: number) {
+    const query = {
+      include: [
+        {
+          model: VideoChannelModel.unscoped(),
+          required: true,
+          where: {
+            accountId
+          }
+        }
+      ]
+    }
+
+    return VideoChannelSyncModel.unscoped().count(query)
+  }
+
+  static loadWithChannel (id: number): Promise<MChannelSyncChannel> {
+    return VideoChannelSyncModel.findByPk(id)
+  }
+
+  static async listSyncs (): Promise<MChannelSync[]> {
+    const query = {
+      include: [
+        {
+          model: VideoChannelModel.unscoped(),
+          required: true,
+          include: [
+            {
+              model: AccountModel.unscoped(),
+              required: true,
+              include: [ {
+                attributes: [],
+                model: UserModel.unscoped(),
+                required: true,
+                where: {
+                  videoQuota: {
+                    [Op.ne]: 0
+                  },
+                  videoQuotaDaily: {
+                    [Op.ne]: 0
+                  }
+                }
+              } ]
+            }
+          ]
+        }
+      ]
+    }
+    return VideoChannelSyncModel.unscoped().findAll(query)
+  }
+
+  toFormattedJSON (this: MChannelSyncFormattable): VideoChannelSync {
+    return {
+      id: this.id,
+      state: {
+        id: this.state,
+        label: VIDEO_CHANNEL_SYNC_STATE[this.state]
+      },
+      externalChannelUrl: this.externalChannelUrl,
+      createdAt: this.createdAt.toISOString(),
+      channel: this.VideoChannel.toFormattedSummaryJSON(),
+      lastSyncAt: this.lastSyncAt?.toISOString()
+    }
+  }
+}

server/models/video/video-import.ts

@@ -1,4 +1,4 @@
-import { WhereOptions } from 'sequelize'
+import { Op, WhereOptions } from 'sequelize'
 import {
   AfterUpdate,
   AllowNull,
@@ -161,6 +161,28 @@ export class VideoImportModel extends Model<Partial<AttributesOnly<VideoImportMo
     ]).then(([ total, data ]) => ({ total, data }))
   }
 
+  static async urlAlreadyImported (channelId: number, targetUrl: string): Promise<boolean> {
+    const element = await VideoImportModel.unscoped().findOne({
+      where: {
+        targetUrl,
+        state: {
+          [Op.in]: [ VideoImportState.PENDING, VideoImportState.PROCESSING, VideoImportState.SUCCESS ]
+        }
+      },
+      include: [
+        {
+          model: VideoModel,
+          required: true,
+          where: {
+            channelId
+          }
+        }
+      ]
+    })
+
+    return !!element
+  }
+
   getTargetIdentifier () {
     return this.targetUrl || this.magnetUri || this.torrentName
   }

server/tests/api/check-params/config.ts

@@ -1,7 +1,8 @@
 /* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */
 
 import 'mocha'
-import { omit } from 'lodash'
+import { merge, omit } from 'lodash'
+import { CustomConfig, HttpStatusCode } from '@shared/models'
 import {
   cleanupTests,
   createSingleServer,
@@ -11,7 +12,6 @@ import {
   PeerTubeServer,
   setAccessTokensToServers
 } from '@shared/server-commands'
-import { CustomConfig, HttpStatusCode } from '@shared/models'
 
 describe('Test config API validators', function () {
   const path = '/api/v1/config/custom'
@@ -162,6 +162,10 @@ describe('Test config API validators', function () {
         torrent: {
           enabled: false
         }
+      },
+      videoChannelSynchronization: {
+        enabled: false,
+        maxPerUser: 10
       }
     },
     trending: {
@@ -346,7 +350,26 @@ describe('Test config API validators', function () {
       })
     })
 
-    it('Should success with the correct parameters', async function () {
+    it('Should fail with a disabled http upload & enabled sync', async function () {
+      const newUpdateParams: CustomConfig = merge({}, updateParams, {
+        import: {
+          videos: {
+            http: { enabled: false }
+          },
+          videoChannelSynchronization: { enabled: true }
+        }
+      })
+
+      await makePutBodyRequest({
+        url: server.url,
+        path,
+        fields: newUpdateParams,
+        token: server.accessToken,
+        expectedStatus: HttpStatusCode.BAD_REQUEST_400
+      })
+    })
+
+    it('Should succeed with the correct parameters', async function () {
       await makePutBodyRequest({
         url: server.url,
         path,

server/tests/api/check-params/index.ts

@@ -27,6 +27,7 @@ import './video-channels'
 import './video-comments'
 import './video-files'
 import './video-imports'
+import './video-channel-syncs'
 import './video-playlists'
 import './video-source'
 import './video-studio'

server/tests/api/check-params/upload-quota.ts

@@ -70,7 +70,7 @@ describe('Test upload quota', function () {
     })
 
     it('Should fail to import with HTTP/Torrent/magnet', async function () {
-      this.timeout(120000)
+      this.timeout(120_000)
 
       const baseAttributes = {
         channelId: server.store.channel.id,

server/tests/api/check-params/video-channel-syncs.ts

@@ -0,0 +1,318 @@
+import { checkBadCountPagination, checkBadSortPagination, checkBadStartPagination, FIXTURE_URLS } from '@server/tests/shared'
+import { HttpStatusCode, VideoChannelSyncCreate } from '@shared/models'
+import {
+  ChannelSyncsCommand,
+  createSingleServer,
+  makePostBodyRequest,
+  PeerTubeServer,
+  setAccessTokensToServers,
+  setDefaultVideoChannel
+} from '@shared/server-commands'
+
+describe('Test video channel sync API validator', () => {
+  const path = '/api/v1/video-channel-syncs'
+  let server: PeerTubeServer
+  let command: ChannelSyncsCommand
+  let rootChannelId: number
+  let rootChannelSyncId: number
+  const userInfo = {
+    accessToken: '',
+    username: 'user1',
+    id: -1,
+    channelId: -1,
+    syncId: -1
+  }
+
+  async function withChannelSyncDisabled<T> (callback: () => Promise<T>): Promise<void> {
+    try {
+      await server.config.disableChannelSync()
+      await callback()
+    } finally {
+      await server.config.enableChannelSync()
+    }
+  }
+
+  async function withMaxSyncsPerUser<T> (maxSync: number, callback: () => Promise<T>): Promise<void> {
+    const origConfig = await server.config.getCustomConfig()
+
+    await server.config.updateExistingSubConfig({
+      newConfig: {
+        import: {
+          videoChannelSynchronization: {
+            maxPerUser: maxSync
+          }
+        }
+      }
+    })
+
+    try {
+      await callback()
+    } finally {
+      await server.config.updateCustomConfig({ newCustomConfig: origConfig })
+    }
+  }
+
+  before(async function () {
+    this.timeout(30_000)
+
+    server = await createSingleServer(1)
+
+    await setAccessTokensToServers([ server ])
+    await setDefaultVideoChannel([ server ])
+
+    command = server.channelSyncs
+
+    rootChannelId = server.store.channel.id
+
+    {
+      userInfo.accessToken = await server.users.generateUserAndToken(userInfo.username)
+
+      const { videoChannels, id: userId } = await server.users.getMyInfo({ token: userInfo.accessToken })
+      userInfo.id = userId
+      userInfo.channelId = videoChannels[0].id
+    }
+
+    await server.config.enableChannelSync()
+  })
+
+  describe('When creating a sync', function () {
+    let baseCorrectParams: VideoChannelSyncCreate
+
+    before(function () {
+      baseCorrectParams = {
+        externalChannelUrl: FIXTURE_URLS.youtubeChannel,
+        videoChannelId: rootChannelId
+      }
+    })
+
+    it('Should fail when sync is disabled', async function () {
+      await withChannelSyncDisabled(async () => {
+        await command.create({
+          token: server.accessToken,
+          attributes: baseCorrectParams,
+          expectedStatus: HttpStatusCode.FORBIDDEN_403
+        })
+      })
+    })
+
+    it('Should fail with nothing', async function () {
+      const fields = {}
+      await makePostBodyRequest({
+        url: server.url,
+        path,
+        token: server.accessToken,
+        fields,
+        expectedStatus: HttpStatusCode.BAD_REQUEST_400
+      })
+    })
+
+    it('Should fail with no authentication', async function () {
+      await command.create({
+        token: null,
+        attributes: baseCorrectParams,
+        expectedStatus: HttpStatusCode.UNAUTHORIZED_401
+      })
+    })
+
+    it('Should fail without a target url', async function () {
+      const attributes: VideoChannelSyncCreate = {
+        ...baseCorrectParams,
+        externalChannelUrl: null
+      }
+      await command.create({
+        token: server.accessToken,
+        attributes,
+        expectedStatus: HttpStatusCode.BAD_REQUEST_400
+      })
+    })
+
+    it('Should fail without a channelId', async function () {
+      const attributes: VideoChannelSyncCreate = {
+        ...baseCorrectParams,
+        videoChannelId: null
+      }
+      await command.create({
+        token: server.accessToken,
+        attributes,
+        expectedStatus: HttpStatusCode.BAD_REQUEST_400
+      })
+    })
+
+    it('Should fail with a channelId refering nothing', async function () {
+      const attributes: VideoChannelSyncCreate = {
+        ...baseCorrectParams,
+        videoChannelId: 42
+      }
+      await command.create({
+        token: server.accessToken,
+        attributes,
+        expectedStatus: HttpStatusCode.NOT_FOUND_404
+      })
+    })
+
+    it('Should fail to create a sync when the user does not own the channel', async function () {
+      await command.create({
+        token: userInfo.accessToken,
+        attributes: baseCorrectParams,
+        expectedStatus: HttpStatusCode.FORBIDDEN_403
+      })
+    })
+
+    it('Should succeed to create a sync with root and for another user\'s channel', async function () {
+      const { videoChannelSync } = await command.create({
+        token: server.accessToken,
+        attributes: {
+          ...baseCorrectParams,
+          videoChannelId: userInfo.channelId
+        },
+        expectedStatus: HttpStatusCode.OK_200
+      })
+      userInfo.syncId = videoChannelSync.id
+    })
+
+    it('Should succeed with the correct parameters', async function () {
+      const { videoChannelSync } = await command.create({
+        token: server.accessToken,
+        attributes: baseCorrectParams,
+        expectedStatus: HttpStatusCode.OK_200
+      })
+      rootChannelSyncId = videoChannelSync.id
+    })
+
+    it('Should fail when the user exceeds allowed number of synchronizations', async function () {
+      await withMaxSyncsPerUser(1, async () => {
+        await command.create({
+          token: server.accessToken,
+          attributes: {
+            ...baseCorrectParams,
+            videoChannelId: userInfo.channelId
+          },
+          expectedStatus: HttpStatusCode.BAD_REQUEST_400
+        })
+      })
+    })
+  })
+
+  describe('When listing my channel syncs', function () {
+    const myPath = '/api/v1/accounts/root/video-channel-syncs'
+
+    it('Should fail with a bad start pagination', async function () {
+      await checkBadStartPagination(server.url, myPath, server.accessToken)
+    })
+
+    it('Should fail with a bad count pagination', async function () {
+      await checkBadCountPagination(server.url, myPath, server.accessToken)
+    })
+
+    it('Should fail with an incorrect sort', async function () {
+      await checkBadSortPagination(server.url, myPath, server.accessToken)
+    })
+
+    it('Should succeed with the correct parameters', async function () {
+      await command.listByAccount({
+        accountName: 'root',
+        token: server.accessToken,
+        expectedStatus: HttpStatusCode.OK_200
+      })
+    })
+
+    it('Should fail with no authentication', async function () {
+      await command.listByAccount({
+        accountName: 'root',
+        token: null,
+        expectedStatus: HttpStatusCode.UNAUTHORIZED_401
+      })
+    })
+
+    it('Should fail when a simple user lists another user\'s synchronizations', async function () {
+      await command.listByAccount({
+        accountName: 'root',
+        token: userInfo.accessToken,
+        expectedStatus: HttpStatusCode.FORBIDDEN_403
+      })
+    })
+
+    it('Should succeed when root lists another user\'s synchronizations', async function () {
+      await command.listByAccount({
+        accountName: userInfo.username,
+        token: server.accessToken,
+        expectedStatus: HttpStatusCode.OK_200
+      })
+    })
+
+    it('Should succeed even with synchronization disabled', async function () {
+      await withChannelSyncDisabled(async function () {
+        await command.listByAccount({
+          accountName: 'root',
+          token: server.accessToken,
+          expectedStatus: HttpStatusCode.OK_200
+        })
+      })
+    })
+  })
+
+  describe('When triggering deletion', function () {
+    it('should fail with no authentication', async function () {
+      await command.delete({
+        channelSyncId: userInfo.syncId,
+        token: null,
+        expectedStatus: HttpStatusCode.UNAUTHORIZED_401
+      })
+    })
+
+    it('Should fail when channelSyncId does not refer to any sync', async function () {
+      await command.delete({
+        channelSyncId: 42,
+        token: server.accessToken,
+        expectedStatus: HttpStatusCode.NOT_FOUND_404
+      })
+    })
+
+    it('Should fail when sync is not owned by the user', async function () {
+      await command.delete({
+        channelSyncId: rootChannelSyncId,
+        token: userInfo.accessToken,
+        expectedStatus: HttpStatusCode.FORBIDDEN_403
+      })
+    })
+
+    it('Should succeed when root delete a sync they do not own', async function () {
+      await command.delete({
+        channelSyncId: userInfo.syncId,
+        token: server.accessToken,
+        expectedStatus: HttpStatusCode.NO_CONTENT_204
+      })
+    })
+
+    it('should succeed when user delete a sync they own', async function () {
+      const { videoChannelSync } = await command.create({
+        attributes: {
+          externalChannelUrl: FIXTURE_URLS.youtubeChannel,
+          videoChannelId: userInfo.channelId
+        },
+        token: server.accessToken,
+        expectedStatus: HttpStatusCode.OK_200
+      })
+
+      await command.delete({
+        channelSyncId: videoChannelSync.id,
+        token: server.accessToken,
+        expectedStatus: HttpStatusCode.NO_CONTENT_204
+      })
+    })
+
+    it('Should succeed even when synchronization is disabled', async function () {
+      await withChannelSyncDisabled(async function () {
+        await command.delete({
+          channelSyncId: rootChannelSyncId,
+          token: server.accessToken,
+          expectedStatus: HttpStatusCode.NO_CONTENT_204
+        })
+      })
+    })
+  })
+
+  after(async function () {
+    await server?.kill()
+  })
+})

server/tests/api/check-params/video-channels.ts

@@ -3,8 +3,8 @@
 import 'mocha'
 import * as chai from 'chai'
 import { omit } from 'lodash'
-import { checkBadCountPagination, checkBadSortPagination, checkBadStartPagination } from '@server/tests/shared'
+import { checkBadCountPagination, checkBadSortPagination, checkBadStartPagination, FIXTURE_URLS } from '@server/tests/shared'
-import { buildAbsoluteFixturePath } from '@shared/core-utils'
+import { areHttpImportTestsDisabled, buildAbsoluteFixturePath } from '@shared/core-utils'
 import { HttpStatusCode, VideoChannelUpdate } from '@shared/models'
 import {
   ChannelsCommand,
@@ -23,7 +23,13 @@ const expect = chai.expect
 describe('Test video channels API validator', function () {
   const videoChannelPath = '/api/v1/video-channels'
   let server: PeerTubeServer
-  let accessTokenUser: string
+  const userInfo = {
+    accessToken: '',
+    channelName: 'fake_channel',
+    id: -1,
+    videoQuota: -1,
+    videoQuotaDaily: -1
+  }
   let command: ChannelsCommand
 
   // ---------------------------------------------------------------
@@ -35,14 +41,15 @@ describe('Test video channels API validator', function () {
 
     await setAccessTokensToServers([ server ])
 
-    const user = {
+    const userCreds = {
       username: 'fake',
       password: 'fake_password'
     }
 
     {
-      await server.users.create({ username: user.username, password: user.password })
+      const user = await server.users.create({ username: userCreds.username, password: userCreds.password })
-      accessTokenUser = await server.login.getAccessToken(user)
+      userInfo.id = user.id
+      userInfo.accessToken = await server.login.getAccessToken(userCreds)
     }
 
     command = server.channels
@@ -191,7 +198,7 @@ describe('Test video channels API validator', function () {
       await makePutBodyRequest({
         url: server.url,
         path,
-        token: accessTokenUser,
+        token: userInfo.accessToken,
         fields: baseCorrectParams,
         expectedStatus: HttpStatusCode.FORBIDDEN_403
       })
@@ -339,7 +346,7 @@ describe('Test video channels API validator', function () {
     })
 
     it('Should fail with a another user', async function () {
-      await makeGetRequest({ url: server.url, path, token: accessTokenUser, expectedStatus: HttpStatusCode.FORBIDDEN_403 })
+      await makeGetRequest({ url: server.url, path, token: userInfo.accessToken, expectedStatus: HttpStatusCode.FORBIDDEN_403 })
     })
 
     it('Should succeed with the correct params', async function () {
@@ -347,13 +354,122 @@ describe('Test video channels API validator', function () {
     })
   })
 
+  describe('When triggering full synchronization', function () {
+
+    it('Should fail when HTTP upload is disabled', async function () {
+      await server.config.disableImports()
+
+      await command.importVideos({
+        channelName: 'super_channel',
+        externalChannelUrl: FIXTURE_URLS.youtubeChannel,
+        token: server.accessToken,
+        expectedStatus: HttpStatusCode.FORBIDDEN_403
+      })
+
+      await server.config.enableImports()
+    })
+
+    it('Should fail when externalChannelUrl is not provided', async function () {
+      await command.importVideos({
+        channelName: 'super_channel',
+        externalChannelUrl: null,
+        token: server.accessToken,
+        expectedStatus: HttpStatusCode.BAD_REQUEST_400
+      })
+    })
+
+    it('Should fail when externalChannelUrl is malformed', async function () {
+      await command.importVideos({
+        channelName: 'super_channel',
+        externalChannelUrl: 'not-a-url',
+        token: server.accessToken,
+        expectedStatus: HttpStatusCode.BAD_REQUEST_400
+      })
+    })
+
+    it('Should fail with no authentication', async function () {
+      await command.importVideos({
+        channelName: 'super_channel',
+        externalChannelUrl: FIXTURE_URLS.youtubeChannel,
+        token: null,
+        expectedStatus: HttpStatusCode.UNAUTHORIZED_401
+      })
+    })
+
+    it('Should fail when sync is not owned by the user', async function () {
+      await command.importVideos({
+        channelName: 'super_channel',
+        externalChannelUrl: FIXTURE_URLS.youtubeChannel,
+        token: userInfo.accessToken,
+        expectedStatus: HttpStatusCode.FORBIDDEN_403
+      })
+    })
+
+    it('Should fail when the user has no quota', async function () {
+      await server.users.update({
+        userId: userInfo.id,
+        videoQuota: 0
+      })
+
+      await command.importVideos({
+        channelName: 'fake_channel',
+        externalChannelUrl: FIXTURE_URLS.youtubeChannel,
+        token: userInfo.accessToken,
+        expectedStatus: HttpStatusCode.PAYLOAD_TOO_LARGE_413
+      })
+
+      await server.users.update({
+        userId: userInfo.id,
+        videoQuota: userInfo.videoQuota
+      })
+    })
+
+    it('Should fail when the user has no daily quota', async function () {
+      await server.users.update({
+        userId: userInfo.id,
+        videoQuotaDaily: 0
+      })
+
+      await command.importVideos({
+        channelName: 'fake_channel',
+        externalChannelUrl: FIXTURE_URLS.youtubeChannel,
+        token: userInfo.accessToken,
+        expectedStatus: HttpStatusCode.PAYLOAD_TOO_LARGE_413
+      })
+
+      await server.users.update({
+        userId: userInfo.id,
+        videoQuotaDaily: userInfo.videoQuotaDaily
+      })
+    })
+
+    it('Should succeed when sync is run by its owner', async function () {
+      if (!areHttpImportTestsDisabled()) return
+
+      await command.importVideos({
+        channelName: 'fake_channel',
+        externalChannelUrl: FIXTURE_URLS.youtubeChannel,
+        token: userInfo.accessToken
+      })
+    })
+
+    it('Should succeed when sync is run with root and for another user\'s channel', async function () {
+      if (!areHttpImportTestsDisabled()) return
+
+      await command.importVideos({
+        channelName: 'fake_channel',
+        externalChannelUrl: FIXTURE_URLS.youtubeChannel
+      })
+    })
+  })
+
   describe('When deleting a video channel', function () {
     it('Should fail with a non authenticated user', async function () {
       await command.delete({ token: 'coucou', channelName: 'super_channel', expectedStatus: HttpStatusCode.UNAUTHORIZED_401 })
     })
 
     it('Should fail with another authenticated user', async function () {
-      await command.delete({ token: accessTokenUser, channelName: 'super_channel', expectedStatus: HttpStatusCode.FORBIDDEN_403 })
+      await command.delete({ token: userInfo.accessToken, channelName: 'super_channel', expectedStatus: HttpStatusCode.FORBIDDEN_403 })
     })
 
     it('Should fail with an unknown video channel id', async function () {

server/tests/api/check-params/video-imports.ts

@@ -88,7 +88,13 @@ describe('Test video imports API validator', function () {
 
     it('Should fail with nothing', async function () {
       const fields = {}
-      await makePostBodyRequest({ url: server.url, path, token: server.accessToken, fields })
+      await makePostBodyRequest({
+        url: server.url,
+        path,
+        token: server.accessToken,
+        fields,
+        expectedStatus: HttpStatusCode.BAD_REQUEST_400
+      })
     })
 
     it('Should fail without a target url', async function () {

server/tests/api/server/config.ts

@@ -368,6 +368,10 @@ const newCustomConfig: CustomConfig = {
       torrent: {
         enabled: false
       }
+    },
+    videoChannelSynchronization: {
+      enabled: false,
+      maxPerUser: 10
     }
   },
   trending: {

server/tests/api/videos/channel-import-videos.ts

@@ -0,0 +1,50 @@
+import { expect } from 'chai'
+import { FIXTURE_URLS } from '@server/tests/shared'
+import { areHttpImportTestsDisabled } from '@shared/core-utils'
+import {
+  createSingleServer,
+  getServerImportConfig,
+  PeerTubeServer,
+  setAccessTokensToServers,
+  setDefaultVideoChannel,
+  waitJobs
+} from '@shared/server-commands'
+
+describe('Test videos import in a channel', function () {
+  if (areHttpImportTestsDisabled()) return
+
+  function runSuite (mode: 'youtube-dl' | 'yt-dlp') {
+
+    describe('Import using ' + mode, function () {
+      let server: PeerTubeServer
+
+      before(async function () {
+        this.timeout(120_000)
+
+        server = await createSingleServer(1, getServerImportConfig(mode))
+
+        await setAccessTokensToServers([ server ])
+        await setDefaultVideoChannel([ server ])
+
+        await server.config.enableChannelSync()
+      })
+
+      it('Should import a whole channel', async function () {
+        this.timeout(240_000)
+
+        await server.channels.importVideos({ channelName: server.store.channel.name, externalChannelUrl: FIXTURE_URLS.youtubeChannel })
+        await waitJobs(server)
+
+        const videos = await server.videos.listByChannel({ handle: server.store.channel.name })
+        expect(videos.total).to.equal(2)
+      })
+
+      after(async function () {
+        await server?.kill()
+      })
+    })
+  }
+
+  runSuite('yt-dlp')
+  runSuite('youtube-dl')
+})

server/tests/api/videos/index.ts

@@ -4,6 +4,8 @@ import './single-server'
 import './video-captions'
 import './video-change-ownership'
 import './video-channels'
+import './channel-import-videos'
+import './video-channel-syncs'
 import './video-comments'
 import './video-description'
 import './video-files'

server/tests/api/videos/video-channel-syncs.ts

@@ -0,0 +1,226 @@
+import 'mocha'
+import { expect } from 'chai'
+import { FIXTURE_URLS } from '@server/tests/shared'
+import { areHttpImportTestsDisabled } from '@shared/core-utils'
+import { HttpStatusCode, VideoChannelSyncState, VideoInclude, VideoPrivacy } from '@shared/models'
+import {
+  ChannelSyncsCommand,
+  createSingleServer,
+  getServerImportConfig,
+  PeerTubeServer,
+  setAccessTokensToServers,
+  setDefaultAccountAvatar,
+  setDefaultChannelAvatar,
+  setDefaultVideoChannel,
+  waitJobs
+} from '@shared/server-commands'
+
+describe('Test channel synchronizations', function () {
+  if (areHttpImportTestsDisabled()) return
+
+  function runSuite (mode: 'youtube-dl' | 'yt-dlp') {
+
+    describe('Sync using ' + mode, function () {
+      let server: PeerTubeServer
+      let command: ChannelSyncsCommand
+      let startTestDate: Date
+      const userInfo = {
+        accessToken: '',
+        username: 'user1',
+        channelName: 'user1_channel',
+        channelId: -1,
+        syncId: -1
+      }
+
+      async function changeDateForSync (channelSyncId: number, newDate: string) {
+        await server.sql.updateQuery(
+          `UPDATE "videoChannelSync" ` +
+          `SET "createdAt"='${newDate}', "lastSyncAt"='${newDate}' ` +
+          `WHERE id=${channelSyncId}`
+        )
+      }
+
+      before(async function () {
+        this.timeout(120_000)
+
+        startTestDate = new Date()
+
+        server = await createSingleServer(1, getServerImportConfig(mode))
+
+        await setAccessTokensToServers([ server ])
+        await setDefaultVideoChannel([ server ])
+        await setDefaultChannelAvatar([ server ])
+        await setDefaultAccountAvatar([ server ])
+
+        await server.config.enableChannelSync()
+
+        command = server.channelSyncs
+
+        {
+          userInfo.accessToken = await server.users.generateUserAndToken(userInfo.username)
+
+          const { videoChannels } = await server.users.getMyInfo({ token: userInfo.accessToken })
+          userInfo.channelId = videoChannels[0].id
+        }
+      })
+
+      it('Should fetch the latest channel videos of a remote channel', async function () {
+        this.timeout(120_000)
+
+        {
+          const { video } = await server.imports.importVideo({
+            attributes: {
+              channelId: server.store.channel.id,
+              privacy: VideoPrivacy.PUBLIC,
+              targetUrl: FIXTURE_URLS.youtube
+            }
+          })
+
+          expect(video.name).to.equal('small video - youtube')
+
+          const { total } = await server.videos.listByChannel({ handle: 'root_channel', include: VideoInclude.NOT_PUBLISHED_STATE })
+          expect(total).to.equal(1)
+        }
+
+        const { videoChannelSync } = await command.create({
+          attributes: {
+            externalChannelUrl: FIXTURE_URLS.youtubeChannel,
+            videoChannelId: server.store.channel.id
+          },
+          token: server.accessToken,
+          expectedStatus: HttpStatusCode.OK_200
+        })
+
+        // Ensure any missing video not already fetched will be considered as new
+        await changeDateForSync(videoChannelSync.id, '1970-01-01')
+
+        await server.debug.sendCommand({
+          body: {
+            command: 'process-video-channel-sync-latest'
+          }
+        })
+
+        {
+          await waitJobs(server)
+
+          const { total, data } = await server.videos.listByChannel({ handle: 'root_channel', include: VideoInclude.NOT_PUBLISHED_STATE })
+          expect(total).to.equal(2)
+          expect(data[0].name).to.equal('test')
+        }
+      })
+
+      it('Should add another synchronization', async function () {
+        const externalChannelUrl = FIXTURE_URLS.youtubeChannel + '?foo=bar'
+
+        const { videoChannelSync } = await command.create({
+          attributes: {
+            externalChannelUrl,
+            videoChannelId: server.store.channel.id
+          },
+          token: server.accessToken,
+          expectedStatus: HttpStatusCode.OK_200
+        })
+
+        expect(videoChannelSync.externalChannelUrl).to.equal(externalChannelUrl)
+        expect(videoChannelSync.channel).to.include({
+          id: server.store.channel.id,
+          name: 'root_channel'
+        })
+        expect(videoChannelSync.state.id).to.equal(VideoChannelSyncState.WAITING_FIRST_RUN)
+        expect(new Date(videoChannelSync.createdAt)).to.be.above(startTestDate).and.to.be.at.most(new Date())
+      })
+
+      it('Should add a synchronization for another user', async function () {
+        const { videoChannelSync } = await command.create({
+          attributes: {
+            externalChannelUrl: FIXTURE_URLS.youtubeChannel + '?baz=qux',
+            videoChannelId: userInfo.channelId
+          },
+          token: userInfo.accessToken
+        })
+        userInfo.syncId = videoChannelSync.id
+      })
+
+      it('Should not import a channel if not asked', async function () {
+        await waitJobs(server)
+
+        const { data } = await command.listByAccount({ accountName: userInfo.username })
+
+        expect(data[0].state).to.contain({
+          id: VideoChannelSyncState.WAITING_FIRST_RUN,
+          label: 'Waiting first run'
+        })
+      })
+
+      it('Should only fetch the videos newer than the creation date', async function () {
+        this.timeout(120_000)
+
+        await changeDateForSync(userInfo.syncId, '2019-03-01')
+
+        await server.debug.sendCommand({
+          body: {
+            command: 'process-video-channel-sync-latest'
+          }
+        })
+
+        await waitJobs(server)
+
+        const { data, total } = await server.videos.listByChannel({
+          handle: userInfo.channelName,
+          include: VideoInclude.NOT_PUBLISHED_STATE
+        })
+
+        expect(total).to.equal(1)
+        expect(data[0].name).to.equal('test')
+      })
+
+      it('Should list channel synchronizations', async function () {
+        // Root
+        {
+          const { total, data } = await command.listByAccount({ accountName: 'root' })
+          expect(total).to.equal(2)
+
+          expect(data[0]).to.deep.contain({
+            externalChannelUrl: FIXTURE_URLS.youtubeChannel,
+            state: {
+              id: VideoChannelSyncState.SYNCED,
+              label: 'Synchronized'
+            }
+          })
+
+          expect(new Date(data[0].lastSyncAt)).to.be.greaterThan(startTestDate)
+
+          expect(data[0].channel).to.contain({ id: server.store.channel.id })
+          expect(data[1]).to.contain({ externalChannelUrl: FIXTURE_URLS.youtubeChannel + '?foo=bar' })
+        }
+
+        // User
+        {
+          const { total, data } = await command.listByAccount({ accountName: userInfo.username })
+          expect(total).to.equal(1)
+          expect(data[0]).to.deep.contain({
+            externalChannelUrl: FIXTURE_URLS.youtubeChannel + '?baz=qux',
+            state: {
+              id: VideoChannelSyncState.SYNCED,
+              label: 'Synchronized'
+            }
+          })
+        }
+      })
+
+      it('Should remove user\'s channel synchronizations', async function () {
+        await command.delete({ channelSyncId: userInfo.syncId })
+
+        const { total } = await command.listByAccount({ accountName: userInfo.username })
+        expect(total).to.equal(0)
+      })
+
+      after(async function () {
+        await server?.kill()
+      })
+    })
+  }
+
+  runSuite('youtube-dl')
+  runSuite('yt-dlp')
+})

server/tests/api/videos/video-imports.ts

@@ -12,6 +12,7 @@ import {
   createMultipleServers,
   createSingleServer,
   doubleFollow,
+  getServerImportConfig,
   PeerTubeServer,
   setAccessTokensToServers,
   setDefaultVideoChannel,
@@ -84,24 +85,9 @@ describe('Test video imports', function () {
       let servers: PeerTubeServer[] = []
 
       before(async function () {
-        this.timeout(30_000)
+        this.timeout(60_000)
 
-        // Run servers
+        servers = await createMultipleServers(2, getServerImportConfig(mode))
-        servers = await createMultipleServers(2, {
-          import: {
-            videos: {
-              http: {
-                youtube_dl_release: {
-                  url: mode === 'youtube-dl'
-                    ? 'https://yt-dl.org/downloads/latest/youtube-dl'
-                    : 'https://api.github.com/repos/yt-dlp/yt-dlp/releases',
-
-                  name: mode
-                }
-              }
-            }
-          }
-        })
 
         await setAccessTokensToServers(servers)
         await setDefaultVideoChannel(servers)

server/tests/shared/tests.ts

@@ -16,6 +16,8 @@ const FIXTURE_URLS = {
    */
   youtubeHDR: 'https://www.youtube.com/watch?v=RQgnBB9z_N4',
 
+  youtubeChannel: 'https://youtube.com/channel/UCtnlZdXv3-xQzxiqfn6cjIA',
+
   // eslint-disable-next-line max-len
   magnet: 'magnet:?xs=https%3A%2F%2Fpeertube2.cpy.re%2Flazy-static%2Ftorrents%2Fb209ca00-c8bb-4b2b-b421-1ede169f3dbc-720.torrent&xt=urn:btih:0f498834733e8057ed5c6f2ee2b4efd8d84a76ee&dn=super+peertube2+video&tr=https%3A%2F%2Fpeertube2.cpy.re%2Ftracker%2Fannounce&tr=wss%3A%2F%2Fpeertube2.cpy.re%3A443%2Ftracker%2Fsocket&ws=https%3A%2F%2Fpeertube2.cpy.re%2Fstatic%2Fwebseed%2Fb209ca00-c8bb-4b2b-b421-1ede169f3dbc-720.mp4',
 

server/types/express.d.ts

@@ -8,6 +8,7 @@ import {
   MActorFollowActorsDefault,
   MActorUrl,
   MChannelBannerAccountDefault,
+  MChannelSyncChannel,
   MStreamingPlaylist,
   MVideoChangeOwnershipFull,
   MVideoFile,
@@ -145,6 +146,7 @@ declare module 'express' {
       videoStreamingPlaylist?: MStreamingPlaylist
 
       videoChannel?: MChannelBannerAccountDefault
+      videoChannelSync?: MChannelSyncChannel
 
       videoPlaylistFull?: MVideoPlaylistFull
       videoPlaylistSummary?: MVideoPlaylistFullSummary
@@ -194,6 +196,7 @@ declare module 'express' {
       plugin?: MPlugin
 
       localViewerFull?: MLocalVideoViewerWithWatchSections
+
     }
   }
 }

server/types/models/video/index.ts

@@ -8,6 +8,7 @@ export * from './video'
 export * from './video-blacklist'
 export * from './video-caption'
 export * from './video-change-ownership'
+export * from './video-channel-sync'
 export * from './video-channels'
 export * from './video-comment'
 export * from './video-file'

server/types/models/video/video-channel-sync.ts

@@ -0,0 +1,17 @@
+import { VideoChannelSyncModel } from '@server/models/video/video-channel-sync'
+import { FunctionProperties, PickWith } from '@shared/typescript-utils'
+import { MChannelAccountDefault, MChannelFormattable } from './video-channels'
+
+type Use<K extends keyof VideoChannelSyncModel, M> = PickWith<VideoChannelSyncModel, K, M>
+
+export type MChannelSync = Omit<VideoChannelSyncModel, 'VideoChannel'>
+
+export type MChannelSyncChannel =
+  MChannelSync &
+  Use<'VideoChannel', MChannelAccountDefault> &
+  FunctionProperties<VideoChannelSyncModel>
+
+export type MChannelSyncFormattable =
+  FunctionProperties<MChannelSyncChannel> &
+  Use<'VideoChannel', MChannelFormattable> &
+  MChannelSync

shared/models/server/custom-config.model.ts

@@ -165,6 +165,10 @@ export interface CustomConfig {
         enabled: boolean
       }
     }
+    videoChannelSynchronization: {
+      enabled: boolean
+      maxPerUser: number
+    }
   }
 
   trending: {

shared/models/server/debug.model.ts

@@ -4,5 +4,8 @@ export interface Debug {
 }
 
 export interface SendDebugCommand {
-  command: 'remove-dandling-resumable-uploads' | 'process-video-views-buffer' | 'process-video-viewers'
+  command: 'remove-dandling-resumable-uploads'
+  | 'process-video-views-buffer'
+  | 'process-video-viewers'
+  | 'process-video-channel-sync-latest'
 }

shared/models/server/job.model.ts

@@ -25,6 +25,8 @@ export type JobType =
   | 'manage-video-torrent'
   | 'move-to-object-storage'
   | 'video-studio-edition'
+  | 'video-channel-import'
+  | 'after-video-channel-import'
   | 'notify'
   | 'federate-video'
 
@@ -82,20 +84,32 @@ export type VideoFileImportPayload = {
   filePath: string
 }
 
+// ---------------------------------------------------------------------------
+
 export type VideoImportTorrentPayloadType = 'magnet-uri' | 'torrent-file'
 export type VideoImportYoutubeDLPayloadType = 'youtube-dl'
 
-export type VideoImportYoutubeDLPayload = {
+export interface VideoImportYoutubeDLPayload {
   type: VideoImportYoutubeDLPayloadType
   videoImportId: number
 
   fileExt?: string
 }
-export type VideoImportTorrentPayload = {
+
+export interface VideoImportTorrentPayload {
   type: VideoImportTorrentPayloadType
   videoImportId: number
 }
-export type VideoImportPayload = VideoImportYoutubeDLPayload | VideoImportTorrentPayload
+
+export type VideoImportPayload = (VideoImportYoutubeDLPayload | VideoImportTorrentPayload) & {
+  preventException: boolean
+}
+
+export interface VideoImportPreventExceptionResult {
+  resultType: 'success' | 'error'
+}
+
+// ---------------------------------------------------------------------------
 
 export type VideoRedundancyPayload = {
   videoId: number
@@ -219,6 +233,17 @@ export interface VideoStudioEditionPayload {
 
 // ---------------------------------------------------------------------------
 
+export interface VideoChannelImportPayload {
+  externalChannelUrl: string
+  videoChannelId: number
+}
+
+export interface AfterVideoChannelImportPayload {
+  channelSyncId: number
+}
+
+// ---------------------------------------------------------------------------
+
 export type NotifyPayload =
   {
     action: 'new-video'

shared/models/server/server-config.model.ts

@@ -188,6 +188,9 @@ export interface ServerConfig {
         enabled: boolean
       }
     }
+    videoChannelSynchronization: {
+      enabled: boolean
+    }
   }
 
   autoBlacklist: {

shared/models/videos/channel-sync/index.ts

@@ -0,0 +1,3 @@
+export * from './video-channel-sync-state.enum'
+export * from './video-channel-sync.model'
+export * from './video-channel-sync-create.model'

shared/models/videos/channel-sync/video-channel-sync-create.model.ts

@@ -0,0 +1,4 @@
+export interface VideoChannelSyncCreate {
+  externalChannelUrl: string
+  videoChannelId: number
+}

shared/models/videos/channel-sync/video-channel-sync-state.enum.ts

@@ -0,0 +1,6 @@
+export const enum VideoChannelSyncState {
+  WAITING_FIRST_RUN = 1,
+  PROCESSING = 2,
+  SYNCED = 3,
+  FAILED = 4
+}

shared/models/videos/channel-sync/video-channel-sync.model.ts

@@ -0,0 +1,14 @@
+import { VideoChannelSummary } from '../channel/video-channel.model'
+import { VideoConstant } from '../video-constant.model'
+import { VideoChannelSyncState } from './video-channel-sync-state.enum'
+
+export interface VideoChannelSync {
+  id: number
+
+  externalChannelUrl: string
+
+  createdAt: string
+  channel: VideoChannelSummary
+  state: VideoConstant<VideoChannelSyncState>
+  lastSyncAt: string
+}

shared/models/videos/index.ts

@@ -11,6 +11,7 @@ export * from './playlist'
 export * from './rate'
 export * from './stats'
 export * from './transcoding'
+export * from './channel-sync'
 
 export * from './nsfw-policy.type'
 

shared/server-commands/server/config-command.ts

@@ -18,17 +18,25 @@ export class ConfigCommand extends AbstractCommand {
     }
   }
 
+  disableImports () {
+    return this.setImportsEnabled(false)
+  }
+
   enableImports () {
+    return this.setImportsEnabled(true)
+  }
+
+  private setImportsEnabled (enabled: boolean) {
     return this.updateExistingSubConfig({
       newConfig: {
         import: {
           videos: {
             http: {
-              enabled: true
+              enabled
             },
 
             torrent: {
-              enabled: true
+              enabled
             }
           }
         }
@@ -36,6 +44,26 @@ export class ConfigCommand extends AbstractCommand {
     })
   }
 
+  private setChannelSyncEnabled (enabled: boolean) {
+    return this.updateExistingSubConfig({
+      newConfig: {
+        import: {
+          videoChannelSynchronization: {
+            enabled
+          }
+        }
+      }
+    })
+  }
+
+  enableChannelSync () {
+    return this.setChannelSyncEnabled(true)
+  }
+
+  disableChannelSync () {
+    return this.setChannelSyncEnabled(false)
+  }
+
   enableLive (options: {
     allowReplay?: boolean
     transcoding?: boolean
@@ -356,6 +384,10 @@ export class ConfigCommand extends AbstractCommand {
           torrent: {
             enabled: false
           }
+        },
+        videoChannelSynchronization: {
+          enabled: false,
+          maxPerUser: 10
         }
       },
       trending: {

shared/server-commands/server/server.ts

@@ -19,6 +19,7 @@ import {
   CaptionsCommand,
   ChangeOwnershipCommand,
   ChannelsCommand,
+  ChannelSyncsCommand,
   HistoryCommand,
   ImportsCommand,
   LiveCommand,
@@ -118,6 +119,7 @@ export class PeerTubeServer {
   playlists?: PlaylistsCommand
   history?: HistoryCommand
   imports?: ImportsCommand
+  channelSyncs?: ChannelSyncsCommand
   streamingPlaylists?: StreamingPlaylistsCommand
   channels?: ChannelsCommand
   comments?: CommentsCommand
@@ -390,6 +392,7 @@ export class PeerTubeServer {
     this.playlists = new PlaylistsCommand(this)
     this.history = new HistoryCommand(this)
     this.imports = new ImportsCommand(this)
+    this.channelSyncs = new ChannelSyncsCommand(this)
     this.streamingPlaylists = new StreamingPlaylistsCommand(this)
     this.channels = new ChannelsCommand(this)
     this.comments = new CommentsCommand(this)

shared/server-commands/server/servers.ts

@@ -39,11 +39,30 @@ async function cleanupTests (servers: PeerTubeServer[]) {
   return Promise.all(p)
 }
 
+function getServerImportConfig (mode: 'youtube-dl' | 'yt-dlp') {
+  return {
+    import: {
+      videos: {
+        http: {
+          youtube_dl_release: {
+            url: mode === 'youtube-dl'
+              ? 'https://yt-dl.org/downloads/latest/youtube-dl'
+              : 'https://api.github.com/repos/yt-dlp/yt-dlp/releases',
+
+            name: mode
+          }
+        }
+      }
+    }
+  }
+}
+
 // ---------------------------------------------------------------------------
 
 export {
   createSingleServer,
   createMultipleServers,
   cleanupTests,
-  killallServers
+  killallServers,
+  getServerImportConfig
 }

shared/server-commands/videos/channel-syncs-command.ts

@@ -0,0 +1,55 @@
+import { HttpStatusCode, ResultList, VideoChannelSync, VideoChannelSyncCreate } from '@shared/models'
+import { pick } from '@shared/core-utils'
+import { unwrapBody } from '../requests'
+import { AbstractCommand, OverrideCommandOptions } from '../shared'
+
+export class ChannelSyncsCommand extends AbstractCommand {
+  private static readonly API_PATH = '/api/v1/video-channel-syncs'
+
+  listByAccount (options: OverrideCommandOptions & {
+    accountName: string
+    start?: number
+    count?: number
+    sort?: string
+  }) {
+    const { accountName, sort = 'createdAt' } = options
+
+    const path = `/api/v1/accounts/${accountName}/video-channel-syncs`
+
+    return this.getRequestBody<ResultList<VideoChannelSync>>({
+      ...options,
+
+      path,
+      query: { sort, ...pick(options, [ 'start', 'count' ]) },
+      implicitToken: true,
+      defaultExpectedStatus: HttpStatusCode.OK_200
+    })
+  }
+
+  async create (options: OverrideCommandOptions & {
+    attributes: VideoChannelSyncCreate
+  }) {
+    return unwrapBody<{ videoChannelSync: VideoChannelSync }>(this.postBodyRequest({
+      ...options,
+
+      path: ChannelSyncsCommand.API_PATH,
+      fields: options.attributes,
+      implicitToken: true,
+      defaultExpectedStatus: HttpStatusCode.OK_200
+    }))
+  }
+
+  delete (options: OverrideCommandOptions & {
+    channelSyncId: number
+  }) {
+    const path = `${ChannelSyncsCommand.API_PATH}/${options.channelSyncId}`
+
+    return this.deleteRequest({
+      ...options,
+
+      path,
+      implicitToken: true,
+      defaultExpectedStatus: HttpStatusCode.NO_CONTENT_204
+    })
+  }
+}

shared/server-commands/videos/channels-command.ts

@@ -181,4 +181,22 @@ export class ChannelsCommand extends AbstractCommand {
       defaultExpectedStatus: HttpStatusCode.OK_200
     })
   }
+
+  importVideos (options: OverrideCommandOptions & {
+    channelName: string
+    externalChannelUrl: string
+  }) {
+    const { channelName, externalChannelUrl } = options
+
+    const path = `/api/v1/video-channels/${channelName}/import-videos`
+
+    return this.postBodyRequest({
+      ...options,
+
+      path,
+      fields: { externalChannelUrl },
+      implicitToken: true,
+      defaultExpectedStatus: HttpStatusCode.NO_CONTENT_204
+    })
+  }
 }

shared/server-commands/videos/index.ts

@@ -3,6 +3,7 @@ export * from './captions-command'
 export * from './change-ownership-command'
 export * from './channels'
 export * from './channels-command'
+export * from './channel-syncs-command'
 export * from './comments-command'
 export * from './history-command'
 export * from './imports-command'

support/doc/api/openapi.yaml

@@ -254,6 +254,8 @@ tags:
       download it for you, saving you as much bandwidth and avoiding any instability or limitation your network might have.
   - name: Video Imports
     description: Operations dealing with listing, adding and removing video imports.
+  - name: Channels Sync
+    description: Operations dealing with synchronizing PeerTube user's channel with channels of other platforms
   - name: Video Captions
     description: Operations dealing with listing, adding and removing closed captions of a video.
   - name: Video Channels
@@ -327,6 +329,7 @@ x-tagGroups:
       - Video Transcoding
       - Live Videos
       - Feeds
+      - Channels Sync
   - name: Search
     tags:
       - Search
@@ -3050,7 +3053,7 @@ paths:
       tags:
         - Video Channels
       responses:
-        '204':
+        '200':
           description: successful operation
           content:
             application/json:
@@ -3288,6 +3291,59 @@ paths:
         '204':
           description: successful operation
 
+  '/video-channel-syncs':
+    post:
+      summary: Create a synchronization for a video channel
+      operationId: addVideoChannelSync
+      security:
+        - OAuth2: []
+      tags:
+        - Channels Sync
+      requestBody:
+        content:
+          application/json:
+            schema:
+              $ref: '#/components/schemas/VideoChannelSyncCreate'
+      responses:
+        '200':
+          description: successful operation
+          content:
+            application/json:
+              schema:
+                type: object
+                properties:
+                  videoChannelSync:
+                    $ref: "#/components/schemas/VideoChannelSync"
+
+  '/video-channel-syncs/{channelSyncId}':
+    delete:
+      summary: Delete a video channel synchronization
+      operationId: delVideoChannelSync
+      security:
+        - OAuth2: []
+      tags:
+        - Channels Sync
+      parameters:
+        - $ref: '#/components/parameters/channelSyncId'
+      responses:
+        '204':
+          description: successful operation
+
+  '/video-channel-syncs/{channelSyncId}/sync':
+    post:
+      summary: Triggers the channel synchronization job, fetching all the videos from the remote channel
+      operationId: triggerVideoChannelSync
+      security:
+        - OAuth2: []
+      tags:
+        - Channels Sync
+      parameters:
+        - $ref: '#/components/parameters/channelSyncId'
+      responses:
+        '204':
+          description: successful operation
+
+
   /video-playlists/privacies:
     get:
       summary: List available playlist privacy policies
@@ -3659,6 +3715,26 @@ paths:
               schema:
                 $ref: '#/components/schemas/VideoChannelList'
 
+  '/accounts/{name}/video-channel-syncs':
+    get:
+      summary: List the synchronizations of video channels of an account
+      tags:
+        - Video Channels
+        - Channels Sync
+        - Accounts
+      parameters:
+        - $ref: '#/components/parameters/name'
+        - $ref: '#/components/parameters/start'
+        - $ref: '#/components/parameters/count'
+        - $ref: '#/components/parameters/sort'
+      responses:
+        '200':
+          description: successful operation
+          content:
+            application/json:
+              schema:
+                $ref: '#/components/schemas/VideoChannelSyncList'
+
   '/accounts/{name}/ratings':
     get:
       summary: List ratings of an account
@@ -5141,6 +5217,13 @@ components:
       schema:
         type: string
         example: my_username | my_username@example.com
+    channelSyncId:
+      name: channelSyncId
+      in: path
+      required: true
+      description: Channel Sync id
+      schema:
+        $ref: '#/components/schemas/Abuse/properties/id'
     subscriptionHandle:
       name: subscriptionHandle
       in: path
@@ -5347,6 +5430,7 @@ components:
           - activitypub-refresher
           - video-redundancy
           - video-live-ending
+          - video-channel-import
     followState:
       name: state
       in: query
@@ -6497,6 +6581,11 @@ components:
                   properties:
                     enabled:
                       type: boolean
+            videoChannelSynchronization:
+              type: object
+              properties:
+                enabled:
+                  type: boolean
         autoBlacklist:
           type: object
           properties:
@@ -6861,6 +6950,11 @@ components:
                   properties:
                     enabled:
                       type: boolean
+            video_channel_synchronization:
+              type: object
+              properties:
+                enabled:
+                  type: boolean
         autoBlacklist:
           type: object
           properties:
@@ -6953,6 +7047,7 @@ components:
             - videos-views-stats
             - activitypub-refresher
             - video-redundancy
+            - video-channel-import
         data:
           type: object
           additionalProperties: true
@@ -7473,6 +7568,7 @@ components:
                   type: integer
                 uuid:
                   $ref: '#/components/schemas/UUIDv4'
+
     VideoChannelCreate:
       allOf:
         - $ref: '#/components/schemas/VideoChannel'
@@ -7503,6 +7599,51 @@ components:
               - $ref: '#/components/schemas/VideoChannel'
               - $ref: '#/components/schemas/Actor'
 
+    VideoChannelSync:
+      type: object
+      properties:
+        id:
+          $ref: '#/components/schemas/id'
+        state:
+          type: object
+          properties:
+            id:
+              type: integer
+              example: 2
+            label:
+              type: string
+              example: PROCESSING
+        externalChannelUrl:
+          type: string
+          example: 'https://youtube.com/c/UC_myfancychannel'
+        createdAt:
+          type: string
+          format: date-time
+        lastSyncAt:
+          type: string
+          format: date-time
+          nullable: true
+        channel:
+          $ref: '#/components/schemas/VideoChannel'
+    VideoChannelSyncList:
+      type: object
+      properties:
+        total:
+          type: integer
+          example: 1
+        data:
+          type: array
+          items:
+            allOf:
+              - $ref: '#/components/schemas/VideoChannelSync'
+    VideoChannelSyncCreate:
+      type: object
+      properties:
+        externalChannelUrl:
+          type: string
+          example: https://youtube.com/c/UC_myfancychannel
+        videoChannelId:
+          $ref: '#/components/schemas/id'
     MRSSPeerLink:
       type: object
       xml: