Add import http enabled configuration

client/src/app/+admin/config/edit-custom-config/edit-custom-config.component.html

@@ -101,6 +101,13 @@
         </div>
       </div>
 
+      <div i18n class="inner-form-title">Import</div>
+
+      <my-peertube-checkbox
+        inputName="importVideosHttpEnabled" formControlName="importVideosHttpEnabled"
+        i18n-labelText labelText="Video import with HTTP enabled"
+      ></my-peertube-checkbox>
+
       <div i18n class="inner-form-title">Administrator</div>
 
       <div class="form-group">

client/src/app/+admin/config/edit-custom-config/edit-custom-config.component.ts

@@ -71,6 +71,7 @@ export class EditCustomConfigComponent extends FormReactive implements OnInit {
       cacheCaptionsSize: this.customConfigValidatorsService.CACHE_CAPTIONS_SIZE,
       signupEnabled: null,
       signupLimit: this.customConfigValidatorsService.SIGNUP_LIMIT,
+      importVideosHttpEnabled: null,
       adminEmail: this.customConfigValidatorsService.ADMIN_EMAIL,
       userVideoQuota: this.userValidatorsService.USER_VIDEO_QUOTA,
       transcodingThreads: this.customConfigValidatorsService.TRANSCODING_THREADS,
@@ -183,6 +184,13 @@ export class EditCustomConfigComponent extends FormReactive implements OnInit {
           '720p': this.form.value[this.getResolutionKey('720p')],
           '1080p': this.form.value[this.getResolutionKey('1080p')]
         }
+      },
+      import: {
+        videos: {
+          http: {
+            enabled: this.form.value['importVideosHttpEnabled']
+          }
+        }
       }
     }
 
@@ -222,7 +230,8 @@ export class EditCustomConfigComponent extends FormReactive implements OnInit {
       transcodingThreads: this.customConfig.transcoding.threads,
       transcodingEnabled: this.customConfig.transcoding.enabled,
       customizationJavascript: this.customConfig.instance.customizations.javascript,
-      customizationCSS: this.customConfig.instance.customizations.css
+      customizationCSS: this.customConfig.instance.customizations.css,
+      importVideosHttpEnabled: this.customConfig.import.videos.http.enabled
     }
 
     for (const resolution of this.resolutions) {

client/src/app/+my-account/my-account.component.html

@@ -6,7 +6,7 @@
 
     <a i18n routerLink="/my-account/videos" routerLinkActive="active" class="title-page">My videos</a>
 
-    <a i18n routerLink="/my-account/video-imports" routerLinkActive="active" class="title-page">My video imports</a>
+    <a *ngIf="isVideoImportEnabled()" i18n routerLink="/my-account/video-imports" routerLinkActive="active" class="title-page">My video imports</a>
   </div>
 
   <div class="margin-content">

client/src/app/+my-account/my-account.component.ts

@@ -1,7 +1,17 @@
 import { Component } from '@angular/core'
+import { ServerService } from '@app/core'
 
 @Component({
   selector: 'my-my-account',
   templateUrl: './my-account.component.html'
 })
-export class MyAccountComponent {}
+export class MyAccountComponent {
+
+  constructor (
+    private serverService: ServerService
+  ) {}
+
+  isVideoImportEnabled () {
+    return this.serverService.getConfig().import.video.http.enabled
+  }
+}

client/src/app/core/server/server.service.ts

@@ -68,6 +68,13 @@ export class ServerService {
     },
     user: {
       videoQuota: -1
+    },
+    import: {
+      video: {
+        http: {
+          enabled: false
+        }
+      }
     }
   }
   private videoCategories: Array<VideoConstant<string>> = []

client/src/app/videos/+video-edit/video-add.component.html

@@ -10,7 +10,7 @@
       <my-video-upload #videoUpload (firstStepDone)="onFirstStepDone('upload', $event)"></my-video-upload>
     </tab>
 
-    <tab i18n-heading heading="Import your video">
+    <tab *ngIf="isVideoImportEnabled()" i18n-heading heading="Import your video">
       <my-video-import #videoImport (firstStepDone)="onFirstStepDone('import', $event)"></my-video-import>
     </tab>
   </tabset>

client/src/app/videos/+video-edit/video-add.component.ts

@@ -2,6 +2,7 @@ import { Component, ViewChild } from '@angular/core'
 import { CanComponentDeactivate } from '@app/shared/guards/can-deactivate-guard.service'
 import { VideoImportComponent } from '@app/videos/+video-edit/video-import.component'
 import { VideoUploadComponent } from '@app/videos/+video-edit/video-upload.component'
+import { ServerService } from '@app/core'
 
 @Component({
   selector: 'my-videos-add',
@@ -15,6 +16,10 @@ export class VideoAddComponent implements CanComponentDeactivate {
   secondStepType: 'upload' | 'import'
   videoName: string
 
+  constructor (
+    private serverService: ServerService
+  ) {}
+
   onFirstStepDone (type: 'upload' | 'import', videoName: string) {
     this.secondStepType = type
     this.videoName = videoName
@@ -26,4 +31,8 @@ export class VideoAddComponent implements CanComponentDeactivate {
 
     return { canDeactivate: true }
   }
+
+  isVideoImportEnabled () {
+    return this.serverService.getConfig().import.video.http.enabled
+  }
 }

client/src/app/videos/+video-edit/video-import.component.ts

@@ -97,8 +97,11 @@ export class VideoImportComponent extends FormReactive implements OnInit, CanCom
       channelId: this.firstStepChannelId
     }
 
+    this.loadingBar.start()
+
     this.videoImportService.importVideo(this.targetUrl, videoUpdate).subscribe(
       res => {
+        this.loadingBar.complete()
         this.firstStepDone.emit(res.video.name)
         this.isImportingVideo = false
         this.hasImportedVideo = true
@@ -113,6 +116,7 @@ export class VideoImportComponent extends FormReactive implements OnInit, CanCom
       },
 
       err => {
+        this.loadingBar.complete()
         this.isImportingVideo = false
         this.notificationsService.error(this.i18n('Error'), err.message)
       }

config/default.yaml

@@ -96,7 +96,7 @@ import:
   # Add ability for your users to import remote videos (from YouTube, torrent...)
   videos:
     http: # Classic HTTP or all sites supported by youtube-dl https://rg3.github.io/youtube-dl/supportedsites.html
-      enabled: true
+      enabled: false
 
 instance:
   name: 'PeerTube'

config/production.yaml.example

@@ -106,6 +106,12 @@ transcoding:
     720p: false
     1080p: false
 
+import:
+  # Add ability for your users to import remote videos (from YouTube, torrent...)
+  videos:
+    http: # Classic HTTP or all sites supported by youtube-dl https://rg3.github.io/youtube-dl/supportedsites.html
+      enabled: false
+
 # Instance settings
 instance:
   name: 'PeerTube'

config/test.yaml

@@ -40,5 +40,10 @@ transcoding:
     720p: true
     1080p: true
 
+import:
+  videos:
+    http:
+      enabled: true
+
 instance:
   default_nsfw_policy: 'display'

server/controllers/api/config.ts

@@ -65,6 +65,13 @@ async function getConfig (req: express.Request, res: express.Response, next: exp
     transcoding: {
       enabledResolutions
     },
+    import: {
+      video: {
+        http: {
+          enabled: CONFIG.IMPORT.VIDEOS.HTTP.ENABLED
+        }
+      }
+    },
     avatar: {
       file: {
         size: {
@@ -225,6 +232,13 @@ function customConfig (): CustomConfig {
         '720p': CONFIG.TRANSCODING.RESOLUTIONS[ '720p' ],
         '1080p': CONFIG.TRANSCODING.RESOLUTIONS[ '1080p' ]
       }
+    },
+    import: {
+      videos: {
+        http: {
+          enabled: CONFIG.IMPORT.VIDEOS.HTTP.ENABLED
+        }
+      }
     }
   }
 }

server/controllers/api/users.ts

@@ -68,7 +68,6 @@ usersRouter.get('/me/video-quota-used',
   asyncMiddleware(getUserVideoQuotaUsed)
 )
 
-
 usersRouter.get('/me/videos/imports',
   authenticate,
   paginationValidator,

server/controllers/api/videos/import.ts

@@ -77,7 +77,7 @@ async function addVideoImport (req: express.Request, res: express.Response) {
   video.url = getVideoActivityPubUrl(video)
 
   // Process thumbnail file?
-  const thumbnailField = req.files['thumbnailfile']
+  const thumbnailField = req.files ? req.files['thumbnailfile'] : undefined
   let downloadThumbnail = true
   if (thumbnailField) {
     const thumbnailPhysicalFile = thumbnailField[ 0 ]
@@ -86,7 +86,7 @@ async function addVideoImport (req: express.Request, res: express.Response) {
   }
 
   // Process preview file?
-  const previewField = req.files['previewfile']
+  const previewField = req.files ? req.files['previewfile'] : undefined
   let downloadPreview = true
   if (previewField) {
     const previewPhysicalFile = previewField[0]

server/helpers/audit-logger.ts

@@ -246,11 +246,9 @@ class CustomConfigAuditView extends EntityAuditView {
     const resolutionsDict = infos.transcoding.resolutions
     const resolutionsArray = []
     Object.entries(resolutionsDict).forEach(([resolution, isEnabled]) => {
-      if (isEnabled) {
+      if (isEnabled) resolutionsArray.push(resolution)
-        resolutionsArray.push(resolution)
-      }
     })
-    infos.transcoding.resolutions = resolutionsArray
+    Object.assign({}, infos, { transcoding: { resolutions: resolutionsArray } })
     super(customConfigKeysToKeep, 'config', infos)
   }
 }

server/initializers/checker.ts

@@ -51,6 +51,7 @@ function checkMissedConfig () {
     'cache.previews.size', 'admin.email',
     'signup.enabled', 'signup.limit', 'signup.filters.cidr.whitelist', 'signup.filters.cidr.blacklist',
     'transcoding.enabled', 'transcoding.threads',
+    'import.videos.http.enabled',
     'instance.name', 'instance.short_description', 'instance.description', 'instance.terms', 'instance.default_client_route',
     'instance.default_nsfw_policy', 'instance.robots',
     'services.twitter.username', 'services.twitter.whitelisted'

server/initializers/constants.ts

@@ -206,6 +206,13 @@ const CONFIG = {
       get '1080p' () { return config.get<boolean>('transcoding.resolutions.1080p') }
     }
   },
+  IMPORT: {
+    VIDEOS: {
+      HTTP: {
+        get ENABLED () { return config.get<boolean>('import.videos.http.enabled') }
+      }
+    }
+  },
   CACHE: {
     PREVIEWS: {
       get SIZE () { return config.get<number>('cache.previews.size') }

server/middlewares/validators/config.ts

@@ -24,6 +24,7 @@ const customConfigUpdateValidator = [
   body('transcoding.resolutions.480p').isBoolean().withMessage('Should have a valid transcoding 480p resolution enabled boolean'),
   body('transcoding.resolutions.720p').isBoolean().withMessage('Should have a valid transcoding 720p resolution enabled boolean'),
   body('transcoding.resolutions.1080p').isBoolean().withMessage('Should have a valid transcoding 1080p resolution enabled boolean'),
+  body('import.videos.http.enabled').isBoolean().withMessage('Should have a valid import video http enabled boolean'),
 
   async (req: express.Request, res: express.Response, next: express.NextFunction) => {
     logger.debug('Checking customConfigUpdateValidator parameters', { parameters: req.body })

server/middlewares/validators/video-imports.ts

@@ -7,6 +7,7 @@ import { getCommonVideoAttributes } from './videos'
 import { isVideoImportTargetUrlValid } from '../../helpers/custom-validators/video-imports'
 import { cleanUpReqFiles } from '../../helpers/utils'
 import { isVideoChannelOfAccountExist, isVideoNameValid } from '../../helpers/custom-validators/videos'
+import { CONFIG } from '../../initializers/constants'
 
 const videoImportAddValidator = getCommonVideoAttributes().concat([
   body('targetUrl').custom(isVideoImportTargetUrlValid).withMessage('Should have a valid video import target URL'),
@@ -23,6 +24,14 @@ const videoImportAddValidator = getCommonVideoAttributes().concat([
     const user = res.locals.oauth.token.User
 
     if (areValidationErrors(req, res)) return cleanUpReqFiles(req)
+
+    if (CONFIG.IMPORT.VIDEOS.HTTP.ENABLED !== true) {
+      cleanUpReqFiles(req)
+      return res.status(409)
+        .json({ error: 'Import is not enabled on this instance.' })
+        .end()
+    }
+
     if (!await isVideoChannelOfAccountExist(req.body.channelId, user, res)) return cleanUpReqFiles(req)
 
     return next()

server/tests/api/check-params/config.ts

@@ -60,6 +60,13 @@ describe('Test config API validators', function () {
         '720p': false,
         '1080p': false
       }
+    },
+    import: {
+      videos: {
+        http: {
+          enabled: false
+        }
+      }
     }
   }
 

server/tests/api/check-params/index.ts

@@ -10,4 +10,5 @@ import './video-captions'
 import './video-channels'
 import './video-comments'
 import './videos'
+import './video-imports'
 import './search'

server/tests/api/check-params/video-imports.ts

@@ -0,0 +1,246 @@
+/* tslint:disable:no-unused-expression */
+
+import { omit } from 'lodash'
+import 'mocha'
+import { join } from 'path'
+import { VideoPrivacy } from '../../../../shared/models/videos/video-privacy.enum'
+import {
+  createUser,
+  flushTests,
+  getMyUserInformation,
+  immutableAssign,
+  killallServers,
+  makeGetRequest,
+  makePostBodyRequest,
+  makeUploadRequest,
+  runServer,
+  ServerInfo,
+  setAccessTokensToServers,
+  userLogin
+} from '../../utils'
+import { checkBadCountPagination, checkBadSortPagination, checkBadStartPagination } from '../../utils/requests/check-api-params'
+
+describe('Test video imports API validator', function () {
+  const path = '/api/v1/videos/imports'
+  let server: ServerInfo
+  let userAccessToken = ''
+  let accountName: string
+  let channelId: number
+  let channelUUID: string
+
+  // ---------------------------------------------------------------
+
+  before(async function () {
+    this.timeout(30000)
+
+    await flushTests()
+
+    server = await runServer(1)
+
+    await setAccessTokensToServers([ server ])
+
+    const username = 'user1'
+    const password = 'my super password'
+    await createUser(server.url, server.accessToken, username, password)
+    userAccessToken = await userLogin(server, { username, password })
+
+    {
+      const res = await getMyUserInformation(server.url, server.accessToken)
+      channelId = res.body.videoChannels[ 0 ].id
+      channelUUID = res.body.videoChannels[ 0 ].uuid
+      accountName = res.body.account.name + '@' + res.body.account.host
+    }
+  })
+
+  describe('When listing my video imports', function () {
+    const myPath = '/api/v1/users/me/videos/imports'
+
+    it('Should fail with a bad start pagination', async function () {
+      await checkBadStartPagination(server.url, myPath, server.accessToken)
+    })
+
+    it('Should fail with a bad count pagination', async function () {
+      await checkBadCountPagination(server.url, myPath, server.accessToken)
+    })
+
+    it('Should fail with an incorrect sort', async function () {
+      await checkBadSortPagination(server.url, myPath, server.accessToken)
+    })
+
+    it('Should success with the correct parameters', async function () {
+      await makeGetRequest({ url: server.url, path: myPath, statusCodeExpected: 200, token: server.accessToken })
+    })
+  })
+
+  describe('When adding a video import', function () {
+    let baseCorrectParams
+
+    before(function () {
+      baseCorrectParams = {
+        targetUrl: 'https://youtu.be/msX3jv1XdvM',
+        name: 'my super name',
+        category: 5,
+        licence: 1,
+        language: 'pt',
+        nsfw: false,
+        commentsEnabled: true,
+        waitTranscoding: true,
+        description: 'my super description',
+        support: 'my super support text',
+        tags: [ 'tag1', 'tag2' ],
+        privacy: VideoPrivacy.PUBLIC,
+        channelId: channelId
+      }
+    })
+
+    it('Should fail with nothing', async function () {
+      const fields = {}
+      await makePostBodyRequest({ url: server.url, path, token: server.accessToken, fields })
+    })
+
+    it('Should fail with a long name', async function () {
+      const fields = immutableAssign(baseCorrectParams, { name: 'super'.repeat(65) })
+
+      await makePostBodyRequest({ url: server.url, path, token: server.accessToken, fields })
+    })
+
+    it('Should fail with a bad category', async function () {
+      const fields = immutableAssign(baseCorrectParams, { category: 125 })
+
+      await makePostBodyRequest({ url: server.url, path, token: server.accessToken, fields })
+    })
+
+    it('Should fail with a bad licence', async function () {
+      const fields = immutableAssign(baseCorrectParams, { licence: 125 })
+
+      await makePostBodyRequest({ url: server.url, path, token: server.accessToken, fields })
+    })
+
+    it('Should fail with a bad language', async function () {
+      const fields = immutableAssign(baseCorrectParams, { language: 'a'.repeat(15) })
+
+      await makePostBodyRequest({ url: server.url, path, token: server.accessToken, fields })
+    })
+
+    it('Should fail with a long description', async function () {
+      const fields = immutableAssign(baseCorrectParams, { description: 'super'.repeat(2500) })
+
+      await makePostBodyRequest({ url: server.url, path, token: server.accessToken, fields })
+    })
+
+    it('Should fail with a long support text', async function () {
+      const fields = immutableAssign(baseCorrectParams, { support: 'super'.repeat(150) })
+
+      await makePostBodyRequest({ url: server.url, path, token: server.accessToken, fields })
+    })
+
+    it('Should fail without a channel', async function () {
+      const fields = omit(baseCorrectParams, 'channelId')
+
+      await makePostBodyRequest({ url: server.url, path, token: server.accessToken, fields })
+    })
+
+    it('Should fail with a bad channel', async function () {
+      const fields = immutableAssign(baseCorrectParams, { channelId: 545454 })
+
+      await makePostBodyRequest({ url: server.url, path, token: server.accessToken, fields })
+    })
+
+    it('Should fail with another user channel', async function () {
+      const user = {
+        username: 'fake',
+        password: 'fake_password'
+      }
+      await createUser(server.url, server.accessToken, user.username, user.password)
+
+      const accessTokenUser = await userLogin(server, user)
+      const res = await getMyUserInformation(server.url, accessTokenUser)
+      const customChannelId = res.body.videoChannels[0].id
+
+      const fields = immutableAssign(baseCorrectParams, { channelId: customChannelId })
+
+      await makePostBodyRequest({ url: server.url, path, token: userAccessToken, fields })
+    })
+
+    it('Should fail with too many tags', async function () {
+      const fields = immutableAssign(baseCorrectParams, { tags: [ 'tag1', 'tag2', 'tag3', 'tag4', 'tag5', 'tag6' ] })
+
+      await makePostBodyRequest({ url: server.url, path, token: server.accessToken, fields })
+    })
+
+    it('Should fail with a tag length too low', async function () {
+      const fields = immutableAssign(baseCorrectParams, { tags: [ 'tag1', 't' ] })
+
+      await makePostBodyRequest({ url: server.url, path, token: server.accessToken, fields })
+    })
+
+    it('Should fail with a tag length too big', async function () {
+      const fields = immutableAssign(baseCorrectParams, { tags: [ 'tag1', 'my_super_tag_too_long_long_long_long_long_long' ] })
+
+      await makePostBodyRequest({ url: server.url, path, token: server.accessToken, fields })
+    })
+
+    it('Should fail with an incorrect thumbnail file', async function () {
+      const fields = baseCorrectParams
+      const attaches = {
+        'thumbnailfile': join(__dirname, '..', '..', 'fixtures', 'avatar.png')
+      }
+
+      await makeUploadRequest({ url: server.url, path, token: server.accessToken, fields, attaches })
+    })
+
+    it('Should fail with a big thumbnail file', async function () {
+      const fields = baseCorrectParams
+      const attaches = {
+        'thumbnailfile': join(__dirname, '..', '..', 'fixtures', 'avatar-big.png')
+      }
+
+      await makeUploadRequest({ url: server.url, path, token: server.accessToken, fields, attaches })
+    })
+
+    it('Should fail with an incorrect preview file', async function () {
+      const fields = baseCorrectParams
+      const attaches = {
+        'previewfile': join(__dirname, '..', '..', 'fixtures', 'avatar.png')
+      }
+
+      await makeUploadRequest({ url: server.url, path, token: server.accessToken, fields, attaches })
+    })
+
+    it('Should fail with a big preview file', async function () {
+      const fields = baseCorrectParams
+      const attaches = {
+        'previewfile': join(__dirname, '..', '..', 'fixtures', 'avatar-big.png')
+      }
+
+      await makeUploadRequest({ url: server.url, path, token: server.accessToken, fields, attaches })
+    })
+
+    it('Should succeed with the correct parameters', async function () {
+      this.timeout(10000)
+
+      const fields = baseCorrectParams
+
+      {
+        await makePostBodyRequest({
+          url: server.url,
+          path,
+          token: server.accessToken,
+          fields,
+          statusCodeExpected: 200
+        })
+      }
+    })
+
+    it('Should forbid importing')
+  })
+
+  after(async function () {
+    killallServers([ server ])
+
+    // Keep the logs if the test failed
+    if (this['ok']) {
+      await flushTests()
+    }
+  })
+})

server/tests/api/server/config.ts

@@ -44,6 +44,7 @@ function checkInitialConfig (data: CustomConfig) {
   expect(data.transcoding.resolutions['480p']).to.be.true
   expect(data.transcoding.resolutions['720p']).to.be.true
   expect(data.transcoding.resolutions['1080p']).to.be.true
+  expect(data.import.videos.http.enabled).to.be.true
 }
 
 function checkUpdatedConfig (data: CustomConfig) {
@@ -70,6 +71,7 @@ function checkUpdatedConfig (data: CustomConfig) {
   expect(data.transcoding.resolutions['480p']).to.be.true
   expect(data.transcoding.resolutions['720p']).to.be.false
   expect(data.transcoding.resolutions['1080p']).to.be.false
+  expect(data.import.videos.http.enabled).to.be.false
 }
 
 describe('Test config', function () {
@@ -160,6 +162,13 @@ describe('Test config', function () {
           '720p': false,
           '1080p': false
         }
+      },
+      import: {
+        videos: {
+          http: {
+            enabled: false
+          }
+        }
       }
     }
     await updateCustomConfig(server.url, server.accessToken, newCustomConfig)

server/tests/client.ts

@@ -164,6 +164,13 @@ describe('Test a client controllers', function () {
           '720p': false,
           '1080p': false
         }
+      },
+      import: {
+        videos: {
+          http: {
+            enabled: false
+          }
+        }
       }
     }
     await updateCustomConfig(server.url, server.accessToken, newCustomConfig)

shared/models/server/custom-config.model.ts

@@ -55,4 +55,12 @@ export interface CustomConfig {
       '1080p': boolean
     }
   }
+
+  import: {
+    videos: {
+      http: {
+        enabled: boolean
+      }
+    }
+  }
 }

shared/models/server/server-config.model.ts

@@ -1,4 +1,5 @@
 import { NSFWPolicyType } from '../videos/nsfw-policy.type'
+import { CONFIG } from '../../../server/initializers'
 
 export interface ServerConfig {
   serverVersion: string
@@ -23,6 +24,14 @@ export interface ServerConfig {
     enabledResolutions: number[]
   }
 
+  import: {
+    video: {
+      http: {
+        enabled: boolean
+      }
+    }
+  }
+
   avatar: {
     file: {
       size: {