From 3a1157a68a70e90df21f743ea15154805649b8e5 Mon Sep 17 00:00:00 2001 From: John Livingston <38844060+JohnXLivingston@users.noreply.github.com> Date: Fri, 3 Dec 2021 10:14:01 +0100 Subject: [PATCH 01/28] CLI: plugins install command accept a --plugin-version parameter. (#4599) * CLI: plugins install command accept a --plugin-version parameter. * Unit tests for plugins install --plugin-version. * Fix linting. * Styling Co-authored-by: Chocobozzz --- server/controllers/api/plugins.ts | 7 ++++++- server/middlewares/validators/plugins.ts | 6 ++++++ server/tests/cli/peertube.ts | 19 +++++++++++++++++++ server/tools/peertube-plugins.ts | 3 ++- shared/extra-utils/server/plugins-command.ts | 5 +++-- .../server/api/install-plugin.model.ts | 1 + 6 files changed, 37 insertions(+), 4 deletions(-) diff --git a/server/controllers/api/plugins.ts b/server/controllers/api/plugins.ts index 2de7fe41f..de9e055dc 100644 --- a/server/controllers/api/plugins.ts +++ b/server/controllers/api/plugins.ts @@ -144,8 +144,13 @@ async function installPlugin (req: express.Request, res: express.Response) { const fromDisk = !!body.path const toInstall = body.npmName || body.path + + const pluginVersion = body.pluginVersion && body.npmName + ? body.pluginVersion + : undefined + try { - const plugin = await PluginManager.Instance.install(toInstall, undefined, fromDisk) + const plugin = await PluginManager.Instance.install(toInstall, pluginVersion, fromDisk) return res.json(plugin.toFormattedJSON()) } catch (err) { diff --git a/server/middlewares/validators/plugins.ts b/server/middlewares/validators/plugins.ts index 21171af23..c1e9ebefb 100644 --- a/server/middlewares/validators/plugins.ts +++ b/server/middlewares/validators/plugins.ts @@ -116,6 +116,9 @@ const installOrUpdatePluginValidator = [ body('npmName') .optional() .custom(isNpmPluginNameValid).withMessage('Should have a valid npm name'), + body('pluginVersion') + .optional() + .custom(isPluginVersionValid).withMessage('Should have a valid plugin version'), body('path') .optional() .custom(isSafePath).withMessage('Should have a valid safe path'), @@ -129,6 +132,9 @@ const installOrUpdatePluginValidator = [ if (!body.path && !body.npmName) { return res.fail({ message: 'Should have either a npmName or a path' }) } + if (body.pluginVersion && !body.npmName) { + return res.fail({ message: 'Should have a npmName when specifying a pluginVersion' }) + } return next() } diff --git a/server/tests/cli/peertube.ts b/server/tests/cli/peertube.ts index f2a984962..3ac440f84 100644 --- a/server/tests/cli/peertube.ts +++ b/server/tests/cli/peertube.ts @@ -207,6 +207,25 @@ describe('Test CLI wrapper', function () { expect(res).to.not.contain('peertube-plugin-hello-world') }) + + it('Should install a plugin in requested version', async function () { + this.timeout(60000) + + await cliCommand.execWithEnv(`${cmd} plugins install --npm-name peertube-plugin-hello-world --plugin-version 0.0.17`) + }) + + it('Should list installed plugins, in correct version', async function () { + const res = await cliCommand.execWithEnv(`${cmd} plugins list`) + + expect(res).to.contain('peertube-plugin-hello-world') + expect(res).to.contain('0.0.17') + }) + + it('Should uninstall the plugin again', async function () { + const res = await cliCommand.execWithEnv(`${cmd} plugins uninstall --npm-name peertube-plugin-hello-world`) + + expect(res).to.not.contain('peertube-plugin-hello-world') + }) }) describe('Manage video redundancies', function () { diff --git a/server/tools/peertube-plugins.ts b/server/tools/peertube-plugins.ts index ae625114d..9dd3f08c9 100644 --- a/server/tools/peertube-plugins.ts +++ b/server/tools/peertube-plugins.ts @@ -31,6 +31,7 @@ program .option('-p, --password ', 'Password') .option('-P --path ', 'Install from a path') .option('-n, --npm-name ', 'Install from npm') + .option('--plugin-version ', 'Specify the plugin version to install (only available when installing from npm)') .action((options, command) => installPluginCLI(command, options)) program @@ -109,7 +110,7 @@ async function installPluginCLI (command: Command, options: OptionValues) { await assignToken(server, username, password) try { - await server.plugins.install({ npmName: options.npmName, path: options.path }) + await server.plugins.install({ npmName: options.npmName, path: options.path, pluginVersion: options.pluginVersion }) } catch (err) { console.error('Cannot install plugin.', err) process.exit(-1) diff --git a/shared/extra-utils/server/plugins-command.ts b/shared/extra-utils/server/plugins-command.ts index b944475a2..9bf24afff 100644 --- a/shared/extra-utils/server/plugins-command.ts +++ b/shared/extra-utils/server/plugins-command.ts @@ -158,15 +158,16 @@ export class PluginsCommand extends AbstractCommand { install (options: OverrideCommandOptions & { path?: string npmName?: string + pluginVersion?: string }) { - const { npmName, path } = options + const { npmName, path, pluginVersion } = options const apiPath = '/api/v1/plugins/install' return this.postBodyRequest({ ...options, path: apiPath, - fields: { npmName, path }, + fields: { npmName, path, pluginVersion }, implicitToken: true, defaultExpectedStatus: HttpStatusCode.OK_200 }) diff --git a/shared/models/plugins/server/api/install-plugin.model.ts b/shared/models/plugins/server/api/install-plugin.model.ts index 5a268ebe1..a1d009a00 100644 --- a/shared/models/plugins/server/api/install-plugin.model.ts +++ b/shared/models/plugins/server/api/install-plugin.model.ts @@ -1,4 +1,5 @@ export interface InstallOrUpdatePlugin { npmName?: string + pluginVersion?: string path?: string } From b202d007cc553915b77ba9fa76b6d01fa678d845 Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Tue, 30 Nov 2021 09:01:37 +0100 Subject: [PATCH 02/28] Bumped to version v4.0.0-rc.1 --- .github/workflows/codeql.yml | 71 ++++++++++++++++++++++ .github/workflows/codeql/codeql-config.yml | 4 ++ .github/workflows/stats.yml | 2 +- .github/workflows/test.yml | 2 +- 4 files changed, 77 insertions(+), 2 deletions(-) create mode 100644 .github/workflows/codeql.yml create mode 100644 .github/workflows/codeql/codeql-config.yml diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml new file mode 100644 index 000000000..20803bd86 --- /dev/null +++ b/.github/workflows/codeql.yml @@ -0,0 +1,71 @@ +# For most projects, this workflow file will not need changing; you simply need +# to commit it to your repository. +# +# You may wish to alter this file to override the set of languages analyzed, +# or to provide custom queries or build logic. +# +# ******** NOTE ******** +# We have attempted to detect the languages in your repository. Please check +# the `language` matrix defined below to confirm you have the correct set of +# supported CodeQL languages. +# +name: "CodeQL" + +on: + push: + branches: [ next ] + pull_request: + # The branches below must be a subset of the branches above + branches: [ next ] + schedule: + - cron: '36 9 * * 5' + +jobs: + analyze: + name: Analyze + runs-on: ubuntu-latest + permissions: + actions: read + contents: read + security-events: write + + strategy: + fail-fast: false + matrix: + language: [ 'javascript' ] + # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python', 'ruby' ] + # Learn more about CodeQL language support at https://git.io/codeql-language-support + + steps: + - name: Checkout repository + uses: actions/checkout@v2 + + # Initializes the CodeQL tools for scanning. + - name: Initialize CodeQL + uses: github/codeql-action/init@v1 + with: + languages: ${{ matrix.language }} + config-file: ./.github/workflows/codeql/codeql-config.yml + # If you wish to specify custom queries, you can do so here or in a config file. + # By default, queries listed here will override any specified in a config file. + # Prefix the list here with "+" to use these queries and those in the config file. + # queries: ./path/to/local/query, your-org/your-repo/queries@main + + # Autobuild attempts to build any compiled languages (C/C++, C#, or Java). + # If this step fails, then you should remove it and run the build manually (see below) + - name: Autobuild + uses: github/codeql-action/autobuild@v1 + + # ℹī¸ Command-line programs to run using the OS shell. + # 📚 https://git.io/JvXDl + + # ✏ī¸ If the Autobuild fails above, remove it and uncomment the following three lines + # and modify them (or add more) to build your code if your project + # uses a compiled language + + #- run: | + # make bootstrap + # make release + + - name: Perform CodeQL Analysis + uses: github/codeql-action/analyze@v1 diff --git a/.github/workflows/codeql/codeql-config.yml b/.github/workflows/codeql/codeql-config.yml new file mode 100644 index 000000000..8b771ae99 --- /dev/null +++ b/.github/workflows/codeql/codeql-config.yml @@ -0,0 +1,4 @@ +name: "PeerTube CodeQL config" + +paths-ignore: + - server/tests diff --git a/.github/workflows/stats.yml b/.github/workflows/stats.yml index e211f6a3b..d6213deed 100644 --- a/.github/workflows/stats.yml +++ b/.github/workflows/stats.yml @@ -1,4 +1,4 @@ -name: "Stats" +name: Stats" on: push: diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 78a9a28c0..0366ac49f 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -1,4 +1,4 @@ -name: Test Suite +name: Test on: push: From f3fbbf01402fc7491e044f160edca06ef793eb50 Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Tue, 30 Nov 2021 16:00:59 +0100 Subject: [PATCH 03/28] Add docker github action --- .github/workflows/docker.yml | 70 ++++++++++++++++++++++++++++++++++++ .github/workflows/stats.yml | 2 +- .gitlab-ci.yml | 44 ----------------------- 3 files changed, 71 insertions(+), 45 deletions(-) create mode 100644 .github/workflows/docker.yml diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml new file mode 100644 index 000000000..b4ff1a452 --- /dev/null +++ b/.github/workflows/docker.yml @@ -0,0 +1,70 @@ +name: Docker + +on: + push: + branches: + - 'master' + schedule: + - cron: '0 3 * * *' + +jobs: + generate-matrix: + name: Generate matrix for Docker build + runs-on: ubuntu-latest + outputs: + matrix: ${{ steps.set-matrix.outputs.matrix }} + steps: + - name: Checkout + uses: actions/checkout@v2 + with: + ref: master + - name: Set matrix for build + id: set-matrix + run: | + # FIXME: https://github.com/actions/checkout/issues/290 + git fetch --force --tags + + one="{ \"file\": \"./support/docker/production/Dockerfile.bullseye\", \"ref\": \"develop\", \"tags\": \"chocobozzz/peertube-test:develop-bullseye\" }" + two="{ \"file\": \"./support/docker/production/Dockerfile.buster\", \"ref\": \"master\", \"tags\": \"chocobozzz/peertube-test:production-buster,chocobozzz/peertube-test:$(git describe --abbrev=0)-buster\" }" + three="{ \"file\": \"./support/docker/production/Dockerfile.nginx\", \"ref\": \"master\", \"tags\": \"chocobozzz/peertube-webserver-test:latest\" }" + + matrix="[$one,$two,$three]" + echo ::set-output name=matrix::{\"include\":$(echo $matrix)} + + docker: + runs-on: ubuntu-latest + + needs: generate-matrix + + strategy: + matrix: ${{ fromJson(needs.generate-matrix.outputs.matrix) }} + fail-fast: false + + steps: + - + name: Set up QEMU + uses: docker/setup-qemu-action@v1 + - + name: Set up Docker Buildx + uses: docker/setup-buildx-action@v1 + - + name: Login to DockerHub + uses: docker/login-action@v1 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + + - + name: Checkout develop + uses: actions/checkout@v2 + with: + ref: ${{ matrix.ref }} + - + name: Docker build + uses: docker/build-push-action@v2 + with: + context: '.' + platforms: linux/amd64,linux/arm64 + push: true + file: ${{ matrix.file }} + tags: ${{ matrix.tags }} diff --git a/.github/workflows/stats.yml b/.github/workflows/stats.yml index d6213deed..e50c14720 100644 --- a/.github/workflows/stats.yml +++ b/.github/workflows/stats.yml @@ -1,4 +1,4 @@ -name: Stats" +name: Stats on: push: diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index ad94c8cab..66c6df761 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -38,47 +38,3 @@ build-nightly: - eval `ssh-agent -s` - if [ ! -z ${DEPLOYEMENT_KEY+x} ]; then ssh-add <(echo "${DEPLOYEMENT_KEY}"); fi - if [ ! -z ${DEPLOYEMENT_KEY+x} ]; then scp ./peertube-nightly-* ${DEPLOYEMENT_USER}@${DEPLOYEMENT_HOST}:../../web/nightly; fi - -.docker: &docker - stage: docker-nightly - cache: {} - image: - name: gcr.io/kaniko-project/executor:debug - entrypoint: [""] - before_script: - - mkdir -p /kaniko/.docker - - echo "{\"auths\":{\"$CI_REGISTRY\":{\"auth\":\"$CI_REGISTRY_AUTH\",\"email\":\"$CI_REGISTRY_EMAIL\"}}}" > /kaniko/.docker/config.json - script: - - /kaniko/executor --context $CI_PROJECT_DIR --dockerfile $DOCKERFILE --destination $DOCKER_IMAGE_NAME - -build-docker-develop: - <<: *docker - only: - - schedules - variables: - DOCKER_IMAGE_NAME: chocobozzz/peertube:develop-bullseye - DOCKERFILE: $CI_PROJECT_DIR/support/docker/production/Dockerfile.bullseye - -build-docker-webserver: - <<: *docker - only: - - schedules - variables: - DOCKER_IMAGE_NAME: chocobozzz/peertube-webserver - DOCKERFILE: $CI_PROJECT_DIR/support/docker/production/Dockerfile.nginx - -build-docker-tag: - <<: *docker - only: - - tags - variables: - DOCKER_IMAGE_NAME: chocobozzz/peertube:$CI_COMMIT_TAG-bullseye - DOCKERFILE: $CI_PROJECT_DIR/support/docker/production/Dockerfile.bullseye - -build-docker-master: - <<: *docker - only: - - master - variables: - DOCKER_IMAGE_NAME: chocobozzz/peertube:production-bullseye - DOCKERFILE: $CI_PROJECT_DIR/support/docker/production/Dockerfile.bullseye From 51e9e152f7df003c65c16f822669a0674efcaf03 Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Wed, 1 Dec 2021 14:14:58 +0100 Subject: [PATCH 04/28] Build nightly using github action --- .github/actions/reusable-deploy/action.yml | 46 +++++++++++++ .../action.yml | 31 +++++++++ .../reusable-prepare-peertube-run/action.yml | 16 +++++ .github/workflows/benchmark.yml | 69 +++---------------- .github/workflows/codeql.yml | 2 +- .github/workflows/nightly.yml | 36 ++++++++++ .github/workflows/stats.yml | 48 +++---------- .github/workflows/test.yml | 25 +------ .gitlab-ci.yml | 40 ----------- support/doc/development/ci.md | 40 +++++++++++ 10 files changed, 191 insertions(+), 162 deletions(-) create mode 100644 .github/actions/reusable-deploy/action.yml create mode 100644 .github/actions/reusable-prepare-peertube-build/action.yml create mode 100644 .github/actions/reusable-prepare-peertube-run/action.yml create mode 100644 .github/workflows/nightly.yml delete mode 100644 .gitlab-ci.yml create mode 100644 support/doc/development/ci.md diff --git a/.github/actions/reusable-deploy/action.yml b/.github/actions/reusable-deploy/action.yml new file mode 100644 index 000000000..bc69a2e43 --- /dev/null +++ b/.github/actions/reusable-deploy/action.yml @@ -0,0 +1,46 @@ +name: "Reusable deploy on builds.joinpeertube.org" + +description: "Reusable deploy on builds.joinpeertube.org" + +inputs: + source: + required: true + description: "Source file/files/directory/directories to deploy" + destination: + required: true + description: "Destination directory on builds.joinpeertube.org" + knownHosts: + required: true + description: "Known hosts" + deployKey: + required: true + description: "Deploy key" + deployUser: + required: true + description: "Deploy user" + deployHost: + required: true + description: "Deploy host" + + +runs: + using: "composite" + + steps: + - name: "Deploy" + shell: bash + run: | + mkdir -p ~/.ssh + chmod 700 ~/.ssh + + echo "Adding ssh key to known hosts" + echo -e "${{ inputs.knownHosts }}" > ~/.ssh/known_hosts; + + eval `ssh-agent -s` + + echo "Adding ssh deploy key" + ssh-add <(echo "${{ inputs.deployKey }}"); + + echo "Uploading files" + + scp ${{ inputs.source }} ${{ inputs.deployUser }}@${{ inputs.deployHost }}:../../web/${{ inputs.destination }}; diff --git a/.github/actions/reusable-prepare-peertube-build/action.yml b/.github/actions/reusable-prepare-peertube-build/action.yml new file mode 100644 index 000000000..41ebf71c5 --- /dev/null +++ b/.github/actions/reusable-prepare-peertube-build/action.yml @@ -0,0 +1,31 @@ +name: "Reusable prepare PeerTube build" + +description: "Reusable prepare PeerTube build" + +inputs: + node-version: + required: true + description: 'NodeJS version' + +runs: + using: "composite" + + steps: + - name: Use Node.js + uses: actions/setup-node@v1 + with: + node-version: ${{ inputs.node-version }} + + - name: Cache Node.js modules + uses: actions/cache@v2 + with: + path: | + **/node_modules + key: ${{ runner.OS }}-node-${{ hashFiles('**/yarn.lock') }} + restore-keys: | + ${{ runner.OS }}-node- + ${{ runner.OS }}- + + - name: Install dependencies + shell: bash + run: yarn install --frozen-lockfile diff --git a/.github/actions/reusable-prepare-peertube-run/action.yml b/.github/actions/reusable-prepare-peertube-run/action.yml new file mode 100644 index 000000000..1a6cd2cfd --- /dev/null +++ b/.github/actions/reusable-prepare-peertube-run/action.yml @@ -0,0 +1,16 @@ +name: "Reusable prepare PeerTube run" +description: "Reusable prepare PeerTube run" + +runs: + using: "composite" + + steps: + - name: Setup system dependencies + shell: bash + run: | + sudo apt-get install postgresql-client-common redis-tools parallel + wget --quiet --no-check-certificate "https://download.cpy.re/ffmpeg/ffmpeg-release-4.3.1-64bit-static.tar.xz" + tar xf ffmpeg-release-4.3.1-64bit-static.tar.xz + mkdir -p $HOME/bin + cp ffmpeg-*/{ffmpeg,ffprobe} $HOME/bin + echo "$HOME/bin" >> $GITHUB_PATH diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index 86f675432..f02b88a42 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -29,48 +29,15 @@ jobs: env: PGUSER: peertube PGHOST: localhost - NODE_PENDING_JOB_WAIT: 500 steps: - uses: actions/checkout@v2 - - name: Use Node.js - uses: actions/setup-node@v1 + - uses: './.github/actions/reusable-prepare-peertube-build' with: node-version: '12.x' - - name: Setup system dependencies - run: | - sudo apt-get install postgresql-client-common redis-tools parallel - wget --quiet --no-check-certificate "https://download.cpy.re/ffmpeg/ffmpeg-release-4.3.1-64bit-static.tar.xz" - tar xf ffmpeg-release-4.3.1-64bit-static.tar.xz - mkdir -p $HOME/bin - cp ffmpeg-*/{ffmpeg,ffprobe} $HOME/bin - echo "$HOME/bin" >> $GITHUB_PATH - - - name: Cache Node.js modules - uses: actions/cache@v2 - with: - path: | - **/node_modules - key: ${{ runner.OS }}-node-${{ hashFiles('**/yarn.lock') }} - restore-keys: | - ${{ runner.OS }}-node- - ${{ runner.OS }}- - - - name: Cache fixtures - uses: actions/cache@v2 - with: - path: | - fixtures - key: ${{ runner.OS }}-fixtures-${{ matrix.test_suite }}-${{ hashFiles('fixtures/*') }} - restore-keys: | - ${{ runner.OS }}-fixtures-${{ matrix.test_suite }}- - ${{ runner.OS }}-fixtures- - ${{ runner.OS }}- - - - name: Install dependencies - run: yarn install --frozen-lockfile + - uses: './.github/actions/reusable-prepare-peertube-run' - name: Build run: | @@ -111,27 +78,11 @@ jobs: cat benchmark.json build-time.json startup-time.json - name: Upload benchmark result - env: - STATS_DEPLOYEMENT_KNOWN_HOSTS: ${{ secrets.STATS_DEPLOYEMENT_KNOWN_HOSTS }} - STATS_DEPLOYEMENT_KEY: ${{ secrets.STATS_DEPLOYEMENT_KEY }} - STATS_DEPLOYEMENT_USER: ${{ secrets.STATS_DEPLOYEMENT_USER }} - STATS_DEPLOYEMENT_HOST: ${{ secrets.STATS_DEPLOYEMENT_HOST }} - run: | - mkdir -p ~/.ssh - chmod 700 ~/.ssh - if [ ! -z ${STATS_DEPLOYEMENT_KNOWN_HOSTS+x} ]; then - echo "Adding ssh key to known hosts" - echo -e "${STATS_DEPLOYEMENT_KNOWN_HOSTS}" > ~/.ssh/known_hosts; - fi - - eval `ssh-agent -s` - - if [ ! -z ${STATS_DEPLOYEMENT_KEY+x} ]; then - echo "Adding ssh reployement key" - ssh-add <(echo "${STATS_DEPLOYEMENT_KEY}"); - fi - - if [ ! -z ${STATS_DEPLOYEMENT_KEY+x} ]; then - echo "Uploading files" - scp benchmark.json build-time.json startup-time.json ${STATS_DEPLOYEMENT_USER}@${STATS_DEPLOYEMENT_HOST}:../../web/peertube-stats; - fi + uses: './.github/actions/reusable-deploy.yml' + with: + source: benchmark.json build-time.json startup-time.json + destination: peertube-stats + knownHosts: ${{ secrets.STATS_DEPLOYEMENT_KNOWN_HOSTS }} + deployKey: ${{ secrets.STATS_DEPLOYEMENT_KEY }} + deployUser: ${{ secrets.STATS_DEPLOYEMENT_USER }} + deployHost: ${{ secrets.STATS_DEPLOYEMENT_HOST }} diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index 20803bd86..5c0c93886 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -13,7 +13,7 @@ name: "CodeQL" on: push: - branches: [ next ] + branches: [ develop, next ] pull_request: # The branches below must be a subset of the branches above branches: [ next ] diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml new file mode 100644 index 000000000..156b7143a --- /dev/null +++ b/.github/workflows/nightly.yml @@ -0,0 +1,36 @@ +name: Nightly + +on: + push: + branches: + - 'next' + schedule: + - cron: '0 3 * * *' + +jobs: + + nightly: + runs-on: ubuntu-latest + + steps: + - + name: Checkout develop + uses: actions/checkout@v2 + with: + ref: next + + - uses: './.github/actions/reusable-prepare-peertube-build' + with: + node-version: '14.x' + + - name: Build + run: npm run nightly + + - uses: './.github/actions/reusable-deploy' + with: + source: ./peertube-nightly-* + destination: nightly-test + knownHosts: ${{ secrets.STATS_DEPLOYEMENT_KNOWN_HOSTS }} + deployKey: ${{ secrets.STATS_DEPLOYEMENT_KEY }} + deployUser: ${{ secrets.STATS_DEPLOYEMENT_USER }} + deployHost: ${{ secrets.STATS_DEPLOYEMENT_HOST }} diff --git a/.github/workflows/stats.yml b/.github/workflows/stats.yml index e50c14720..c87e6fb77 100644 --- a/.github/workflows/stats.yml +++ b/.github/workflows/stats.yml @@ -20,24 +20,10 @@ jobs: steps: - uses: actions/checkout@v2 - - name: Use Node.js - uses: actions/setup-node@v1 + - uses: './.github/actions/reusable-prepare-peertube-build' with: node-version: '14.x' - - name: Cache Node.js modules - uses: actions/cache@v2 - with: - path: | - **/node_modules - key: ${{ runner.OS }}-node-${{ hashFiles('**/yarn.lock') }} - restore-keys: | - ${{ runner.OS }}-node- - ${{ runner.OS }}- - - - name: Install dependencies - run: yarn install --frozen-lockfile - - name: Angular bundlewatch uses: jackyef/bundlewatch-gh-action@master with: @@ -73,27 +59,11 @@ jobs: - name: Upload stats if: github.event_name != 'pull_request' - env: - STATS_DEPLOYEMENT_KNOWN_HOSTS: ${{ secrets.STATS_DEPLOYEMENT_KNOWN_HOSTS }} - STATS_DEPLOYEMENT_KEY: ${{ secrets.STATS_DEPLOYEMENT_KEY }} - STATS_DEPLOYEMENT_USER: ${{ secrets.STATS_DEPLOYEMENT_USER }} - STATS_DEPLOYEMENT_HOST: ${{ secrets.STATS_DEPLOYEMENT_HOST }} - run: | - mkdir -p ~/.ssh - chmod 700 ~/.ssh - if [ ! -z ${STATS_DEPLOYEMENT_KNOWN_HOSTS+x} ]; then - echo "Adding ssh key to known hosts" - echo -e "${STATS_DEPLOYEMENT_KNOWN_HOSTS}" > ~/.ssh/known_hosts; - fi - - eval `ssh-agent -s` - - if [ ! -z ${STATS_DEPLOYEMENT_KEY+x} ]; then - echo "Adding ssh reployement key" - ssh-add <(echo "${STATS_DEPLOYEMENT_KEY}"); - fi - - if [ ! -z ${STATS_DEPLOYEMENT_KEY+x} ]; then - echo "Uploading files" - scp lighthouse.json client-build-stats.json scc.json ${STATS_DEPLOYEMENT_USER}@${STATS_DEPLOYEMENT_HOST}:../../web/peertube-stats; - fi + uses: './.github/actions/reusable-deploy' + with: + source: lighthouse.json client-build-stats.json scc.json + destination: peertube-stats + knownHosts: ${{ secrets.STATS_DEPLOYEMENT_KNOWN_HOSTS }} + deployKey: ${{ secrets.STATS_DEPLOYEMENT_KEY }} + deployUser: ${{ secrets.STATS_DEPLOYEMENT_USER }} + deployHost: ${{ secrets.STATS_DEPLOYEMENT_HOST }} diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 0366ac49f..030ec3790 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -50,29 +50,11 @@ jobs: steps: - uses: actions/checkout@v2 - - name: Use Node.js - uses: actions/setup-node@v1 + - uses: './.github/actions/reusable-prepare-peertube-build' with: node-version: '12.x' - - name: Setup system dependencies - run: | - sudo apt-get install postgresql-client-common redis-tools parallel - wget --quiet --no-check-certificate "https://download.cpy.re/ffmpeg/ffmpeg-release-4.3.1-64bit-static.tar.xz" - tar xf ffmpeg-release-4.3.1-64bit-static.tar.xz - mkdir -p $HOME/bin - cp ffmpeg-*/{ffmpeg,ffprobe} $HOME/bin - echo "$HOME/bin" >> $GITHUB_PATH - - - name: Cache Node.js modules - uses: actions/cache@v2 - with: - path: | - **/node_modules - key: ${{ runner.OS }}-node-${{ hashFiles('**/yarn.lock') }} - restore-keys: | - ${{ runner.OS }}-node- - ${{ runner.OS }}- + - uses: './.github/actions/reusable-prepare-peertube-run' - name: Cache fixtures uses: actions/cache@v2 @@ -85,9 +67,6 @@ jobs: ${{ runner.OS }}-fixtures- ${{ runner.OS }}- - - name: Install dependencies - run: yarn install --frozen-lockfile - - name: Set env test variable (schedule) if: github.event_name != 'schedule' run: | diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml deleted file mode 100644 index 66c6df761..000000000 --- a/.gitlab-ci.yml +++ /dev/null @@ -1,40 +0,0 @@ -image: chocobozzz/peertube-ci:14 - -stages: - - clients - - docker-nightly - -cache: - key: yarn - paths: - - .yarn-cache - - cached-fixtures - -# build-openapi-clients: -# stage: clients -# only: -# refs: -# - master -# - schedules -# changes: -# - support/doc/api/openapi.yaml -# script: -# - apt-get update -qq -# - apt-get -yqqq install openjdk-8-jre -# - yarn install --pure-lockfile -# - scripts/openapi-peertube-version.sh -# - scripts/openapi-clients.sh - -build-nightly: - stage: docker-nightly - only: - - schedules - script: - - yarn install --pure-lockfile --cache-folder .yarn-cache - - npm run nightly - - mkdir "${HOME}/.ssh" - - chmod 700 "${HOME}/.ssh" - - if [ ! -z ${DEPLOYEMENT_KNOWN_HOSTS+x} ]; then echo -e "${DEPLOYEMENT_KNOWN_HOSTS}" > ${HOME}/.ssh/known_hosts; fi - - eval `ssh-agent -s` - - if [ ! -z ${DEPLOYEMENT_KEY+x} ]; then ssh-add <(echo "${DEPLOYEMENT_KEY}"); fi - - if [ ! -z ${DEPLOYEMENT_KEY+x} ]; then scp ./peertube-nightly-* ${DEPLOYEMENT_USER}@${DEPLOYEMENT_HOST}:../../web/nightly; fi diff --git a/support/doc/development/ci.md b/support/doc/development/ci.md new file mode 100644 index 000000000..7d6eef197 --- /dev/null +++ b/support/doc/development/ci.md @@ -0,0 +1,40 @@ +# Continuous integration + +PeerTube uses Github Actions as a CI platform. +CI tasks are described in `.github/workflows`. + +## benchmark.yml + +*Scheduled* + +Run various benchmarks (build, API etc) and upload results on https://builds.joinpeertube.org/peertube-stats/ to be publicly consumed. + +## codeql.yml + +*Scheduled, on push on develop and on pull request* + +Run CodeQL task to throw code security issues in Github. https://lgtm.com/projects/g/Chocobozzz/PeerTube can also be used. + +## docker.yml + +*Scheduled and on push on master* + +Build `chocobozzz/peertube-webserver:latest`, `chocobozzz/peertube:production-...`, `chocobozzz/peertube:v-...` (only latest PeerTube tag) and `chocobozzz/peertube:develop-...` Docker images. Scheduled to automatically upgrade image software (Debian security issues etc). + +## nightly.yml + +*Scheduled* + +Build PeerTube nightly build (`develop` branch) and upload the release on https://builds.joinpeertube.org/nightly. + +## stats.yml + +*On push on develop* + +Create various PeerTube stats (line of codes, build size, lighthouse report) and upload results on https://builds.joinpeertube.org/peertube-stats/ to be publicly consumed. + +## test.yml + +*Scheduled, on push and pull request* + +Run PeerTube lint and tests. From 22f25c740bf3eb24d9988ab6add8e1a6aec1b0bd Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Thu, 2 Dec 2021 13:45:32 +0100 Subject: [PATCH 05/28] Recover HLS video with duration inconsistency With audio longer than video --- client/src/assets/player/p2p-media-loader/hls-plugin.ts | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/client/src/assets/player/p2p-media-loader/hls-plugin.ts b/client/src/assets/player/p2p-media-loader/hls-plugin.ts index 71c31696a..2aa691f05 100644 --- a/client/src/assets/player/p2p-media-loader/hls-plugin.ts +++ b/client/src/assets/player/p2p-media-loader/hls-plugin.ts @@ -146,7 +146,9 @@ class Html5Hlsjs { } duration () { - return this._duration || this.videoElement.duration || 0 + if (!isNaN(this.videoElement.duration)) return this.videoElement.duration + + return this._duration || 0 } seekable () { @@ -366,6 +368,7 @@ class Html5Hlsjs { this.isLive = data.details.live this.dvrDuration = data.details.totalduration + this._duration = this.isLive ? Infinity : data.details.totalduration }) From 8406a9e8eec261e563d99c92f8a18b6bd3e46e0f Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Thu, 2 Dec 2021 14:34:00 +0100 Subject: [PATCH 06/28] Fix forgot password button role --- client/src/app/+login/login.component.html | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/client/src/app/+login/login.component.html b/client/src/app/+login/login.component.html index 90eea1505..531b06dc9 100644 --- a/client/src/app/+login/login.component.html +++ b/client/src/app/+login/login.component.html @@ -48,7 +48,8 @@
- I forgot my password + I forgot my password + -
- -

- MODERATION -

- - -
-
+ -
-
+ +
+
+ +

Hardware information

+ -
-
- -

Terms

- - -
-
- -
- -

- OTHER INFORMATION -

- - -
-
- -

Hardware information

- - -
+
+
-
+

FEATURES

-
+
- Instance + Instance - PeerTube + PeerTube - Network + Network
diff --git a/client/src/app/menu/menu.component.html b/client/src/app/menu/menu.component.html index 46dd807ec..55fc27b5f 100644 --- a/client/src/app/menu/menu.component.html +++ b/client/src/app/menu/menu.component.html @@ -30,7 +30,10 @@
- + Interface: {{ currentInterfaceLanguage }} diff --git a/shared/models/plugins/client/plugin-selector-id.type.ts b/shared/models/plugins/client/plugin-selector-id.type.ts index b74dffbef..8d23314b5 100644 --- a/shared/models/plugins/client/plugin-selector-id.type.ts +++ b/shared/models/plugins/client/plugin-selector-id.type.ts @@ -1 +1,10 @@ -export type PluginSelectorId = 'login-form' +export type PluginSelectorId = + 'login-form' | + 'menu-user-dropdown-language-item' | + 'about-instance-features' | + 'about-instance-statistics' | + 'about-instance-moderation' | + 'about-menu-instance' | + 'about-menu-peertube' | + 'about-menu-network' | + 'about-instance-other-information' From 9213df1d79f9dcbf6d51d27434a98375f7b95411 Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Fri, 3 Dec 2021 10:49:36 +0100 Subject: [PATCH 11/28] Fix comment --- server/lib/plugins/plugin-manager.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/server/lib/plugins/plugin-manager.ts b/server/lib/plugins/plugin-manager.ts index 8add72a85..6c2f4764e 100644 --- a/server/lib/plugins/plugin-manager.ts +++ b/server/lib/plugins/plugin-manager.ts @@ -312,12 +312,12 @@ export class PluginManager implements ServerHook { logger.error('Cannot install plugin %s, removing it...', toInstall, { err: rootErr }) try { - // await this.uninstall(npmName) + await this.uninstall(npmName) } catch (err) { logger.error('Cannot uninstall plugin %s after failed installation.', toInstall, { err }) try { - // await removeNpmPlugin(npmName) + await removeNpmPlugin(npmName) } catch (err) { logger.error('Cannot remove plugin %s after failed installation.', toInstall, { err }) } From 390366472d7239864060753c63513d0f316ca468 Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Fri, 3 Dec 2021 13:35:23 +0100 Subject: [PATCH 12/28] Fix benchmark ci --- .github/workflows/benchmark.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index f02b88a42..7e8259d27 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -78,7 +78,7 @@ jobs: cat benchmark.json build-time.json startup-time.json - name: Upload benchmark result - uses: './.github/actions/reusable-deploy.yml' + uses: './.github/actions/reusable-deploy' with: source: benchmark.json build-time.json startup-time.json destination: peertube-stats From 6d472b4046084e5f477124b11ac62dca9f6a1a63 Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Fri, 3 Dec 2021 14:00:40 +0100 Subject: [PATCH 13/28] Decrease time to cleanup resumable uploads --- server/controllers/api/videos/upload.ts | 9 ++--- server/initializers/constants.ts | 2 +- ...ve-dangling-resumable-uploads-scheduler.ts | 35 ++++--------------- server/lib/uploadx.ts | 10 ++++++ 4 files changed, 21 insertions(+), 35 deletions(-) create mode 100644 server/lib/uploadx.ts diff --git a/server/controllers/api/videos/upload.ts b/server/controllers/api/videos/upload.ts index 6773b500f..c827f6bf0 100644 --- a/server/controllers/api/videos/upload.ts +++ b/server/controllers/api/videos/upload.ts @@ -8,6 +8,7 @@ import { createTorrentAndSetInfoHash } from '@server/helpers/webtorrent' import { getLocalVideoActivityPubUrl } from '@server/lib/activitypub/url' import { generateWebTorrentVideoFilename } from '@server/lib/paths' import { Redis } from '@server/lib/redis' +import { uploadx } from '@server/lib/uploadx' import { addMoveToObjectStorageJob, addOptimizeOrMergeAudioJob, @@ -19,7 +20,6 @@ import { VideoPathManager } from '@server/lib/video-path-manager' import { buildNextVideoState } from '@server/lib/video-state' import { openapiOperationDoc } from '@server/middlewares/doc' import { MVideo, MVideoFile, MVideoFullLight } from '@server/types/models' -import { Uploadx } from '@uploadx/core' import { VideoCreate, VideoState } from '../../../../shared' import { HttpStatusCode } from '../../../../shared/models' import { auditLoggerFactory, getAuditIdFromRes, VideoAuditView } from '../../../helpers/audit-logger' @@ -41,8 +41,8 @@ import { authenticate, videosAddLegacyValidator, videosAddResumableInitValidator, - videosResumableUploadIdValidator, - videosAddResumableValidator + videosAddResumableValidator, + videosResumableUploadIdValidator } from '../../../middlewares' import { ScheduleVideoUpdateModel } from '../../../models/video/schedule-video-update' import { VideoModel } from '../../../models/video/video' @@ -52,9 +52,6 @@ const lTags = loggerTagsFactory('api', 'video') const auditLogger = auditLoggerFactory('videos') const uploadRouter = express.Router() -const uploadx = new Uploadx({ directory: getResumableUploadPath() }) -uploadx.getUserId = (_, res: express.Response) => res.locals.oauth?.token.user.id - const reqVideoFileAdd = createReqFiles( [ 'videofile', 'thumbnailfile', 'previewfile' ], Object.assign({}, MIMETYPES.VIDEO.MIMETYPE_EXT, MIMETYPES.IMAGE.MIMETYPE_EXT), diff --git a/server/initializers/constants.ts b/server/initializers/constants.ts index b8633e83e..c61c01d62 100644 --- a/server/initializers/constants.ts +++ b/server/initializers/constants.ts @@ -223,7 +223,7 @@ const SCHEDULER_INTERVALS_MS = { REMOVE_OLD_VIEWS: 60000 * 60 * 24, // 1 day REMOVE_OLD_HISTORY: 60000 * 60 * 24, // 1 day UPDATE_INBOX_STATS: 1000 * 60, // 1 minute - REMOVE_DANGLING_RESUMABLE_UPLOADS: 60000 * 60 * 16 // 16 hours + REMOVE_DANGLING_RESUMABLE_UPLOADS: 60000 * 60 // 1 hour } // --------------------------------------------------------------------------- diff --git a/server/lib/schedulers/remove-dangling-resumable-uploads-scheduler.ts b/server/lib/schedulers/remove-dangling-resumable-uploads-scheduler.ts index d6e561cad..61e93eafa 100644 --- a/server/lib/schedulers/remove-dangling-resumable-uploads-scheduler.ts +++ b/server/lib/schedulers/remove-dangling-resumable-uploads-scheduler.ts @@ -1,9 +1,7 @@ -import { map } from 'bluebird' -import { readdir, remove, stat } from 'fs-extra' + import { logger, loggerTagsFactory } from '@server/helpers/logger' -import { getResumableUploadPath } from '@server/helpers/upload' import { SCHEDULER_INTERVALS_MS } from '@server/initializers/constants' -import { METAFILE_EXTNAME } from '@uploadx/core' +import { uploadx } from '../uploadx' import { AbstractScheduler } from './abstract-scheduler' const lTags = loggerTagsFactory('scheduler', 'resumable-upload', 'cleaner') @@ -22,36 +20,17 @@ export class RemoveDanglingResumableUploadsScheduler extends AbstractScheduler { } protected async internalExecute () { - const path = getResumableUploadPath() - const files = await readdir(path) + logger.debug('Removing dangling resumable uploads', lTags()) - const metafiles = files.filter(f => f.endsWith(METAFILE_EXTNAME)) - - if (metafiles.length === 0) return - - logger.debug('Reading resumable video upload folder %s with %d files', path, metafiles.length, lTags()) + const now = new Date().getTime() try { - await map(metafiles, metafile => { - return this.deleteIfOlderThan(metafile, this.lastExecutionTimeMs) - }, { concurrency: 5 }) + // Remove files that were not updated since the last execution + await uploadx.storage.purge(now - this.lastExecutionTimeMs) } catch (error) { logger.error('Failed to handle file during resumable video upload folder cleanup', { error, ...lTags() }) } finally { - this.lastExecutionTimeMs = new Date().getTime() - } - } - - private async deleteIfOlderThan (metafile: string, olderThan: number) { - const metafilePath = getResumableUploadPath(metafile) - const statResult = await stat(metafilePath) - - // Delete uploads that started since a long time - if (statResult.ctimeMs < olderThan) { - await remove(metafilePath) - - const datafile = metafilePath.replace(new RegExp(`${METAFILE_EXTNAME}$`), '') - await remove(datafile) + this.lastExecutionTimeMs = now } } diff --git a/server/lib/uploadx.ts b/server/lib/uploadx.ts new file mode 100644 index 000000000..11b1044db --- /dev/null +++ b/server/lib/uploadx.ts @@ -0,0 +1,10 @@ +import express from 'express' +import { getResumableUploadPath } from '@server/helpers/upload' +import { Uploadx } from '@uploadx/core' + +const uploadx = new Uploadx({ directory: getResumableUploadPath() }) +uploadx.getUserId = (_, res: express.Response) => res.locals.oauth?.token.user.id + +export { + uploadx +} From e78acf81ff00568555e9823c1d9b01c7616b7272 Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Fri, 3 Dec 2021 14:09:52 +0100 Subject: [PATCH 14/28] Fix additional extensions admin config description --- .../edit-custom-config/edit-vod-transcoding.component.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/client/src/app/+admin/config/edit-custom-config/edit-vod-transcoding.component.html b/client/src/app/+admin/config/edit-custom-config/edit-vod-transcoding.component.html index 048f7908d..1158f027b 100644 --- a/client/src/app/+admin/config/edit-custom-config/edit-vod-transcoding.component.html +++ b/client/src/app/+admin/config/edit-custom-config/edit-vod-transcoding.component.html @@ -51,7 +51,7 @@ i18n-labelText labelText="Allow additional extensions" > - Allows users to upload {{ additionalVideoExtensions }} videos. + Allows users to upload videos with additional extensions than .mp4, .ogv and .webm (for example: .avi, .mov, .mkv etc).
From 453537426a4e172967320cfac4bb1f53c28d94f5 Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Fri, 3 Dec 2021 14:23:45 +0100 Subject: [PATCH 15/28] Fix too long filename video upload --- .../video-upload.component.ts | 21 ++++++++++++++++--- .../middlewares/validators/videos/videos.ts | 2 +- 2 files changed, 19 insertions(+), 4 deletions(-) diff --git a/client/src/app/+videos/+video-edit/video-add-components/video-upload.component.ts b/client/src/app/+videos/+video-edit/video-add-components/video-upload.component.ts index 76205db44..28d7ec458 100644 --- a/client/src/app/+videos/+video-edit/video-add-components/video-upload.component.ts +++ b/client/src/app/+videos/+video-edit/video-add-components/video-upload.component.ts @@ -1,4 +1,6 @@ +import { truncate } from 'lodash-es' import { UploadState, UploadxOptions, UploadxService } from 'ngx-uploadx' +import { isIOS } from 'src/assets/player/utils' import { HttpErrorResponse, HttpEventType, HttpHeaders } from '@angular/common/http' import { AfterViewInit, Component, ElementRef, EventEmitter, OnDestroy, OnInit, Output, ViewChild } from '@angular/core' import { Router } from '@angular/router' @@ -10,7 +12,6 @@ import { LoadingBarService } from '@ngx-loading-bar/core' import { HttpStatusCode, VideoCreateResult, VideoPrivacy } from '@shared/models' import { UploaderXFormData } from './uploaderx-form-data' import { VideoSend } from './video-send' -import { isIOS } from 'src/assets/player/utils' @Component({ selector: 'my-video-upload', @@ -281,6 +282,7 @@ export class VideoUploadComponent extends VideoSend implements OnInit, OnDestroy channelId: this.firstStepChannelId, nsfw: this.serverConfig.instance.isNSFW, privacy: this.highestPrivacy.toString(), + name: this.buildVideoFilename(file.name), filename: file.name, previewfile: previewfile as any } @@ -311,8 +313,7 @@ export class VideoUploadComponent extends VideoSend implements OnInit, OnDestroy } private closeFirstStep (filename: string) { - const nameWithoutExtension = filename.replace(/\.[^/.]+$/, '') - const name = nameWithoutExtension.length < 3 ? filename : nameWithoutExtension + const name = this.buildVideoFilename(filename) this.form.patchValue({ name, @@ -369,4 +370,18 @@ export class VideoUploadComponent extends VideoSend implements OnInit, OnDestroy return extensions.some(e => filename.endsWith(e)) } + + private buildVideoFilename (filename: string) { + const nameWithoutExtension = filename.replace(/\.[^/.]+$/, '') + let name = nameWithoutExtension.length < 3 + ? filename + : nameWithoutExtension + + const videoNameMaxSize = 110 + if (name.length > videoNameMaxSize) { + name = truncate(name, { length: videoNameMaxSize, omission: '' }) + } + + return name + } } diff --git a/server/middlewares/validators/videos/videos.ts b/server/middlewares/validators/videos/videos.ts index 4916decbf..3ebdbc33d 100644 --- a/server/middlewares/validators/videos/videos.ts +++ b/server/middlewares/validators/videos/videos.ts @@ -211,7 +211,7 @@ const videosAddResumableInitValidator = getCommonVideoEditAttributes().concat([ const videoFileMetadata = { mimetype: req.headers['x-upload-content-type'] as string, size: +req.headers['x-upload-content-length'], - originalname: req.body.name + originalname: req.body.filename } const user = res.locals.oauth.token.User From 025d858e7997d6b9895fa6b7beaf23bf5ff814af Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Fri, 3 Dec 2021 14:40:29 +0100 Subject: [PATCH 16/28] Fix pending transcoding with failed job --- .../job-queue/handlers/video-transcoding.ts | 27 ++++++++++--------- server/models/video/video-job-info.ts | 6 +++-- 2 files changed, 18 insertions(+), 15 deletions(-) diff --git a/server/lib/job-queue/handlers/video-transcoding.ts b/server/lib/job-queue/handlers/video-transcoding.ts index 0d80eb6c5..0edcdcba3 100644 --- a/server/lib/job-queue/handlers/video-transcoding.ts +++ b/server/lib/job-queue/handlers/video-transcoding.ts @@ -53,6 +53,7 @@ async function processVideoTranscoding (job: Job) { if (!handler) { await moveToFailedTranscodingState(video) + await VideoJobInfoModel.decrease(video.uuid, 'pendingTranscode') throw new Error('Cannot find transcoding handler for ' + payload.type) } @@ -62,12 +63,20 @@ async function processVideoTranscoding (job: Job) { } catch (error) { await moveToFailedTranscodingState(video) + await VideoJobInfoModel.decrease(video.uuid, 'pendingTranscode') + throw error } return video } +// --------------------------------------------------------------------------- + +export { + processVideoTranscoding +} + // --------------------------------------------------------------------------- // Job handlers // --------------------------------------------------------------------------- @@ -119,7 +128,7 @@ async function handleWebTorrentMergeAudioJob (job: Job, payload: MergeAudioTrans logger.info('Merge audio transcoding job for %s ended.', video.uuid, lTags(video.uuid)) - await onVideoFileOptimizer(video, payload, 'video', user) + await onVideoFirstWebTorrentTranscoding(video, payload, 'video', user) } async function handleWebTorrentOptimizeJob (job: Job, payload: OptimizeTranscodingPayload, video: MVideoFullLight, user: MUserId) { @@ -129,7 +138,7 @@ async function handleWebTorrentOptimizeJob (job: Job, payload: OptimizeTranscodi logger.info('Optimize transcoding job for %s ended.', video.uuid, lTags(video.uuid)) - await onVideoFileOptimizer(video, payload, transcodeType, user) + await onVideoFirstWebTorrentTranscoding(video, payload, transcodeType, user) } // --------------------------------------------------------------------------- @@ -159,7 +168,7 @@ async function onHlsPlaylistGeneration (video: MVideoFullLight, user: MUser, pay await retryTransactionWrapper(moveToNextState, video, payload.isNewVideo) } -async function onVideoFileOptimizer ( +async function onVideoFirstWebTorrentTranscoding ( videoArg: MVideoWithFile, payload: OptimizeTranscodingPayload | MergeAudioTranscodingPayload, transcodeType: TranscodeOptionsType, @@ -211,6 +220,8 @@ async function onNewWebTorrentFileResolution ( await retryTransactionWrapper(moveToNextState, video, payload.isNewVideo) } +// --------------------------------------------------------------------------- + async function createHlsJobIfEnabled (user: MUserId, payload: { videoUUID: string resolution: number @@ -241,16 +252,6 @@ async function createHlsJobIfEnabled (user: MUserId, payload: { return true } -// --------------------------------------------------------------------------- - -export { - processVideoTranscoding, - createHlsJobIfEnabled, - onNewWebTorrentFileResolution -} - -// --------------------------------------------------------------------------- - async function createLowerResolutionsJobs (options: { video: MVideoFullLight user: MUserId diff --git a/server/models/video/video-job-info.ts b/server/models/video/video-job-info.ts index cb1f3f2f0..7da5128d7 100644 --- a/server/models/video/video-job-info.ts +++ b/server/models/video/video-job-info.ts @@ -3,6 +3,8 @@ import { AllowNull, BelongsTo, Column, CreatedAt, Default, ForeignKey, IsInt, Mo import { AttributesOnly } from '@shared/core-utils' import { VideoModel } from './video' +export type VideoJobInfoColumnType = 'pendingMove' | 'pendingTranscode' + @Table({ tableName: 'videoJobInfo', indexes: [ @@ -57,7 +59,7 @@ export class VideoJobInfoModel extends Model { + static async increaseOrCreate (videoUUID: string, column: VideoJobInfoColumnType): Promise { const options = { type: QueryTypes.SELECT as QueryTypes.SELECT, bind: { videoUUID } } const [ { pendingMove } ] = await VideoJobInfoModel.sequelize.query<{ pendingMove: number }>(` @@ -79,7 +81,7 @@ export class VideoJobInfoModel extends Model { + static async decrease (videoUUID: string, column: VideoJobInfoColumnType): Promise { const options = { type: QueryTypes.SELECT as QueryTypes.SELECT, bind: { videoUUID } } const [ { pendingMove } ] = await VideoJobInfoModel.sequelize.query<{ pendingMove: number }>(` From 5bd427e01c89e178ceaba357ab1f392553c259bb Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Fri, 3 Dec 2021 16:04:45 +0100 Subject: [PATCH 17/28] Fix search on ios --- client/src/app/header/search-typeahead.component.html | 6 +++--- client/src/app/header/search-typeahead.component.ts | 7 ++++++- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/client/src/app/header/search-typeahead.component.html b/client/src/app/header/search-typeahead.component.html index 0ced249a7..9b21d3f86 100644 --- a/client/src/app/header/search-typeahead.component.html +++ b/client/src/app/header/search-typeahead.component.html @@ -1,7 +1,7 @@
    • diff --git a/client/src/app/header/search-typeahead.component.ts b/client/src/app/header/search-typeahead.component.ts index e10baea2b..0794ec8f4 100644 --- a/client/src/app/header/search-typeahead.component.ts +++ b/client/src/app/header/search-typeahead.component.ts @@ -152,7 +152,7 @@ export class SearchTypeaheadComponent implements OnInit, AfterViewChecked, OnDes } } - onSuggestionlicked (payload: SuggestionPayload) { + onSuggestionClicked (payload: SuggestionPayload) { this.doSearch(this.buildSearchTarget(payload)) } @@ -170,6 +170,11 @@ export class SearchTypeaheadComponent implements OnInit, AfterViewChecked, OnDes this.keyboardEventsManager.onKeydown(event) break + + case 'Enter': + event.stopPropagation() + this.doSearch() + break } } From 0bc53e20296de68288481e2375b297626087174b Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Fri, 3 Dec 2021 17:04:47 +0100 Subject: [PATCH 18/28] Add ability to redirect users on external auth --- .../edit-basic-configuration.component.html | 32 ++++++++++++++++++- .../edit-basic-configuration.component.ts | 4 +++ .../edit-custom-config.component.ts | 12 +++++++ client/src/app/+login/login.component.ts | 5 +-- client/src/app/menu/menu.component.html | 4 ++- client/src/app/menu/menu.component.ts | 10 ++++++ client/src/root-helpers/plugins-manager.ts | 6 ++++ config/default.yaml | 6 ++++ config/production.yaml.example | 6 ++++ server/controllers/api/config.ts | 12 +++++++ server/initializers/checker-before-init.ts | 1 + server/initializers/config.ts | 5 +++ server/lib/server-config-manager.ts | 5 +++ server/tests/api/check-params/config.ts | 12 +++++++ server/tests/api/server/config.ts | 18 +++++++++++ shared/extra-utils/server/config-command.ts | 12 +++++++ shared/models/server/custom-config.model.ts | 14 ++++++++ shared/models/server/server-config.model.ts | 6 ++++ 18 files changed, 166 insertions(+), 4 deletions(-) diff --git a/client/src/app/+admin/config/edit-custom-config/edit-basic-configuration.component.html b/client/src/app/+admin/config/edit-custom-config/edit-basic-configuration.component.html index 318c8e2c2..c9533208a 100644 --- a/client/src/app/+admin/config/edit-custom-config/edit-basic-configuration.component.html +++ b/client/src/app/+admin/config/edit-custom-config/edit-basic-configuration.component.html @@ -56,6 +56,36 @@
    + + + + +
    + +
    +     +     + + + +
    + + + ⚠ī¸ You don't have any external auth plugin enabled. + ⚠ī¸ You have multiple external auth plugins enabled. + + +
    +     +     +     +
    @@ -276,7 +306,7 @@
    VIDEO CHANNELS
    - +
    diff --git a/client/src/app/+admin/config/edit-custom-config/edit-basic-configuration.component.ts b/client/src/app/+admin/config/edit-custom-config/edit-basic-configuration.component.ts index 7a8258820..81457bd36 100644 --- a/client/src/app/+admin/config/edit-custom-config/edit-basic-configuration.component.ts +++ b/client/src/app/+admin/config/edit-custom-config/edit-basic-configuration.component.ts @@ -36,6 +36,10 @@ export class EditBasicConfigurationComponent implements OnInit, OnChanges { } } + countExternalAuth () { + return this.serverConfig.plugin.registeredExternalAuths.length + } + getVideoQuotaOptions () { return this.configService.videoQuotaOptions } diff --git a/client/src/app/+admin/config/edit-custom-config/edit-custom-config.component.ts b/client/src/app/+admin/config/edit-custom-config/edit-custom-config.component.ts index fdb0a7532..f2eaa3033 100644 --- a/client/src/app/+admin/config/edit-custom-config/edit-custom-config.component.ts +++ b/client/src/app/+admin/config/edit-custom-config/edit-custom-config.component.ts @@ -106,6 +106,18 @@ export class EditCustomConfigComponent extends FormReactive implements OnInit { whitelisted: null } }, + client: { + videos: { + miniature: { + preferAuthorDisplayName: null + } + }, + menu: { + login: { + redirectOnSingleExternalAuth: null + } + } + }, cache: { previews: { size: CACHE_PREVIEWS_SIZE_VALIDATOR diff --git a/client/src/app/+login/login.component.ts b/client/src/app/+login/login.component.ts index 1fa4bd3b5..648b8db36 100644 --- a/client/src/app/+login/login.component.ts +++ b/client/src/app/+login/login.component.ts @@ -1,4 +1,4 @@ -import { environment } from 'src/environments/environment' + import { AfterViewInit, Component, ElementRef, OnInit, ViewChild } from '@angular/core' import { ActivatedRoute } from '@angular/router' import { AuthService, Notifier, RedirectService, UserService } from '@app/core' @@ -7,6 +7,7 @@ import { LOGIN_PASSWORD_VALIDATOR, LOGIN_USERNAME_VALIDATOR } from '@app/shared/ import { FormReactive, FormValidatorService } from '@app/shared/shared-forms' import { InstanceAboutAccordionComponent } from '@app/shared/shared-instance' import { NgbAccordion, NgbModal, NgbModalRef } from '@ng-bootstrap/ng-bootstrap' +import { PluginsManager } from '@root-helpers/plugins-manager' import { RegisteredExternalAuthConfig, ServerConfig } from '@shared/models' @Component({ @@ -98,7 +99,7 @@ export class LoginComponent extends FormReactive implements OnInit, AfterViewIni } getAuthHref (auth: RegisteredExternalAuthConfig) { - return environment.apiUrl + `/plugins/${auth.name}/${auth.version}/auth/${auth.authName}` + return PluginsManager.getExternalAuthHref(auth) } login () { diff --git a/client/src/app/menu/menu.component.html b/client/src/app/menu/menu.component.html index 55fc27b5f..9ea991042 100644 --- a/client/src/app/menu/menu.component.html +++ b/client/src/app/menu/menu.component.html @@ -99,7 +99,9 @@
    diff --git a/client/src/app/menu/menu.component.ts b/client/src/app/menu/menu.component.ts index 97f07c956..bcc884878 100644 --- a/client/src/app/menu/menu.component.ts +++ b/client/src/app/menu/menu.component.ts @@ -21,6 +21,7 @@ import { LanguageChooserComponent } from '@app/menu/language-chooser.component' import { QuickSettingsModalComponent } from '@app/modal/quick-settings-modal.component' import { PeertubeModalService } from '@app/shared/shared-main/peertube-modal/peertube-modal.service' import { NgbDropdown } from '@ng-bootstrap/ng-bootstrap' +import { PluginsManager } from '@root-helpers/plugins-manager' import { HTMLServerConfig, ServerConfig, UserRight, VideoConstant } from '@shared/models' const logger = debug('peertube:menu:MenuComponent') @@ -129,6 +130,15 @@ export class MenuComponent implements OnInit { .subscribe(() => this.openQuickSettings()) } + getExternalLoginHref () { + if (this.serverConfig.client.menu.login.redirectOnSingleExternalAuth !== true) return undefined + + const externalAuths = this.serverConfig.plugin.registeredExternalAuths + if (externalAuths.length !== 1) return undefined + + return PluginsManager.getExternalAuthHref(externalAuths[0]) + } + isRegistrationAllowed () { if (!this.serverConfig) return false diff --git a/client/src/root-helpers/plugins-manager.ts b/client/src/root-helpers/plugins-manager.ts index a1b763ff2..9cba63373 100644 --- a/client/src/root-helpers/plugins-manager.ts +++ b/client/src/root-helpers/plugins-manager.ts @@ -15,6 +15,7 @@ import { RegisterClientHookOptions, RegisterClientSettingsScript, RegisterClientVideoFieldOptions, + RegisteredExternalAuthConfig, ServerConfigPlugin } from '../../../shared/models' import { environment } from '../environments/environment' @@ -78,6 +79,11 @@ class PluginsManager { return isTheme ? '/themes' : '/plugins' } + static getExternalAuthHref (auth: RegisteredExternalAuthConfig) { + return environment.apiUrl + `/plugins/${auth.name}/${auth.version}/auth/${auth.authName}` + + } + loadPluginsList (config: HTMLServerConfig) { for (const plugin of config.plugin.registered) { this.addPlugin(plugin) diff --git a/config/default.yaml b/config/default.yaml index b9c725cea..074951117 100644 --- a/config/default.yaml +++ b/config/default.yaml @@ -82,6 +82,12 @@ client: # By default PeerTube client displays author username prefer_author_display_name: false + menu: + login: + # If you enable only one external auth plugin + # You can automatically redirect your users on this external platform when they click on the login button + redirect_on_single_external_auth: false + # From the project root directory storage: tmp: 'storage/tmp/' # Use to download data (imports etc), store uploaded files before and during processing... diff --git a/config/production.yaml.example b/config/production.yaml.example index d67349c1d..e38b79587 100644 --- a/config/production.yaml.example +++ b/config/production.yaml.example @@ -80,6 +80,12 @@ client: # By default PeerTube client displays author username prefer_author_display_name: false + menu: + login: + # If you enable only one external auth plugin + # You can automatically redirect your users on this external platform when they click on the login button + redirect_on_single_external_auth: false + # From the project root directory storage: tmp: '/var/www/peertube/storage/tmp/' # Use to download data (imports etc), store uploaded files before and during processing... diff --git a/server/controllers/api/config.ts b/server/controllers/api/config.ts index 805ad99c7..b253db397 100644 --- a/server/controllers/api/config.ts +++ b/server/controllers/api/config.ts @@ -169,6 +169,18 @@ function customConfig (): CustomConfig { whitelisted: CONFIG.SERVICES.TWITTER.WHITELISTED } }, + client: { + videos: { + miniature: { + preferAuthorDisplayName: CONFIG.CLIENT.VIDEOS.MINIATURE.PREFER_AUTHOR_DISPLAY_NAME + } + }, + menu: { + login: { + redirectOnSingleExternalAuth: CONFIG.CLIENT.MENU.LOGIN.REDIRECT_ON_SINGLE_EXTERNAL_AUTH + } + } + }, cache: { previews: { size: CONFIG.CACHE.PREVIEWS.SIZE diff --git a/server/initializers/checker-before-init.ts b/server/initializers/checker-before-init.ts index 51c396548..c85c389cd 100644 --- a/server/initializers/checker-before-init.ts +++ b/server/initializers/checker-before-init.ts @@ -33,6 +33,7 @@ function checkMissedConfig () { 'transcoding.resolutions.2160p', 'import.videos.http.enabled', 'import.videos.torrent.enabled', 'import.videos.concurrency', 'auto_blacklist.videos.of_users.enabled', 'trending.videos.interval_days', + 'client.videos.miniature.prefer_author_display_name', 'client.menu.login.redirect_on_single_external_auth', 'instance.name', 'instance.short_description', 'instance.description', 'instance.terms', 'instance.default_client_route', 'instance.is_nsfw', 'instance.default_nsfw_policy', 'instance.robots', 'instance.securitytxt', 'services.twitter.username', 'services.twitter.whitelisted', diff --git a/server/initializers/config.ts b/server/initializers/config.ts index f3a7c6b6b..eb848be6b 100644 --- a/server/initializers/config.ts +++ b/server/initializers/config.ts @@ -63,6 +63,11 @@ const CONFIG = { MINIATURE: { get PREFER_AUTHOR_DISPLAY_NAME () { return config.get('client.videos.miniature.prefer_author_display_name') } } + }, + MENU: { + LOGIN: { + get REDIRECT_ON_SINGLE_EXTERNAL_AUTH () { return config.get('client.menu.login.redirect_on_single_external_auth') } + } } }, diff --git a/server/lib/server-config-manager.ts b/server/lib/server-config-manager.ts index bdf6492f9..6aa459f82 100644 --- a/server/lib/server-config-manager.ts +++ b/server/lib/server-config-manager.ts @@ -47,6 +47,11 @@ class ServerConfigManager { miniature: { preferAuthorDisplayName: CONFIG.CLIENT.VIDEOS.MINIATURE.PREFER_AUTHOR_DISPLAY_NAME } + }, + menu: { + login: { + redirectOnSingleExternalAuth: CONFIG.CLIENT.MENU.LOGIN.REDIRECT_ON_SINGLE_EXTERNAL_AUTH + } } }, diff --git a/server/tests/api/check-params/config.ts b/server/tests/api/check-params/config.ts index d0cd7722b..a6e87730a 100644 --- a/server/tests/api/check-params/config.ts +++ b/server/tests/api/check-params/config.ts @@ -54,6 +54,18 @@ describe('Test config API validators', function () { whitelisted: true } }, + client: { + videos: { + miniature: { + preferAuthorDisplayName: false + } + }, + menu: { + login: { + redirectOnSingleExternalAuth: false + } + } + }, cache: { previews: { size: 2 diff --git a/server/tests/api/server/config.ts b/server/tests/api/server/config.ts index ea524723c..96ec17b0f 100644 --- a/server/tests/api/server/config.ts +++ b/server/tests/api/server/config.ts @@ -43,6 +43,9 @@ function checkInitialConfig (server: PeerTubeServer, data: CustomConfig) { expect(data.services.twitter.username).to.equal('@Chocobozzz') expect(data.services.twitter.whitelisted).to.be.false + expect(data.client.videos.miniature.preferAuthorDisplayName).to.be.false + expect(data.client.menu.login.redirectOnSingleExternalAuth).to.be.false + expect(data.cache.previews.size).to.equal(1) expect(data.cache.captions.size).to.equal(1) expect(data.cache.torrents.size).to.equal(1) @@ -138,6 +141,9 @@ function checkUpdatedConfig (data: CustomConfig) { expect(data.services.twitter.username).to.equal('@Kuja') expect(data.services.twitter.whitelisted).to.be.true + expect(data.client.videos.miniature.preferAuthorDisplayName).to.be.true + expect(data.client.menu.login.redirectOnSingleExternalAuth).to.be.true + expect(data.cache.previews.size).to.equal(2) expect(data.cache.captions.size).to.equal(3) expect(data.cache.torrents.size).to.equal(4) @@ -246,6 +252,18 @@ const newCustomConfig: CustomConfig = { whitelisted: true } }, + client: { + videos: { + miniature: { + preferAuthorDisplayName: true + } + }, + menu: { + login: { + redirectOnSingleExternalAuth: true + } + } + }, cache: { previews: { size: 2 diff --git a/shared/extra-utils/server/config-command.ts b/shared/extra-utils/server/config-command.ts index 7a768b4df..a061ca89e 100644 --- a/shared/extra-utils/server/config-command.ts +++ b/shared/extra-utils/server/config-command.ts @@ -194,6 +194,18 @@ export class ConfigCommand extends AbstractCommand { whitelisted: true } }, + client: { + videos: { + miniature: { + preferAuthorDisplayName: false + } + }, + menu: { + login: { + redirectOnSingleExternalAuth: false + } + } + }, cache: { previews: { size: 2 diff --git a/shared/models/server/custom-config.model.ts b/shared/models/server/custom-config.model.ts index 3ed932494..52d3d9588 100644 --- a/shared/models/server/custom-config.model.ts +++ b/shared/models/server/custom-config.model.ts @@ -52,6 +52,20 @@ export interface CustomConfig { } } + client: { + videos: { + miniature: { + preferAuthorDisplayName: boolean + } + } + + menu: { + login: { + redirectOnSingleExternalAuth: boolean + } + } + } + cache: { previews: { size: number diff --git a/shared/models/server/server-config.model.ts b/shared/models/server/server-config.model.ts index e75eefd47..9f17276e0 100644 --- a/shared/models/server/server-config.model.ts +++ b/shared/models/server/server-config.model.ts @@ -39,6 +39,12 @@ export interface ServerConfig { preferAuthorDisplayName: boolean } } + + menu: { + login: { + redirectOnSingleExternalAuth: boolean + } + } } webadmin: { From 14a07e01769b585fc341b524b93be31c6c8acb56 Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Mon, 6 Dec 2021 08:57:10 +0100 Subject: [PATCH 19/28] Increase yarn timeout It can be slow when building docker using qemu --- support/docker/production/Dockerfile.bullseye | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/support/docker/production/Dockerfile.bullseye b/support/docker/production/Dockerfile.bullseye index 7b2650538..ec06d6b1d 100644 --- a/support/docker/production/Dockerfile.bullseye +++ b/support/docker/production/Dockerfile.bullseye @@ -21,10 +21,10 @@ WORKDIR /app USER peertube -RUN yarn install --pure-lockfile \ +RUN yarn install --pure-lockfile --network-timeout 600000 \ && npm run build -- $NPM_RUN_BUILD_OPTS \ && rm -r ./node_modules ./client/node_modules \ - && yarn install --pure-lockfile --production \ + && yarn install --pure-lockfile --production --network-timeout 600000 \ && yarn cache clean USER root From 33675a4775d1e1e9dcb58e7e53f5027b81262622 Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Mon, 6 Dec 2021 10:04:31 +0100 Subject: [PATCH 20/28] Increase timeout --- server/tests/api/check-params/plugins.ts | 2 +- server/tests/api/notifications/user-notifications.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/server/tests/api/check-params/plugins.ts b/server/tests/api/check-params/plugins.ts index 33f84ecbc..2c436376c 100644 --- a/server/tests/api/check-params/plugins.ts +++ b/server/tests/api/check-params/plugins.ts @@ -30,7 +30,7 @@ describe('Test server plugins API validators', function () { // --------------------------------------------------------------- before(async function () { - this.timeout(30000) + this.timeout(60000) server = await createSingleServer(1) diff --git a/server/tests/api/notifications/user-notifications.ts b/server/tests/api/notifications/user-notifications.ts index 468efdf35..9af20843e 100644 --- a/server/tests/api/notifications/user-notifications.ts +++ b/server/tests/api/notifications/user-notifications.ts @@ -267,7 +267,7 @@ describe('Test user notifications', function () { }) it('Should send a notification when an imported video is transcoded', async function () { - this.timeout(50000) + this.timeout(120000) const name = 'video import ' + buildUUID() From 80badf493afca026bc542260f353210e605a1715 Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Mon, 6 Dec 2021 16:53:00 +0100 Subject: [PATCH 21/28] Add mute status in account and channel pages --- .../src/app/+accounts/accounts.component.html | 6 +- .../src/app/+accounts/accounts.component.scss | 16 ++- .../src/app/+accounts/accounts.component.ts | 9 +- .../video-channels.component.html | 6 +- .../video-channels.component.ts | 14 ++- .../+video-channels/video-channels.module.ts | 6 +- .../shared-main/account/account.model.ts | 9 +- .../account-block-badges.component.html | 4 + .../account-block-badges.component.scss | 9 ++ .../account-block-badges.component.ts | 11 ++ .../shared-moderation/blocklist.service.ts | 20 +++- .../src/app/shared/shared-moderation/index.ts | 1 + .../shared-moderation.module.ts | 7 +- .../user-moderation-dropdown.component.ts | 4 +- server/controllers/api/blocklist.ts | 108 ++++++++++++++++++ server/controllers/api/index.ts | 2 + .../controllers/api/users/my-subscriptions.ts | 21 ++-- server/helpers/actors.ts | 17 +++ server/lib/blocklist.ts | 4 +- .../shared/comment/comment-mention.ts | 4 +- server/middlewares/validators/blocklist.ts | 29 ++++- server/models/account/account-blocklist.ts | 40 ++++++- server/models/server/server-blocklist.ts | 19 ++- server/tests/api/check-params/blocklist.ts | 72 ++++++++++++ server/tests/api/moderation/blocklist.ts | 107 +++++++++++++++++ shared/extra-utils/users/blocklist-command.ts | 25 +++- .../models/moderation/block-status.model.ts | 15 +++ shared/models/moderation/index.ts | 1 + 28 files changed, 533 insertions(+), 53 deletions(-) create mode 100644 client/src/app/shared/shared-moderation/account-block-badges.component.html create mode 100644 client/src/app/shared/shared-moderation/account-block-badges.component.scss create mode 100644 client/src/app/shared/shared-moderation/account-block-badges.component.ts create mode 100644 server/controllers/api/blocklist.ts create mode 100644 server/helpers/actors.ts create mode 100644 shared/models/moderation/block-status.model.ts diff --git a/client/src/app/+accounts/accounts.component.html b/client/src/app/+accounts/accounts.component.html index 245edfd58..144545250 100644 --- a/client/src/app/+accounts/accounts.component.html +++ b/client/src/app/+accounts/accounts.component.html @@ -19,10 +19,8 @@ > Banned - Muted - Instance muted - Muted by your instance - Instance muted by your instance + +
    diff --git a/client/src/app/+accounts/accounts.component.scss b/client/src/app/+accounts/accounts.component.scss index cdd00487b..5043b98c4 100644 --- a/client/src/app/+accounts/accounts.component.scss +++ b/client/src/app/+accounts/accounts.component.scss @@ -30,16 +30,10 @@ } } -my-user-moderation-dropdown, -.badge { - @include margin-left(10px); +my-user-moderation-dropdown { + margin: 0 10px; - position: relative; - top: 3px; -} - -.badge { - font-size: 13px; + height: fit-content; } .copy-button { @@ -64,6 +58,10 @@ my-user-moderation-dropdown, @include avatar-row-responsive(var(--myImgMargin), var(--myGreyFontSize)); } +.actor-display-name { + align-items: center; +} + .description { grid-column: 1 / 3; max-width: 1000px; diff --git a/client/src/app/+accounts/accounts.component.ts b/client/src/app/+accounts/accounts.component.ts index 3cb117fcc..460f1dbf9 100644 --- a/client/src/app/+accounts/accounts.component.ts +++ b/client/src/app/+accounts/accounts.component.ts @@ -12,7 +12,7 @@ import { VideoChannelService, VideoService } from '@app/shared/shared-main' -import { AccountReportComponent } from '@app/shared/shared-moderation' +import { AccountReportComponent, BlocklistService } from '@app/shared/shared-moderation' import { HttpStatusCode, User, UserRight } from '@shared/models' @Component({ @@ -52,6 +52,7 @@ export class AccountsComponent implements OnInit, OnDestroy { private authService: AuthService, private videoService: VideoService, private markdown: MarkdownService, + private blocklist: BlocklistService, private screenService: ScreenService ) { } @@ -159,6 +160,7 @@ export class AccountsComponent implements OnInit, OnDestroy { this.updateModerationActions() this.loadUserIfNeeded(account) this.loadAccountVideosCount() + this.loadAccountBlockStatus() } private showReportModal () { @@ -217,4 +219,9 @@ export class AccountsComponent implements OnInit, OnDestroy { this.accountVideosCount = res.total }) } + + private loadAccountBlockStatus () { + this.blocklist.getStatus({ accounts: [ this.account.nameWithHostForced ], hosts: [ this.account.host ] }) + .subscribe(status => this.account.updateBlockStatus(status)) + } } diff --git a/client/src/app/+video-channels/video-channels.component.html b/client/src/app/+video-channels/video-channels.component.html index 064fbb6f5..aec2e373c 100644 --- a/client/src/app/+video-channels/video-channels.component.html +++ b/client/src/app/+video-channels/video-channels.component.html @@ -23,14 +23,16 @@
    OWNER ACCOUNT
    - +

    - {{ videoChannel.ownerAccount.displayName }} + {{ ownerAccount.displayName }}

    @{{ videoChannel.ownerBy }}
    + +
    diff --git a/client/src/app/+video-channels/video-channels.component.ts b/client/src/app/+video-channels/video-channels.component.ts index 272fc41d9..ebb991f4e 100644 --- a/client/src/app/+video-channels/video-channels.component.ts +++ b/client/src/app/+video-channels/video-channels.component.ts @@ -4,7 +4,8 @@ import { catchError, distinctUntilChanged, map, switchMap } from 'rxjs/operators import { Component, OnDestroy, OnInit, ViewChild } from '@angular/core' import { ActivatedRoute } from '@angular/router' import { AuthService, MarkdownService, Notifier, RestExtractor, ScreenService } from '@app/core' -import { ListOverflowItem, VideoChannel, VideoChannelService, VideoService } from '@app/shared/shared-main' +import { Account, ListOverflowItem, VideoChannel, VideoChannelService, VideoService } from '@app/shared/shared-main' +import { BlocklistService } from '@app/shared/shared-moderation' import { SupportModalComponent } from '@app/shared/shared-support-modal' import { SubscribeButtonComponent } from '@app/shared/shared-user-subscription' import { HttpStatusCode } from '@shared/models' @@ -18,6 +19,7 @@ export class VideoChannelsComponent implements OnInit, OnDestroy { @ViewChild('supportModal') supportModal: SupportModalComponent videoChannel: VideoChannel + ownerAccount: Account hotkeys: Hotkey[] links: ListOverflowItem[] = [] isChannelManageable = false @@ -38,7 +40,8 @@ export class VideoChannelsComponent implements OnInit, OnDestroy { private restExtractor: RestExtractor, private hotkeysService: HotkeysService, private screenService: ScreenService, - private markdown: MarkdownService + private markdown: MarkdownService, + private blocklist: BlocklistService ) { } ngOnInit () { @@ -58,8 +61,10 @@ export class VideoChannelsComponent implements OnInit, OnDestroy { // After the markdown renderer to avoid layout changes this.videoChannel = videoChannel + this.ownerAccount = new Account(this.videoChannel.ownerAccount) this.loadChannelVideosCount() + this.loadOwnerBlockStatus() }) this.hotkeys = [ @@ -125,4 +130,9 @@ export class VideoChannelsComponent implements OnInit, OnDestroy { sort: '-publishedAt' }).subscribe(res => this.channelVideosCount = res.total) } + + private loadOwnerBlockStatus () { + this.blocklist.getStatus({ accounts: [ this.ownerAccount.nameWithHostForced ], hosts: [ this.ownerAccount.host ] }) + .subscribe(status => this.ownerAccount.updateBlockStatus(status)) + } } diff --git a/client/src/app/+video-channels/video-channels.module.ts b/client/src/app/+video-channels/video-channels.module.ts index 35c39cc2e..76aaecf83 100644 --- a/client/src/app/+video-channels/video-channels.module.ts +++ b/client/src/app/+video-channels/video-channels.module.ts @@ -2,15 +2,16 @@ import { NgModule } from '@angular/core' import { SharedFormModule } from '@app/shared/shared-forms' import { SharedGlobalIconModule } from '@app/shared/shared-icons' import { SharedMainModule } from '@app/shared/shared-main' +import { SharedModerationModule } from '@app/shared/shared-moderation' import { SharedSupportModal } from '@app/shared/shared-support-modal' import { SharedUserSubscriptionModule } from '@app/shared/shared-user-subscription' import { SharedVideoMiniatureModule } from '@app/shared/shared-video-miniature' import { SharedVideoPlaylistModule } from '@app/shared/shared-video-playlist' +import { SharedActorImageModule } from '../shared/shared-actor-image/shared-actor-image.module' import { VideoChannelPlaylistsComponent } from './video-channel-playlists/video-channel-playlists.component' import { VideoChannelVideosComponent } from './video-channel-videos/video-channel-videos.component' import { VideoChannelsRoutingModule } from './video-channels-routing.module' import { VideoChannelsComponent } from './video-channels.component' -import { SharedActorImageModule } from '../shared/shared-actor-image/shared-actor-image.module' @NgModule({ imports: [ @@ -23,7 +24,8 @@ import { SharedActorImageModule } from '../shared/shared-actor-image/shared-acto SharedUserSubscriptionModule, SharedGlobalIconModule, SharedSupportModal, - SharedActorImageModule + SharedActorImageModule, + SharedModerationModule ], declarations: [ diff --git a/client/src/app/shared/shared-main/account/account.model.ts b/client/src/app/shared/shared-main/account/account.model.ts index 92606e7fa..8b78d01a6 100644 --- a/client/src/app/shared/shared-main/account/account.model.ts +++ b/client/src/app/shared/shared-main/account/account.model.ts @@ -1,4 +1,4 @@ -import { Account as ServerAccount, ActorImage } from '@shared/models' +import { Account as ServerAccount, ActorImage, BlockStatus } from '@shared/models' import { Actor } from './actor.model' export class Account extends Actor implements ServerAccount { @@ -49,4 +49,11 @@ export class Account extends Actor implements ServerAccount { resetAvatar () { this.avatar = null } + + updateBlockStatus (blockStatus: BlockStatus) { + this.mutedByInstance = blockStatus.accounts[this.nameWithHostForced].blockedByServer + this.mutedByUser = blockStatus.accounts[this.nameWithHostForced].blockedByUser + this.mutedServerByUser = blockStatus.hosts[this.host].blockedByUser + this.mutedServerByInstance = blockStatus.hosts[this.host].blockedByServer + } } diff --git a/client/src/app/shared/shared-moderation/account-block-badges.component.html b/client/src/app/shared/shared-moderation/account-block-badges.component.html new file mode 100644 index 000000000..feac707c2 --- /dev/null +++ b/client/src/app/shared/shared-moderation/account-block-badges.component.html @@ -0,0 +1,4 @@ +Muted +Instance muted +Muted by your instance +Instance muted by your instance diff --git a/client/src/app/shared/shared-moderation/account-block-badges.component.scss b/client/src/app/shared/shared-moderation/account-block-badges.component.scss new file mode 100644 index 000000000..ccc3666aa --- /dev/null +++ b/client/src/app/shared/shared-moderation/account-block-badges.component.scss @@ -0,0 +1,9 @@ +@use '_variables' as *; +@use '_mixins' as *; + +.badge { + @include margin-right(10px); + + height: fit-content; + font-size: 12px; +} diff --git a/client/src/app/shared/shared-moderation/account-block-badges.component.ts b/client/src/app/shared/shared-moderation/account-block-badges.component.ts new file mode 100644 index 000000000..a72601118 --- /dev/null +++ b/client/src/app/shared/shared-moderation/account-block-badges.component.ts @@ -0,0 +1,11 @@ +import { Component, Input } from '@angular/core' +import { Account } from '../shared-main' + +@Component({ + selector: 'my-account-block-badges', + styleUrls: [ './account-block-badges.component.scss' ], + templateUrl: './account-block-badges.component.html' +}) +export class AccountBlockBadgesComponent { + @Input() account: Account +} diff --git a/client/src/app/shared/shared-moderation/blocklist.service.ts b/client/src/app/shared/shared-moderation/blocklist.service.ts index db2a8c584..f4836c6c4 100644 --- a/client/src/app/shared/shared-moderation/blocklist.service.ts +++ b/client/src/app/shared/shared-moderation/blocklist.service.ts @@ -3,7 +3,7 @@ import { catchError, map } from 'rxjs/operators' import { HttpClient, HttpParams } from '@angular/common/http' import { Injectable } from '@angular/core' import { RestExtractor, RestPagination, RestService } from '@app/core' -import { AccountBlock as AccountBlockServer, ResultList, ServerBlock } from '@shared/models' +import { AccountBlock as AccountBlockServer, BlockStatus, ResultList, ServerBlock } from '@shared/models' import { environment } from '../../../environments/environment' import { Account } from '../shared-main' import { AccountBlock } from './account-block.model' @@ -12,6 +12,7 @@ export enum BlocklistComponentType { Account, Instance } @Injectable() export class BlocklistService { + static BASE_BLOCKLIST_URL = environment.apiUrl + '/api/v1/blocklist' static BASE_USER_BLOCKLIST_URL = environment.apiUrl + '/api/v1/users/me/blocklist' static BASE_SERVER_BLOCKLIST_URL = environment.apiUrl + '/api/v1/server/blocklist' @@ -21,6 +22,23 @@ export class BlocklistService { private restService: RestService ) { } + /** ********************* Blocklist status ***********************/ + + getStatus (options: { + accounts?: string[] + hosts?: string[] + }) { + const { accounts, hosts } = options + + let params = new HttpParams() + + if (accounts) params = this.restService.addArrayParams(params, 'accounts', accounts) + if (hosts) params = this.restService.addArrayParams(params, 'hosts', hosts) + + return this.authHttp.get(BlocklistService.BASE_BLOCKLIST_URL + '/status', { params }) + .pipe(catchError(err => this.restExtractor.handleError(err))) + } + /** ********************* User -> Account blocklist ***********************/ getUserAccountBlocklist (options: { pagination: RestPagination, sort: SortMeta, search?: string }) { diff --git a/client/src/app/shared/shared-moderation/index.ts b/client/src/app/shared/shared-moderation/index.ts index 41c910ffe..da85b2299 100644 --- a/client/src/app/shared/shared-moderation/index.ts +++ b/client/src/app/shared/shared-moderation/index.ts @@ -1,6 +1,7 @@ export * from './report-modals' export * from './abuse.service' +export * from './account-block-badges.component' export * from './account-block.model' export * from './account-blocklist.component' export * from './batch-domains-modal.component' diff --git a/client/src/app/shared/shared-moderation/shared-moderation.module.ts b/client/src/app/shared/shared-moderation/shared-moderation.module.ts index 95213e2bd..7cadda67c 100644 --- a/client/src/app/shared/shared-moderation/shared-moderation.module.ts +++ b/client/src/app/shared/shared-moderation/shared-moderation.module.ts @@ -13,6 +13,7 @@ import { UserBanModalComponent } from './user-ban-modal.component' import { UserModerationDropdownComponent } from './user-moderation-dropdown.component' import { VideoBlockComponent } from './video-block.component' import { VideoBlockService } from './video-block.service' +import { AccountBlockBadgesComponent } from './account-block-badges.component' import { SharedActorImageModule } from '../shared-actor-image/shared-actor-image.module' @NgModule({ @@ -31,7 +32,8 @@ import { SharedActorImageModule } from '../shared-actor-image/shared-actor-image VideoReportComponent, BatchDomainsModalComponent, CommentReportComponent, - AccountReportComponent + AccountReportComponent, + AccountBlockBadgesComponent ], exports: [ @@ -41,7 +43,8 @@ import { SharedActorImageModule } from '../shared-actor-image/shared-actor-image VideoReportComponent, BatchDomainsModalComponent, CommentReportComponent, - AccountReportComponent + AccountReportComponent, + AccountBlockBadgesComponent ], providers: [ diff --git a/client/src/app/shared/shared-moderation/user-moderation-dropdown.component.ts b/client/src/app/shared/shared-moderation/user-moderation-dropdown.component.ts index b18d861d6..e2cd2cdc1 100644 --- a/client/src/app/shared/shared-moderation/user-moderation-dropdown.component.ts +++ b/client/src/app/shared/shared-moderation/user-moderation-dropdown.component.ts @@ -289,13 +289,13 @@ export class UserModerationDropdownComponent implements OnInit, OnChanges { { label: $localize`Mute the instance`, description: $localize`Hide any content from that instance for you.`, - isDisplayed: ({ account }) => !account.userId && account.mutedServerByInstance === false, + isDisplayed: ({ account }) => !account.userId && account.mutedServerByUser === false, handler: ({ account }) => this.blockServerByUser(account.host) }, { label: $localize`Unmute the instance`, description: $localize`Show back content from that instance for you.`, - isDisplayed: ({ account }) => !account.userId && account.mutedServerByInstance === true, + isDisplayed: ({ account }) => !account.userId && account.mutedServerByUser === true, handler: ({ account }) => this.unblockServerByUser(account.host) }, { diff --git a/server/controllers/api/blocklist.ts b/server/controllers/api/blocklist.ts new file mode 100644 index 000000000..1e936ad10 --- /dev/null +++ b/server/controllers/api/blocklist.ts @@ -0,0 +1,108 @@ +import express from 'express' +import { handleToNameAndHost } from '@server/helpers/actors' +import { AccountBlocklistModel } from '@server/models/account/account-blocklist' +import { getServerActor } from '@server/models/application/application' +import { ServerBlocklistModel } from '@server/models/server/server-blocklist' +import { MActorAccountId, MUserAccountId } from '@server/types/models' +import { BlockStatus } from '@shared/models' +import { asyncMiddleware, blocklistStatusValidator, optionalAuthenticate } from '../../middlewares' +import { logger } from '@server/helpers/logger' + +const blocklistRouter = express.Router() + +blocklistRouter.get('/status', + optionalAuthenticate, + blocklistStatusValidator, + asyncMiddleware(getBlocklistStatus) +) + +// --------------------------------------------------------------------------- + +export { + blocklistRouter +} + +// --------------------------------------------------------------------------- + +async function getBlocklistStatus (req: express.Request, res: express.Response) { + const hosts = req.query.hosts as string[] + const accounts = req.query.accounts as string[] + const user = res.locals.oauth?.token.User + + const serverActor = await getServerActor() + + const byAccountIds = [ serverActor.Account.id ] + if (user) byAccountIds.push(user.Account.id) + + const status: BlockStatus = { + accounts: {}, + hosts: {} + } + + const baseOptions = { + byAccountIds, + user, + serverActor, + status + } + + await Promise.all([ + populateServerBlocklistStatus({ ...baseOptions, hosts }), + populateAccountBlocklistStatus({ ...baseOptions, accounts }) + ]) + + return res.json(status) +} + +async function populateServerBlocklistStatus (options: { + byAccountIds: number[] + user?: MUserAccountId + serverActor: MActorAccountId + hosts: string[] + status: BlockStatus +}) { + const { byAccountIds, user, serverActor, hosts, status } = options + + if (!hosts || hosts.length === 0) return + + const serverBlocklistStatus = await ServerBlocklistModel.getBlockStatus(byAccountIds, hosts) + + logger.debug('Got server blocklist status.', { serverBlocklistStatus, byAccountIds, hosts }) + + for (const host of hosts) { + const block = serverBlocklistStatus.find(b => b.host === host) + + status.hosts[host] = getStatus(block, serverActor, user) + } +} + +async function populateAccountBlocklistStatus (options: { + byAccountIds: number[] + user?: MUserAccountId + serverActor: MActorAccountId + accounts: string[] + status: BlockStatus +}) { + const { byAccountIds, user, serverActor, accounts, status } = options + + if (!accounts || accounts.length === 0) return + + const accountBlocklistStatus = await AccountBlocklistModel.getBlockStatus(byAccountIds, accounts) + + logger.debug('Got account blocklist status.', { accountBlocklistStatus, byAccountIds, accounts }) + + for (const account of accounts) { + const sanitizedHandle = handleToNameAndHost(account) + + const block = accountBlocklistStatus.find(b => b.name === sanitizedHandle.name && b.host === sanitizedHandle.host) + + status.accounts[sanitizedHandle.handle] = getStatus(block, serverActor, user) + } +} + +function getStatus (block: { accountId: number }, serverActor: MActorAccountId, user?: MUserAccountId) { + return { + blockedByServer: !!(block && block.accountId === serverActor.Account.id), + blockedByUser: !!(block && user && block.accountId === user.Account.id) + } +} diff --git a/server/controllers/api/index.ts b/server/controllers/api/index.ts index 9949b378a..5f49336b1 100644 --- a/server/controllers/api/index.ts +++ b/server/controllers/api/index.ts @@ -6,6 +6,7 @@ import { badRequest } from '../../helpers/express-utils' import { CONFIG } from '../../initializers/config' import { abuseRouter } from './abuse' import { accountsRouter } from './accounts' +import { blocklistRouter } from './blocklist' import { bulkRouter } from './bulk' import { configRouter } from './config' import { customPageRouter } from './custom-page' @@ -49,6 +50,7 @@ apiRouter.use('/search', searchRouter) apiRouter.use('/overviews', overviewsRouter) apiRouter.use('/plugins', pluginRouter) apiRouter.use('/custom-pages', customPageRouter) +apiRouter.use('/blocklist', blocklistRouter) apiRouter.use('/ping', pong) apiRouter.use('/*', badRequest) diff --git a/server/controllers/api/users/my-subscriptions.ts b/server/controllers/api/users/my-subscriptions.ts index 6799ca8c5..fb1f68635 100644 --- a/server/controllers/api/users/my-subscriptions.ts +++ b/server/controllers/api/users/my-subscriptions.ts @@ -1,5 +1,6 @@ import 'multer' import express from 'express' +import { handlesToNameAndHost } from '@server/helpers/actors' import { pickCommonVideoQuery } from '@server/helpers/query' import { sendUndoFollow } from '@server/lib/activitypub/send' import { guessAdditionalAttributesFromQuery } from '@server/models/video/formatter/video-format-utils' @@ -7,7 +8,6 @@ import { VideoChannelModel } from '@server/models/video/video-channel' import { HttpStatusCode } from '../../../../shared/models/http/http-error-codes' import { buildNSFWFilter, getCountVideos } from '../../../helpers/express-utils' import { getFormattedObjects } from '../../../helpers/utils' -import { WEBSERVER } from '../../../initializers/constants' import { sequelizeTypescript } from '../../../initializers/database' import { JobQueue } from '../../../lib/job-queue' import { @@ -89,28 +89,23 @@ async function areSubscriptionsExist (req: express.Request, res: express.Respons const uris = req.query.uris as string[] const user = res.locals.oauth.token.User - const handles = uris.map(u => { - let [ name, host ] = u.split('@') - if (host === WEBSERVER.HOST) host = null + const sanitizedHandles = handlesToNameAndHost(uris) - return { name, host, uri: u } - }) - - const results = await ActorFollowModel.listSubscriptionsOf(user.Account.Actor.id, handles) + const results = await ActorFollowModel.listSubscriptionsOf(user.Account.Actor.id, sanitizedHandles) const existObject: { [id: string ]: boolean } = {} - for (const handle of handles) { + for (const sanitizedHandle of sanitizedHandles) { const obj = results.find(r => { const server = r.ActorFollowing.Server - return r.ActorFollowing.preferredUsername === handle.name && + return r.ActorFollowing.preferredUsername === sanitizedHandle.name && ( - (!server && !handle.host) || - (server.host === handle.host) + (!server && !sanitizedHandle.host) || + (server.host === sanitizedHandle.host) ) }) - existObject[handle.uri] = obj !== undefined + existObject[sanitizedHandle.handle] = obj !== undefined } return res.json(existObject) diff --git a/server/helpers/actors.ts b/server/helpers/actors.ts new file mode 100644 index 000000000..c31fe6f8e --- /dev/null +++ b/server/helpers/actors.ts @@ -0,0 +1,17 @@ +import { WEBSERVER } from '@server/initializers/constants' + +function handleToNameAndHost (handle: string) { + let [ name, host ] = handle.split('@') + if (host === WEBSERVER.HOST) host = null + + return { name, host, handle } +} + +function handlesToNameAndHost (handles: string[]) { + return handles.map(h => handleToNameAndHost(h)) +} + +export { + handleToNameAndHost, + handlesToNameAndHost +} diff --git a/server/lib/blocklist.ts b/server/lib/blocklist.ts index d6b684015..98273a6ea 100644 --- a/server/lib/blocklist.ts +++ b/server/lib/blocklist.ts @@ -40,12 +40,12 @@ async function isBlockedByServerOrAccount (targetAccount: MAccountServer, userAc if (userAccount) sourceAccounts.push(userAccount.id) - const accountMutedHash = await AccountBlocklistModel.isAccountMutedByMulti(sourceAccounts, targetAccount.id) + const accountMutedHash = await AccountBlocklistModel.isAccountMutedByAccounts(sourceAccounts, targetAccount.id) if (accountMutedHash[serverAccountId] || (userAccount && accountMutedHash[userAccount.id])) { return true } - const instanceMutedHash = await ServerBlocklistModel.isServerMutedByMulti(sourceAccounts, targetAccount.Actor.serverId) + const instanceMutedHash = await ServerBlocklistModel.isServerMutedByAccounts(sourceAccounts, targetAccount.Actor.serverId) if (instanceMutedHash[serverAccountId] || (userAccount && instanceMutedHash[userAccount.id])) { return true } diff --git a/server/lib/notifier/shared/comment/comment-mention.ts b/server/lib/notifier/shared/comment/comment-mention.ts index 4f84d8dea..765cbaad9 100644 --- a/server/lib/notifier/shared/comment/comment-mention.ts +++ b/server/lib/notifier/shared/comment/comment-mention.ts @@ -47,8 +47,8 @@ export class CommentMention extends AbstractNotification u.Account.id).concat([ this.serverAccountId ]) - this.accountMutedHash = await AccountBlocklistModel.isAccountMutedByMulti(sourceAccounts, this.payload.accountId) - this.instanceMutedHash = await ServerBlocklistModel.isServerMutedByMulti(sourceAccounts, this.payload.Account.Actor.serverId) + this.accountMutedHash = await AccountBlocklistModel.isAccountMutedByAccounts(sourceAccounts, this.payload.accountId) + this.instanceMutedHash = await ServerBlocklistModel.isServerMutedByAccounts(sourceAccounts, this.payload.Account.Actor.serverId) } log () { diff --git a/server/middlewares/validators/blocklist.ts b/server/middlewares/validators/blocklist.ts index b7749e204..12980ced4 100644 --- a/server/middlewares/validators/blocklist.ts +++ b/server/middlewares/validators/blocklist.ts @@ -1,8 +1,10 @@ import express from 'express' -import { body, param } from 'express-validator' +import { body, param, query } from 'express-validator' +import { areValidActorHandles } from '@server/helpers/custom-validators/activitypub/actor' +import { toArray } from '@server/helpers/custom-validators/misc' import { getServerActor } from '@server/models/application/application' import { HttpStatusCode } from '../../../shared/models/http/http-error-codes' -import { isHostValid } from '../../helpers/custom-validators/servers' +import { isEachUniqueHostValid, isHostValid } from '../../helpers/custom-validators/servers' import { logger } from '../../helpers/logger' import { WEBSERVER } from '../../initializers/constants' import { AccountBlocklistModel } from '../../models/account/account-blocklist' @@ -123,6 +125,26 @@ const unblockServerByServerValidator = [ } ] +const blocklistStatusValidator = [ + query('hosts') + .optional() + .customSanitizer(toArray) + .custom(isEachUniqueHostValid).withMessage('Should have a valid hosts array'), + + query('accounts') + .optional() + .customSanitizer(toArray) + .custom(areValidActorHandles).withMessage('Should have a valid accounts array'), + + (req: express.Request, res: express.Response, next: express.NextFunction) => { + logger.debug('Checking blocklistStatusValidator parameters', { query: req.query }) + + if (areValidationErrors(req, res)) return + + return next() + } +] + // --------------------------------------------------------------------------- export { @@ -131,7 +153,8 @@ export { unblockAccountByAccountValidator, unblockServerByAccountValidator, unblockAccountByServerValidator, - unblockServerByServerValidator + unblockServerByServerValidator, + blocklistStatusValidator } // --------------------------------------------------------------------------- diff --git a/server/models/account/account-blocklist.ts b/server/models/account/account-blocklist.ts index b2375b006..21983428a 100644 --- a/server/models/account/account-blocklist.ts +++ b/server/models/account/account-blocklist.ts @@ -1,11 +1,12 @@ -import { Op } from 'sequelize' +import { Op, QueryTypes } from 'sequelize' import { BelongsTo, Column, CreatedAt, ForeignKey, Model, Scopes, Table, UpdatedAt } from 'sequelize-typescript' +import { handlesToNameAndHost } from '@server/helpers/actors' import { MAccountBlocklist, MAccountBlocklistAccounts, MAccountBlocklistFormattable } from '@server/types/models' import { AttributesOnly } from '@shared/core-utils' import { AccountBlock } from '../../../shared/models' import { ActorModel } from '../actor/actor' import { ServerModel } from '../server/server' -import { getSort, searchAttribute } from '../utils' +import { createSafeIn, getSort, searchAttribute } from '../utils' import { AccountModel } from './account' enum ScopeNames { @@ -77,7 +78,7 @@ export class AccountBlocklistModel extends Model entries.map(e => `${e.BlockedAccount.Actor.preferredUsername}@${e.BlockedAccount.Actor.Server.host}`)) } + static getBlockStatus (byAccountIds: number[], handles: string[]): Promise<{ name: string, host: string, accountId: number }[]> { + const sanitizedHandles = handlesToNameAndHost(handles) + + const localHandles = sanitizedHandles.filter(h => !h.host) + .map(h => h.name) + + const remoteHandles = sanitizedHandles.filter(h => !!h.host) + .map(h => ([ h.name, h.host ])) + + const handlesWhere: string[] = [] + + if (localHandles.length !== 0) { + handlesWhere.push(`("actor"."preferredUsername" IN (:localHandles) AND "server"."id" IS NULL)`) + } + + if (remoteHandles.length !== 0) { + handlesWhere.push(`(("actor"."preferredUsername", "server"."host") IN (:remoteHandles))`) + } + + const rawQuery = `SELECT "accountBlocklist"."accountId", "actor"."preferredUsername" AS "name", "server"."host" ` + + `FROM "accountBlocklist" ` + + `INNER JOIN "account" ON "account"."id" = "accountBlocklist"."targetAccountId" ` + + `INNER JOIN "actor" ON "actor"."id" = "account"."actorId" ` + + `LEFT JOIN "server" ON "server"."id" = "actor"."serverId" ` + + `WHERE "accountBlocklist"."accountId" IN (${createSafeIn(AccountBlocklistModel.sequelize, byAccountIds)}) ` + + `AND (${handlesWhere.join(' OR ')})` + + return AccountBlocklistModel.sequelize.query(rawQuery, { + type: QueryTypes.SELECT as QueryTypes.SELECT, + replacements: { byAccountIds, localHandles, remoteHandles } + }) + } + toFormattedJSON (this: MAccountBlocklistFormattable): AccountBlock { return { byAccount: this.ByAccount.toFormattedJSON(), diff --git a/server/models/server/server-blocklist.ts b/server/models/server/server-blocklist.ts index b3579d589..092998db3 100644 --- a/server/models/server/server-blocklist.ts +++ b/server/models/server/server-blocklist.ts @@ -1,10 +1,10 @@ -import { Op } from 'sequelize' +import { Op, QueryTypes } from 'sequelize' import { BelongsTo, Column, CreatedAt, ForeignKey, Model, Scopes, Table, UpdatedAt } from 'sequelize-typescript' import { MServerBlocklist, MServerBlocklistAccountServer, MServerBlocklistFormattable } from '@server/types/models' import { AttributesOnly } from '@shared/core-utils' import { ServerBlock } from '@shared/models' import { AccountModel } from '../account/account' -import { getSort, searchAttribute } from '../utils' +import { createSafeIn, getSort, searchAttribute } from '../utils' import { ServerModel } from './server' enum ScopeNames { @@ -76,7 +76,7 @@ export class ServerBlocklistModel extends Model entries.map(e => e.BlockedServer.host)) } + static getBlockStatus (byAccountIds: number[], hosts: string[]): Promise<{ host: string, accountId: number }[]> { + const rawQuery = `SELECT "server"."host", "serverBlocklist"."accountId" ` + + `FROM "serverBlocklist" ` + + `INNER JOIN "server" ON "server"."id" = "serverBlocklist"."targetServerId" ` + + `WHERE "server"."host" IN (:hosts) ` + + `AND "serverBlocklist"."accountId" IN (${createSafeIn(ServerBlocklistModel.sequelize, byAccountIds)})` + + return ServerBlocklistModel.sequelize.query(rawQuery, { + type: QueryTypes.SELECT as QueryTypes.SELECT, + replacements: { hosts } + }) + } + static listForApi (parameters: { start: number count: number diff --git a/server/tests/api/check-params/blocklist.ts b/server/tests/api/check-params/blocklist.ts index 7d5fae5cf..f72a892e2 100644 --- a/server/tests/api/check-params/blocklist.ts +++ b/server/tests/api/check-params/blocklist.ts @@ -481,6 +481,78 @@ describe('Test blocklist API validators', function () { }) }) + describe('When getting blocklist status', function () { + const path = '/api/v1/blocklist/status' + + it('Should fail with a bad token', async function () { + await makeGetRequest({ + url: server.url, + path, + token: 'false', + expectedStatus: HttpStatusCode.UNAUTHORIZED_401 + }) + }) + + it('Should fail with a bad accounts field', async function () { + await makeGetRequest({ + url: server.url, + path, + query: { + accounts: 1 + }, + expectedStatus: HttpStatusCode.BAD_REQUEST_400 + }) + + await makeGetRequest({ + url: server.url, + path, + query: { + accounts: [ 1 ] + }, + expectedStatus: HttpStatusCode.BAD_REQUEST_400 + }) + }) + + it('Should fail with a bad hosts field', async function () { + await makeGetRequest({ + url: server.url, + path, + query: { + hosts: 1 + }, + expectedStatus: HttpStatusCode.BAD_REQUEST_400 + }) + + await makeGetRequest({ + url: server.url, + path, + query: { + hosts: [ 1 ] + }, + expectedStatus: HttpStatusCode.BAD_REQUEST_400 + }) + }) + + it('Should succeed with the correct parameters', async function () { + await makeGetRequest({ + url: server.url, + path, + query: {}, + expectedStatus: HttpStatusCode.OK_200 + }) + + await makeGetRequest({ + url: server.url, + path, + query: { + hosts: [ 'example.com' ], + accounts: [ 'john@example.com' ] + }, + expectedStatus: HttpStatusCode.OK_200 + }) + }) + }) + after(async function () { await cleanupTests(servers) }) diff --git a/server/tests/api/moderation/blocklist.ts b/server/tests/api/moderation/blocklist.ts index 089af8b15..b3fd8ecac 100644 --- a/server/tests/api/moderation/blocklist.ts +++ b/server/tests/api/moderation/blocklist.ts @@ -254,6 +254,45 @@ describe('Test blocklist', function () { } }) + it('Should get blocked status', async function () { + const remoteHandle = 'user2@' + servers[1].host + const localHandle = 'user1@' + servers[0].host + const unknownHandle = 'user5@' + servers[0].host + + { + const status = await command.getStatus({ accounts: [ remoteHandle ] }) + expect(Object.keys(status.accounts)).to.have.lengthOf(1) + expect(status.accounts[remoteHandle].blockedByUser).to.be.false + expect(status.accounts[remoteHandle].blockedByServer).to.be.false + + expect(Object.keys(status.hosts)).to.have.lengthOf(0) + } + + { + const status = await command.getStatus({ token: servers[0].accessToken, accounts: [ remoteHandle ] }) + expect(Object.keys(status.accounts)).to.have.lengthOf(1) + expect(status.accounts[remoteHandle].blockedByUser).to.be.true + expect(status.accounts[remoteHandle].blockedByServer).to.be.false + + expect(Object.keys(status.hosts)).to.have.lengthOf(0) + } + + { + const status = await command.getStatus({ token: servers[0].accessToken, accounts: [ localHandle, remoteHandle, unknownHandle ] }) + expect(Object.keys(status.accounts)).to.have.lengthOf(3) + + for (const handle of [ localHandle, remoteHandle ]) { + expect(status.accounts[handle].blockedByUser).to.be.true + expect(status.accounts[handle].blockedByServer).to.be.false + } + + expect(status.accounts[unknownHandle].blockedByUser).to.be.false + expect(status.accounts[unknownHandle].blockedByServer).to.be.false + + expect(Object.keys(status.hosts)).to.have.lengthOf(0) + } + }) + it('Should not allow a remote blocked user to comment my videos', async function () { this.timeout(60000) @@ -434,6 +473,35 @@ describe('Test blocklist', function () { expect(block.blockedServer.host).to.equal('localhost:' + servers[1].port) }) + it('Should get blocklist status', async function () { + const blockedServer = servers[1].host + const notBlockedServer = 'example.com' + + { + const status = await command.getStatus({ hosts: [ blockedServer, notBlockedServer ] }) + expect(Object.keys(status.accounts)).to.have.lengthOf(0) + + expect(Object.keys(status.hosts)).to.have.lengthOf(2) + expect(status.hosts[blockedServer].blockedByUser).to.be.false + expect(status.hosts[blockedServer].blockedByServer).to.be.false + + expect(status.hosts[notBlockedServer].blockedByUser).to.be.false + expect(status.hosts[notBlockedServer].blockedByServer).to.be.false + } + + { + const status = await command.getStatus({ token: servers[0].accessToken, hosts: [ blockedServer, notBlockedServer ] }) + expect(Object.keys(status.accounts)).to.have.lengthOf(0) + + expect(Object.keys(status.hosts)).to.have.lengthOf(2) + expect(status.hosts[blockedServer].blockedByUser).to.be.true + expect(status.hosts[blockedServer].blockedByServer).to.be.false + + expect(status.hosts[notBlockedServer].blockedByUser).to.be.false + expect(status.hosts[notBlockedServer].blockedByServer).to.be.false + } + }) + it('Should unblock the remote server', async function () { await command.removeFromMyBlocklist({ server: 'localhost:' + servers[1].port }) }) @@ -575,6 +643,27 @@ describe('Test blocklist', function () { } }) + it('Should get blocked status', async function () { + const remoteHandle = 'user2@' + servers[1].host + const localHandle = 'user1@' + servers[0].host + const unknownHandle = 'user5@' + servers[0].host + + for (const token of [ undefined, servers[0].accessToken ]) { + const status = await command.getStatus({ token, accounts: [ localHandle, remoteHandle, unknownHandle ] }) + expect(Object.keys(status.accounts)).to.have.lengthOf(3) + + for (const handle of [ localHandle, remoteHandle ]) { + expect(status.accounts[handle].blockedByUser).to.be.false + expect(status.accounts[handle].blockedByServer).to.be.true + } + + expect(status.accounts[unknownHandle].blockedByUser).to.be.false + expect(status.accounts[unknownHandle].blockedByServer).to.be.false + + expect(Object.keys(status.hosts)).to.have.lengthOf(0) + } + }) + it('Should unblock the remote account', async function () { await command.removeFromServerBlocklist({ account: 'user2@localhost:' + servers[1].port }) }) @@ -620,6 +709,7 @@ describe('Test blocklist', function () { }) describe('When managing server blocklist', function () { + it('Should list all videos', async function () { for (const token of [ userModeratorToken, servers[0].accessToken ]) { await checkAllVideos(servers[0], token) @@ -713,6 +803,23 @@ describe('Test blocklist', function () { expect(block.blockedServer.host).to.equal('localhost:' + servers[1].port) }) + it('Should get blocklist status', async function () { + const blockedServer = servers[1].host + const notBlockedServer = 'example.com' + + for (const token of [ undefined, servers[0].accessToken ]) { + const status = await command.getStatus({ token, hosts: [ blockedServer, notBlockedServer ] }) + expect(Object.keys(status.accounts)).to.have.lengthOf(0) + + expect(Object.keys(status.hosts)).to.have.lengthOf(2) + expect(status.hosts[blockedServer].blockedByUser).to.be.false + expect(status.hosts[blockedServer].blockedByServer).to.be.true + + expect(status.hosts[notBlockedServer].blockedByUser).to.be.false + expect(status.hosts[notBlockedServer].blockedByServer).to.be.false + } + }) + it('Should unblock the remote server', async function () { await command.removeFromServerBlocklist({ server: 'localhost:' + servers[1].port }) }) diff --git a/shared/extra-utils/users/blocklist-command.ts b/shared/extra-utils/users/blocklist-command.ts index 14491a1ae..2e7ed074d 100644 --- a/shared/extra-utils/users/blocklist-command.ts +++ b/shared/extra-utils/users/blocklist-command.ts @@ -1,6 +1,6 @@ /* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ -import { AccountBlock, HttpStatusCode, ResultList, ServerBlock } from '@shared/models' +import { AccountBlock, BlockStatus, HttpStatusCode, ResultList, ServerBlock } from '@shared/models' import { AbstractCommand, OverrideCommandOptions } from '../shared' type ListBlocklistOptions = OverrideCommandOptions & { @@ -37,6 +37,29 @@ export class BlocklistCommand extends AbstractCommand { // --------------------------------------------------------------------------- + getStatus (options: OverrideCommandOptions & { + accounts?: string[] + hosts?: string[] + }) { + const { accounts, hosts } = options + + const path = '/api/v1/blocklist/status' + + return this.getRequestBody({ + ...options, + + path, + query: { + accounts, + hosts + }, + implicitToken: false, + defaultExpectedStatus: HttpStatusCode.OK_200 + }) + } + + // --------------------------------------------------------------------------- + addToMyBlocklist (options: OverrideCommandOptions & { account?: string server?: string diff --git a/shared/models/moderation/block-status.model.ts b/shared/models/moderation/block-status.model.ts new file mode 100644 index 000000000..597312757 --- /dev/null +++ b/shared/models/moderation/block-status.model.ts @@ -0,0 +1,15 @@ +export interface BlockStatus { + accounts: { + [ handle: string ]: { + blockedByServer: boolean + blockedByUser?: boolean + } + } + + hosts: { + [ host: string ]: { + blockedByServer: boolean + blockedByUser?: boolean + } + } +} diff --git a/shared/models/moderation/index.ts b/shared/models/moderation/index.ts index 8b6042e97..f8e6d351c 100644 --- a/shared/models/moderation/index.ts +++ b/shared/models/moderation/index.ts @@ -1,3 +1,4 @@ export * from './abuse' +export * from './block-status.model' export * from './account-block.model' export * from './server-block.model' From ae9809a7d021fcaf01ee482a9c6c59fdd50125a6 Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Tue, 7 Dec 2021 11:05:45 +0100 Subject: [PATCH 22/28] Fix report modal error --- client/src/app/+accounts/accounts.component.html | 2 +- client/src/app/+accounts/accounts.component.ts | 2 +- .../report-modals/account-report.component.ts | 13 +++++++------ 3 files changed, 9 insertions(+), 8 deletions(-) diff --git a/client/src/app/+accounts/accounts.component.html b/client/src/app/+accounts/accounts.component.html index 245edfd58..0bb24de2e 100644 --- a/client/src/app/+accounts/accounts.component.html +++ b/client/src/app/+accounts/accounts.component.html @@ -85,5 +85,5 @@
    - + diff --git a/client/src/app/+accounts/accounts.component.ts b/client/src/app/+accounts/accounts.component.ts index 3cb117fcc..0dcbc250a 100644 --- a/client/src/app/+accounts/accounts.component.ts +++ b/client/src/app/+accounts/accounts.component.ts @@ -162,7 +162,7 @@ export class AccountsComponent implements OnInit, OnDestroy { } private showReportModal () { - this.accountReportModal.show() + this.accountReportModal.show(this.account) } private loadUserIfNeeded (account: Account) { diff --git a/client/src/app/shared/shared-moderation/report-modals/account-report.component.ts b/client/src/app/shared/shared-moderation/report-modals/account-report.component.ts index 4ec02f11a..78c9b3382 100644 --- a/client/src/app/shared/shared-moderation/report-modals/account-report.component.ts +++ b/client/src/app/shared/shared-moderation/report-modals/account-report.component.ts @@ -1,5 +1,5 @@ import { mapValues, pickBy } from 'lodash-es' -import { Component, Input, OnInit, ViewChild } from '@angular/core' +import { Component, OnInit, ViewChild } from '@angular/core' import { Notifier } from '@app/core' import { ABUSE_REASON_VALIDATOR } from '@app/shared/form-validators/abuse-validators' import { FormReactive, FormValidatorService } from '@app/shared/shared-forms' @@ -16,13 +16,12 @@ import { AbuseService } from '../abuse.service' styleUrls: [ './report.component.scss' ] }) export class AccountReportComponent extends FormReactive implements OnInit { - @Input() account: Account = null - @ViewChild('modal', { static: true }) modal: NgbModal error: string = null predefinedReasons: { id: AbusePredefinedReasonsString, label: string, description?: string, help?: string }[] = [] modalTitle: string + account: Account = null private openedModal: NgbModalRef @@ -48,8 +47,6 @@ export class AccountReportComponent extends FormReactive implements OnInit { } ngOnInit () { - this.modalTitle = $localize`Report ${this.account.displayName}` - this.buildForm({ reason: ABUSE_REASON_VALIDATOR, predefinedReasons: mapValues(abusePredefinedReasonsMap, r => null) @@ -58,7 +55,11 @@ export class AccountReportComponent extends FormReactive implements OnInit { this.predefinedReasons = this.abuseService.getPrefefinedReasons('account') } - show () { + show (account: Account) { + this.account = account + + this.modalTitle = $localize`Report ${this.account.displayName}` + this.openedModal = this.modalService.open(this.modal, { centered: true, keyboard: false, size: 'lg' }) } From 6e14e40053961cf0b2f55710847f21a3e1b5b627 Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Tue, 7 Dec 2021 11:06:35 +0100 Subject: [PATCH 23/28] Fix iframe allow-popups for oembed --- server/controllers/services.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/controllers/services.ts b/server/controllers/services.ts index 8a8a95486..9151e1b04 100644 --- a/server/controllers/services.ts +++ b/server/controllers/services.ts @@ -98,7 +98,7 @@ function buildOEmbed (options: { thumbnailUrl = undefined } - const html = `` const json: any = { From bdf70330e4d74091ccc9d64534d3f780d5ee8ef4 Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Tue, 7 Dec 2021 11:30:53 +0100 Subject: [PATCH 24/28] Add account block status in openapi --- support/doc/api/openapi.yaml | 56 ++++++++++++++++++++++++++++++++++++ 1 file changed, 56 insertions(+) diff --git a/support/doc/api/openapi.yaml b/support/doc/api/openapi.yaml index cfba7b361..1d5581072 100644 --- a/support/doc/api/openapi.yaml +++ b/support/doc/api/openapi.yaml @@ -3792,6 +3792,39 @@ paths: '500': description: search index unavailable + /blocklist/status: + get: + tags: + - Account Blocks + - Server Blocks + summary: Get block status of accounts/hosts + parameters: + - + name: 'accounts' + in: query + description: 'Check if these accounts are blocked' + example: [ 'goofy@example.com', 'donald@example.com' ] + schema: + type: array + items: + type: string + - + name: 'hosts' + in: query + description: 'Check if these hosts are blocked' + example: [ 'example.com' ] + schema: + type: array + items: + type: string + responses: + '200': + description: successful operation + content: + 'application/json': + schema: + $ref: '#/components/schemas/BlockStatus' + /server/blocklist/accounts: get: tags: @@ -5134,6 +5167,29 @@ components: label: type: string + BlockStatus: + properties: + accounts: + type: object + additionalProperties: + x-additionalPropertiesName: account + type: object + properties: + blockedByServer: + type: boolean + blockedByUser: + type: boolean + hosts: + type: object + additionalProperties: + x-additionalPropertiesName: host + type: object + properties: + blockedByServer: + type: boolean + blockedByUser: + type: boolean + NSFWPolicy: type: string enum: From 51d811751f1a299c61d64e11fbe600edeb6d586d Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Tue, 7 Dec 2021 11:32:19 +0100 Subject: [PATCH 25/28] Fix live duration in player --- client/src/assets/player/p2p-media-loader/hls-plugin.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/client/src/assets/player/p2p-media-loader/hls-plugin.ts b/client/src/assets/player/p2p-media-loader/hls-plugin.ts index 2aa691f05..421ce4934 100644 --- a/client/src/assets/player/p2p-media-loader/hls-plugin.ts +++ b/client/src/assets/player/p2p-media-loader/hls-plugin.ts @@ -146,6 +146,7 @@ class Html5Hlsjs { } duration () { + if (this._duration === Infinity) return Infinity if (!isNaN(this.videoElement.duration)) return this.videoElement.duration return this._duration || 0 From 4c6757f2fd2111e51e51e1b0d8f21f3b5e16ff40 Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Tue, 7 Dec 2021 13:45:01 +0100 Subject: [PATCH 26/28] Fix live log tags --- server/lib/live/shared/muxing-session.ts | 43 ++++++++++++++++-------- 1 file changed, 29 insertions(+), 14 deletions(-) diff --git a/server/lib/live/shared/muxing-session.ts b/server/lib/live/shared/muxing-session.ts index c71f4e25f..eccaefcfa 100644 --- a/server/lib/live/shared/muxing-session.ts +++ b/server/lib/live/shared/muxing-session.ts @@ -135,13 +135,20 @@ class MuxingSession extends EventEmitter { }) : getLiveMuxingCommand(this.inputUrl, outPath, this.streamingPlaylist.playlistFilename) - logger.info('Running live muxing/transcoding for %s.', this.videoUUID, this.lTags) + logger.info('Running live muxing/transcoding for %s.', this.videoUUID, this.lTags()) this.watchTSFiles(outPath) this.watchMasterFile(outPath) + let ffmpegShellCommand: string + this.ffmpegCommand.on('start', cmdline => { + ffmpegShellCommand = cmdline + + logger.debug('Running ffmpeg command for live', { ffmpegShellCommand, ...this.lTags() }) + }) + this.ffmpegCommand.on('error', (err, stdout, stderr) => { - this.onFFmpegError(err, stdout, stderr, outPath) + this.onFFmpegError({ err, stdout, stderr, outPath, ffmpegShellCommand }) }) this.ffmpegCommand.on('end', () => this.onFFmpegEnded(outPath)) @@ -161,19 +168,27 @@ class MuxingSession extends EventEmitter { this.hasClientSocketInBadHealthWithCache.clear() } - private onFFmpegError (err: any, stdout: string, stderr: string, outPath: string) { + private onFFmpegError (options: { + err: any + stdout: string + stderr: string + outPath: string + ffmpegShellCommand: string + }) { + const { err, stdout, stderr, outPath, ffmpegShellCommand } = options + this.onFFmpegEnded(outPath) // Don't care that we killed the ffmpeg process if (err?.message?.includes('Exiting normally')) return - logger.error('Live transcoding error.', { err, stdout, stderr, ...this.lTags }) + logger.error('Live transcoding error.', { err, stdout, stderr, ffmpegShellCommand, ...this.lTags() }) this.emit('ffmpeg-error', ({ sessionId: this.sessionId })) } private onFFmpegEnded (outPath: string) { - logger.info('RTMP transmuxing for video %s ended. Scheduling cleanup', this.inputUrl, this.lTags) + logger.info('RTMP transmuxing for video %s ended. Scheduling cleanup', this.inputUrl, this.lTags()) setTimeout(() => { // Wait latest segments generation, and close watchers @@ -188,7 +203,7 @@ class MuxingSession extends EventEmitter { .catch(err => { logger.error( 'Cannot close watchers of %s or process remaining hash segments.', outPath, - { err, ...this.lTags } + { err, ...this.lTags() } ) }) @@ -203,7 +218,7 @@ class MuxingSession extends EventEmitter { this.emit('master-playlist-created', { videoId: this.videoId }) this.masterWatcher.close() - .catch(err => logger.error('Cannot close master watcher of %s.', outPath, { err, ...this.lTags })) + .catch(err => logger.error('Cannot close master watcher of %s.', outPath, { err, ...this.lTags() })) }) } @@ -215,7 +230,7 @@ class MuxingSession extends EventEmitter { const playlistIdMatcher = /^([\d+])-/ const addHandler = async segmentPath => { - logger.debug('Live add handler of %s.', segmentPath, this.lTags) + logger.debug('Live add handler of %s.', segmentPath, this.lTags()) const playlistId = basename(segmentPath).match(playlistIdMatcher)[0] @@ -259,7 +274,7 @@ class MuxingSession extends EventEmitter { return canUpload !== true } catch (err) { - logger.error('Cannot stat %s or check quota of %d.', segmentPath, this.user.id, { err, ...this.lTags }) + logger.error('Cannot stat %s or check quota of %d.', segmentPath, this.user.id, { err, ...this.lTags() }) } } @@ -277,7 +292,7 @@ class MuxingSession extends EventEmitter { }) VideoFileModel.customUpsert(file, 'streaming-playlist', null) - .catch(err => logger.error('Cannot create file for live streaming.', { err, ...this.lTags })) + .catch(err => logger.error('Cannot create file for live streaming.', { err, ...this.lTags() })) } } @@ -313,14 +328,14 @@ class MuxingSession extends EventEmitter { if (this.saveReplay) { await this.addSegmentToReplay(hlsVideoPath, previousSegment) } - }).catch(err => logger.error('Cannot process segments in %s', hlsVideoPath, { err, ...this.lTags })) + }).catch(err => logger.error('Cannot process segments in %s', hlsVideoPath, { err, ...this.lTags() })) } private hasClientSocketInBadHealth (sessionId: string) { const rtmpSession = this.context.sessions.get(sessionId) if (!rtmpSession) { - logger.warn('Cannot get session %s to check players socket health.', sessionId, this.lTags) + logger.warn('Cannot get session %s to check players socket health.', sessionId, this.lTags()) return } @@ -328,7 +343,7 @@ class MuxingSession extends EventEmitter { const playerSession = this.context.sessions.get(playerSessionId) if (!playerSession) { - logger.error('Cannot get player session %s to check socket health.', playerSession, this.lTags) + logger.error('Cannot get player session %s to check socket health.', playerSession, this.lTags()) continue } @@ -349,7 +364,7 @@ class MuxingSession extends EventEmitter { await appendFile(dest, data) } catch (err) { - logger.error('Cannot copy segment %s to replay directory.', segmentPath, { err, ...this.lTags }) + logger.error('Cannot copy segment %s to replay directory.', segmentPath, { err, ...this.lTags() }) } } } From 7e44bd4a6f39373ef1751d4d19cdcf7de65bdcf9 Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Tue, 7 Dec 2021 13:46:07 +0100 Subject: [PATCH 27/28] Fix services tests --- server/tests/api/server/services.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/server/tests/api/server/services.ts b/server/tests/api/server/services.ts index 3a87df981..823630ae4 100644 --- a/server/tests/api/server/services.ts +++ b/server/tests/api/server/services.ts @@ -56,7 +56,7 @@ describe('Test services', function () { const oembedUrl = server.url + basePath + video.uuid + suffix const res = await server.services.getOEmbed({ oembedUrl }) - const expectedHtml = '' const expectedThumbnailUrl = 'http://localhost:' + server.port + video.previewPath @@ -79,7 +79,7 @@ describe('Test services', function () { const oembedUrl = server.url + basePath + playlistUUID + suffix const res = await server.services.getOEmbed({ oembedUrl }) - const expectedHtml = '' @@ -103,7 +103,7 @@ describe('Test services', function () { const maxWidth = 50 const res = await server.services.getOEmbed({ oembedUrl, format, maxHeight, maxWidth }) - const expectedHtml = '' From f307255e694d290c346749bff68545d5dfc581cf Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Tue, 7 Dec 2021 15:20:03 +0100 Subject: [PATCH 28/28] Display server logs when server run fails --- shared/extra-utils/server/server.ts | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/shared/extra-utils/server/server.ts b/shared/extra-utils/server/server.ts index 31224ebe9..9da293877 100644 --- a/shared/extra-utils/server/server.ts +++ b/shared/extra-utils/server/server.ts @@ -220,10 +220,11 @@ export class PeerTubeServer { return new Promise((res, rej) => { const self = this + let aggregatedLogs = '' this.app = fork(join(root(), 'dist', 'server.js'), options.peertubeArgs || [], forkOptions) - const onPeerTubeExit = () => rej(new Error('Process exited')) + const onPeerTubeExit = () => rej(new Error('Process exited:\n' + aggregatedLogs)) const onParentExit = () => { if (!this.app || !this.app.pid) return @@ -238,10 +239,13 @@ export class PeerTubeServer { this.app.stdout.on('data', function onStdout (data) { let dontContinue = false + const log: string = data.toString() + aggregatedLogs += log + // Capture things if we want to for (const key of Object.keys(regexps)) { const regexp = regexps[key] - const matches = data.toString().match(regexp) + const matches = log.match(regexp) if (matches !== null) { if (key === 'client_id') self.store.client.id = matches[1] else if (key === 'client_secret') self.store.client.secret = matches[1] @@ -252,7 +256,7 @@ export class PeerTubeServer { // Check if all required sentences are here for (const key of Object.keys(serverRunString)) { - if (data.toString().indexOf(key) !== -1) serverRunString[key] = true + if (log.includes(key)) serverRunString[key] = true if (serverRunString[key] === false) dontContinue = true } @@ -260,7 +264,7 @@ export class PeerTubeServer { if (dontContinue === true) return if (options.hideLogs === false) { - console.log(data.toString()) + console.log(log) } else { process.removeListener('exit', onParentExit) self.app.stdout.removeListener('data', onStdout)