borekon/PeerTube_original
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Process broadcast requests in parallel

Browse Source
This commit is contained in:
Chocobozzz 2018-04-18 16:04:49 +02:00
parent 5350fd8e5b
commit f55e5a7bf8
No known key found for this signature in database
GPG Key ID: 583A612D890159BE
3 changed files with 13 additions and 13 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
server.ts
View File

@ -215,7 +215,8 @@ async function startApplication () {
Redis.Instance.init() Redis.Instance.init()
// Make server listening // Make server listening
server.listen(port, hostname) server.listen(port, hostname, () => {
logger.info('Server listening on %s:%d', hostname, port) logger.info('Server listening on %s:%d', hostname, port)
logger.info('Web server: %s', CONFIG.WEBSERVER.URL) logger.info('Web server: %s', CONFIG.WEBSERVER.URL)
})
} }

2
server/initializers/constants.ts
View File

@ -77,6 +77,7 @@ const JOB_CONCURRENCY: { [ id in JobType ]: number } = {
'video-file': 1, 'video-file': 1,
'email': 5 'email': 5
} }
const BROADCAST_CONCURRENCY = 5 // How many requests in parallel we do in activitypub-http-broadcast job
// 2 days // 2 days
const JOB_COMPLETED_LIFETIME = 60000 * 60 * 24 * 2 const JOB_COMPLETED_LIFETIME = 60000 * 60 * 24 * 2
@ -463,6 +464,7 @@ export {
LAST_MIGRATION_VERSION, LAST_MIGRATION_VERSION,
OAUTH_LIFETIME, OAUTH_LIFETIME,
OPENGRAPH_AND_OEMBED_COMMENT, OPENGRAPH_AND_OEMBED_COMMENT,
BROADCAST_CONCURRENCY,
PAGINATION_COUNT_DEFAULT, PAGINATION_COUNT_DEFAULT,
ACTOR_FOLLOW_SCORE, ACTOR_FOLLOW_SCORE,
PREVIEWS_SIZE, PREVIEWS_SIZE,

17
server/lib/job-queue/handlers/activitypub-http-broadcast.ts
View File

@ -1,8 +1,10 @@
import * as kue from 'kue' import * as kue from 'kue'
import * as Bluebird from 'bluebird'
import { logger } from '../../../helpers/logger' import { logger } from '../../../helpers/logger'
import { doRequest } from '../../../helpers/requests' import { doRequest } from '../../../helpers/requests'
import { ActorFollowModel } from '../../../models/activitypub/actor-follow' import { ActorFollowModel } from '../../../models/activitypub/actor-follow'
import { buildSignedRequestOptions, computeBody } from './utils/activitypub-http-utils' import { buildSignedRequestOptions, computeBody } from './utils/activitypub-http-utils'
import { BROADCAST_CONCURRENCY } from '../../../initializers'
export type ActivitypubHttpBroadcastPayload = { export type ActivitypubHttpBroadcastPayload = {
uris: string[] uris: string[]
@ -28,16 +30,11 @@ async function processActivityPubHttpBroadcast (job: kue.Job) {
const badUrls: string[] = [] const badUrls: string[] = []
const goodUrls: string[] = [] const goodUrls: string[] = []
for (const uri of payload.uris) { await Bluebird.map(payload.uris, uri => {
options.uri = uri return doRequest(Object.assign({}, options, { uri }))
.then(() => goodUrls.push(uri))
try { .catch(() => badUrls.push(uri))
await doRequest(options) }, { concurrency: BROADCAST_CONCURRENCY })
goodUrls.push(uri)
} catch (err) {
badUrls.push(uri)
}
}
return ActorFollowModel.updateActorFollowsScore(goodUrls, badUrls, undefined) return ActorFollowModel.updateActorFollowsScore(goodUrls, badUrls, undefined)
} }