diff --git a/.config/example.yml b/.config/example.yml index c037a280b6..c82e744ee1 100644 --- a/.config/example.yml +++ b/.config/example.yml @@ -287,5 +287,10 @@ checkActivityPubGetSignature: false # Upload or download file size limits (bytes) #maxFileSize: 262144000 +# timeout and maximum size for imports (e.g. note imports) +#import: +# downloadTimeout: 30 +# maxFileSize: 262144000 + # PID File of master process #pidFile: /tmp/misskey.pid diff --git a/packages/backend/src/config.ts b/packages/backend/src/config.ts index 7eb5b86b18..250b5c0536 100644 --- a/packages/backend/src/config.ts +++ b/packages/backend/src/config.ts @@ -97,6 +97,12 @@ type Source = { perChannelMaxNoteCacheCount?: number; perUserNotificationsMaxCount?: number; deactivateAntennaThreshold?: number; + + import?: { + downloadTimeout: number; + maxFileSize: number; + }; + pidFile: string; }; @@ -177,6 +183,12 @@ export type Config = { perChannelMaxNoteCacheCount: number; perUserNotificationsMaxCount: number; deactivateAntennaThreshold: number; + + import: { + downloadTimeout: number; + maxFileSize: number; + } | undefined; + pidFile: string; }; @@ -284,6 +296,7 @@ export function loadConfig(): Config { perChannelMaxNoteCacheCount: config.perChannelMaxNoteCacheCount ?? 1000, perUserNotificationsMaxCount: config.perUserNotificationsMaxCount ?? 500, deactivateAntennaThreshold: config.deactivateAntennaThreshold ?? (1000 * 60 * 60 * 24 * 7), + import: config.import, pidFile: config.pidFile, }; } @@ -427,4 +440,5 @@ function applyEnvOverrides(config: Source) { _apply_top([['clusterLimit', 'deliverJobConcurrency', 'inboxJobConcurrency', 'relashionshipJobConcurrency', 'deliverJobPerSec', 'inboxJobPerSec', 'relashionshipJobPerSec', 'deliverJobMaxAttempts', 'inboxJobMaxAttempts']]); _apply_top([['outgoingAddress', 'outgoingAddressFamily', 'proxy', 'proxySmtp', 'mediaProxy', 'videoThumbnailGenerator']]); _apply_top([['maxFileSize', 'maxNoteLength', 'pidFile']]); + _apply_top(['import', ['downloadTimeout', 'maxFileSize']]); } diff --git a/packages/backend/src/core/DownloadService.ts b/packages/backend/src/core/DownloadService.ts index 21ae798f9f..83452845d4 100644 --- a/packages/backend/src/core/DownloadService.ts +++ b/packages/backend/src/core/DownloadService.ts @@ -35,14 +35,14 @@ export class DownloadService { } @bindThis - public async downloadUrl(url: string, path: string): Promise<{ + public async downloadUrl(url: string, path: string, options: { timeout?: number, operationTimeout?: number, maxSize?: number} = {} ): Promise<{ filename: string; }> { this.logger.info(`Downloading ${chalk.cyan(url)} to ${chalk.cyanBright(path)} ...`); - const timeout = 30 * 1000; - const operationTimeout = 60 * 1000; - const maxSize = this.config.maxFileSize ?? 262144000; + const timeout = options.timeout ?? 30 * 1000; + const operationTimeout = options.operationTimeout ?? 60 * 1000; + const maxSize = options.maxSize ?? this.config.maxFileSize ?? 262144000; const urlObj = new URL(url); let filename = urlObj.pathname.split('/').pop() ?? 'untitled'; diff --git a/packages/backend/src/queue/processors/ImportNotesProcessorService.ts b/packages/backend/src/queue/processors/ImportNotesProcessorService.ts index 7cef858c51..58a0ea10ad 100644 --- a/packages/backend/src/queue/processors/ImportNotesProcessorService.ts +++ b/packages/backend/src/queue/processors/ImportNotesProcessorService.ts @@ -19,12 +19,16 @@ import { IdService } from '@/core/IdService.js'; import { QueueLoggerService } from '../QueueLoggerService.js'; import type * as Bull from 'bullmq'; import type { DbNoteImportToDbJobData, DbNoteImportJobData, DbNoteWithParentImportToDbJobData } from '../types.js'; +import type { Config } from '@/config.js'; @Injectable() export class ImportNotesProcessorService { private logger: Logger; constructor( + @Inject(DI.config) + private config: Config, + @Inject(DI.usersRepository) private usersRepository: UsersRepository, @@ -73,6 +77,11 @@ export class ImportNotesProcessorService { } } + @bindThis + private downloadUrl(url: string, path:string): Promise<{filename: string}> { + return this.downloadService.downloadUrl(url, path, { operationTimeout: this.config.import?.downloadTimeout, maxSize: this.config.import?.maxFileSize }); + } + @bindThis private async recreateChain(idFieldPath: string[], replyFieldPath: string[], arr: any[], includeOrphans: boolean): Promise { type NotesMap = { @@ -176,7 +185,7 @@ export class ImportNotesProcessorService { try { await fsp.writeFile(destPath, '', 'binary'); - await this.downloadService.downloadUrl(file.url, destPath); + await this.downloadUrl(file.url, destPath); } catch (e) { // TODO: 何度か再試行 if (e instanceof Error || typeof e === 'string') { this.logger.error(e); @@ -206,7 +215,7 @@ export class ImportNotesProcessorService { try { await fsp.writeFile(destPath, '', 'binary'); - await this.downloadService.downloadUrl(file.url, destPath); + await this.downloadUrl(file.url, destPath); } catch (e) { // TODO: 何度か再試行 if (e instanceof Error || typeof e === 'string') { this.logger.error(e); @@ -239,7 +248,7 @@ export class ImportNotesProcessorService { try { await fsp.writeFile(destPath, '', 'binary'); - await this.downloadService.downloadUrl(file.url, destPath); + await this.downloadUrl(file.url, destPath); } catch (e) { // TODO: 何度か再試行 if (e instanceof Error || typeof e === 'string') { this.logger.error(e); @@ -297,7 +306,7 @@ export class ImportNotesProcessorService { try { await fsp.writeFile(path, '', 'utf-8'); - await this.downloadService.downloadUrl(file.url, path); + await this.downloadUrl(file.url, path); } catch (e) { // TODO: 何度か再試行 if (e instanceof Error || typeof e === 'string') { this.logger.error(e); @@ -349,7 +358,7 @@ export class ImportNotesProcessorService { if (!exists) { try { - await this.downloadService.downloadUrl(file.url, filePath); + await this.downloadUrl(file.url, filePath); } catch (e) { // TODO: 何度か再試行 this.logger.error(e instanceof Error ? e : new Error(e as string)); } @@ -488,7 +497,7 @@ export class ImportNotesProcessorService { if (!exists) { try { - await this.downloadService.downloadUrl(file.url, filePath); + await this.downloadUrl(file.url, filePath); } catch (e) { // TODO: 何度か再試行 this.logger.error(e instanceof Error ? e : new Error(e as string)); }