diff --git a/.config/ci.yml b/.config/ci.yml index c381d21d92..44092d3662 100644 --- a/.config/ci.yml +++ b/.config/ci.yml @@ -106,7 +106,7 @@ redis: # ┌───────────────────────────┐ #───┘ MeiliSearch configuration └───────────────────────────── -# You can set scope to local (default value) or global +# You can set scope to local (default value) or global # (include notes from remote). #meilisearch: @@ -198,13 +198,18 @@ proxyRemoteFiles: true # https://example.com/thumbnail.webp?thumbnail=1&url=https%3A%2F%2Fstorage.example.com%2Fpath%2Fto%2Fvideo.mp4 #videoThumbnailGenerator: https://example.com -# Sign to ActivityPub GET request (default: true) +# Sign outgoing ActivityPub GET request (default: true) signToActivityPubGet: true +# Sign outgoing ActivityPub Activities (default: true) +# Linked Data signatures are cryptographic signatures attached to each activity to provide proof of authenticity. +# When using authorized fetch, this is often undesired as any signed activity can be forwarded to a blocked instance by relays and other instances. +# This setting allows admins to disable LD signatures for increased privacy, at the expense of fewer relayed activities and additional inbound fetch (GET) requests. +attachLdSignatureForRelays: true # check that inbound ActivityPub GET requests are signed ("authorized fetch") checkActivityPubGetSignature: false # For security reasons, uploading attachments from the intranet is prohibited, -# but exceptions can be made from the following settings. Default value is "undefined". +# but exceptions can be made from the following settings. Default value is "undefined". # Read changelog to learn more (Improvements of 12.90.0 (2021/09/04)). #allowedPrivateNetworks: [ # '127.0.0.1/32' diff --git a/.config/docker_example.yml b/.config/docker_example.yml index c33b158afa..de95f1b21a 100644 --- a/.config/docker_example.yml +++ b/.config/docker_example.yml @@ -273,8 +273,13 @@ proxyRemoteFiles: true # https://example.com/thumbnail.webp?thumbnail=1&url=https%3A%2F%2Fstorage.example.com%2Fpath%2Fto%2Fvideo.mp4 #videoThumbnailGenerator: https://example.com -# Sign to ActivityPub GET request (default: true) +# Sign outgoing ActivityPub GET request (default: true) signToActivityPubGet: true +# Sign outgoing ActivityPub Activities (default: true) +# Linked Data signatures are cryptographic signatures attached to each activity to provide proof of authenticity. +# When using authorized fetch, this is often undesired as any signed activity can be forwarded to a blocked instance by relays and other instances. +# This setting allows admins to disable LD signatures for increased privacy, at the expense of fewer relayed activities and additional inbound fetch (GET) requests. +attachLdSignatureForRelays: true # check that inbound ActivityPub GET requests are signed ("authorized fetch") checkActivityPubGetSignature: false diff --git a/.config/example.yml b/.config/example.yml index ae55b983bb..21e85b7b89 100644 --- a/.config/example.yml +++ b/.config/example.yml @@ -285,8 +285,13 @@ proxyRemoteFiles: true # https://example.com/thumbnail.webp?thumbnail=1&url=https%3A%2F%2Fstorage.example.com%2Fpath%2Fto%2Fvideo.mp4 #videoThumbnailGenerator: https://example.com -# Sign to ActivityPub GET request (default: true) +# Sign outgoing ActivityPub GET request (default: true) signToActivityPubGet: true +# Sign outgoing ActivityPub Activities (default: true) +# Linked Data signatures are cryptographic signatures attached to each activity to provide proof of authenticity. +# When using authorized fetch, this is often undesired as any signed activity can be forwarded to a blocked instance by relays and other instances. +# This setting allows admins to disable LD signatures for increased privacy, at the expense of fewer relayed activities and additional inbound fetch (GET) requests. +attachLdSignatureForRelays: true # check that inbound ActivityPub GET requests are signed ("authorized fetch") checkActivityPubGetSignature: false diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 7a017ca7cc..4ac34b726f 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -111,6 +111,15 @@ If your language is not listed in Crowdin, please open an issue. ![Crowdin](https://d322cqt584bo4o.cloudfront.net/misskey/localized.svg) +## Icon Font (Shark Font) +Sharkey has its own Icon Font called Shark Font which can be found at https://activitypub.software/TransFem-org/shark-font +Build Instructions can all be found over there in the `README`. + +If you have an Icon Suggestion or want to add an Icon please open an issue/merge request over at that repo. + +When Updating the Font make sure to copy **all generated files** from the `dest` folder into `packages/backend/assets/fonts/sharkey-icons` +For the CSS simply copy the file content and replace the old content in `style.css` and for the WOFF, TTF and SVG simply replace them. + ## Development ### Setup Before developing, you have to set up environment. Misskey requires Redis, PostgreSQL, and FFmpeg. diff --git a/chart/files/default.yml b/chart/files/default.yml index 2e1381ec57..aab7ed6ce1 100644 --- a/chart/files/default.yml +++ b/chart/files/default.yml @@ -208,8 +208,13 @@ id: "aidx" # Media Proxy #mediaProxy: https://example.com/proxy -# Sign to ActivityPub GET request (default: true) +# Sign outgoing ActivityPub GET request (default: true) signToActivityPubGet: true +# Sign outgoing ActivityPub Activities (default: true) +# Linked Data signatures are cryptographic signatures attached to each activity to provide proof of authenticity. +# When using authorized fetch, this is often undesired as any signed activity can be forwarded to a blocked instance by relays and other instances. +# This setting allows admins to disable LD signatures for increased privacy, at the expense of fewer relayed activities and additional inbound fetch (GET) requests. +attachLdSignatureForRelays: true # check that inbound ActivityPub GET requests are signed ("authorized fetch") checkActivityPubGetSignature: false diff --git a/packages/backend/assets/fonts/sharkey-icons/custom-sharkey-icons.svg b/packages/backend/assets/fonts/sharkey-icons/custom-sharkey-icons.svg deleted file mode 100644 index 9d21137072..0000000000 Binary files a/packages/backend/assets/fonts/sharkey-icons/custom-sharkey-icons.svg and /dev/null differ diff --git a/packages/backend/assets/fonts/sharkey-icons/custom-sharkey-icons.ttf b/packages/backend/assets/fonts/sharkey-icons/custom-sharkey-icons.ttf deleted file mode 100644 index a2601e0f1b..0000000000 Binary files a/packages/backend/assets/fonts/sharkey-icons/custom-sharkey-icons.ttf and /dev/null differ diff --git a/packages/backend/assets/fonts/sharkey-icons/custom-sharkey-icons.woff b/packages/backend/assets/fonts/sharkey-icons/custom-sharkey-icons.woff deleted file mode 100644 index d9f471fa35..0000000000 Binary files a/packages/backend/assets/fonts/sharkey-icons/custom-sharkey-icons.woff and /dev/null differ diff --git a/packages/backend/assets/fonts/sharkey-icons/shark-font.svg b/packages/backend/assets/fonts/sharkey-icons/shark-font.svg new file mode 100644 index 0000000000..b67bd7f7d8 Binary files /dev/null and b/packages/backend/assets/fonts/sharkey-icons/shark-font.svg differ diff --git a/packages/backend/assets/fonts/sharkey-icons/shark-font.ttf b/packages/backend/assets/fonts/sharkey-icons/shark-font.ttf new file mode 100644 index 0000000000..0fdf04df08 Binary files /dev/null and b/packages/backend/assets/fonts/sharkey-icons/shark-font.ttf differ diff --git a/packages/backend/assets/fonts/sharkey-icons/shark-font.woff b/packages/backend/assets/fonts/sharkey-icons/shark-font.woff new file mode 100644 index 0000000000..993666bc3a Binary files /dev/null and b/packages/backend/assets/fonts/sharkey-icons/shark-font.woff differ diff --git a/packages/backend/assets/fonts/sharkey-icons/style.css b/packages/backend/assets/fonts/sharkey-icons/style.css index 7fb0f94504..7168702e5a 100644 --- a/packages/backend/assets/fonts/sharkey-icons/style.css +++ b/packages/backend/assets/fonts/sharkey-icons/style.css @@ -1,31 +1,114 @@ -@charset "UTF-8"; - @font-face { - font-family: "custom-sharkey-icons"; - src: url("./custom-sharkey-icons.woff") format("woff"), - url("./custom-sharkey-icons.ttf") format("truetype"), - url("./custom-sharkey-icons.svg#custom-sharkey-icons") format("svg"); - font-weight: normal; + font-display: auto; + font-family: "shark-font"; font-style: normal; - font-display: block; + font-weight: normal; + + src: url("./shark-font.woff?1722899913909") format("woff"), url("./shark-font.ttf?1722899913909") format("truetype"), url("./shark-font.svg?1722899913909#shark-font") format("svg"); } .sk-icons { - font-family: "custom-sharkey-icons" !important; - font-style: normal; + display: inline-block; + font-family: "shark-font"; font-weight: normal; + font-style: normal; font-variant: normal; - text-transform: none; + text-rendering: auto; line-height: 1; - speak: none; - -webkit-font-smoothing: antialiased; -moz-osx-font-smoothing: grayscale; + -webkit-font-smoothing: antialiased; } -.sk-icons.sk-shark:before { - content: "\61"; +.sk-icons-lg { + font-size: 1.33333em; + line-height: 0.75em; + vertical-align: -0.0667em; } -.sk-icons.sk-misskey:before { - content: "\62"; +.sk-icons-xs { + font-size: 0.75em; } + +.sk-icons-sm { + font-size: 0.875em; +} + +.sk-icons-1x { + font-size: 1em; +} + +.sk-icons-2x { + font-size: 2em; +} + +.sk-icons-3x { + font-size: 3em; +} + +.sk-icons-4x { + font-size: 4em; +} + +.sk-icons-5x { + font-size: 5em; +} + +.sk-icons-6x { + font-size: 6em; +} + +.sk-icons-7x { + font-size: 7em; +} + +.sk-icons-8x { + font-size: 8em; +} + +.sk-icons-9x { + font-size: 9em; +} + +.sk-icons-10x { + font-size: 10em; +} + +.sk-icons-fw { + text-align: center; + width: 1.25em; +} + +.sk-icons-border { + border: solid 0.08em #eee; + border-radius: 0.1em; + padding: 0.2em 0.25em 0.15em; +} + +.sk-icons-pull-left { + float: left; +} + +.sk-icons-pull-right { + float: right; +} + +.sk-icons.sk-icons-pull-left { + margin-right: 0.3em; +} + +.sk-icons.sk-icons-pull-right { + margin-left: 0.3em; +} + + +.sk-icons.sk-foldermove::before { + content: "\ea01"; +} + +.sk-icons.sk-misskey::before { + content: "\ea02"; +} + +.sk-icons.sk-shark::before { + content: "\ea03"; +} \ No newline at end of file diff --git a/packages/backend/scripts/check_connect.js b/packages/backend/scripts/check_connect.js index ba25fd416c..d4bf4baf43 100644 --- a/packages/backend/scripts/check_connect.js +++ b/packages/backend/scripts/check_connect.js @@ -5,11 +5,33 @@ import Redis from 'ioredis'; import { loadConfig } from '../built/config.js'; +import { createPostgresDataSource } from '../built/postgres.js'; const config = loadConfig(); -const redis = new Redis(config.redis); -redis.on('connect', () => redis.disconnect()); -redis.on('error', (e) => { - throw e; -}); +// createPostgresDataSource handels primaries and replicas automatically. +// usually, it only opens connections first use, so we force it using +// .initialize() +createPostgresDataSource(config) + .initialize() + .then(c => { c.destroy() }) + .catch(e => { throw e }); + + +// Connect to all redis servers +function connectToRedis(redisOptions) { + const redis = new Redis(redisOptions); + redis.on('connect', () => redis.disconnect()); + redis.on('error', (e) => { + throw e; + }); +} + +// If not all of these are defined, the default one gets reused. +// so we use a Set to only try connecting once to each **uniq** redis. +(new Set([ + config.redis, + config.redisForPubsub, + config.redisForJobQueue, + config.redisForTimelines, +])).forEach(connectToRedis); diff --git a/packages/backend/src/config.ts b/packages/backend/src/config.ts index 64f2d2ac65..3f9967ad19 100644 --- a/packages/backend/src/config.ts +++ b/packages/backend/src/config.ts @@ -96,6 +96,7 @@ type Source = { customMOTD?: string[]; signToActivityPubGet?: boolean; + attachLdSignatureForRelays?: boolean; checkActivityPubGetSignature?: boolean; perChannelMaxNoteCacheCount?: number; @@ -162,6 +163,7 @@ export type Config = { proxyRemoteFiles: boolean | undefined; customMOTD: string[] | undefined; signToActivityPubGet: boolean; + attachLdSignatureForRelays: boolean; checkActivityPubGetSignature: boolean | undefined; version: string; @@ -303,6 +305,7 @@ export function loadConfig(): Config { proxyRemoteFiles: config.proxyRemoteFiles, customMOTD: config.customMOTD, signToActivityPubGet: config.signToActivityPubGet ?? true, + attachLdSignatureForRelays: config.attachLdSignatureForRelays ?? true, checkActivityPubGetSignature: config.checkActivityPubGetSignature, mediaProxy: externalMediaProxy ?? internalMediaProxy, externalMediaProxyEnabled: externalMediaProxy !== null && externalMediaProxy !== internalMediaProxy, diff --git a/packages/backend/src/core/AvatarDecorationService.ts b/packages/backend/src/core/AvatarDecorationService.ts index 8b54bbe012..4efd6122b1 100644 --- a/packages/backend/src/core/AvatarDecorationService.ts +++ b/packages/backend/src/core/AvatarDecorationService.ts @@ -29,7 +29,7 @@ export class AvatarDecorationService implements OnApplicationShutdown { private moderationLogService: ModerationLogService, private globalEventService: GlobalEventService, ) { - this.cache = new MemorySingleCache(1000 * 60 * 30); + this.cache = new MemorySingleCache(1000 * 60 * 30); // 30s this.redisForSub.on('message', this.onMessage); } diff --git a/packages/backend/src/core/CacheService.ts b/packages/backend/src/core/CacheService.ts index d008e7ec52..6725ebe75b 100644 --- a/packages/backend/src/core/CacheService.ts +++ b/packages/backend/src/core/CacheService.ts @@ -56,10 +56,10 @@ export class CacheService implements OnApplicationShutdown { ) { //this.onMessage = this.onMessage.bind(this); - this.userByIdCache = new MemoryKVCache(Infinity); - this.localUserByNativeTokenCache = new MemoryKVCache(Infinity); - this.localUserByIdCache = new MemoryKVCache(Infinity); - this.uriPersonCache = new MemoryKVCache(Infinity); + this.userByIdCache = new MemoryKVCache(1000 * 60 * 5); // 5m + this.localUserByNativeTokenCache = new MemoryKVCache(1000 * 60 * 5); // 5m + this.localUserByIdCache = new MemoryKVCache(1000 * 60 * 5); // 5m + this.uriPersonCache = new MemoryKVCache(1000 * 60 * 5); // 5m this.userProfileCache = new RedisKVCache(this.redisClient, 'userProfile', { lifetime: 1000 * 60 * 30, // 30m @@ -135,14 +135,14 @@ export class CacheService implements OnApplicationShutdown { if (user == null) { this.userByIdCache.delete(body.id); this.localUserByIdCache.delete(body.id); - for (const [k, v] of this.uriPersonCache.cache.entries()) { + for (const [k, v] of this.uriPersonCache.entries) { if (v.value?.id === body.id) { this.uriPersonCache.delete(k); } } } else { this.userByIdCache.set(user.id, user); - for (const [k, v] of this.uriPersonCache.cache.entries()) { + for (const [k, v] of this.uriPersonCache.entries) { if (v.value?.id === user.id) { this.uriPersonCache.set(k, user); } diff --git a/packages/backend/src/core/CustomEmojiService.ts b/packages/backend/src/core/CustomEmojiService.ts index 1eff2fdbff..dd5f79a22b 100644 --- a/packages/backend/src/core/CustomEmojiService.ts +++ b/packages/backend/src/core/CustomEmojiService.ts @@ -26,7 +26,7 @@ const parseEmojiStrRegexp = /^([-\w]+)(?:@([\w.-]+))?$/; @Injectable() export class CustomEmojiService implements OnApplicationShutdown { - private cache: MemoryKVCache; + private emojisCache: MemoryKVCache; public localEmojisCache: RedisSingleCache>; constructor( @@ -49,7 +49,7 @@ export class CustomEmojiService implements OnApplicationShutdown { private globalEventService: GlobalEventService, private driveService: DriveService, ) { - this.cache = new MemoryKVCache(1000 * 60 * 60 * 12); + this.emojisCache = new MemoryKVCache(1000 * 60 * 60 * 12); // 12h this.localEmojisCache = new RedisSingleCache>(this.redisClient, 'localEmojis', { lifetime: 1000 * 60 * 30, // 30m @@ -142,6 +142,13 @@ export class CustomEmojiService implements OnApplicationShutdown { this.localEmojisCache.refresh(); + if (data.driveFile != null) { + const file = await this.driveFilesRepository.findOneBy({ url: emoji.originalUrl, userHost: emoji.host ? emoji.host : IsNull() }); + if (file && file.id != data.driveFile.id) { + await this.driveService.deleteFile(file, false, moderator ? moderator : undefined); + } + } + const packed = await this.emojiEntityService.packDetailed(emoji.id); if (emoji.name === data.name) { @@ -350,14 +357,14 @@ export class CustomEmojiService implements OnApplicationShutdown { if (name == null) return null; if (host == null) return null; - const newHost = host === this.config.host ? null : host; + const newHost = host === this.config.host ? null : host; const queryOrNull = async () => (await this.emojisRepository.findOneBy({ name, host: newHost ?? IsNull(), })) ?? null; - const emoji = await this.cache.fetch(`${name} ${host}`, queryOrNull); + const emoji = await this.emojisCache.fetch(`${name} ${host}`, queryOrNull); if (emoji == null) return null; return emoji.publicUrl || emoji.originalUrl; // || emoji.originalUrl してるのは後方互換性のため（publicUrlはstringなので??はだめ） @@ -384,7 +391,7 @@ export class CustomEmojiService implements OnApplicationShutdown { */ @bindThis public async prefetchEmojis(emojis: { name: string; host: string | null; }[]): Promise { - const notCachedEmojis = emojis.filter(emoji => this.cache.get(`${emoji.name} ${emoji.host}`) == null); + const notCachedEmojis = emojis.filter(emoji => this.emojisCache.get(`${emoji.name} ${emoji.host}`) == null); const emojisQuery: any[] = []; const hosts = new Set(notCachedEmojis.map(e => e.host)); for (const host of hosts) { @@ -399,7 +406,7 @@ export class CustomEmojiService implements OnApplicationShutdown { select: ['name', 'host', 'originalUrl', 'publicUrl'], }) : []; for (const emoji of _emojis) { - this.cache.set(`${emoji.name} ${emoji.host}`, emoji); + this.emojisCache.set(`${emoji.name} ${emoji.host}`, emoji); } } @@ -424,7 +431,7 @@ export class CustomEmojiService implements OnApplicationShutdown { @bindThis public dispose(): void { - this.cache.dispose(); + this.emojisCache.dispose(); } @bindThis diff --git a/packages/backend/src/core/MfmService.ts b/packages/backend/src/core/MfmService.ts index 5619ea4d7b..048910cb52 100644 --- a/packages/backend/src/core/MfmService.ts +++ b/packages/backend/src/core/MfmService.ts @@ -6,7 +6,7 @@ import { URL } from 'node:url'; import { Inject, Injectable } from '@nestjs/common'; import * as parse5 from 'parse5'; -import { Window, XMLSerializer } from 'happy-dom'; +import { Window, DocumentFragment, XMLSerializer } from 'happy-dom'; import { DI } from '@/di-symbols.js'; import type { Config } from '@/config.js'; import { intersperse } from '@/misc/prelude/array.js'; @@ -478,6 +478,8 @@ export class MfmService { const doc = window.document; + const body = doc.createElement('p'); + async function appendChildren(children: mfm.MfmNode[], targetElement: any): Promise { if (children) { for (const child of await Promise.all(children.map(async (x) => await (handlers as any)[x.type](x)))) targetElement.appendChild(child); @@ -656,7 +658,7 @@ export class MfmService { }, }; - await appendChildren(nodes, doc.body); + await appendChildren(nodes, body); if (quoteUri !== null) { const a = doc.createElement('a'); @@ -670,9 +672,15 @@ export class MfmService { quote.innerHTML += 'RE: '; quote.appendChild(a); - doc.body.appendChild(quote); + body.appendChild(quote); } - return inline ? doc.body.innerHTML : `

${doc.body.innerHTML}

`; + let result = new XMLSerializer().serializeToString(body); + + if (inline) { + result = result.replace(/^

/,'').replace(/<\/p>$/,''); + } + + return result; } } diff --git a/packages/backend/src/core/RelayService.ts b/packages/backend/src/core/RelayService.ts index 8dd3d64f5b..db32114346 100644 --- a/packages/backend/src/core/RelayService.ts +++ b/packages/backend/src/core/RelayService.ts @@ -35,7 +35,7 @@ export class RelayService { private createSystemUserService: CreateSystemUserService, private apRendererService: ApRendererService, ) { - this.relaysCache = new MemorySingleCache(1000 * 60 * 10); + this.relaysCache = new MemorySingleCache(1000 * 60 * 10); // 10m } @bindThis diff --git a/packages/backend/src/core/RoleService.ts b/packages/backend/src/core/RoleService.ts index 64a81bbc15..7984dc5627 100644 --- a/packages/backend/src/core/RoleService.ts +++ b/packages/backend/src/core/RoleService.ts @@ -131,10 +131,8 @@ export class RoleService implements OnApplicationShutdown, OnModuleInit { private moderationLogService: ModerationLogService, private fanoutTimelineService: FanoutTimelineService, ) { - //this.onMessage = this.onMessage.bind(this); - - this.rolesCache = new MemorySingleCache(1000 * 60 * 60 * 1); - this.roleAssignmentByUserIdCache = new MemoryKVCache(1000 * 60 * 60 * 1); + this.rolesCache = new MemorySingleCache(1000 * 60 * 60); // 1h + this.roleAssignmentByUserIdCache = new MemoryKVCache(1000 * 60 * 5); // 5m this.redisForSub.on('message', this.onMessage); } diff --git a/packages/backend/src/core/UserKeypairService.ts b/packages/backend/src/core/UserKeypairService.ts index 51ac99179a..92d61cd103 100644 --- a/packages/backend/src/core/UserKeypairService.ts +++ b/packages/backend/src/core/UserKeypairService.ts @@ -25,7 +25,7 @@ export class UserKeypairService implements OnApplicationShutdown { ) { this.cache = new RedisKVCache(this.redisClient, 'userKeypair', { lifetime: 1000 * 60 * 60 * 24, // 24h - memoryCacheLifetime: Infinity, + memoryCacheLifetime: 1000 * 60 * 60, // 1h fetcher: (key) => this.userKeypairsRepository.findOneByOrFail({ userId: key }), toRedisConverter: (value) => JSON.stringify(value), fromRedisConverter: (value) => JSON.parse(value), diff --git a/packages/backend/src/core/activitypub/ApDbResolverService.ts b/packages/backend/src/core/activitypub/ApDbResolverService.ts index 44680a2ed5..062af39732 100644 --- a/packages/backend/src/core/activitypub/ApDbResolverService.ts +++ b/packages/backend/src/core/activitypub/ApDbResolverService.ts @@ -54,8 +54,8 @@ export class ApDbResolverService implements OnApplicationShutdown { private cacheService: CacheService, private apPersonService: ApPersonService, ) { - this.publicKeyCache = new MemoryKVCache(Infinity); - this.publicKeyByUserIdCache = new MemoryKVCache(Infinity); + this.publicKeyCache = new MemoryKVCache(1000 * 60 * 60 * 12); // 12h + this.publicKeyByUserIdCache = new MemoryKVCache(1000 * 60 * 60 * 12); // 12h } @bindThis diff --git a/packages/backend/src/core/activitypub/ApRendererService.ts b/packages/backend/src/core/activitypub/ApRendererService.ts index 7e9a762c8d..55d1054de9 100644 --- a/packages/backend/src/core/activitypub/ApRendererService.ts +++ b/packages/backend/src/core/activitypub/ApRendererService.ts @@ -792,6 +792,13 @@ export class ApRendererService { @bindThis public async attachLdSignature(activity: any, user: { id: MiUser['id']; host: null; }): Promise { + // Linked Data signatures are cryptographic signatures attached to each activity to provide proof of authenticity. + // When using authorized fetch, this is often undesired as any signed activity can be forwarded to a blocked instance by relays and other instances. + // This setting allows admins to disable LD signatures for increased privacy, at the expense of fewer relayed activities and additional inbound fetch (GET) requests. + if (!this.config.attachLdSignatureForRelays) { + return activity; + } + const keypair = await this.userKeypairService.getUserKeypair(user.id); const jsonLd = this.jsonLdService.use(); diff --git a/packages/backend/src/misc/cache.ts b/packages/backend/src/misc/cache.ts index bba64a06ef..d968069ca3 100644 --- a/packages/backend/src/misc/cache.ts +++ b/packages/backend/src/misc/cache.ts @@ -7,23 +7,23 @@ import * as Redis from 'ioredis'; import { bindThis } from '@/decorators.js'; export class RedisKVCache { - private redisClient: Redis.Redis; - private name: string; - private lifetime: number; - private memoryCache: MemoryKVCache; - private fetcher: (key: string) => Promise; - private toRedisConverter: (value: T) => string; - private fromRedisConverter: (value: string) => T | undefined; + private readonly lifetime: number; + private readonly memoryCache: MemoryKVCache; + private readonly fetcher: (key: string) => Promise; + private readonly toRedisConverter: (value: T) => string; + private readonly fromRedisConverter: (value: string) => T | undefined; - constructor(redisClient: RedisKVCache['redisClient'], name: RedisKVCache['name'], opts: { - lifetime: RedisKVCache['lifetime']; - memoryCacheLifetime: number; - fetcher: RedisKVCache['fetcher']; - toRedisConverter: RedisKVCache['toRedisConverter']; - fromRedisConverter: RedisKVCache['fromRedisConverter']; - }) { - this.redisClient = redisClient; - this.name = name; + constructor( + private redisClient: Redis.Redis, + private name: string, + opts: { + lifetime: RedisKVCache['lifetime']; + memoryCacheLifetime: number; + fetcher: RedisKVCache['fetcher']; + toRedisConverter: RedisKVCache['toRedisConverter']; + fromRedisConverter: RedisKVCache['fromRedisConverter']; + }, + ) { this.lifetime = opts.lifetime; this.memoryCache = new MemoryKVCache(opts.memoryCacheLifetime); this.fetcher = opts.fetcher; @@ -55,7 +55,13 @@ export class RedisKVCache { const cached = await this.redisClient.get(`kvcache:${this.name}:${key}`); if (cached == null) return undefined; - return this.fromRedisConverter(cached); + + const value = this.fromRedisConverter(cached); + if (value !== undefined) { + this.memoryCache.set(key, value); + } + + return value; } @bindThis @@ -77,14 +83,14 @@ export class RedisKVCache { // Cache MISS const value = await this.fetcher(key); - this.set(key, value); + await this.set(key, value); return value; } @bindThis public async refresh(key: string) { const value = await this.fetcher(key); - this.set(key, value); + await this.set(key, value); // TODO: イベント発行して他プロセスのメモリキャッシュも更新できるようにする } @@ -101,23 +107,23 @@ export class RedisKVCache { } export class RedisSingleCache { - private redisClient: Redis.Redis; - private name: string; - private lifetime: number; - private memoryCache: MemorySingleCache; - private fetcher: () => Promise; - private toRedisConverter: (value: T) => string; - private fromRedisConverter: (value: string) => T | undefined; + private readonly lifetime: number; + private readonly memoryCache: MemorySingleCache; + private readonly fetcher: () => Promise; + private readonly toRedisConverter: (value: T) => string; + private readonly fromRedisConverter: (value: string) => T | undefined; - constructor(redisClient: RedisSingleCache['redisClient'], name: RedisSingleCache['name'], opts: { - lifetime: RedisSingleCache['lifetime']; - memoryCacheLifetime: number; - fetcher: RedisSingleCache['fetcher']; - toRedisConverter: RedisSingleCache['toRedisConverter']; - fromRedisConverter: RedisSingleCache['fromRedisConverter']; - }) { - this.redisClient = redisClient; - this.name = name; + constructor( + private redisClient: Redis.Redis, + private name: string, + opts: { + lifetime: number; + memoryCacheLifetime: number; + fetcher: RedisSingleCache['fetcher']; + toRedisConverter: RedisSingleCache['toRedisConverter']; + fromRedisConverter: RedisSingleCache['fromRedisConverter']; + }, + ) { this.lifetime = opts.lifetime; this.memoryCache = new MemorySingleCache(opts.memoryCacheLifetime); this.fetcher = opts.fetcher; @@ -149,7 +155,13 @@ export class RedisSingleCache { const cached = await this.redisClient.get(`singlecache:${this.name}`); if (cached == null) return undefined; - return this.fromRedisConverter(cached); + + const value = this.fromRedisConverter(cached); + if (value !== undefined) { + this.memoryCache.set(value); + } + + return value; } @bindThis @@ -171,14 +183,14 @@ export class RedisSingleCache { // Cache MISS const value = await this.fetcher(); - this.set(value); + await this.set(value); return value; } @bindThis public async refresh() { const value = await this.fetcher(); - this.set(value); + await this.set(value); // TODO: イベント発行して他プロセスのメモリキャッシュも更新できるようにする } @@ -187,22 +199,12 @@ export class RedisSingleCache { // TODO: メモリ節約のためあまり参照されないキーを定期的に削除できるようにする？ export class MemoryKVCache { - /** - * データを持つマップ - * @deprecated これを直接操作するべきではない - */ - public cache: Map; - private lifetime: number; - private gcIntervalHandle: NodeJS.Timeout; + private readonly cache = new Map(); + private readonly gcIntervalHandle = setInterval(() => this.gc(), 1000 * 60 * 3); // 3m - constructor(lifetime: MemoryKVCache['lifetime']) { - this.cache = new Map(); - this.lifetime = lifetime; - - this.gcIntervalHandle = setInterval(() => { - this.gc(); - }, 1000 * 60 * 3); - } + constructor( + private readonly lifetime: number, + ) {} @bindThis /** @@ -287,10 +289,14 @@ export class MemoryKVCache { @bindThis public gc(): void { const now = Date.now(); + for (const [key, { date }] of this.cache.entries()) { - if ((now - date) > this.lifetime) { - this.cache.delete(key); - } + // The map is ordered from oldest to youngest. + // We can stop once we find an entry that's still active, because all following entries must *also* be active. + const age = now - date; + if (age < this.lifetime) break; + + this.cache.delete(key); } } @@ -298,16 +304,19 @@ export class MemoryKVCache { public dispose(): void { clearInterval(this.gcIntervalHandle); } + + public get entries() { + return this.cache.entries(); + } } export class MemorySingleCache { private cachedAt: number | null = null; private value: T | undefined; - private lifetime: number; - constructor(lifetime: MemorySingleCache['lifetime']) { - this.lifetime = lifetime; - } + constructor( + private lifetime: number, + ) {} @bindThis public set(value: T): void { diff --git a/packages/backend/src/queue/processors/DeliverProcessorService.ts b/packages/backend/src/queue/processors/DeliverProcessorService.ts index d665945861..4076e9da90 100644 --- a/packages/backend/src/queue/processors/DeliverProcessorService.ts +++ b/packages/backend/src/queue/processors/DeliverProcessorService.ts @@ -45,7 +45,7 @@ export class DeliverProcessorService { private queueLoggerService: QueueLoggerService, ) { this.logger = this.queueLoggerService.logger.createSubLogger('deliver'); - this.suspendedHostsCache = new MemorySingleCache(1000 * 60 * 60); + this.suspendedHostsCache = new MemorySingleCache(1000 * 60 * 60); // 1h } @bindThis diff --git a/packages/backend/src/server/NodeinfoServerService.ts b/packages/backend/src/server/NodeinfoServerService.ts index 716bb0944b..bc8d3c0411 100644 --- a/packages/backend/src/server/NodeinfoServerService.ts +++ b/packages/backend/src/server/NodeinfoServerService.ts @@ -135,7 +135,7 @@ export class NodeinfoServerService { return document; }; - const cache = new MemorySingleCache>>(1000 * 60 * 10); + const cache = new MemorySingleCache>>(1000 * 60 * 10); // 10m fastify.get(nodeinfo2_1path, async (request, reply) => { const base = await cache.fetch(() => nodeinfo2(21)); diff --git a/packages/backend/src/server/api/AuthenticateService.ts b/packages/backend/src/server/api/AuthenticateService.ts index ddef8db987..690ff2e022 100644 --- a/packages/backend/src/server/api/AuthenticateService.ts +++ b/packages/backend/src/server/api/AuthenticateService.ts @@ -37,7 +37,7 @@ export class AuthenticateService implements OnApplicationShutdown { private cacheService: CacheService, ) { - this.appCache = new MemoryKVCache(Infinity); + this.appCache = new MemoryKVCache(1000 * 60 * 60 * 24 * 7); // 1w } @bindThis diff --git a/packages/backend/src/server/web/UrlPreviewService.ts b/packages/backend/src/server/web/UrlPreviewService.ts index 96038d9c1e..ef804b5bfd 100644 --- a/packages/backend/src/server/web/UrlPreviewService.ts +++ b/packages/backend/src/server/web/UrlPreviewService.ts @@ -38,8 +38,8 @@ export class UrlPreviewService { ) { this.logger = this.loggerService.getLogger('url-preview'); this.previewCache = new RedisKVCache(this.redisClient, 'summaly', { - lifetime: 1000 * 86400, - memoryCacheLifetime: 1000 * 10 * 60, + lifetime: 1000 * 60 * 60 * 24, // 1d + memoryCacheLifetime: 1000 * 60 * 10, // 10m fetcher: (key: string) => { throw new Error('the UrlPreview cache should never fetch'); }, toRedisConverter: (value) => JSON.stringify(value), fromRedisConverter: (value) => JSON.parse(value),