From aa8296ef116233f43048ba533a4b9033c3d414b7 Mon Sep 17 00:00:00 2001 From: syuilo <4439005+syuilo@users.noreply.github.com> Date: Thu, 19 Sep 2024 20:52:17 +0900 Subject: [PATCH 01/17] wip --- locales/index.d.ts | 12 +++++ locales/ja-JP.yml | 1 + packages/backend/src/core/ReactionService.ts | 29 ++++++----- .../src/core/entities/NoteEntityService.ts | 52 ++++++++++++++++--- packages/backend/src/models/Meta.ts | 13 +++++ 5 files changed, 89 insertions(+), 18 deletions(-) diff --git a/locales/index.d.ts b/locales/index.d.ts index bd2421a5ca58..798cb89f8336 100644 --- a/locales/index.d.ts +++ b/locales/index.d.ts @@ -2384,6 +2384,14 @@ export interface Locale extends ILocale { * スクラッチパッドは、AiScriptの実験環境を提供します。Misskeyと対話するコードの記述、実行、結果の確認ができます。 */ "scratchpadDescription": string; + /** + * UIインスペクター + */ + "uiInspector": string; + /** + * メモリ上に存在しているUIコンポーネントのインスタンスの一覧を見ることができます。UIコンポーネントはUi:C:系関数により生成されます。 + */ + "uiInspectorDescription": string; /** * 出力 */ @@ -5575,6 +5583,10 @@ export interface Locale extends ILocale { * 有効にすると、タイムラインがキャッシュされていない場合にDBへ追加で問い合わせを行うフォールバック処理を行います。無効にすると、フォールバック処理を行わないことでさらにサーバーの負荷を軽減することができますが、タイムラインが取得できる範囲に制限が生じます。 */ "fanoutTimelineDbFallbackDescription": string; + /** + * 有効にすると、リアクション作成時のパフォーマンスが大幅に向上し、データベースへの負荷を軽減することが可能です。ただし、Redisのメモリ使用量は増加します。 + */ + "reactionsBufferingDescription": string; /** * 問い合わせ先URL */ diff --git a/locales/ja-JP.yml b/locales/ja-JP.yml index 2a5b530c9f56..726e4f4ef450 100644 --- a/locales/ja-JP.yml +++ b/locales/ja-JP.yml @@ -1411,6 +1411,7 @@ _serverSettings: fanoutTimelineDescription: "有効にすると、各種タイムラインを取得する際のパフォーマンスが大幅に向上し、データベースへの負荷を軽減することが可能です。ただし、Redisのメモリ使用量は増加します。サーバーのメモリ容量が少ない場合、または動作が不安定な場合は無効にすることができます。" fanoutTimelineDbFallback: "データベースへのフォールバック" fanoutTimelineDbFallbackDescription: "有効にすると、タイムラインがキャッシュされていない場合にDBへ追加で問い合わせを行うフォールバック処理を行います。無効にすると、フォールバック処理を行わないことでさらにサーバーの負荷を軽減することができますが、タイムラインが取得できる範囲に制限が生じます。" + reactionsBufferingDescription: "有効にすると、リアクション作成時のパフォーマンスが大幅に向上し、データベースへの負荷を軽減することが可能です。ただし、Redisのメモリ使用量は増加します。" inquiryUrl: "問い合わせ先URL" inquiryUrlDescription: "サーバー運営者へのお問い合わせフォームのURLや、運営者の連絡先等が記載されたWebページのURLを指定します。" diff --git a/packages/backend/src/core/ReactionService.ts b/packages/backend/src/core/ReactionService.ts index 371207c33a7d..882513878a99 100644 --- a/packages/backend/src/core/ReactionService.ts +++ b/packages/backend/src/core/ReactionService.ts @@ -72,7 +72,7 @@ const decodeCustomEmojiRegexp = /^:([\w+-]+)(?:@([\w.-]+))?:$/; export class ReactionService { constructor( @Inject(DI.redis) - private redisClient: Redis.Redis, + private redisClient: Redis.Redis, // TODO: 専用のRedisインスタンスにする @Inject(DI.usersRepository) private usersRepository: UsersRepository, @@ -174,7 +174,6 @@ export class ReactionService { reaction, }; - // Create reaction try { await this.noteReactionsRepository.insert(record); } catch (e) { @@ -197,17 +196,23 @@ export class ReactionService { } } + const rbt = true; + // Increment reactions count - const sql = `jsonb_set("reactions", '{${reaction}}', (COALESCE("reactions"->>'${reaction}', '0')::int + 1)::text::jsonb)`; - await this.notesRepository.createQueryBuilder().update() - .set({ - reactions: () => sql, - ...(note.reactionAndUserPairCache.length < PER_NOTE_REACTION_USER_PAIR_CACHE_MAX ? { - reactionAndUserPairCache: () => `array_append("reactionAndUserPairCache", '${user.id}/${reaction}')`, - } : {}), - }) - .where('id = :id', { id: note.id }) - .execute(); + if (rbt) { + this.redisClient.hincrby(`reactionsBuffer:${note.id}`, reaction, 1); + } else { + const sql = `jsonb_set("reactions", '{${reaction}}', (COALESCE("reactions"->>'${reaction}', '0')::int + 1)::text::jsonb)`; + await this.notesRepository.createQueryBuilder().update() + .set({ + reactions: () => sql, + ...(note.reactionAndUserPairCache.length < PER_NOTE_REACTION_USER_PAIR_CACHE_MAX ? { + reactionAndUserPairCache: () => `array_append("reactionAndUserPairCache", '${user.id}/${reaction}')`, + } : {}), + }) + .where('id = :id', { id: note.id }) + .execute(); + } // 30%の確率、セルフではない、3日以内に投稿されたノートの場合ハイライト用ランキング更新 if ( diff --git a/packages/backend/src/core/entities/NoteEntityService.ts b/packages/backend/src/core/entities/NoteEntityService.ts index 2cd092231cf5..97eb75901c56 100644 --- a/packages/backend/src/core/entities/NoteEntityService.ts +++ b/packages/backend/src/core/entities/NoteEntityService.ts @@ -6,6 +6,7 @@ import { Inject, Injectable } from '@nestjs/common'; import { In } from 'typeorm'; import { ModuleRef } from '@nestjs/core'; +import * as Redis from 'ioredis'; import { DI } from '@/di-symbols.js'; import type { Packed } from '@/misc/json-schema.js'; import { awaitAll } from '@/misc/prelude/await-all.js'; @@ -22,6 +23,18 @@ import type { ReactionService } from '../ReactionService.js'; import type { UserEntityService } from './UserEntityService.js'; import type { DriveFileEntityService } from './DriveFileEntityService.js'; +function mergeReactions(src: Record, delta: Record) { + const reactions = { ...src }; + for (const [name, count] of Object.entries(delta)) { + if (reactions[name] != null) { + reactions[name] += count; + } else { + reactions[name] = count; + } + } + return reactions; +} + @Injectable() export class NoteEntityService implements OnModuleInit { private userEntityService: UserEntityService; @@ -34,6 +47,9 @@ export class NoteEntityService implements OnModuleInit { constructor( private moduleRef: ModuleRef, + @Inject(DI.redis) + private redisClient: Redis.Redis, // TODO: 専用のRedisインスタンスにする + @Inject(DI.usersRepository) private usersRepository: UsersRepository, @@ -287,6 +303,7 @@ export class NoteEntityService implements OnModuleInit { skipHide?: boolean; withReactionAndUserPairCache?: boolean; _hint_?: { + reactionsDeltas: Map>; myReactions: Map; packedFiles: Map | null>; packedUsers: Map> @@ -315,7 +332,7 @@ export class NoteEntityService implements OnModuleInit { : await this.channelsRepository.findOneBy({ id: note.channelId }) : null; - const reactionEmojiNames = Object.keys(note.reactions) + const reactionEmojiNames = Object.keys(mergeReactions(note.reactions, opts._hint_?.reactionsDeltas.get(note.id) ?? {})) .filter(x => x.startsWith(':') && x.includes('@') && !x.includes('@.')) // リモートカスタム絵文字のみ .map(x => this.reactionService.decodeReaction(x).reaction.replaceAll(':', '')); const packedFiles = options?._hint_?.packedFiles; @@ -334,8 +351,8 @@ export class NoteEntityService implements OnModuleInit { visibleUserIds: note.visibility === 'specified' ? note.visibleUserIds : undefined, renoteCount: note.renoteCount, repliesCount: note.repliesCount, - reactionCount: Object.values(note.reactions).reduce((a, b) => a + b, 0), - reactions: this.reactionService.convertLegacyReactions(note.reactions), + reactionCount: Object.values(mergeReactions(note.reactions, opts._hint_?.reactionsDeltas.get(note.id) ?? {})).reduce((a, b) => a + b, 0), + reactions: mergeReactions(this.reactionService.convertLegacyReactions(note.reactions), opts._hint_?.reactionsDeltas.get(note.id) ?? {}), reactionEmojis: this.customEmojiService.populateEmojis(reactionEmojiNames, host), reactionAndUserPairCache: opts.withReactionAndUserPairCache ? note.reactionAndUserPairCache : undefined, emojis: host != null ? this.customEmojiService.populateEmojis(note.emojis, host) : undefined, @@ -376,7 +393,7 @@ export class NoteEntityService implements OnModuleInit { poll: note.hasPoll ? this.populatePoll(note, meId) : undefined, - ...(meId && Object.keys(note.reactions).length > 0 ? { + ...(meId && Object.keys(mergeReactions(note.reactions, opts._hint_?.reactionsDeltas.get(note.id) ?? {})).length > 0 ? { myReaction: this.populateMyReaction(note, meId, options?._hint_), } : {}), } : {}), @@ -400,6 +417,28 @@ export class NoteEntityService implements OnModuleInit { ) { if (notes.length === 0) return []; + const reactionsDeltas = new Map>(); + + const rbt = true; + + if (rbt) { + const pipeline = this.redisClient.pipeline(); + for (const note of notes) { + pipeline.hgetall(`reactionsBuffer:${note.id}`); + } + const results = await pipeline.exec(); + + for (let i = 0; i < notes.length; i++) { + const note = notes[i]; + const result = results![i][1]; + const delta = {}; + for (const [name, count] of Object.entries(result)) { + delta[name] = parseInt(count); + } + reactionsDeltas.set(note.id, delta); + } + } + const meId = me ? me.id : null; const myReactionsMap = new Map(); if (meId) { @@ -410,7 +449,7 @@ export class NoteEntityService implements OnModuleInit { for (const note of notes) { if (note.renote && (note.text == null && note.fileIds.length === 0)) { // pure renote - const reactionsCount = Object.values(note.renote.reactions).reduce((a, b) => a + b, 0); + const reactionsCount = Object.values(mergeReactions(note.renote.reactions, reactionsDeltas.get(note.renote.id) ?? {})).reduce((a, b) => a + b, 0); if (reactionsCount === 0) { myReactionsMap.set(note.renote.id, null); } else if (reactionsCount <= note.renote.reactionAndUserPairCache.length) { @@ -421,7 +460,7 @@ export class NoteEntityService implements OnModuleInit { } } else { if (note.id < oldId) { - const reactionsCount = Object.values(note.reactions).reduce((a, b) => a + b, 0); + const reactionsCount = Object.values(mergeReactions(note.reactions, reactionsDeltas.get(note.id) ?? {})).reduce((a, b) => a + b, 0); if (reactionsCount === 0) { myReactionsMap.set(note.id, null); } else if (reactionsCount <= note.reactionAndUserPairCache.length) { @@ -461,6 +500,7 @@ export class NoteEntityService implements OnModuleInit { return await Promise.all(notes.map(n => this.pack(n, me, { ...options, _hint_: { + reactionsDeltas, myReactions: myReactionsMap, packedFiles, packedUsers, diff --git a/packages/backend/src/models/Meta.ts b/packages/backend/src/models/Meta.ts index 70d41801b5ee..6ec4da187386 100644 --- a/packages/backend/src/models/Meta.ts +++ b/packages/backend/src/models/Meta.ts @@ -589,6 +589,19 @@ export class MiMeta { }) public perUserListTimelineCacheMax: number; + // @Column('boolean', { + // default: true, + // }) + // public enableReactionsBuffering: boolean; + // + // /** + // * 分 + // */ + // @Column('integer', { + // default: 180, + // }) + // public reactionsBufferingTtl: number; + @Column('integer', { default: 0, }) From c7a0929072f96892d4efa09d0da6fed632e0b8fa Mon Sep 17 00:00:00 2001 From: syuilo <4439005+syuilo@users.noreply.github.com> Date: Fri, 20 Sep 2024 09:18:50 +0900 Subject: [PATCH 02/17] wip --- packages/backend/src/core/CoreModule.ts | 6 ++ packages/backend/src/core/ReactionService.ts | 4 +- .../src/core/ReactionsBufferingService.ts | 62 +++++++++++++++++++ .../src/core/entities/NoteEntityService.ts | 34 +++------- 4 files changed, 81 insertions(+), 25 deletions(-) create mode 100644 packages/backend/src/core/ReactionsBufferingService.ts diff --git a/packages/backend/src/core/CoreModule.ts b/packages/backend/src/core/CoreModule.ts index 674241ac120b..3b3c35f97671 100644 --- a/packages/backend/src/core/CoreModule.ts +++ b/packages/backend/src/core/CoreModule.ts @@ -50,6 +50,7 @@ import { PollService } from './PollService.js'; import { PushNotificationService } from './PushNotificationService.js'; import { QueryService } from './QueryService.js'; import { ReactionService } from './ReactionService.js'; +import { ReactionsBufferingService } from './ReactionsBufferingService.js'; import { RelayService } from './RelayService.js'; import { RoleService } from './RoleService.js'; import { S3Service } from './S3Service.js'; @@ -193,6 +194,7 @@ const $ProxyAccountService: Provider = { provide: 'ProxyAccountService', useExis const $PushNotificationService: Provider = { provide: 'PushNotificationService', useExisting: PushNotificationService }; const $QueryService: Provider = { provide: 'QueryService', useExisting: QueryService }; const $ReactionService: Provider = { provide: 'ReactionService', useExisting: ReactionService }; +const $ReactionsBufferingService: Provider = { provide: 'ReactionsBufferingService', useExisting: ReactionsBufferingService }; const $RelayService: Provider = { provide: 'RelayService', useExisting: RelayService }; const $RoleService: Provider = { provide: 'RoleService', useExisting: RoleService }; const $S3Service: Provider = { provide: 'S3Service', useExisting: S3Service }; @@ -342,6 +344,7 @@ const $ApQuestionService: Provider = { provide: 'ApQuestionService', useExisting PushNotificationService, QueryService, ReactionService, + ReactionsBufferingService, RelayService, RoleService, S3Service, @@ -487,6 +490,7 @@ const $ApQuestionService: Provider = { provide: 'ApQuestionService', useExisting $PushNotificationService, $QueryService, $ReactionService, + $ReactionsBufferingService, $RelayService, $RoleService, $S3Service, @@ -633,6 +637,7 @@ const $ApQuestionService: Provider = { provide: 'ApQuestionService', useExisting PushNotificationService, QueryService, ReactionService, + ReactionsBufferingService, RelayService, RoleService, S3Service, @@ -777,6 +782,7 @@ const $ApQuestionService: Provider = { provide: 'ApQuestionService', useExisting $PushNotificationService, $QueryService, $ReactionService, + $ReactionsBufferingService, $RelayService, $RoleService, $S3Service, diff --git a/packages/backend/src/core/ReactionService.ts b/packages/backend/src/core/ReactionService.ts index 882513878a99..595da8c7224e 100644 --- a/packages/backend/src/core/ReactionService.ts +++ b/packages/backend/src/core/ReactionService.ts @@ -30,6 +30,7 @@ import { RoleService } from '@/core/RoleService.js'; import { FeaturedService } from '@/core/FeaturedService.js'; import { trackPromise } from '@/misc/promise-tracker.js'; import { isQuote, isRenote } from '@/misc/is-renote.js'; +import { ReactionsBufferingService } from '@/core/ReactionsBufferingService.js'; const FALLBACK = '\u2764'; const PER_NOTE_REACTION_USER_PAIR_CACHE_MAX = 16; @@ -93,6 +94,7 @@ export class ReactionService { private userEntityService: UserEntityService, private noteEntityService: NoteEntityService, private userBlockingService: UserBlockingService, + private reactionsBufferingService: ReactionsBufferingService, private idService: IdService, private featuredService: FeaturedService, private globalEventService: GlobalEventService, @@ -200,7 +202,7 @@ export class ReactionService { // Increment reactions count if (rbt) { - this.redisClient.hincrby(`reactionsBuffer:${note.id}`, reaction, 1); + this.reactionsBufferingService.create(note, reaction); } else { const sql = `jsonb_set("reactions", '{${reaction}}', (COALESCE("reactions"->>'${reaction}', '0')::int + 1)::text::jsonb)`; await this.notesRepository.createQueryBuilder().update() diff --git a/packages/backend/src/core/ReactionsBufferingService.ts b/packages/backend/src/core/ReactionsBufferingService.ts new file mode 100644 index 000000000000..d179abc2efb7 --- /dev/null +++ b/packages/backend/src/core/ReactionsBufferingService.ts @@ -0,0 +1,62 @@ +/* + * SPDX-FileCopyrightText: syuilo and misskey-project + * SPDX-License-Identifier: AGPL-3.0-only + */ + +import { Inject, Injectable } from '@nestjs/common'; +import * as Redis from 'ioredis'; +import { DI } from '@/di-symbols.js'; +import type { MiNote } from '@/models/Note.js'; +import { bindThis } from '@/decorators.js'; + +@Injectable() +export class ReactionsBufferingService { + constructor( + @Inject(DI.redis) + private redisClient: Redis.Redis, // TODO: 専用のRedisインスタンスにする + ) { + } + + @bindThis + public async create(note: MiNote, reaction: string) { + this.redisClient.hincrby(`reactionsBuffer:${note.id}`, reaction, 1); + } + + @bindThis + public async delete(note: MiNote, reaction: string) { + this.redisClient.hincrby(`reactionsBuffer:${note.id}`, reaction, -1); + } + + @bindThis + public async get(note: MiNote) { + const result = await this.redisClient.hgetall(`reactionsBuffer:${note.id}`); + const delta = {}; + for (const [name, count] of Object.entries(result)) { + delta[name] = parseInt(count); + } + return delta; + } + + @bindThis + public async getMany(notes: MiNote[]) { + const deltas = new Map>(); + + const pipeline = this.redisClient.pipeline(); + for (const note of notes) { + pipeline.hgetall(`reactionsBuffer:${note.id}`); + } + const results = await pipeline.exec(); + + for (let i = 0; i < notes.length; i++) { + const note = notes[i]; + const result = results![i][1]; + const delta = {}; + for (const [name, count] of Object.entries(result)) { + delta[name] = parseInt(count); + } + deltas.set(note.id, delta); + } + + return deltas; + } +} diff --git a/packages/backend/src/core/entities/NoteEntityService.ts b/packages/backend/src/core/entities/NoteEntityService.ts index 97eb75901c56..9066bb8b5c82 100644 --- a/packages/backend/src/core/entities/NoteEntityService.ts +++ b/packages/backend/src/core/entities/NoteEntityService.ts @@ -17,6 +17,7 @@ import type { UsersRepository, NotesRepository, FollowingsRepository, PollsRepos import { bindThis } from '@/decorators.js'; import { DebounceLoader } from '@/misc/loader.js'; import { IdService } from '@/core/IdService.js'; +import { ReactionsBufferingService } from '@/core/ReactionsBufferingService.js'; import type { OnModuleInit } from '@nestjs/common'; import type { CustomEmojiService } from '../CustomEmojiService.js'; import type { ReactionService } from '../ReactionService.js'; @@ -41,6 +42,7 @@ export class NoteEntityService implements OnModuleInit { private driveFileEntityService: DriveFileEntityService; private customEmojiService: CustomEmojiService; private reactionService: ReactionService; + private reactionsBufferingService: ReactionsBufferingService; private idService: IdService; private noteLoader = new DebounceLoader(this.findNoteOrFail); @@ -75,6 +77,7 @@ export class NoteEntityService implements OnModuleInit { //private driveFileEntityService: DriveFileEntityService, //private customEmojiService: CustomEmojiService, //private reactionService: ReactionService, + //private reactionsBufferingService: ReactionsBufferingService, ) { } @@ -83,6 +86,7 @@ export class NoteEntityService implements OnModuleInit { this.driveFileEntityService = this.moduleRef.get('DriveFileEntityService'); this.customEmojiService = this.moduleRef.get('CustomEmojiService'); this.reactionService = this.moduleRef.get('ReactionService'); + this.reactionsBufferingService = this.moduleRef.get('ReactionsBufferingService'); this.idService = this.moduleRef.get('IdService'); } @@ -319,6 +323,7 @@ export class NoteEntityService implements OnModuleInit { const meId = me ? me.id : null; const note = typeof src === 'object' ? src : await this.noteLoader.load(src); const host = note.userHost; + const reactions = mergeReactions(note.reactions, opts._hint_?.reactionsDeltas.get(note.id) ?? {}); let text = note.text; @@ -332,7 +337,7 @@ export class NoteEntityService implements OnModuleInit { : await this.channelsRepository.findOneBy({ id: note.channelId }) : null; - const reactionEmojiNames = Object.keys(mergeReactions(note.reactions, opts._hint_?.reactionsDeltas.get(note.id) ?? {})) + const reactionEmojiNames = Object.keys(reactions) .filter(x => x.startsWith(':') && x.includes('@') && !x.includes('@.')) // リモートカスタム絵文字のみ .map(x => this.reactionService.decodeReaction(x).reaction.replaceAll(':', '')); const packedFiles = options?._hint_?.packedFiles; @@ -351,8 +356,8 @@ export class NoteEntityService implements OnModuleInit { visibleUserIds: note.visibility === 'specified' ? note.visibleUserIds : undefined, renoteCount: note.renoteCount, repliesCount: note.repliesCount, - reactionCount: Object.values(mergeReactions(note.reactions, opts._hint_?.reactionsDeltas.get(note.id) ?? {})).reduce((a, b) => a + b, 0), - reactions: mergeReactions(this.reactionService.convertLegacyReactions(note.reactions), opts._hint_?.reactionsDeltas.get(note.id) ?? {}), + reactionCount: Object.values(reactions).reduce((a, b) => a + b, 0), + reactions: reactions, reactionEmojis: this.customEmojiService.populateEmojis(reactionEmojiNames, host), reactionAndUserPairCache: opts.withReactionAndUserPairCache ? note.reactionAndUserPairCache : undefined, emojis: host != null ? this.customEmojiService.populateEmojis(note.emojis, host) : undefined, @@ -393,7 +398,7 @@ export class NoteEntityService implements OnModuleInit { poll: note.hasPoll ? this.populatePoll(note, meId) : undefined, - ...(meId && Object.keys(mergeReactions(note.reactions, opts._hint_?.reactionsDeltas.get(note.id) ?? {})).length > 0 ? { + ...(meId && Object.keys(reactions).length > 0 ? { myReaction: this.populateMyReaction(note, meId, options?._hint_), } : {}), } : {}), @@ -417,27 +422,8 @@ export class NoteEntityService implements OnModuleInit { ) { if (notes.length === 0) return []; - const reactionsDeltas = new Map>(); - const rbt = true; - - if (rbt) { - const pipeline = this.redisClient.pipeline(); - for (const note of notes) { - pipeline.hgetall(`reactionsBuffer:${note.id}`); - } - const results = await pipeline.exec(); - - for (let i = 0; i < notes.length; i++) { - const note = notes[i]; - const result = results![i][1]; - const delta = {}; - for (const [name, count] of Object.entries(result)) { - delta[name] = parseInt(count); - } - reactionsDeltas.set(note.id, delta); - } - } + const reactionsDeltas = rbt ? await this.reactionsBufferingService.getMany(notes) : new Map(); const meId = me ? me.id : null; const myReactionsMap = new Map(); From e9ba46c84ea4159ef6d721a03eb1497234e58e3b Mon Sep 17 00:00:00 2001 From: syuilo <4439005+syuilo@users.noreply.github.com> Date: Fri, 20 Sep 2024 09:36:08 +0900 Subject: [PATCH 03/17] Update ReactionsBufferingService.ts --- .../src/core/ReactionsBufferingService.ts | 23 +++++++++++++++---- 1 file changed, 19 insertions(+), 4 deletions(-) diff --git a/packages/backend/src/core/ReactionsBufferingService.ts b/packages/backend/src/core/ReactionsBufferingService.ts index d179abc2efb7..f5beca993f00 100644 --- a/packages/backend/src/core/ReactionsBufferingService.ts +++ b/packages/backend/src/core/ReactionsBufferingService.ts @@ -9,6 +9,8 @@ import { DI } from '@/di-symbols.js'; import type { MiNote } from '@/models/Note.js'; import { bindThis } from '@/decorators.js'; +const REDIS_PREFIX = 'reactionsBuffer'; + @Injectable() export class ReactionsBufferingService { constructor( @@ -19,17 +21,17 @@ export class ReactionsBufferingService { @bindThis public async create(note: MiNote, reaction: string) { - this.redisClient.hincrby(`reactionsBuffer:${note.id}`, reaction, 1); + this.redisClient.hincrby(`${REDIS_PREFIX}:${note.id}`, reaction, 1); } @bindThis public async delete(note: MiNote, reaction: string) { - this.redisClient.hincrby(`reactionsBuffer:${note.id}`, reaction, -1); + this.redisClient.hincrby(`${REDIS_PREFIX}:${note.id}`, reaction, -1); } @bindThis public async get(note: MiNote) { - const result = await this.redisClient.hgetall(`reactionsBuffer:${note.id}`); + const result = await this.redisClient.hgetall(`${REDIS_PREFIX}:${note.id}`); const delta = {}; for (const [name, count] of Object.entries(result)) { delta[name] = parseInt(count); @@ -43,7 +45,7 @@ export class ReactionsBufferingService { const pipeline = this.redisClient.pipeline(); for (const note of notes) { - pipeline.hgetall(`reactionsBuffer:${note.id}`); + pipeline.hgetall(`${REDIS_PREFIX}:${note.id}`); } const results = await pipeline.exec(); @@ -59,4 +61,17 @@ export class ReactionsBufferingService { return deltas; } + + @bindThis + public async bake() { + const bufferedNoteIds = []; + let cursor = '0'; + do { + const result = await this.redisClient.scan(cursor, 'MATCH', `${REDIS_PREFIX}:*`, 'COUNT', '1000'); + cursor = result[0]; + bufferedNoteIds.push(...result[1]); + } while (cursor !== '0'); + + console.log(bufferedNoteIds); + } } From 59b8ca7e4efa71cbd120643633ab0d2bdf5eb74a Mon Sep 17 00:00:00 2001 From: syuilo <4439005+syuilo@users.noreply.github.com> Date: Fri, 20 Sep 2024 09:37:01 +0900 Subject: [PATCH 04/17] Update ReactionsBufferingService.ts --- packages/backend/src/core/ReactionsBufferingService.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/backend/src/core/ReactionsBufferingService.ts b/packages/backend/src/core/ReactionsBufferingService.ts index f5beca993f00..e507495b898d 100644 --- a/packages/backend/src/core/ReactionsBufferingService.ts +++ b/packages/backend/src/core/ReactionsBufferingService.ts @@ -62,6 +62,7 @@ export class ReactionsBufferingService { return deltas; } + // TODO: scanは重い可能性があるので、別途 bufferedNoteIds を直接Redis上に持っておいてもいいかもしれない @bindThis public async bake() { const bufferedNoteIds = []; From 09b1773e2fcb3445ecd8e8f899401ff6a9cadecd Mon Sep 17 00:00:00 2001 From: syuilo <4439005+syuilo@users.noreply.github.com> Date: Fri, 20 Sep 2024 10:19:51 +0900 Subject: [PATCH 05/17] wip --- .../src/core/ReactionsBufferingService.ts | 45 +++++++++++++++---- .../src/core/entities/NoteEntityService.ts | 7 +-- 2 files changed, 38 insertions(+), 14 deletions(-) diff --git a/packages/backend/src/core/ReactionsBufferingService.ts b/packages/backend/src/core/ReactionsBufferingService.ts index e507495b898d..47379886a07c 100644 --- a/packages/backend/src/core/ReactionsBufferingService.ts +++ b/packages/backend/src/core/ReactionsBufferingService.ts @@ -8,6 +8,7 @@ import * as Redis from 'ioredis'; import { DI } from '@/di-symbols.js'; import type { MiNote } from '@/models/Note.js'; import { bindThis } from '@/decorators.js'; +import type { NotesRepository } from '@/models/_.js'; const REDIS_PREFIX = 'reactionsBuffer'; @@ -16,6 +17,9 @@ export class ReactionsBufferingService { constructor( @Inject(DI.redis) private redisClient: Redis.Redis, // TODO: 専用のRedisインスタンスにする + + @Inject(DI.notesRepository) + private notesRepository: NotesRepository, ) { } @@ -30,8 +34,8 @@ export class ReactionsBufferingService { } @bindThis - public async get(note: MiNote) { - const result = await this.redisClient.hgetall(`${REDIS_PREFIX}:${note.id}`); + public async get(noteId: MiNote['id']) { + const result = await this.redisClient.hgetall(`${REDIS_PREFIX}:${noteId}`); const delta = {}; for (const [name, count] of Object.entries(result)) { delta[name] = parseInt(count); @@ -40,23 +44,23 @@ export class ReactionsBufferingService { } @bindThis - public async getMany(notes: MiNote[]) { + public async getMany(noteIds: MiNote['id'][]) { const deltas = new Map>(); const pipeline = this.redisClient.pipeline(); - for (const note of notes) { - pipeline.hgetall(`${REDIS_PREFIX}:${note.id}`); + for (const noteId of noteIds) { + pipeline.hgetall(`${REDIS_PREFIX}:${noteId}`); } const results = await pipeline.exec(); - for (let i = 0; i < notes.length; i++) { - const note = notes[i]; + for (let i = 0; i < noteIds.length; i++) { + const noteId = noteIds[i]; const result = results![i][1]; const delta = {}; for (const [name, count] of Object.entries(result)) { delta[name] = parseInt(count); } - deltas.set(note.id, delta); + deltas.set(noteId, delta); } return deltas; @@ -74,5 +78,30 @@ export class ReactionsBufferingService { } while (cursor !== '0'); console.log(bufferedNoteIds); + + const deltas = await this.getMany(bufferedNoteIds); + + console.log(deltas); + + // clear + const pipeline = this.redisClient.pipeline(); + for (const noteId of bufferedNoteIds) { + pipeline.del(`${REDIS_PREFIX}:${noteId}`); + } + await pipeline.exec(); + + // TODO: SQL一個にまとめたい + for (const [noteId, delta] of deltas) { + const sqls = [] as string[]; + for (const [reaction, count] of Object.entries(delta)) { + sqls.push(`jsonb_set("reactions", '{${reaction}}', (COALESCE("reactions"->>'${reaction}', '0')::int + ${count})::text::jsonb)`); + } + this.notesRepository.createQueryBuilder().update() + .set({ + reactions: () => sqls.join(' || '), + }) + .where('id = :id', { id: noteId }) + .execute(); + } } } diff --git a/packages/backend/src/core/entities/NoteEntityService.ts b/packages/backend/src/core/entities/NoteEntityService.ts index 9066bb8b5c82..e4a55ef4d053 100644 --- a/packages/backend/src/core/entities/NoteEntityService.ts +++ b/packages/backend/src/core/entities/NoteEntityService.ts @@ -6,13 +6,11 @@ import { Inject, Injectable } from '@nestjs/common'; import { In } from 'typeorm'; import { ModuleRef } from '@nestjs/core'; -import * as Redis from 'ioredis'; import { DI } from '@/di-symbols.js'; import type { Packed } from '@/misc/json-schema.js'; import { awaitAll } from '@/misc/prelude/await-all.js'; import type { MiUser } from '@/models/User.js'; import type { MiNote } from '@/models/Note.js'; -import type { MiNoteReaction } from '@/models/NoteReaction.js'; import type { UsersRepository, NotesRepository, FollowingsRepository, PollsRepository, PollVotesRepository, NoteReactionsRepository, ChannelsRepository } from '@/models/_.js'; import { bindThis } from '@/decorators.js'; import { DebounceLoader } from '@/misc/loader.js'; @@ -49,9 +47,6 @@ export class NoteEntityService implements OnModuleInit { constructor( private moduleRef: ModuleRef, - @Inject(DI.redis) - private redisClient: Redis.Redis, // TODO: 専用のRedisインスタンスにする - @Inject(DI.usersRepository) private usersRepository: UsersRepository, @@ -423,7 +418,7 @@ export class NoteEntityService implements OnModuleInit { if (notes.length === 0) return []; const rbt = true; - const reactionsDeltas = rbt ? await this.reactionsBufferingService.getMany(notes) : new Map(); + const reactionsDeltas = rbt ? await this.reactionsBufferingService.getMany(notes.map(x => x.id)) : new Map(); const meId = me ? me.id : null; const myReactionsMap = new Map(); From 4777eecf017c48746f14766106f586c4e502328c Mon Sep 17 00:00:00 2001 From: syuilo <4439005+syuilo@users.noreply.github.com> Date: Fri, 20 Sep 2024 10:27:38 +0900 Subject: [PATCH 06/17] wip --- packages/backend/src/core/QueueService.ts | 6 ++++ .../src/core/ReactionsBufferingService.ts | 12 +++---- .../backend/src/queue/QueueProcessorModule.ts | 2 ++ .../src/queue/QueueProcessorService.ts | 3 ++ .../BakeBufferedReactionsProcessorService.ts | 32 +++++++++++++++++++ 5 files changed, 49 insertions(+), 6 deletions(-) create mode 100644 packages/backend/src/queue/processors/BakeBufferedReactionsProcessorService.ts diff --git a/packages/backend/src/core/QueueService.ts b/packages/backend/src/core/QueueService.ts index ddb90a051fc7..f35e456556df 100644 --- a/packages/backend/src/core/QueueService.ts +++ b/packages/backend/src/core/QueueService.ts @@ -87,6 +87,12 @@ export class QueueService { repeat: { pattern: '*/5 * * * *' }, removeOnComplete: true, }); + + this.systemQueue.add('bakeBufferedReactions', { + }, { + repeat: { pattern: '0 0 * * *' }, + removeOnComplete: true, + }); } @bindThis diff --git a/packages/backend/src/core/ReactionsBufferingService.ts b/packages/backend/src/core/ReactionsBufferingService.ts index 47379886a07c..9a9f78481bd6 100644 --- a/packages/backend/src/core/ReactionsBufferingService.ts +++ b/packages/backend/src/core/ReactionsBufferingService.ts @@ -34,9 +34,9 @@ export class ReactionsBufferingService { } @bindThis - public async get(noteId: MiNote['id']) { + public async get(noteId: MiNote['id']): Promise> { const result = await this.redisClient.hgetall(`${REDIS_PREFIX}:${noteId}`); - const delta = {}; + const delta = {} as Record; for (const [name, count] of Object.entries(result)) { delta[name] = parseInt(count); } @@ -44,7 +44,7 @@ export class ReactionsBufferingService { } @bindThis - public async getMany(noteIds: MiNote['id'][]) { + public async getMany(noteIds: MiNote['id'][]): Promise>> { const deltas = new Map>(); const pipeline = this.redisClient.pipeline(); @@ -55,8 +55,8 @@ export class ReactionsBufferingService { for (let i = 0; i < noteIds.length; i++) { const noteId = noteIds[i]; - const result = results![i][1]; - const delta = {}; + const result = results![i][1] as Record; + const delta = {} as Record; for (const [name, count] of Object.entries(result)) { delta[name] = parseInt(count); } @@ -68,7 +68,7 @@ export class ReactionsBufferingService { // TODO: scanは重い可能性があるので、別途 bufferedNoteIds を直接Redis上に持っておいてもいいかもしれない @bindThis - public async bake() { + public async bake(): Promise { const bufferedNoteIds = []; let cursor = '0'; do { diff --git a/packages/backend/src/queue/QueueProcessorModule.ts b/packages/backend/src/queue/QueueProcessorModule.ts index a1fd38fcc58c..0027b5ef3d7a 100644 --- a/packages/backend/src/queue/QueueProcessorModule.ts +++ b/packages/backend/src/queue/QueueProcessorModule.ts @@ -14,6 +14,7 @@ import { InboxProcessorService } from './processors/InboxProcessorService.js'; import { UserWebhookDeliverProcessorService } from './processors/UserWebhookDeliverProcessorService.js'; import { SystemWebhookDeliverProcessorService } from './processors/SystemWebhookDeliverProcessorService.js'; import { CheckExpiredMutingsProcessorService } from './processors/CheckExpiredMutingsProcessorService.js'; +import { BakeBufferedReactionsProcessorService } from './processors/BakeBufferedReactionsProcessorService.js'; import { CleanChartsProcessorService } from './processors/CleanChartsProcessorService.js'; import { CleanProcessorService } from './processors/CleanProcessorService.js'; import { CleanRemoteFilesProcessorService } from './processors/CleanRemoteFilesProcessorService.js'; @@ -51,6 +52,7 @@ import { RelationshipProcessorService } from './processors/RelationshipProcessor ResyncChartsProcessorService, CleanChartsProcessorService, CheckExpiredMutingsProcessorService, + BakeBufferedReactionsProcessorService, CleanProcessorService, DeleteDriveFilesProcessorService, ExportCustomEmojisProcessorService, diff --git a/packages/backend/src/queue/QueueProcessorService.ts b/packages/backend/src/queue/QueueProcessorService.ts index 7bd74f3210f8..e9e1c4522469 100644 --- a/packages/backend/src/queue/QueueProcessorService.ts +++ b/packages/backend/src/queue/QueueProcessorService.ts @@ -39,6 +39,7 @@ import { TickChartsProcessorService } from './processors/TickChartsProcessorServ import { ResyncChartsProcessorService } from './processors/ResyncChartsProcessorService.js'; import { CleanChartsProcessorService } from './processors/CleanChartsProcessorService.js'; import { CheckExpiredMutingsProcessorService } from './processors/CheckExpiredMutingsProcessorService.js'; +import { BakeBufferedReactionsProcessorService } from './processors/BakeBufferedReactionsProcessorService.js'; import { CleanProcessorService } from './processors/CleanProcessorService.js'; import { AggregateRetentionProcessorService } from './processors/AggregateRetentionProcessorService.js'; import { QueueLoggerService } from './QueueLoggerService.js'; @@ -118,6 +119,7 @@ export class QueueProcessorService implements OnApplicationShutdown { private cleanChartsProcessorService: CleanChartsProcessorService, private aggregateRetentionProcessorService: AggregateRetentionProcessorService, private checkExpiredMutingsProcessorService: CheckExpiredMutingsProcessorService, + private bakeBufferedReactionsProcessorService: BakeBufferedReactionsProcessorService, private cleanProcessorService: CleanProcessorService, ) { this.logger = this.queueLoggerService.logger; @@ -147,6 +149,7 @@ export class QueueProcessorService implements OnApplicationShutdown { case 'cleanCharts': return this.cleanChartsProcessorService.process(); case 'aggregateRetention': return this.aggregateRetentionProcessorService.process(); case 'checkExpiredMutings': return this.checkExpiredMutingsProcessorService.process(); + case 'bakeBufferedReactions': return this.bakeBufferedReactionsProcessorService.process(); case 'clean': return this.cleanProcessorService.process(); default: throw new Error(`unrecognized job type ${job.name} for system`); } diff --git a/packages/backend/src/queue/processors/BakeBufferedReactionsProcessorService.ts b/packages/backend/src/queue/processors/BakeBufferedReactionsProcessorService.ts new file mode 100644 index 000000000000..22bc28669296 --- /dev/null +++ b/packages/backend/src/queue/processors/BakeBufferedReactionsProcessorService.ts @@ -0,0 +1,32 @@ +/* + * SPDX-FileCopyrightText: syuilo and misskey-project + * SPDX-License-Identifier: AGPL-3.0-only + */ + +import { Inject, Injectable } from '@nestjs/common'; +import type Logger from '@/logger.js'; +import { bindThis } from '@/decorators.js'; +import { ReactionsBufferingService } from '@/core/ReactionsBufferingService.js'; +import { QueueLoggerService } from '../QueueLoggerService.js'; +import type * as Bull from 'bullmq'; + +@Injectable() +export class BakeBufferedReactionsProcessorService { + private logger: Logger; + + constructor( + private reactionsBufferingService: ReactionsBufferingService, + private queueLoggerService: QueueLoggerService, + ) { + this.logger = this.queueLoggerService.logger.createSubLogger('bake-buffered-reactions'); + } + + @bindThis + public async process(): Promise { + this.logger.info('Baking buffered reactions...'); + + await this.reactionsBufferingService.bake(); + + this.logger.succ('All buffered reactions baked.'); + } +} From 89846ab47e1ef5b64d8c69da7253b30910f0f66a Mon Sep 17 00:00:00 2001 From: syuilo <4439005+syuilo@users.noreply.github.com> Date: Fri, 20 Sep 2024 10:41:25 +0900 Subject: [PATCH 07/17] wip --- packages/backend/src/core/ReactionService.ts | 5 ++ .../src/pages/admin/other-settings.vue | 72 +++++++++++++++++++ .../frontend/src/pages/admin/settings.vue | 50 ------------- 3 files changed, 77 insertions(+), 50 deletions(-) diff --git a/packages/backend/src/core/ReactionService.ts b/packages/backend/src/core/ReactionService.ts index 595da8c7224e..1c8d62bc2d42 100644 --- a/packages/backend/src/core/ReactionService.ts +++ b/packages/backend/src/core/ReactionService.ts @@ -203,6 +203,11 @@ export class ReactionService { // Increment reactions count if (rbt) { this.reactionsBufferingService.create(note, reaction); + + // for debugging + if (reaction === ':angry_ai:') { + this.reactionsBufferingService.bake(); + } } else { const sql = `jsonb_set("reactions", '{${reaction}}', (COALESCE("reactions"->>'${reaction}', '0')::int + 1)::text::jsonb)`; await this.notesRepository.createQueryBuilder().update() diff --git a/packages/frontend/src/pages/admin/other-settings.vue b/packages/frontend/src/pages/admin/other-settings.vue index 345cf333b51f..72d9e8789162 100644 --- a/packages/frontend/src/pages/admin/other-settings.vue +++ b/packages/frontend/src/pages/admin/other-settings.vue @@ -36,6 +36,55 @@ SPDX-License-Identifier: AGPL-3.0-only + + + + + + + +
+ + + + + + + + + + + + + + + + + + + + + + + + + +
+ + + + + + + + +
+ + + + +
+ @@ -52,11 +101,20 @@ import { fetchInstance } from '@/instance.js'; import { i18n } from '@/i18n.js'; import { definePageMetadata } from '@/scripts/page-metadata.js'; import MkSwitch from '@/components/MkSwitch.vue'; +import MkFolder from '@/components/MkFolder.vue'; +import MkInput from '@/components/MkInput.vue'; const enableServerMachineStats = ref(false); const enableIdenticonGeneration = ref(false); const enableChartsForRemoteUser = ref(false); const enableChartsForFederatedInstances = ref(false); +const enableFanoutTimeline = ref(false); +const enableFanoutTimelineDbFallback = ref(false); +const perLocalUserUserTimelineCacheMax = ref(0); +const perRemoteUserUserTimelineCacheMax = ref(0); +const perUserHomeTimelineCacheMax = ref(0); +const perUserListTimelineCacheMax = ref(0); +const enableReactionsBuffering = ref(false); async function init() { const meta = await misskeyApi('admin/meta'); @@ -64,6 +122,13 @@ async function init() { enableIdenticonGeneration.value = meta.enableIdenticonGeneration; enableChartsForRemoteUser.value = meta.enableChartsForRemoteUser; enableChartsForFederatedInstances.value = meta.enableChartsForFederatedInstances; + enableFanoutTimeline.value = meta.enableFanoutTimeline; + enableFanoutTimelineDbFallback.value = meta.enableFanoutTimelineDbFallback; + perLocalUserUserTimelineCacheMax.value = meta.perLocalUserUserTimelineCacheMax; + perRemoteUserUserTimelineCacheMax.value = meta.perRemoteUserUserTimelineCacheMax; + perUserHomeTimelineCacheMax.value = meta.perUserHomeTimelineCacheMax; + perUserListTimelineCacheMax.value = meta.perUserListTimelineCacheMax; + enableReactionsBuffering.value = meta.enableReactionsBuffering; } function save() { @@ -72,6 +137,13 @@ function save() { enableIdenticonGeneration: enableIdenticonGeneration.value, enableChartsForRemoteUser: enableChartsForRemoteUser.value, enableChartsForFederatedInstances: enableChartsForFederatedInstances.value, + enableFanoutTimeline: enableFanoutTimeline.value, + enableFanoutTimelineDbFallback: enableFanoutTimelineDbFallback.value, + perLocalUserUserTimelineCacheMax: perLocalUserUserTimelineCacheMax.value, + perRemoteUserUserTimelineCacheMax: perRemoteUserUserTimelineCacheMax.value, + perUserHomeTimelineCacheMax: perUserHomeTimelineCacheMax.value, + perUserListTimelineCacheMax: perUserListTimelineCacheMax.value, + enableReactionsBuffering: enableReactionsBuffering.value, }).then(() => { fetchInstance(true); }); diff --git a/packages/frontend/src/pages/admin/settings.vue b/packages/frontend/src/pages/admin/settings.vue index 6f45c212ece1..ffff57b45453 100644 --- a/packages/frontend/src/pages/admin/settings.vue +++ b/packages/frontend/src/pages/admin/settings.vue @@ -96,38 +96,6 @@ SPDX-License-Identifier: AGPL-3.0-only - - - -
- - - - - - - - - - - - - - - - - - - - - - - - - -
- - @@ -236,12 +204,6 @@ const cacheRemoteSensitiveFiles = ref(false); const enableServiceWorker = ref(false); const swPublicKey = ref(null); const swPrivateKey = ref(null); -const enableFanoutTimeline = ref(false); -const enableFanoutTimelineDbFallback = ref(false); -const perLocalUserUserTimelineCacheMax = ref(0); -const perRemoteUserUserTimelineCacheMax = ref(0); -const perUserHomeTimelineCacheMax = ref(0); -const perUserListTimelineCacheMax = ref(0); const notesPerOneAd = ref(0); const urlPreviewEnabled = ref(true); const urlPreviewTimeout = ref(10000); @@ -265,12 +227,6 @@ async function init(): Promise { enableServiceWorker.value = meta.enableServiceWorker; swPublicKey.value = meta.swPublickey; swPrivateKey.value = meta.swPrivateKey; - enableFanoutTimeline.value = meta.enableFanoutTimeline; - enableFanoutTimelineDbFallback.value = meta.enableFanoutTimelineDbFallback; - perLocalUserUserTimelineCacheMax.value = meta.perLocalUserUserTimelineCacheMax; - perRemoteUserUserTimelineCacheMax.value = meta.perRemoteUserUserTimelineCacheMax; - perUserHomeTimelineCacheMax.value = meta.perUserHomeTimelineCacheMax; - perUserListTimelineCacheMax.value = meta.perUserListTimelineCacheMax; notesPerOneAd.value = meta.notesPerOneAd; urlPreviewEnabled.value = meta.urlPreviewEnabled; urlPreviewTimeout.value = meta.urlPreviewTimeout; @@ -295,12 +251,6 @@ async function save() { enableServiceWorker: enableServiceWorker.value, swPublicKey: swPublicKey.value, swPrivateKey: swPrivateKey.value, - enableFanoutTimeline: enableFanoutTimeline.value, - enableFanoutTimelineDbFallback: enableFanoutTimelineDbFallback.value, - perLocalUserUserTimelineCacheMax: perLocalUserUserTimelineCacheMax.value, - perRemoteUserUserTimelineCacheMax: perRemoteUserUserTimelineCacheMax.value, - perUserHomeTimelineCacheMax: perUserHomeTimelineCacheMax.value, - perUserListTimelineCacheMax: perUserListTimelineCacheMax.value, notesPerOneAd: notesPerOneAd.value, urlPreviewEnabled: urlPreviewEnabled.value, urlPreviewTimeout: urlPreviewTimeout.value, From 428ed0f5bde6f239c329a685e6c20c8963565097 Mon Sep 17 00:00:00 2001 From: syuilo <4439005+syuilo@users.noreply.github.com> Date: Fri, 20 Sep 2024 11:57:57 +0900 Subject: [PATCH 08/17] Update ReactionsBufferingService.ts --- .../backend/src/core/ReactionsBufferingService.ts | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/packages/backend/src/core/ReactionsBufferingService.ts b/packages/backend/src/core/ReactionsBufferingService.ts index 9a9f78481bd6..1c8e46ea56c9 100644 --- a/packages/backend/src/core/ReactionsBufferingService.ts +++ b/packages/backend/src/core/ReactionsBufferingService.ts @@ -9,12 +9,16 @@ import { DI } from '@/di-symbols.js'; import type { MiNote } from '@/models/Note.js'; import { bindThis } from '@/decorators.js'; import type { NotesRepository } from '@/models/_.js'; +import type { Config } from '@/config.js'; const REDIS_PREFIX = 'reactionsBuffer'; @Injectable() export class ReactionsBufferingService { constructor( + @Inject(DI.config) + private config: Config, + @Inject(DI.redis) private redisClient: Redis.Redis, // TODO: 専用のRedisインスタンスにする @@ -72,17 +76,15 @@ export class ReactionsBufferingService { const bufferedNoteIds = []; let cursor = '0'; do { - const result = await this.redisClient.scan(cursor, 'MATCH', `${REDIS_PREFIX}:*`, 'COUNT', '1000'); + // https://github.com/redis/ioredis#transparent-key-prefixing + const result = await this.redisClient.scan(cursor, 'MATCH', `${this.config.redis.prefix}:${REDIS_PREFIX}:*`, 'COUNT', '1000'); + console.log(result); cursor = result[0]; - bufferedNoteIds.push(...result[1]); + bufferedNoteIds.push(...result[1].map(x => x.replace(`${this.config.redis.prefix}:${REDIS_PREFIX}:`, ''))); } while (cursor !== '0'); - console.log(bufferedNoteIds); - const deltas = await this.getMany(bufferedNoteIds); - console.log(deltas); - // clear const pipeline = this.redisClient.pipeline(); for (const noteId of bufferedNoteIds) { From e896b08722a8068ac596c8e3d2dcd3fcef5cd419 Mon Sep 17 00:00:00 2001 From: syuilo <4439005+syuilo@users.noreply.github.com> Date: Fri, 20 Sep 2024 12:57:15 +0900 Subject: [PATCH 09/17] wip --- .../1726804538569-reactions-buffering.js | 16 ++++++++++++++++ packages/backend/src/models/Meta.ts | 16 ++++------------ .../src/server/api/endpoints/admin/meta.ts | 5 +++++ .../server/api/endpoints/admin/update-meta.ts | 5 +++++ .../frontend/src/pages/admin/other-settings.vue | 2 +- packages/misskey-js/src/autogen/types.ts | 2 ++ 6 files changed, 33 insertions(+), 13 deletions(-) create mode 100644 packages/backend/migration/1726804538569-reactions-buffering.js diff --git a/packages/backend/migration/1726804538569-reactions-buffering.js b/packages/backend/migration/1726804538569-reactions-buffering.js new file mode 100644 index 000000000000..bc19e9cc8aa6 --- /dev/null +++ b/packages/backend/migration/1726804538569-reactions-buffering.js @@ -0,0 +1,16 @@ +/* + * SPDX-FileCopyrightText: syuilo and misskey-project + * SPDX-License-Identifier: AGPL-3.0-only + */ + +export class ReactionsBuffering1726804538569 { + name = 'ReactionsBuffering1726804538569' + + async up(queryRunner) { + await queryRunner.query(`ALTER TABLE "meta" ADD "enableReactionsBuffering" boolean NOT NULL DEFAULT false`); + } + + async down(queryRunner) { + await queryRunner.query(`ALTER TABLE "meta" DROP COLUMN "enableReactionsBuffering"`); + } +} diff --git a/packages/backend/src/models/Meta.ts b/packages/backend/src/models/Meta.ts index 6ec4da187386..9ab76d373f8c 100644 --- a/packages/backend/src/models/Meta.ts +++ b/packages/backend/src/models/Meta.ts @@ -589,18 +589,10 @@ export class MiMeta { }) public perUserListTimelineCacheMax: number; - // @Column('boolean', { - // default: true, - // }) - // public enableReactionsBuffering: boolean; - // - // /** - // * 分 - // */ - // @Column('integer', { - // default: 180, - // }) - // public reactionsBufferingTtl: number; + @Column('boolean', { + default: false, + }) + public enableReactionsBuffering: boolean; @Column('integer', { default: 0, diff --git a/packages/backend/src/server/api/endpoints/admin/meta.ts b/packages/backend/src/server/api/endpoints/admin/meta.ts index 2e7f73da73b2..29e8bfaf1466 100644 --- a/packages/backend/src/server/api/endpoints/admin/meta.ts +++ b/packages/backend/src/server/api/endpoints/admin/meta.ts @@ -377,6 +377,10 @@ export const meta = { type: 'number', optional: false, nullable: false, }, + enableReactionsBuffering: { + type: 'boolean', + optional: false, nullable: false, + }, notesPerOneAd: { type: 'number', optional: false, nullable: false, @@ -617,6 +621,7 @@ export default class extends Endpoint { // eslint- perRemoteUserUserTimelineCacheMax: instance.perRemoteUserUserTimelineCacheMax, perUserHomeTimelineCacheMax: instance.perUserHomeTimelineCacheMax, perUserListTimelineCacheMax: instance.perUserListTimelineCacheMax, + enableReactionsBuffering: instance.enableReactionsBuffering, notesPerOneAd: instance.notesPerOneAd, summalyProxy: instance.urlPreviewSummaryProxyUrl, urlPreviewEnabled: instance.urlPreviewEnabled, diff --git a/packages/backend/src/server/api/endpoints/admin/update-meta.ts b/packages/backend/src/server/api/endpoints/admin/update-meta.ts index 5efdc9d8c457..865e73f27485 100644 --- a/packages/backend/src/server/api/endpoints/admin/update-meta.ts +++ b/packages/backend/src/server/api/endpoints/admin/update-meta.ts @@ -142,6 +142,7 @@ export const paramDef = { perRemoteUserUserTimelineCacheMax: { type: 'integer' }, perUserHomeTimelineCacheMax: { type: 'integer' }, perUserListTimelineCacheMax: { type: 'integer' }, + enableReactionsBuffering: { type: 'boolean' }, notesPerOneAd: { type: 'integer' }, silencedHosts: { type: 'array', @@ -598,6 +599,10 @@ export default class extends Endpoint { // eslint- set.perUserListTimelineCacheMax = ps.perUserListTimelineCacheMax; } + if (ps.enableReactionsBuffering !== undefined) { + set.enableReactionsBuffering = ps.enableReactionsBuffering; + } + if (ps.notesPerOneAd !== undefined) { set.notesPerOneAd = ps.notesPerOneAd; } diff --git a/packages/frontend/src/pages/admin/other-settings.vue b/packages/frontend/src/pages/admin/other-settings.vue index 72d9e8789162..0163daf1baa2 100644 --- a/packages/frontend/src/pages/admin/other-settings.vue +++ b/packages/frontend/src/pages/admin/other-settings.vue @@ -74,7 +74,7 @@ SPDX-License-Identifier: AGPL-3.0-only - + diff --git a/packages/misskey-js/src/autogen/types.ts b/packages/misskey-js/src/autogen/types.ts index 03828b6552a9..672d75e2671c 100644 --- a/packages/misskey-js/src/autogen/types.ts +++ b/packages/misskey-js/src/autogen/types.ts @@ -5125,6 +5125,7 @@ export type operations = { perRemoteUserUserTimelineCacheMax: number; perUserHomeTimelineCacheMax: number; perUserListTimelineCacheMax: number; + enableReactionsBuffering: boolean; notesPerOneAd: number; backgroundImageUrl: string | null; deeplAuthKey: string | null; @@ -9395,6 +9396,7 @@ export type operations = { perRemoteUserUserTimelineCacheMax?: number; perUserHomeTimelineCacheMax?: number; perUserListTimelineCacheMax?: number; + enableReactionsBuffering?: boolean; notesPerOneAd?: number; silencedHosts?: string[] | null; mediaSilencedHosts?: string[] | null; From 480ec9803a9a130482a83b7664fe77ac4b1f4d20 Mon Sep 17 00:00:00 2001 From: syuilo <4439005+syuilo@users.noreply.github.com> Date: Fri, 20 Sep 2024 13:08:04 +0900 Subject: [PATCH 10/17] wip --- .config/cypress-devcontainer.yml | 8 +++++++ .config/docker_example.yml | 8 +++++++ .config/example.yml | 10 ++++++++ .devcontainer/devcontainer.yml | 8 +++++++ chart/files/default.yml | 8 +++++++ packages/backend/src/GlobalModule.ts | 14 +++++++++-- packages/backend/src/config.ts | 3 +++ packages/backend/src/core/ReactionService.ts | 8 +------ .../src/core/ReactionsBufferingService.ts | 23 +++++++++++-------- .../src/core/entities/NoteEntityService.ts | 10 ++++++-- packages/backend/src/di-symbols.ts | 1 + .../BakeBufferedReactionsProcessorService.ts | 8 +++++++ .../backend/src/server/HealthServerService.ts | 4 ++++ 13 files changed, 93 insertions(+), 20 deletions(-) diff --git a/.config/cypress-devcontainer.yml b/.config/cypress-devcontainer.yml index e8da5f5e276a..91dce3515585 100644 --- a/.config/cypress-devcontainer.yml +++ b/.config/cypress-devcontainer.yml @@ -103,6 +103,14 @@ redis: # #prefix: example-prefix # #db: 1 +#redisForReactions: +# host: redis +# port: 6379 +# #family: 0 # 0=Both, 4=IPv4, 6=IPv6 +# #pass: example-pass +# #prefix: example-prefix +# #db: 1 + # ┌───────────────────────────┐ #───┘ MeiliSearch configuration └───────────────────────────── diff --git a/.config/docker_example.yml b/.config/docker_example.yml index d347882d1a91..3f8e5734ce87 100644 --- a/.config/docker_example.yml +++ b/.config/docker_example.yml @@ -106,6 +106,14 @@ redis: # #prefix: example-prefix # #db: 1 +#redisForReactions: +# host: redis +# port: 6379 +# #family: 0 # 0=Both, 4=IPv4, 6=IPv6 +# #pass: example-pass +# #prefix: example-prefix +# #db: 1 + # ┌───────────────────────────┐ #───┘ MeiliSearch configuration └───────────────────────────── diff --git a/.config/example.yml b/.config/example.yml index b11cbd137328..7080159117ac 100644 --- a/.config/example.yml +++ b/.config/example.yml @@ -172,6 +172,16 @@ redis: # # You can specify more ioredis options... # #username: example-username +#redisForReactions: +# host: localhost +# port: 6379 +# #family: 0 # 0=Both, 4=IPv4, 6=IPv6 +# #pass: example-pass +# #prefix: example-prefix +# #db: 1 +# # You can specify more ioredis options... +# #username: example-username + # ┌───────────────────────────┐ #───┘ MeiliSearch configuration └───────────────────────────── diff --git a/.devcontainer/devcontainer.yml b/.devcontainer/devcontainer.yml index beefcfd0a2d5..3eb4fc28794b 100644 --- a/.devcontainer/devcontainer.yml +++ b/.devcontainer/devcontainer.yml @@ -103,6 +103,14 @@ redis: # #prefix: example-prefix # #db: 1 +#redisForReactions: +# host: redis +# port: 6379 +# #family: 0 # 0=Both, 4=IPv4, 6=IPv6 +# #pass: example-pass +# #prefix: example-prefix +# #db: 1 + # ┌───────────────────────────┐ #───┘ MeiliSearch configuration └───────────────────────────── diff --git a/chart/files/default.yml b/chart/files/default.yml index f98b8ebfee04..4d17131c2546 100644 --- a/chart/files/default.yml +++ b/chart/files/default.yml @@ -124,6 +124,14 @@ redis: # #prefix: example-prefix # #db: 1 +#redisForReactions: +# host: redis +# port: 6379 +# #family: 0 # 0=Both, 4=IPv4, 6=IPv6 +# #pass: example-pass +# #prefix: example-prefix +# #db: 1 + # ┌───────────────────────────┐ #───┘ MeiliSearch configuration └───────────────────────────── diff --git a/packages/backend/src/GlobalModule.ts b/packages/backend/src/GlobalModule.ts index 09971e8ca022..2ecc1f474259 100644 --- a/packages/backend/src/GlobalModule.ts +++ b/packages/backend/src/GlobalModule.ts @@ -78,11 +78,19 @@ const $redisForTimelines: Provider = { inject: [DI.config], }; +const $redisForReactions: Provider = { + provide: DI.redisForReactions, + useFactory: (config: Config) => { + return new Redis.Redis(config.redisForReactions); + }, + inject: [DI.config], +}; + @Global() @Module({ imports: [RepositoryModule], - providers: [$config, $db, $meilisearch, $redis, $redisForPub, $redisForSub, $redisForTimelines], - exports: [$config, $db, $meilisearch, $redis, $redisForPub, $redisForSub, $redisForTimelines, RepositoryModule], + providers: [$config, $db, $meilisearch, $redis, $redisForPub, $redisForSub, $redisForTimelines, $redisForReactions], + exports: [$config, $db, $meilisearch, $redis, $redisForPub, $redisForSub, $redisForTimelines, $redisForReactions, RepositoryModule], }) export class GlobalModule implements OnApplicationShutdown { constructor( @@ -91,6 +99,7 @@ export class GlobalModule implements OnApplicationShutdown { @Inject(DI.redisForPub) private redisForPub: Redis.Redis, @Inject(DI.redisForSub) private redisForSub: Redis.Redis, @Inject(DI.redisForTimelines) private redisForTimelines: Redis.Redis, + @Inject(DI.redisForReactions) private redisForReactions: Redis.Redis, ) { } public async dispose(): Promise { @@ -103,6 +112,7 @@ export class GlobalModule implements OnApplicationShutdown { this.redisForPub.disconnect(), this.redisForSub.disconnect(), this.redisForTimelines.disconnect(), + this.redisForReactions.disconnect(), ]); } diff --git a/packages/backend/src/config.ts b/packages/backend/src/config.ts index cbd6d1c086dc..97ba79c5743b 100644 --- a/packages/backend/src/config.ts +++ b/packages/backend/src/config.ts @@ -49,6 +49,7 @@ type Source = { redisForPubsub?: RedisOptionsSource; redisForJobQueue?: RedisOptionsSource; redisForTimelines?: RedisOptionsSource; + redisForReactions?: RedisOptionsSource; meilisearch?: { host: string; port: string; @@ -171,6 +172,7 @@ export type Config = { redisForPubsub: RedisOptions & RedisOptionsSource; redisForJobQueue: RedisOptions & RedisOptionsSource; redisForTimelines: RedisOptions & RedisOptionsSource; + redisForReactions: RedisOptions & RedisOptionsSource; sentryForBackend: { options: Partial; enableNodeProfiling: boolean; } | undefined; sentryForFrontend: { options: Partial } | undefined; perChannelMaxNoteCacheCount: number; @@ -251,6 +253,7 @@ export function loadConfig(): Config { redisForPubsub: config.redisForPubsub ? convertRedisOptions(config.redisForPubsub, host) : redis, redisForJobQueue: config.redisForJobQueue ? convertRedisOptions(config.redisForJobQueue, host) : redis, redisForTimelines: config.redisForTimelines ? convertRedisOptions(config.redisForTimelines, host) : redis, + redisForReactions: config.redisForReactions ? convertRedisOptions(config.redisForReactions, host) : redis, sentryForBackend: config.sentryForBackend, sentryForFrontend: config.sentryForFrontend, id: config.id, diff --git a/packages/backend/src/core/ReactionService.ts b/packages/backend/src/core/ReactionService.ts index 1c8d62bc2d42..220a4f163e46 100644 --- a/packages/backend/src/core/ReactionService.ts +++ b/packages/backend/src/core/ReactionService.ts @@ -4,7 +4,6 @@ */ import { Inject, Injectable } from '@nestjs/common'; -import * as Redis from 'ioredis'; import { DI } from '@/di-symbols.js'; import type { EmojisRepository, NoteReactionsRepository, UsersRepository, NotesRepository } from '@/models/_.js'; import { IdentifiableError } from '@/misc/identifiable-error.js'; @@ -72,9 +71,6 @@ const decodeCustomEmojiRegexp = /^:([\w+-]+)(?:@([\w.-]+))?:$/; @Injectable() export class ReactionService { constructor( - @Inject(DI.redis) - private redisClient: Redis.Redis, // TODO: 専用のRedisインスタンスにする - @Inject(DI.usersRepository) private usersRepository: UsersRepository, @@ -198,10 +194,8 @@ export class ReactionService { } } - const rbt = true; - // Increment reactions count - if (rbt) { + if (meta.enableReactionsBuffering) { this.reactionsBufferingService.create(note, reaction); // for debugging diff --git a/packages/backend/src/core/ReactionsBufferingService.ts b/packages/backend/src/core/ReactionsBufferingService.ts index 1c8e46ea56c9..2a4a59e30ae1 100644 --- a/packages/backend/src/core/ReactionsBufferingService.ts +++ b/packages/backend/src/core/ReactionsBufferingService.ts @@ -19,8 +19,8 @@ export class ReactionsBufferingService { @Inject(DI.config) private config: Config, - @Inject(DI.redis) - private redisClient: Redis.Redis, // TODO: 専用のRedisインスタンスにする + @Inject(DI.redisForReactions) + private redisForReactions: Redis.Redis, // TODO: 専用のRedisインスタンスにする @Inject(DI.notesRepository) private notesRepository: NotesRepository, @@ -29,17 +29,17 @@ export class ReactionsBufferingService { @bindThis public async create(note: MiNote, reaction: string) { - this.redisClient.hincrby(`${REDIS_PREFIX}:${note.id}`, reaction, 1); + this.redisForReactions.hincrby(`${REDIS_PREFIX}:${note.id}`, reaction, 1); } @bindThis public async delete(note: MiNote, reaction: string) { - this.redisClient.hincrby(`${REDIS_PREFIX}:${note.id}`, reaction, -1); + this.redisForReactions.hincrby(`${REDIS_PREFIX}:${note.id}`, reaction, -1); } @bindThis public async get(noteId: MiNote['id']): Promise> { - const result = await this.redisClient.hgetall(`${REDIS_PREFIX}:${noteId}`); + const result = await this.redisForReactions.hgetall(`${REDIS_PREFIX}:${noteId}`); const delta = {} as Record; for (const [name, count] of Object.entries(result)) { delta[name] = parseInt(count); @@ -51,7 +51,7 @@ export class ReactionsBufferingService { public async getMany(noteIds: MiNote['id'][]): Promise>> { const deltas = new Map>(); - const pipeline = this.redisClient.pipeline(); + const pipeline = this.redisForReactions.pipeline(); for (const noteId of noteIds) { pipeline.hgetall(`${REDIS_PREFIX}:${noteId}`); } @@ -77,8 +77,13 @@ export class ReactionsBufferingService { let cursor = '0'; do { // https://github.com/redis/ioredis#transparent-key-prefixing - const result = await this.redisClient.scan(cursor, 'MATCH', `${this.config.redis.prefix}:${REDIS_PREFIX}:*`, 'COUNT', '1000'); - console.log(result); + const result = await this.redisForReactions.scan( + cursor, + 'MATCH', + `${this.config.redis.prefix}:${REDIS_PREFIX}:*`, + 'COUNT', + '1000'); + cursor = result[0]; bufferedNoteIds.push(...result[1].map(x => x.replace(`${this.config.redis.prefix}:${REDIS_PREFIX}:`, ''))); } while (cursor !== '0'); @@ -86,7 +91,7 @@ export class ReactionsBufferingService { const deltas = await this.getMany(bufferedNoteIds); // clear - const pipeline = this.redisClient.pipeline(); + const pipeline = this.redisForReactions.pipeline(); for (const noteId of bufferedNoteIds) { pipeline.del(`${REDIS_PREFIX}:${noteId}`); } diff --git a/packages/backend/src/core/entities/NoteEntityService.ts b/packages/backend/src/core/entities/NoteEntityService.ts index e4a55ef4d053..8a44d03fc23e 100644 --- a/packages/backend/src/core/entities/NoteEntityService.ts +++ b/packages/backend/src/core/entities/NoteEntityService.ts @@ -16,6 +16,7 @@ import { bindThis } from '@/decorators.js'; import { DebounceLoader } from '@/misc/loader.js'; import { IdService } from '@/core/IdService.js'; import { ReactionsBufferingService } from '@/core/ReactionsBufferingService.js'; +import { MetaService } from '@/core/MetaService.js'; import type { OnModuleInit } from '@nestjs/common'; import type { CustomEmojiService } from '../CustomEmojiService.js'; import type { ReactionService } from '../ReactionService.js'; @@ -42,6 +43,7 @@ export class NoteEntityService implements OnModuleInit { private reactionService: ReactionService; private reactionsBufferingService: ReactionsBufferingService; private idService: IdService; + private metaService: MetaService; private noteLoader = new DebounceLoader(this.findNoteOrFail); constructor( @@ -73,6 +75,8 @@ export class NoteEntityService implements OnModuleInit { //private customEmojiService: CustomEmojiService, //private reactionService: ReactionService, //private reactionsBufferingService: ReactionsBufferingService, + //private idService: IdService, + //private metaService: MetaService, ) { } @@ -83,6 +87,7 @@ export class NoteEntityService implements OnModuleInit { this.reactionService = this.moduleRef.get('ReactionService'); this.reactionsBufferingService = this.moduleRef.get('ReactionsBufferingService'); this.idService = this.moduleRef.get('IdService'); + this.metaService = this.moduleRef.get('MetaService'); } @bindThis @@ -417,8 +422,9 @@ export class NoteEntityService implements OnModuleInit { ) { if (notes.length === 0) return []; - const rbt = true; - const reactionsDeltas = rbt ? await this.reactionsBufferingService.getMany(notes.map(x => x.id)) : new Map(); + const meta = await this.metaService.fetch(); + + const reactionsDeltas = meta.enableReactionsBuffering ? await this.reactionsBufferingService.getMany(notes.map(x => x.id)) : new Map(); const meId = me ? me.id : null; const myReactionsMap = new Map(); diff --git a/packages/backend/src/di-symbols.ts b/packages/backend/src/di-symbols.ts index 271082b4ff35..b6f003c2e6ca 100644 --- a/packages/backend/src/di-symbols.ts +++ b/packages/backend/src/di-symbols.ts @@ -11,6 +11,7 @@ export const DI = { redisForPub: Symbol('redisForPub'), redisForSub: Symbol('redisForSub'), redisForTimelines: Symbol('redisForTimelines'), + redisForReactions: Symbol('redisForReactions'), //#region Repositories usersRepository: Symbol('usersRepository'), diff --git a/packages/backend/src/queue/processors/BakeBufferedReactionsProcessorService.ts b/packages/backend/src/queue/processors/BakeBufferedReactionsProcessorService.ts index 22bc28669296..cd56ba98376f 100644 --- a/packages/backend/src/queue/processors/BakeBufferedReactionsProcessorService.ts +++ b/packages/backend/src/queue/processors/BakeBufferedReactionsProcessorService.ts @@ -7,6 +7,7 @@ import { Inject, Injectable } from '@nestjs/common'; import type Logger from '@/logger.js'; import { bindThis } from '@/decorators.js'; import { ReactionsBufferingService } from '@/core/ReactionsBufferingService.js'; +import { MetaService } from '@/core/MetaService.js'; import { QueueLoggerService } from '../QueueLoggerService.js'; import type * as Bull from 'bullmq'; @@ -16,6 +17,7 @@ export class BakeBufferedReactionsProcessorService { constructor( private reactionsBufferingService: ReactionsBufferingService, + private metaService: MetaService, private queueLoggerService: QueueLoggerService, ) { this.logger = this.queueLoggerService.logger.createSubLogger('bake-buffered-reactions'); @@ -23,6 +25,12 @@ export class BakeBufferedReactionsProcessorService { @bindThis public async process(): Promise { + const meta = await this.metaService.fetch(); + if (!meta.enableReactionsBuffering) { + this.logger.info('Reactions buffering is disabled. Skipping...'); + return; + } + this.logger.info('Baking buffered reactions...'); await this.reactionsBufferingService.bake(); diff --git a/packages/backend/src/server/HealthServerService.ts b/packages/backend/src/server/HealthServerService.ts index 2c3ed85925c9..5980609f02bd 100644 --- a/packages/backend/src/server/HealthServerService.ts +++ b/packages/backend/src/server/HealthServerService.ts @@ -27,6 +27,9 @@ export class HealthServerService { @Inject(DI.redisForTimelines) private redisForTimelines: Redis.Redis, + @Inject(DI.redisForReactions) + private redisForReactions: Redis.Redis, + @Inject(DI.db) private db: DataSource, @@ -43,6 +46,7 @@ export class HealthServerService { this.redisForPub.ping(), this.redisForSub.ping(), this.redisForTimelines.ping(), + this.redisForReactions.ping(), this.db.query('SELECT 1'), ...(this.meilisearch ? [this.meilisearch.health()] : []), ]).then(() => 200, () => 503)); From f514d777ce8e203578d772ad18803d936b92607c Mon Sep 17 00:00:00 2001 From: syuilo <4439005+syuilo@users.noreply.github.com> Date: Fri, 20 Sep 2024 14:56:11 +0900 Subject: [PATCH 11/17] wip --- .../backend/src/core/ReactionsBufferingService.ts | 11 ++++++----- .../backend/src/core/entities/NoteEntityService.ts | 9 ++++++++- 2 files changed, 14 insertions(+), 6 deletions(-) diff --git a/packages/backend/src/core/ReactionsBufferingService.ts b/packages/backend/src/core/ReactionsBufferingService.ts index 2a4a59e30ae1..0514966ee8a6 100644 --- a/packages/backend/src/core/ReactionsBufferingService.ts +++ b/packages/backend/src/core/ReactionsBufferingService.ts @@ -99,13 +99,14 @@ export class ReactionsBufferingService { // TODO: SQL一個にまとめたい for (const [noteId, delta] of deltas) { - const sqls = [] as string[]; - for (const [reaction, count] of Object.entries(delta)) { - sqls.push(`jsonb_set("reactions", '{${reaction}}', (COALESCE("reactions"->>'${reaction}', '0')::int + ${count})::text::jsonb)`); - } + const sql = Object.entries(delta) + .map(([reaction, count]) => + `jsonb_set("reactions", '{${reaction}}', (COALESCE("reactions"->>'${reaction}', '0')::int + ${count})::text::jsonb)`) + .join(' || '); + this.notesRepository.createQueryBuilder().update() .set({ - reactions: () => sqls.join(' || '), + reactions: () => sql, }) .where('id = :id', { id: noteId }) .execute(); diff --git a/packages/backend/src/core/entities/NoteEntityService.ts b/packages/backend/src/core/entities/NoteEntityService.ts index 8a44d03fc23e..913fac1ec068 100644 --- a/packages/backend/src/core/entities/NoteEntityService.ts +++ b/packages/backend/src/core/entities/NoteEntityService.ts @@ -323,7 +323,14 @@ export class NoteEntityService implements OnModuleInit { const meId = me ? me.id : null; const note = typeof src === 'object' ? src : await this.noteLoader.load(src); const host = note.userHost; - const reactions = mergeReactions(note.reactions, opts._hint_?.reactionsDeltas.get(note.id) ?? {}); + + const reactionsDelta = opts._hint_?.reactionsDeltas != null ? (opts._hint_.reactionsDeltas.get(note.id) ?? {}) : await this.reactionsBufferingService.get(note.id); + const reactions = mergeReactions(note.reactions, reactionsDelta); + for (const [name, count] of Object.entries(reactions)) { + if (count <= 0) { + delete reactions[name]; + } + } let text = note.text; From 5c76ad8aff08d96a3b1369c03258471e348c46e7 Mon Sep 17 00:00:00 2001 From: syuilo <4439005+syuilo@users.noreply.github.com> Date: Fri, 20 Sep 2024 15:06:33 +0900 Subject: [PATCH 12/17] Update NoteEntityService.ts --- packages/backend/src/core/entities/NoteEntityService.ts | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/packages/backend/src/core/entities/NoteEntityService.ts b/packages/backend/src/core/entities/NoteEntityService.ts index 913fac1ec068..0d6e73045510 100644 --- a/packages/backend/src/core/entities/NoteEntityService.ts +++ b/packages/backend/src/core/entities/NoteEntityService.ts @@ -406,7 +406,11 @@ export class NoteEntityService implements OnModuleInit { poll: note.hasPoll ? this.populatePoll(note, meId) : undefined, ...(meId && Object.keys(reactions).length > 0 ? { - myReaction: this.populateMyReaction(note, meId, options?._hint_), + myReaction: this.populateMyReaction({ + id: note.id, + reactions: reactions, + reactionAndUserPairCache: note.reactionAndUserPairCache, + }, meId, options?._hint_), } : {}), } : {}), }); From e979038548e6a9c9309ae142935150ca9464fe85 Mon Sep 17 00:00:00 2001 From: syuilo <4439005+syuilo@users.noreply.github.com> Date: Fri, 20 Sep 2024 15:37:42 +0900 Subject: [PATCH 13/17] wip --- packages/backend/src/core/ReactionService.ts | 24 +++++++++------ .../src/core/ReactionsBufferingService.ts | 30 ++++++++++++------- 2 files changed, 34 insertions(+), 20 deletions(-) diff --git a/packages/backend/src/core/ReactionService.ts b/packages/backend/src/core/ReactionService.ts index 220a4f163e46..2c67d16d6fda 100644 --- a/packages/backend/src/core/ReactionService.ts +++ b/packages/backend/src/core/ReactionService.ts @@ -196,7 +196,7 @@ export class ReactionService { // Increment reactions count if (meta.enableReactionsBuffering) { - this.reactionsBufferingService.create(note, reaction); + await this.reactionsBufferingService.create(note.id, user.id, reaction); // for debugging if (reaction === ':angry_ai:') { @@ -310,15 +310,21 @@ export class ReactionService { throw new IdentifiableError('60527ec9-b4cb-4a88-a6bd-32d3ad26817d', 'not reacted'); } + const meta = await this.metaService.fetch(); + // Decrement reactions count - const sql = `jsonb_set("reactions", '{${exist.reaction}}', (COALESCE("reactions"->>'${exist.reaction}', '0')::int - 1)::text::jsonb)`; - await this.notesRepository.createQueryBuilder().update() - .set({ - reactions: () => sql, - reactionAndUserPairCache: () => `array_remove("reactionAndUserPairCache", '${user.id}/${exist.reaction}')`, - }) - .where('id = :id', { id: note.id }) - .execute(); + if (meta.enableReactionsBuffering) { + await this.reactionsBufferingService.delete(note.id, user.id, exist.reaction); + } else { + const sql = `jsonb_set("reactions", '{${exist.reaction}}', (COALESCE("reactions"->>'${exist.reaction}', '0')::int - 1)::text::jsonb)`; + await this.notesRepository.createQueryBuilder().update() + .set({ + reactions: () => sql, + reactionAndUserPairCache: () => `array_remove("reactionAndUserPairCache", '${user.id}/${exist.reaction}')`, + }) + .where('id = :id', { id: note.id }) + .execute(); + } this.globalEventService.publishNoteStream(note.id, 'unreacted', { reaction: this.decodeReaction(exist.reaction).reaction, diff --git a/packages/backend/src/core/ReactionsBufferingService.ts b/packages/backend/src/core/ReactionsBufferingService.ts index 0514966ee8a6..4868903fab9f 100644 --- a/packages/backend/src/core/ReactionsBufferingService.ts +++ b/packages/backend/src/core/ReactionsBufferingService.ts @@ -8,10 +8,11 @@ import * as Redis from 'ioredis'; import { DI } from '@/di-symbols.js'; import type { MiNote } from '@/models/Note.js'; import { bindThis } from '@/decorators.js'; -import type { NotesRepository } from '@/models/_.js'; +import type { MiUser, NotesRepository } from '@/models/_.js'; import type { Config } from '@/config.js'; -const REDIS_PREFIX = 'reactionsBuffer'; +const REDIS_DELTA_PREFIX = 'reactionsBufferDeltas'; +const REDIS_PAIR_PREFIX = 'reactionsBufferPairs'; @Injectable() export class ReactionsBufferingService { @@ -28,18 +29,25 @@ export class ReactionsBufferingService { } @bindThis - public async create(note: MiNote, reaction: string) { - this.redisForReactions.hincrby(`${REDIS_PREFIX}:${note.id}`, reaction, 1); + public async create(noteId: MiNote['id'], userId: MiUser['id'], reaction: string): Promise { + const pipeline = this.redisForReactions.pipeline(); + pipeline.hincrby(`${REDIS_DELTA_PREFIX}:${noteId}`, reaction, 1); + pipeline.lpush(`${REDIS_PAIR_PREFIX}:${noteId}`, `${userId}/${reaction}`); + pipeline.ltrim(`${REDIS_PAIR_PREFIX}:${noteId}`, 0, 32); + await pipeline.exec(); } @bindThis - public async delete(note: MiNote, reaction: string) { - this.redisForReactions.hincrby(`${REDIS_PREFIX}:${note.id}`, reaction, -1); + public async delete(noteId: MiNote['id'], userId: MiUser['id'], reaction: string): Promise { + const pipeline = this.redisForReactions.pipeline(); + pipeline.hincrby(`${REDIS_DELTA_PREFIX}:${noteId}`, reaction, -1); + pipeline.lrem(`${REDIS_PAIR_PREFIX}:${noteId}`, 0, `${userId}/${reaction}`); + await pipeline.exec(); } @bindThis public async get(noteId: MiNote['id']): Promise> { - const result = await this.redisForReactions.hgetall(`${REDIS_PREFIX}:${noteId}`); + const result = await this.redisForReactions.hgetall(`${REDIS_DELTA_PREFIX}:${noteId}`); const delta = {} as Record; for (const [name, count] of Object.entries(result)) { delta[name] = parseInt(count); @@ -53,7 +61,7 @@ export class ReactionsBufferingService { const pipeline = this.redisForReactions.pipeline(); for (const noteId of noteIds) { - pipeline.hgetall(`${REDIS_PREFIX}:${noteId}`); + pipeline.hgetall(`${REDIS_DELTA_PREFIX}:${noteId}`); } const results = await pipeline.exec(); @@ -80,12 +88,12 @@ export class ReactionsBufferingService { const result = await this.redisForReactions.scan( cursor, 'MATCH', - `${this.config.redis.prefix}:${REDIS_PREFIX}:*`, + `${this.config.redis.prefix}:${REDIS_DELTA_PREFIX}:*`, 'COUNT', '1000'); cursor = result[0]; - bufferedNoteIds.push(...result[1].map(x => x.replace(`${this.config.redis.prefix}:${REDIS_PREFIX}:`, ''))); + bufferedNoteIds.push(...result[1].map(x => x.replace(`${this.config.redis.prefix}:${REDIS_DELTA_PREFIX}:`, ''))); } while (cursor !== '0'); const deltas = await this.getMany(bufferedNoteIds); @@ -93,7 +101,7 @@ export class ReactionsBufferingService { // clear const pipeline = this.redisForReactions.pipeline(); for (const noteId of bufferedNoteIds) { - pipeline.del(`${REDIS_PREFIX}:${noteId}`); + pipeline.del(`${REDIS_DELTA_PREFIX}:${noteId}`); } await pipeline.exec(); From 84dee00cd0b7d37e40a1a00c6a8a37e178d2aa76 Mon Sep 17 00:00:00 2001 From: syuilo <4439005+syuilo@users.noreply.github.com> Date: Fri, 20 Sep 2024 16:41:59 +0900 Subject: [PATCH 14/17] wip --- .../src/core/ReactionsBufferingService.ts | 68 ++++++++++++++----- .../src/core/entities/NoteEntityService.ts | 42 ++++++++---- 2 files changed, 78 insertions(+), 32 deletions(-) diff --git a/packages/backend/src/core/ReactionsBufferingService.ts b/packages/backend/src/core/ReactionsBufferingService.ts index 4868903fab9f..1e3554f35e0b 100644 --- a/packages/backend/src/core/ReactionsBufferingService.ts +++ b/packages/backend/src/core/ReactionsBufferingService.ts @@ -46,36 +46,68 @@ export class ReactionsBufferingService { } @bindThis - public async get(noteId: MiNote['id']): Promise> { - const result = await this.redisForReactions.hgetall(`${REDIS_DELTA_PREFIX}:${noteId}`); - const delta = {} as Record; - for (const [name, count] of Object.entries(result)) { - delta[name] = parseInt(count); + public async get(noteId: MiNote['id']): Promise<{ + deltas: Record; + pairs: ([MiUser['id'], string])[]; + }> { + const pipeline = this.redisForReactions.pipeline(); + pipeline.hgetall(`${REDIS_DELTA_PREFIX}:${noteId}`); + pipeline.lrange(`${REDIS_PAIR_PREFIX}:${noteId}`, 0, -1); + const results = await pipeline.exec(); + + const resultDeltas = results![0][1] as Record; + const resultPairs = results![1][1] as string[]; + + const deltas = {} as Record; + for (const [name, count] of Object.entries(resultDeltas)) { + deltas[name] = parseInt(count); } - return delta; + + const pairs = resultPairs.map(x => x.split('/') as [MiUser['id'], string]); + + return { + deltas, + pairs, + }; } @bindThis - public async getMany(noteIds: MiNote['id'][]): Promise>> { - const deltas = new Map>(); + public async getMany(noteIds: MiNote['id'][]): Promise; + pairs: ([MiUser['id'], string])[]; + }>> { + const map = new Map; + pairs: ([MiUser['id'], string])[]; + }>(); const pipeline = this.redisForReactions.pipeline(); for (const noteId of noteIds) { pipeline.hgetall(`${REDIS_DELTA_PREFIX}:${noteId}`); + pipeline.lrange(`${REDIS_PAIR_PREFIX}:${noteId}`, 0, -1); } const results = await pipeline.exec(); + const opsForEachNotes = 2; for (let i = 0; i < noteIds.length; i++) { const noteId = noteIds[i]; - const result = results![i][1] as Record; - const delta = {} as Record; - for (const [name, count] of Object.entries(result)) { - delta[name] = parseInt(count); + const resultDeltas = results![i * opsForEachNotes][1] as Record; + const resultPairs = results![i * opsForEachNotes + 1][1] as string[]; + + const deltas = {} as Record; + for (const [name, count] of Object.entries(resultDeltas)) { + deltas[name] = parseInt(count); } - deltas.set(noteId, delta); + + const pairs = resultPairs.map(x => x.split('/') as [MiUser['id'], string]); + + map.set(noteId, { + deltas, + pairs, + }); } - return deltas; + return map; } // TODO: scanは重い可能性があるので、別途 bufferedNoteIds を直接Redis上に持っておいてもいいかもしれない @@ -96,18 +128,19 @@ export class ReactionsBufferingService { bufferedNoteIds.push(...result[1].map(x => x.replace(`${this.config.redis.prefix}:${REDIS_DELTA_PREFIX}:`, ''))); } while (cursor !== '0'); - const deltas = await this.getMany(bufferedNoteIds); + const bufferedMap = await this.getMany(bufferedNoteIds); // clear const pipeline = this.redisForReactions.pipeline(); for (const noteId of bufferedNoteIds) { pipeline.del(`${REDIS_DELTA_PREFIX}:${noteId}`); + pipeline.del(`${REDIS_PAIR_PREFIX}:${noteId}`); } await pipeline.exec(); // TODO: SQL一個にまとめたい - for (const [noteId, delta] of deltas) { - const sql = Object.entries(delta) + for (const [noteId, buffered] of bufferedMap) { + const sql = Object.entries(buffered.deltas) .map(([reaction, count]) => `jsonb_set("reactions", '{${reaction}}', (COALESCE("reactions"->>'${reaction}', '0')::int + ${count})::text::jsonb)`) .join(' || '); @@ -115,6 +148,7 @@ export class ReactionsBufferingService { this.notesRepository.createQueryBuilder().update() .set({ reactions: () => sql, + // TODO: reactionAndUserPairCache もよしなにベイクする }) .where('id = :id', { id: noteId }) .execute(); diff --git a/packages/backend/src/core/entities/NoteEntityService.ts b/packages/backend/src/core/entities/NoteEntityService.ts index 0d6e73045510..7506d804c3f2 100644 --- a/packages/backend/src/core/entities/NoteEntityService.ts +++ b/packages/backend/src/core/entities/NoteEntityService.ts @@ -307,7 +307,7 @@ export class NoteEntityService implements OnModuleInit { skipHide?: boolean; withReactionAndUserPairCache?: boolean; _hint_?: { - reactionsDeltas: Map>; + bufferdReactions: Map; pairs: ([MiUser['id'], string])[] }> | null; myReactions: Map; packedFiles: Map | null>; packedUsers: Map> @@ -324,14 +324,16 @@ export class NoteEntityService implements OnModuleInit { const note = typeof src === 'object' ? src : await this.noteLoader.load(src); const host = note.userHost; - const reactionsDelta = opts._hint_?.reactionsDeltas != null ? (opts._hint_.reactionsDeltas.get(note.id) ?? {}) : await this.reactionsBufferingService.get(note.id); - const reactions = mergeReactions(note.reactions, reactionsDelta); + const bufferdReactions = opts._hint_?.bufferdReactions != null ? (opts._hint_.bufferdReactions.get(note.id) ?? { deltas: {}, pairs: [] }) : await this.reactionsBufferingService.get(note.id); + const reactions = mergeReactions(note.reactions, bufferdReactions.deltas ?? {}); for (const [name, count] of Object.entries(reactions)) { if (count <= 0) { delete reactions[name]; } } + const reactionAndUserPairCache = note.reactionAndUserPairCache.concat(bufferdReactions.pairs.map(x => x.join('/'))); + let text = note.text; if (note.name && (note.url ?? note.uri)) { @@ -366,7 +368,7 @@ export class NoteEntityService implements OnModuleInit { reactionCount: Object.values(reactions).reduce((a, b) => a + b, 0), reactions: reactions, reactionEmojis: this.customEmojiService.populateEmojis(reactionEmojiNames, host), - reactionAndUserPairCache: opts.withReactionAndUserPairCache ? note.reactionAndUserPairCache : undefined, + reactionAndUserPairCache: opts.withReactionAndUserPairCache ? reactionAndUserPairCache : undefined, emojis: host != null ? this.customEmojiService.populateEmojis(note.emojis, host) : undefined, tags: note.tags.length > 0 ? note.tags : undefined, fileIds: note.fileIds, @@ -409,7 +411,7 @@ export class NoteEntityService implements OnModuleInit { myReaction: this.populateMyReaction({ id: note.id, reactions: reactions, - reactionAndUserPairCache: note.reactionAndUserPairCache, + reactionAndUserPairCache: reactionAndUserPairCache, }, meId, options?._hint_), } : {}), } : {}), @@ -435,7 +437,7 @@ export class NoteEntityService implements OnModuleInit { const meta = await this.metaService.fetch(); - const reactionsDeltas = meta.enableReactionsBuffering ? await this.reactionsBufferingService.getMany(notes.map(x => x.id)) : new Map(); + const bufferdReactions = meta.enableReactionsBuffering ? await this.reactionsBufferingService.getMany(notes.map(x => x.id)) : null; const meId = me ? me.id : null; const myReactionsMap = new Map(); @@ -447,23 +449,33 @@ export class NoteEntityService implements OnModuleInit { for (const note of notes) { if (note.renote && (note.text == null && note.fileIds.length === 0)) { // pure renote - const reactionsCount = Object.values(mergeReactions(note.renote.reactions, reactionsDeltas.get(note.renote.id) ?? {})).reduce((a, b) => a + b, 0); + const reactionsCount = Object.values(mergeReactions(note.renote.reactions, bufferdReactions?.get(note.renote.id)?.deltas ?? {})).reduce((a, b) => a + b, 0); if (reactionsCount === 0) { myReactionsMap.set(note.renote.id, null); - } else if (reactionsCount <= note.renote.reactionAndUserPairCache.length) { - const pair = note.renote.reactionAndUserPairCache.find(p => p.startsWith(meId)); - myReactionsMap.set(note.renote.id, pair ? pair.split('/')[1] : null); + } else if (reactionsCount <= note.renote.reactionAndUserPairCache.length + (bufferdReactions?.get(note.renote.id)?.pairs.length ?? 0)) { + const pairInBuffer = bufferdReactions?.get(note.renote.id)?.pairs.find(p => p[0] === meId); + if (pairInBuffer) { + myReactionsMap.set(note.renote.id, pairInBuffer[1]); + } else { + const pair = note.renote.reactionAndUserPairCache.find(p => p.startsWith(meId)); + myReactionsMap.set(note.renote.id, pair ? pair.split('/')[1] : null); + } } else { idsNeedFetchMyReaction.add(note.renote.id); } } else { if (note.id < oldId) { - const reactionsCount = Object.values(mergeReactions(note.reactions, reactionsDeltas.get(note.id) ?? {})).reduce((a, b) => a + b, 0); + const reactionsCount = Object.values(mergeReactions(note.reactions, bufferdReactions?.get(note.id)?.deltas ?? {})).reduce((a, b) => a + b, 0); if (reactionsCount === 0) { myReactionsMap.set(note.id, null); - } else if (reactionsCount <= note.reactionAndUserPairCache.length) { - const pair = note.reactionAndUserPairCache.find(p => p.startsWith(meId)); - myReactionsMap.set(note.id, pair ? pair.split('/')[1] : null); + } else if (reactionsCount <= note.reactionAndUserPairCache.length + (bufferdReactions?.get(note.id)?.pairs.length ?? 0)) { + const pairInBuffer = bufferdReactions?.get(note.id)?.pairs.find(p => p[0] === meId); + if (pairInBuffer) { + myReactionsMap.set(note.id, pairInBuffer[1]); + } else { + const pair = note.reactionAndUserPairCache.find(p => p.startsWith(meId)); + myReactionsMap.set(note.id, pair ? pair.split('/')[1] : null); + } } else { idsNeedFetchMyReaction.add(note.id); } @@ -498,7 +510,7 @@ export class NoteEntityService implements OnModuleInit { return await Promise.all(notes.map(n => this.pack(n, me, { ...options, _hint_: { - reactionsDeltas, + bufferdReactions, myReactions: myReactionsMap, packedFiles, packedUsers, From cae26a5b2b80a5933222dbaebf1d6dad9e406f94 Mon Sep 17 00:00:00 2001 From: syuilo <4439005+syuilo@users.noreply.github.com> Date: Fri, 20 Sep 2024 17:49:08 +0900 Subject: [PATCH 15/17] wip --- packages/backend/src/core/ReactionService.ts | 2 +- .../src/core/ReactionsBufferingService.ts | 18 +++++++++++------- 2 files changed, 12 insertions(+), 8 deletions(-) diff --git a/packages/backend/src/core/ReactionService.ts b/packages/backend/src/core/ReactionService.ts index 2c67d16d6fda..bf663075cd09 100644 --- a/packages/backend/src/core/ReactionService.ts +++ b/packages/backend/src/core/ReactionService.ts @@ -196,7 +196,7 @@ export class ReactionService { // Increment reactions count if (meta.enableReactionsBuffering) { - await this.reactionsBufferingService.create(note.id, user.id, reaction); + await this.reactionsBufferingService.create(note.id, user.id, reaction, note.reactionAndUserPairCache); // for debugging if (reaction === ':angry_ai:') { diff --git a/packages/backend/src/core/ReactionsBufferingService.ts b/packages/backend/src/core/ReactionsBufferingService.ts index 1e3554f35e0b..f25e5e81d4a3 100644 --- a/packages/backend/src/core/ReactionsBufferingService.ts +++ b/packages/backend/src/core/ReactionsBufferingService.ts @@ -29,11 +29,14 @@ export class ReactionsBufferingService { } @bindThis - public async create(noteId: MiNote['id'], userId: MiUser['id'], reaction: string): Promise { + public async create(noteId: MiNote['id'], userId: MiUser['id'], reaction: string, currentPairs: string[]): Promise { const pipeline = this.redisForReactions.pipeline(); pipeline.hincrby(`${REDIS_DELTA_PREFIX}:${noteId}`, reaction, 1); - pipeline.lpush(`${REDIS_PAIR_PREFIX}:${noteId}`, `${userId}/${reaction}`); - pipeline.ltrim(`${REDIS_PAIR_PREFIX}:${noteId}`, 0, 32); + for (let i = 0; i < currentPairs.length; i++) { + pipeline.zadd(`${REDIS_PAIR_PREFIX}:${noteId}`, i, currentPairs[i]); + } + pipeline.zadd(`${REDIS_PAIR_PREFIX}:${noteId}`, Date.now(), `${userId}/${reaction}`); + pipeline.zremrangebyrank(`${REDIS_PAIR_PREFIX}:${noteId}`, 0, -17); await pipeline.exec(); } @@ -41,7 +44,8 @@ export class ReactionsBufferingService { public async delete(noteId: MiNote['id'], userId: MiUser['id'], reaction: string): Promise { const pipeline = this.redisForReactions.pipeline(); pipeline.hincrby(`${REDIS_DELTA_PREFIX}:${noteId}`, reaction, -1); - pipeline.lrem(`${REDIS_PAIR_PREFIX}:${noteId}`, 0, `${userId}/${reaction}`); + pipeline.zrem(`${REDIS_PAIR_PREFIX}:${noteId}`, `${userId}/${reaction}`); + // TODO: 「消した要素一覧」も持っておかないとcreateされた時に上書きされて復活する await pipeline.exec(); } @@ -52,7 +56,7 @@ export class ReactionsBufferingService { }> { const pipeline = this.redisForReactions.pipeline(); pipeline.hgetall(`${REDIS_DELTA_PREFIX}:${noteId}`); - pipeline.lrange(`${REDIS_PAIR_PREFIX}:${noteId}`, 0, -1); + pipeline.zrange(`${REDIS_PAIR_PREFIX}:${noteId}`, 0, -1); const results = await pipeline.exec(); const resultDeltas = results![0][1] as Record; @@ -84,7 +88,7 @@ export class ReactionsBufferingService { const pipeline = this.redisForReactions.pipeline(); for (const noteId of noteIds) { pipeline.hgetall(`${REDIS_DELTA_PREFIX}:${noteId}`); - pipeline.lrange(`${REDIS_PAIR_PREFIX}:${noteId}`, 0, -1); + pipeline.zrange(`${REDIS_PAIR_PREFIX}:${noteId}`, 0, -1); } const results = await pipeline.exec(); @@ -148,7 +152,7 @@ export class ReactionsBufferingService { this.notesRepository.createQueryBuilder().update() .set({ reactions: () => sql, - // TODO: reactionAndUserPairCache もよしなにベイクする + reactionAndUserPairCache: buffered.pairs.map(x => x.join('/')), }) .where('id = :id', { id: noteId }) .execute(); From c15028ba84a2b2504cafe093fdbdfafb102cfd09 Mon Sep 17 00:00:00 2001 From: syuilo <4439005+syuilo@users.noreply.github.com> Date: Fri, 20 Sep 2024 17:52:23 +0900 Subject: [PATCH 16/17] wip --- packages/backend/src/const.ts | 2 ++ packages/backend/src/core/ReactionService.ts | 2 +- packages/backend/src/core/ReactionsBufferingService.ts | 3 ++- packages/backend/test/unit/entities/UserEntityService.ts | 4 +++- 4 files changed, 8 insertions(+), 3 deletions(-) diff --git a/packages/backend/src/const.ts b/packages/backend/src/const.ts index a238f4973a95..e3a61861f425 100644 --- a/packages/backend/src/const.ts +++ b/packages/backend/src/const.ts @@ -8,6 +8,8 @@ export const MAX_NOTE_TEXT_LENGTH = 3000; export const USER_ONLINE_THRESHOLD = 1000 * 60 * 10; // 10min export const USER_ACTIVE_THRESHOLD = 1000 * 60 * 60 * 24 * 3; // 3days +export const PER_NOTE_REACTION_USER_PAIR_CACHE_MAX = 16; + //#region hard limits // If you change DB_* values, you must also change the DB schema. diff --git a/packages/backend/src/core/ReactionService.ts b/packages/backend/src/core/ReactionService.ts index bf663075cd09..5993c42a1f4b 100644 --- a/packages/backend/src/core/ReactionService.ts +++ b/packages/backend/src/core/ReactionService.ts @@ -30,9 +30,9 @@ import { FeaturedService } from '@/core/FeaturedService.js'; import { trackPromise } from '@/misc/promise-tracker.js'; import { isQuote, isRenote } from '@/misc/is-renote.js'; import { ReactionsBufferingService } from '@/core/ReactionsBufferingService.js'; +import { PER_NOTE_REACTION_USER_PAIR_CACHE_MAX } from '@/const.js'; const FALLBACK = '\u2764'; -const PER_NOTE_REACTION_USER_PAIR_CACHE_MAX = 16; const legacies: Record = { 'like': '👍', diff --git a/packages/backend/src/core/ReactionsBufferingService.ts b/packages/backend/src/core/ReactionsBufferingService.ts index f25e5e81d4a3..b1a197feeb17 100644 --- a/packages/backend/src/core/ReactionsBufferingService.ts +++ b/packages/backend/src/core/ReactionsBufferingService.ts @@ -10,6 +10,7 @@ import type { MiNote } from '@/models/Note.js'; import { bindThis } from '@/decorators.js'; import type { MiUser, NotesRepository } from '@/models/_.js'; import type { Config } from '@/config.js'; +import { PER_NOTE_REACTION_USER_PAIR_CACHE_MAX } from '@/const.js'; const REDIS_DELTA_PREFIX = 'reactionsBufferDeltas'; const REDIS_PAIR_PREFIX = 'reactionsBufferPairs'; @@ -36,7 +37,7 @@ export class ReactionsBufferingService { pipeline.zadd(`${REDIS_PAIR_PREFIX}:${noteId}`, i, currentPairs[i]); } pipeline.zadd(`${REDIS_PAIR_PREFIX}:${noteId}`, Date.now(), `${userId}/${reaction}`); - pipeline.zremrangebyrank(`${REDIS_PAIR_PREFIX}:${noteId}`, 0, -17); + pipeline.zremrangebyrank(`${REDIS_PAIR_PREFIX}:${noteId}`, 0, -(PER_NOTE_REACTION_USER_PAIR_CACHE_MAX + 1)); await pipeline.exec(); } diff --git a/packages/backend/test/unit/entities/UserEntityService.ts b/packages/backend/test/unit/entities/UserEntityService.ts index ee16d421c4c9..e4f42809f8ec 100644 --- a/packages/backend/test/unit/entities/UserEntityService.ts +++ b/packages/backend/test/unit/entities/UserEntityService.ts @@ -4,10 +4,10 @@ */ import { Test, TestingModule } from '@nestjs/testing'; +import type { MiUser } from '@/models/User.js'; import { UserEntityService } from '@/core/entities/UserEntityService.js'; import { GlobalModule } from '@/GlobalModule.js'; import { CoreModule } from '@/core/CoreModule.js'; -import type { MiUser } from '@/models/User.js'; import { secureRndstr } from '@/misc/secure-rndstr.js'; import { genAidx } from '@/misc/id/aidx.js'; import { @@ -49,6 +49,7 @@ import { ApLoggerService } from '@/core/activitypub/ApLoggerService.js'; import { AccountMoveService } from '@/core/AccountMoveService.js'; import { ReactionService } from '@/core/ReactionService.js'; import { NotificationService } from '@/core/NotificationService.js'; +import { ReactionsBufferingService } from '@/core/ReactionsBufferingService.js'; process.env.NODE_ENV = 'test'; @@ -169,6 +170,7 @@ describe('UserEntityService', () => { ApLoggerService, AccountMoveService, ReactionService, + ReactionsBufferingService, NotificationService, ]; From 0127b9fe55d804bad2b5ed94e91baf6130c5681c Mon Sep 17 00:00:00 2001 From: syuilo <4439005+syuilo@users.noreply.github.com> Date: Fri, 20 Sep 2024 20:22:21 +0900 Subject: [PATCH 17/17] Update CHANGELOG.md --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 35c787d565d5..52a55b0fd5bb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,6 +17,7 @@ - Fix: 設定変更時のリロード確認ダイアログが複数個表示されることがある問題を修正 ### Server +- Feat: Misskey® Reactions Buffering Technology™ (RBT)により、リアクションの作成負荷を低減することが可能に - Fix: アンテナの書き込み時にキーワードが与えられなかった場合のエラーをApiErrorとして投げるように - この変更により、公式フロントエンドでは入力の不備が内部エラーとして報告される代わりに一般的なエラーダイアログで報告されます - Fix: ファイルがサイズの制限を超えてアップロードされた際にエラーを返さなかった問題を修正