Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Misskey® Reactions Buffering Technology™ #14579

Merged
merged 17 commits into from
Sep 20, 2024
8 changes: 8 additions & 0 deletions .config/cypress-devcontainer.yml
Original file line number Diff line number Diff line change
Expand Up @@ -103,6 +103,14 @@ redis:
# #prefix: example-prefix
# #db: 1

#redisForReactions:
# host: redis
# port: 6379
# #family: 0 # 0=Both, 4=IPv4, 6=IPv6
# #pass: example-pass
# #prefix: example-prefix
# #db: 1

# ┌───────────────────────────┐
#───┘ MeiliSearch configuration └─────────────────────────────

Expand Down
8 changes: 8 additions & 0 deletions .config/docker_example.yml
Original file line number Diff line number Diff line change
Expand Up @@ -106,6 +106,14 @@ redis:
# #prefix: example-prefix
# #db: 1

#redisForReactions:
# host: redis
# port: 6379
# #family: 0 # 0=Both, 4=IPv4, 6=IPv6
# #pass: example-pass
# #prefix: example-prefix
# #db: 1

# ┌───────────────────────────┐
#───┘ MeiliSearch configuration └─────────────────────────────

Expand Down
10 changes: 10 additions & 0 deletions .config/example.yml
Original file line number Diff line number Diff line change
Expand Up @@ -172,6 +172,16 @@ redis:
# # You can specify more ioredis options...
# #username: example-username

#redisForReactions:
# host: localhost
# port: 6379
# #family: 0 # 0=Both, 4=IPv4, 6=IPv6
# #pass: example-pass
# #prefix: example-prefix
# #db: 1
# # You can specify more ioredis options...
# #username: example-username

# ┌───────────────────────────┐
#───┘ MeiliSearch configuration └─────────────────────────────

Expand Down
8 changes: 8 additions & 0 deletions .devcontainer/devcontainer.yml
Original file line number Diff line number Diff line change
Expand Up @@ -103,6 +103,14 @@ redis:
# #prefix: example-prefix
# #db: 1

#redisForReactions:
# host: redis
# port: 6379
# #family: 0 # 0=Both, 4=IPv4, 6=IPv6
# #pass: example-pass
# #prefix: example-prefix
# #db: 1

# ┌───────────────────────────┐
#───┘ MeiliSearch configuration └─────────────────────────────

Expand Down
1 change: 1 addition & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@
- Fix: 設定変更時のリロード確認ダイアログが複数個表示されることがある問題を修正

### Server
- Feat: Misskey® Reactions Buffering Technology™ (RBT)により、リアクションの作成負荷を低減することが可能に
- Fix: アンテナの書き込み時にキーワードが与えられなかった場合のエラーをApiErrorとして投げるように
- この変更により、公式フロントエンドでは入力の不備が内部エラーとして報告される代わりに一般的なエラーダイアログで報告されます
- Fix: ファイルがサイズの制限を超えてアップロードされた際にエラーを返さなかった問題を修正
Expand Down
8 changes: 8 additions & 0 deletions chart/files/default.yml
Original file line number Diff line number Diff line change
Expand Up @@ -124,6 +124,14 @@ redis:
# #prefix: example-prefix
# #db: 1

#redisForReactions:
# host: redis
# port: 6379
# #family: 0 # 0=Both, 4=IPv4, 6=IPv6
# #pass: example-pass
# #prefix: example-prefix
# #db: 1

# ┌───────────────────────────┐
#───┘ MeiliSearch configuration └─────────────────────────────

Expand Down
12 changes: 12 additions & 0 deletions locales/index.d.ts
Original file line number Diff line number Diff line change
Expand Up @@ -2384,6 +2384,14 @@ export interface Locale extends ILocale {
* スクラッチパッドは、AiScriptの実験環境を提供します。Misskeyと対話するコードの記述、実行、結果の確認ができます。
*/
"scratchpadDescription": string;
/**
* UIインスペクター
*/
"uiInspector": string;
/**
* メモリ上に存在しているUIコンポーネントのインスタンスの一覧を見ることができます。UIコンポーネントはUi:C:系関数により生成されます。
*/
"uiInspectorDescription": string;
/**
* 出力
*/
Expand Down Expand Up @@ -5575,6 +5583,10 @@ export interface Locale extends ILocale {
* 有効にすると、タイムラインがキャッシュされていない場合にDBへ追加で問い合わせを行うフォールバック処理を行います。無効にすると、フォールバック処理を行わないことでさらにサーバーの負荷を軽減することができますが、タイムラインが取得できる範囲に制限が生じます。
*/
"fanoutTimelineDbFallbackDescription": string;
/**
* 有効にすると、リアクション作成時のパフォーマンスが大幅に向上し、データベースへの負荷を軽減することが可能です。ただし、Redisのメモリ使用量は増加します。
*/
"reactionsBufferingDescription": string;
/**
* 問い合わせ先URL
*/
Expand Down
1 change: 1 addition & 0 deletions locales/ja-JP.yml
Original file line number Diff line number Diff line change
Expand Up @@ -1411,6 +1411,7 @@ _serverSettings:
fanoutTimelineDescription: "有効にすると、各種タイムラインを取得する際のパフォーマンスが大幅に向上し、データベースへの負荷を軽減することが可能です。ただし、Redisのメモリ使用量は増加します。サーバーのメモリ容量が少ない場合、または動作が不安定な場合は無効にすることができます。"
fanoutTimelineDbFallback: "データベースへのフォールバック"
fanoutTimelineDbFallbackDescription: "有効にすると、タイムラインがキャッシュされていない場合にDBへ追加で問い合わせを行うフォールバック処理を行います。無効にすると、フォールバック処理を行わないことでさらにサーバーの負荷を軽減することができますが、タイムラインが取得できる範囲に制限が生じます。"
reactionsBufferingDescription: "有効にすると、リアクション作成時のパフォーマンスが大幅に向上し、データベースへの負荷を軽減することが可能です。ただし、Redisのメモリ使用量は増加します。"
inquiryUrl: "問い合わせ先URL"
inquiryUrlDescription: "サーバー運営者へのお問い合わせフォームのURLや、運営者の連絡先等が記載されたWebページのURLを指定します。"

Expand Down
16 changes: 16 additions & 0 deletions packages/backend/migration/1726804538569-reactions-buffering.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,16 @@
/*
* SPDX-FileCopyrightText: syuilo and misskey-project
* SPDX-License-Identifier: AGPL-3.0-only
*/

export class ReactionsBuffering1726804538569 {
name = 'ReactionsBuffering1726804538569'

async up(queryRunner) {
await queryRunner.query(`ALTER TABLE "meta" ADD "enableReactionsBuffering" boolean NOT NULL DEFAULT false`);
}

async down(queryRunner) {
await queryRunner.query(`ALTER TABLE "meta" DROP COLUMN "enableReactionsBuffering"`);
}
}
14 changes: 12 additions & 2 deletions packages/backend/src/GlobalModule.ts
Original file line number Diff line number Diff line change
Expand Up @@ -78,11 +78,19 @@ const $redisForTimelines: Provider = {
inject: [DI.config],
};

const $redisForReactions: Provider = {
provide: DI.redisForReactions,
useFactory: (config: Config) => {
return new Redis.Redis(config.redisForReactions);
},
inject: [DI.config],
};

@Global()
@Module({
imports: [RepositoryModule],
providers: [$config, $db, $meilisearch, $redis, $redisForPub, $redisForSub, $redisForTimelines],
exports: [$config, $db, $meilisearch, $redis, $redisForPub, $redisForSub, $redisForTimelines, RepositoryModule],
providers: [$config, $db, $meilisearch, $redis, $redisForPub, $redisForSub, $redisForTimelines, $redisForReactions],
exports: [$config, $db, $meilisearch, $redis, $redisForPub, $redisForSub, $redisForTimelines, $redisForReactions, RepositoryModule],
})
export class GlobalModule implements OnApplicationShutdown {
constructor(
Expand All @@ -91,6 +99,7 @@ export class GlobalModule implements OnApplicationShutdown {
@Inject(DI.redisForPub) private redisForPub: Redis.Redis,
@Inject(DI.redisForSub) private redisForSub: Redis.Redis,
@Inject(DI.redisForTimelines) private redisForTimelines: Redis.Redis,
@Inject(DI.redisForReactions) private redisForReactions: Redis.Redis,
) { }

public async dispose(): Promise<void> {
Expand All @@ -103,6 +112,7 @@ export class GlobalModule implements OnApplicationShutdown {
this.redisForPub.disconnect(),
this.redisForSub.disconnect(),
this.redisForTimelines.disconnect(),
this.redisForReactions.disconnect(),
]);
}

Expand Down
3 changes: 3 additions & 0 deletions packages/backend/src/config.ts
Original file line number Diff line number Diff line change
Expand Up @@ -49,6 +49,7 @@ type Source = {
redisForPubsub?: RedisOptionsSource;
redisForJobQueue?: RedisOptionsSource;
redisForTimelines?: RedisOptionsSource;
redisForReactions?: RedisOptionsSource;
meilisearch?: {
host: string;
port: string;
Expand Down Expand Up @@ -171,6 +172,7 @@ export type Config = {
redisForPubsub: RedisOptions & RedisOptionsSource;
redisForJobQueue: RedisOptions & RedisOptionsSource;
redisForTimelines: RedisOptions & RedisOptionsSource;
redisForReactions: RedisOptions & RedisOptionsSource;
sentryForBackend: { options: Partial<Sentry.NodeOptions>; enableNodeProfiling: boolean; } | undefined;
sentryForFrontend: { options: Partial<Sentry.NodeOptions> } | undefined;
perChannelMaxNoteCacheCount: number;
Expand Down Expand Up @@ -251,6 +253,7 @@ export function loadConfig(): Config {
redisForPubsub: config.redisForPubsub ? convertRedisOptions(config.redisForPubsub, host) : redis,
redisForJobQueue: config.redisForJobQueue ? convertRedisOptions(config.redisForJobQueue, host) : redis,
redisForTimelines: config.redisForTimelines ? convertRedisOptions(config.redisForTimelines, host) : redis,
redisForReactions: config.redisForReactions ? convertRedisOptions(config.redisForReactions, host) : redis,
sentryForBackend: config.sentryForBackend,
sentryForFrontend: config.sentryForFrontend,
id: config.id,
Expand Down
2 changes: 2 additions & 0 deletions packages/backend/src/const.ts
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,8 @@ export const MAX_NOTE_TEXT_LENGTH = 3000;
export const USER_ONLINE_THRESHOLD = 1000 * 60 * 10; // 10min
export const USER_ACTIVE_THRESHOLD = 1000 * 60 * 60 * 24 * 3; // 3days

export const PER_NOTE_REACTION_USER_PAIR_CACHE_MAX = 16;

//#region hard limits
// If you change DB_* values, you must also change the DB schema.

Expand Down
6 changes: 6 additions & 0 deletions packages/backend/src/core/CoreModule.ts
Original file line number Diff line number Diff line change
Expand Up @@ -50,6 +50,7 @@ import { PollService } from './PollService.js';
import { PushNotificationService } from './PushNotificationService.js';
import { QueryService } from './QueryService.js';
import { ReactionService } from './ReactionService.js';
import { ReactionsBufferingService } from './ReactionsBufferingService.js';
import { RelayService } from './RelayService.js';
import { RoleService } from './RoleService.js';
import { S3Service } from './S3Service.js';
Expand Down Expand Up @@ -193,6 +194,7 @@ const $ProxyAccountService: Provider = { provide: 'ProxyAccountService', useExis
const $PushNotificationService: Provider = { provide: 'PushNotificationService', useExisting: PushNotificationService };
const $QueryService: Provider = { provide: 'QueryService', useExisting: QueryService };
const $ReactionService: Provider = { provide: 'ReactionService', useExisting: ReactionService };
const $ReactionsBufferingService: Provider = { provide: 'ReactionsBufferingService', useExisting: ReactionsBufferingService };
const $RelayService: Provider = { provide: 'RelayService', useExisting: RelayService };
const $RoleService: Provider = { provide: 'RoleService', useExisting: RoleService };
const $S3Service: Provider = { provide: 'S3Service', useExisting: S3Service };
Expand Down Expand Up @@ -342,6 +344,7 @@ const $ApQuestionService: Provider = { provide: 'ApQuestionService', useExisting
PushNotificationService,
QueryService,
ReactionService,
ReactionsBufferingService,
RelayService,
RoleService,
S3Service,
Expand Down Expand Up @@ -487,6 +490,7 @@ const $ApQuestionService: Provider = { provide: 'ApQuestionService', useExisting
$PushNotificationService,
$QueryService,
$ReactionService,
$ReactionsBufferingService,
$RelayService,
$RoleService,
$S3Service,
Expand Down Expand Up @@ -633,6 +637,7 @@ const $ApQuestionService: Provider = { provide: 'ApQuestionService', useExisting
PushNotificationService,
QueryService,
ReactionService,
ReactionsBufferingService,
RelayService,
RoleService,
S3Service,
Expand Down Expand Up @@ -777,6 +782,7 @@ const $ApQuestionService: Provider = { provide: 'ApQuestionService', useExisting
$PushNotificationService,
$QueryService,
$ReactionService,
$ReactionsBufferingService,
$RelayService,
$RoleService,
$S3Service,
Expand Down
6 changes: 6 additions & 0 deletions packages/backend/src/core/QueueService.ts
Original file line number Diff line number Diff line change
Expand Up @@ -87,6 +87,12 @@ export class QueueService {
repeat: { pattern: '*/5 * * * *' },
removeOnComplete: true,
});

this.systemQueue.add('bakeBufferedReactions', {
}, {
repeat: { pattern: '0 0 * * *' },
removeOnComplete: true,
});
}

@bindThis
Expand Down
60 changes: 36 additions & 24 deletions packages/backend/src/core/ReactionService.ts
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,6 @@
*/

import { Inject, Injectable } from '@nestjs/common';
import * as Redis from 'ioredis';
import { DI } from '@/di-symbols.js';
import type { EmojisRepository, NoteReactionsRepository, UsersRepository, NotesRepository } from '@/models/_.js';
import { IdentifiableError } from '@/misc/identifiable-error.js';
Expand All @@ -30,9 +29,10 @@ import { RoleService } from '@/core/RoleService.js';
import { FeaturedService } from '@/core/FeaturedService.js';
import { trackPromise } from '@/misc/promise-tracker.js';
import { isQuote, isRenote } from '@/misc/is-renote.js';
import { ReactionsBufferingService } from '@/core/ReactionsBufferingService.js';
import { PER_NOTE_REACTION_USER_PAIR_CACHE_MAX } from '@/const.js';

const FALLBACK = '\u2764';
const PER_NOTE_REACTION_USER_PAIR_CACHE_MAX = 16;

const legacies: Record<string, string> = {
'like': '👍',
Expand Down Expand Up @@ -71,9 +71,6 @@ const decodeCustomEmojiRegexp = /^:([\w+-]+)(?:@([\w.-]+))?:$/;
@Injectable()
export class ReactionService {
constructor(
@Inject(DI.redis)
private redisClient: Redis.Redis,

@Inject(DI.usersRepository)
private usersRepository: UsersRepository,

Expand All @@ -93,6 +90,7 @@ export class ReactionService {
private userEntityService: UserEntityService,
private noteEntityService: NoteEntityService,
private userBlockingService: UserBlockingService,
private reactionsBufferingService: ReactionsBufferingService,
private idService: IdService,
private featuredService: FeaturedService,
private globalEventService: GlobalEventService,
Expand Down Expand Up @@ -174,7 +172,6 @@ export class ReactionService {
reaction,
};

// Create reaction
try {
await this.noteReactionsRepository.insert(record);
} catch (e) {
Expand All @@ -198,16 +195,25 @@ export class ReactionService {
}

// Increment reactions count
const sql = `jsonb_set("reactions", '{${reaction}}', (COALESCE("reactions"->>'${reaction}', '0')::int + 1)::text::jsonb)`;
await this.notesRepository.createQueryBuilder().update()
.set({
reactions: () => sql,
...(note.reactionAndUserPairCache.length < PER_NOTE_REACTION_USER_PAIR_CACHE_MAX ? {
reactionAndUserPairCache: () => `array_append("reactionAndUserPairCache", '${user.id}/${reaction}')`,
} : {}),
})
.where('id = :id', { id: note.id })
.execute();
if (meta.enableReactionsBuffering) {
await this.reactionsBufferingService.create(note.id, user.id, reaction, note.reactionAndUserPairCache);

// for debugging
if (reaction === ':angry_ai:') {
this.reactionsBufferingService.bake();
}
} else {
const sql = `jsonb_set("reactions", '{${reaction}}', (COALESCE("reactions"->>'${reaction}', '0')::int + 1)::text::jsonb)`;
await this.notesRepository.createQueryBuilder().update()
.set({
reactions: () => sql,
...(note.reactionAndUserPairCache.length < PER_NOTE_REACTION_USER_PAIR_CACHE_MAX ? {
reactionAndUserPairCache: () => `array_append("reactionAndUserPairCache", '${user.id}/${reaction}')`,
} : {}),
})
.where('id = :id', { id: note.id })
.execute();
}

// 30%の確率、セルフではない、3日以内に投稿されたノートの場合ハイライト用ランキング更新
if (
Expand Down Expand Up @@ -304,15 +310,21 @@ export class ReactionService {
throw new IdentifiableError('60527ec9-b4cb-4a88-a6bd-32d3ad26817d', 'not reacted');
}

const meta = await this.metaService.fetch();

// Decrement reactions count
const sql = `jsonb_set("reactions", '{${exist.reaction}}', (COALESCE("reactions"->>'${exist.reaction}', '0')::int - 1)::text::jsonb)`;
await this.notesRepository.createQueryBuilder().update()
.set({
reactions: () => sql,
reactionAndUserPairCache: () => `array_remove("reactionAndUserPairCache", '${user.id}/${exist.reaction}')`,
})
.where('id = :id', { id: note.id })
.execute();
if (meta.enableReactionsBuffering) {
await this.reactionsBufferingService.delete(note.id, user.id, exist.reaction);
} else {
const sql = `jsonb_set("reactions", '{${exist.reaction}}', (COALESCE("reactions"->>'${exist.reaction}', '0')::int - 1)::text::jsonb)`;
await this.notesRepository.createQueryBuilder().update()
.set({
reactions: () => sql,
reactionAndUserPairCache: () => `array_remove("reactionAndUserPairCache", '${user.id}/${exist.reaction}')`,
})
.where('id = :id', { id: note.id })
.execute();
}

this.globalEventService.publishNoteStream(note.id, 'unreacted', {
reaction: this.decodeReaction(exist.reaction).reaction,
Expand Down
Loading
Loading