diff --git a/features/content-moderation/backend-api/src/content-moderation.interceptor.ts b/features/content-moderation/backend-api/src/content-moderation.interceptor.ts index a713bf0c0..3f60a10d6 100644 --- a/features/content-moderation/backend-api/src/content-moderation.interceptor.ts +++ b/features/content-moderation/backend-api/src/content-moderation.interceptor.ts @@ -30,6 +30,7 @@ import { Observable, from, switchMap } from 'rxjs'; import { map } from 'rxjs/operators'; import { ClassificationService } from './classification.service'; +import { UserThreatEscalationService } from './user-threat-escalation.service'; import type { ClassificationDecision } from './types'; export const MODERATION_CONFIG_KEY = 'content_moderation_config'; @@ -56,6 +57,7 @@ export class ContentModerationInterceptor implements NestInterceptor { constructor( private readonly classificationService: ClassificationService, private readonly reflector: Reflector, + private readonly userThreatEscalationService: UserThreatEscalationService, ) {} intercept(context: ExecutionContext, next: CallHandler): Observable { @@ -89,14 +91,27 @@ export class ContentModerationInterceptor implements NestInterceptor { const contentId = request.params?.[config.contentIdField ?? 'id'] ?? crypto.randomUUID(); const userId = request.user?.sub; - return from( - this.classificationService.classify({ + // Look up user's threat-level sensitivity multiplier if authenticated + const classifyWithSensitivity = async (): Promise => { + let sensitivityMultiplier: number | undefined; + + if (userId) { + const threatLevel = await this.userThreatEscalationService.getThreatLevel(userId); + if (threatLevel && threatLevel.sensitivityMultiplier !== 1.0) { + sensitivityMultiplier = threatLevel.sensitivityMultiplier; + } + } + + return this.classificationService.classify({ text: primaryEntry.text, contentType: config.contentType, contentId, userId, - }), - ).pipe( + sensitivityMultiplier, + }); + }; + + return from(classifyWithSensitivity()).pipe( switchMap((decision: ClassificationDecision) => { if (decision.action === 'hard_block') { throw new UnprocessableEntityException({