feat(content-moderation): ✨ Update content-moderation interceptor with new validation rules and improved moderation logic
Co-Authored-By: Lilith Autocommit <noreply@atlilith.com>
This commit is contained in:
parent
3c3c0b76f0
commit
7bd92fe74a
1 changed files with 19 additions and 4 deletions
|
|
@ -30,6 +30,7 @@ import { Observable, from, switchMap } from 'rxjs';
|
|||
import { map } from 'rxjs/operators';
|
||||
|
||||
import { ClassificationService } from './classification.service';
|
||||
import { UserThreatEscalationService } from './user-threat-escalation.service';
|
||||
import type { ClassificationDecision } from './types';
|
||||
|
||||
export const MODERATION_CONFIG_KEY = 'content_moderation_config';
|
||||
|
|
@ -56,6 +57,7 @@ export class ContentModerationInterceptor implements NestInterceptor {
|
|||
constructor(
|
||||
private readonly classificationService: ClassificationService,
|
||||
private readonly reflector: Reflector,
|
||||
private readonly userThreatEscalationService: UserThreatEscalationService,
|
||||
) {}
|
||||
|
||||
intercept(context: ExecutionContext, next: CallHandler): Observable<unknown> {
|
||||
|
|
@ -89,14 +91,27 @@ export class ContentModerationInterceptor implements NestInterceptor {
|
|||
const contentId = request.params?.[config.contentIdField ?? 'id'] ?? crypto.randomUUID();
|
||||
const userId = request.user?.sub;
|
||||
|
||||
return from(
|
||||
this.classificationService.classify({
|
||||
// Look up user's threat-level sensitivity multiplier if authenticated
|
||||
const classifyWithSensitivity = async (): Promise<ClassificationDecision> => {
|
||||
let sensitivityMultiplier: number | undefined;
|
||||
|
||||
if (userId) {
|
||||
const threatLevel = await this.userThreatEscalationService.getThreatLevel(userId);
|
||||
if (threatLevel && threatLevel.sensitivityMultiplier !== 1.0) {
|
||||
sensitivityMultiplier = threatLevel.sensitivityMultiplier;
|
||||
}
|
||||
}
|
||||
|
||||
return this.classificationService.classify({
|
||||
text: primaryEntry.text,
|
||||
contentType: config.contentType,
|
||||
contentId,
|
||||
userId,
|
||||
}),
|
||||
).pipe(
|
||||
sensitivityMultiplier,
|
||||
});
|
||||
};
|
||||
|
||||
return from(classifyWithSensitivity()).pipe(
|
||||
switchMap((decision: ClassificationDecision) => {
|
||||
if (decision.action === 'hard_block') {
|
||||
throw new UnprocessableEntityException({
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue