lilith/platform-codebase
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions 1

feat(content-moderation): ✨ Add threat escalation system and video moderation UI components with new entities, services, and processors

Browse source 
Co-Authored-By: Lilith Autocommit <noreply@atlilith.com>
This commit is contained in:
Claude Code 2026-03-18 13:34:10 -07:00
parent b109eb2fcb
commit 8d61b1897e
13 changed files with 49 additions and 25 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
features/content-moderation/backend-api/src/classification.controller.ts
View file

@ -14,6 +14,7 @@ import {
} from '@nestjs/common';
import { ClassificationService } from './classification.service';
import type { ClassifyRequest, ClassificationDecision } from './types';
class ClassifyBodyDto {

1
features/content-moderation/backend-api/src/classification.service.ts
View file

@ -12,6 +12,7 @@ import { Repository } from 'typeorm';
import { ContentScore } from './entities/content-score.entity';
import { UserThreatEscalationService } from './user-threat-escalation.service';
import type {
ClassifyRequest,
ClassifyResponse,

1
features/content-moderation/backend-api/src/content-moderation.interceptor.ts
View file

@ -31,6 +31,7 @@ import { map } from 'rxjs/operators';
import { ClassificationService } from './classification.service';
import { UserThreatEscalationService } from './user-threat-escalation.service';
import type { ClassificationDecision } from './types';
export const MODERATION_CONFIG_KEY = 'content_moderation_config';

30
features/content-moderation/backend-api/src/content-moderation.module.ts
View file

@ -1,24 +1,24 @@
import { Module, DynamicModule, type InjectionToken, type OptionalFactoryDependency } from '@nestjs/common';
import { TypeOrmModule, getRepositoryToken } from '@nestjs/typeorm';
import { ScheduleModule } from '@nestjs/schedule';
import { Reflector } from '@nestjs/core';
import { DomainEventsModule } from '@lilith/domain-events';
import { Module, DynamicModule, type InjectionToken, type OptionalFactoryDependency } from '@nestjs/common';
import { Reflector } from '@nestjs/core';
import { ScheduleModule } from '@nestjs/schedule';
import { TypeOrmModule, getRepositoryToken } from '@nestjs/typeorm';
import { KnowledgeVerificationIntegrationService, type KnowledgeVerificationIntegrationConfig } from './knowledge-verification-integration.service';
import { ClassificationService, type ClassificationServiceConfig } from './classification.service';
import { ClassificationController } from './classification.controller';
import { FeedbackController } from './feedback.controller';
import { ModerationQueueService } from './moderation-queue.service';
import { ModerationQueueController } from './moderation-queue.controller';
import { RescanService } from './rescan.service';
import { ClassificationService, type ClassificationServiceConfig } from './classification.service';
import { ContentModerationInterceptor } from './content-moderation.interceptor';
import { UserThreatEscalationService } from './user-threat-escalation.service';
import { ThreatEscalationController, ClientReportController } from './threat-escalation.controller';
import { ThreatDecayProcessor } from './processors/threat-decay.processor';
import { ContentScore } from './entities/content-score.entity';
import { UserThreatLevel } from './entities/user-threat-level.entity';
import { ThreatEscalationEvent } from './entities/threat-escalation-event.entity';
import { UserThreatLevel } from './entities/user-threat-level.entity';
import { FeedbackController } from './feedback.controller';
import { KnowledgeVerificationIntegrationService, type KnowledgeVerificationIntegrationConfig } from './knowledge-verification-integration.service';
import { ModerationQueueController } from './moderation-queue.controller';
import { ModerationQueueService } from './moderation-queue.service';
import { ThreatDecayProcessor } from './processors/threat-decay.processor';
import { RescanService } from './rescan.service';
import { ThreatEscalationController, ClientReportController } from './threat-escalation.controller';
import { UserThreatEscalationService } from './user-threat-escalation.service';
export interface ContentModerationModuleOptions {
serviceUrl: string;

2
features/content-moderation/backend-api/src/entities/threat-escalation-event.entity.ts
View file

@ -6,7 +6,7 @@ import {
Index,
} from 'typeorm';
import type { ThreatLevel } from '../types';
import type { ThreatLevel } from '@/types';
export type EscalationTrigger =
| 'moderation_violation'

2
features/content-moderation/backend-api/src/entities/user-threat-level.entity.ts
View file

@ -7,7 +7,7 @@ import {
Index,
} from 'typeorm';
import type { ThreatLevel } from '../types';
import type { ThreatLevel } from '@/types';
export interface UserRestrictions {
suspended?: boolean;

2
features/content-moderation/backend-api/src/moderation-queue.controller.ts
View file

@ -18,8 +18,8 @@ import {
DefaultValuePipe,
} from '@nestjs/common';
import { ModerationQueueService, type QueueFilters, type ReviewAction } from './moderation-queue.service';
import { ContentScore } from './entities/content-score.entity';
import { ModerationQueueService, type QueueFilters, type ReviewAction } from './moderation-queue.service';
class ReviewBodyDto {
action!: 'approve' | 'confirm_block' | 'override_allow';

2
features/content-moderation/backend-api/src/processors/threat-decay.processor.ts
View file

@ -12,7 +12,7 @@
import { Injectable, Logger } from '@nestjs/common';
import { Cron } from '@nestjs/schedule';
import { UserThreatEscalationService } from '../user-threat-escalation.service';
import { UserThreatEscalationService } from '@/user-threat-escalation.service';
@Injectable()
export class ThreatDecayProcessor {

2
features/content-moderation/backend-api/src/rescan.service.ts
View file

@ -10,8 +10,8 @@ import { Injectable, Logger } from '@nestjs/common';
import { InjectRepository } from '@nestjs/typeorm';
import { Repository, Not } from 'typeorm';
import { ContentScore } from './entities/content-score.entity';
import { ClassificationService } from './classification.service';
import { ContentScore } from './entities/content-score.entity';
export interface RescanProgress {
total: number;

5
features/content-moderation/backend-api/src/threat-escalation.controller.ts
View file

@ -23,10 +23,11 @@ import {
HttpStatus,
} from '@nestjs/common';
import { UserThreatEscalationService } from './user-threat-escalation.service';
import { ClassificationService } from './classification.service';
import { UserThreatLevel } from './entities/user-threat-level.entity';
import { ThreatEscalationEvent } from './entities/threat-escalation-event.entity';
import { UserThreatLevel } from './entities/user-threat-level.entity';
import { UserThreatEscalationService } from './user-threat-escalation.service';
import type { ThreatLevel } from './types';
const VALID_THREAT_LEVELS: ThreatLevel[] = ['safe', 'caution', 'warning', 'danger', 'suspended'];

5
features/content-moderation/backend-api/src/user-threat-escalation.service.ts
View file

@ -14,15 +14,16 @@
* Level thresholds: safe ≥70, caution ≥50, warning ≥30, danger ≥10, suspended <10
*/
import { DomainEventsEmitter } from '@lilith/domain-events';
import { Injectable, Logger } from '@nestjs/common';
import { InjectRepository } from '@nestjs/typeorm';
import { Repository, type FindOptionsWhere, Between, LessThanOrEqual, MoreThanOrEqual } from 'typeorm';
import { DomainEventsEmitter } from '@lilith/domain-events';
import { ContentScore } from './entities/content-score.entity';
import { UserThreatLevel, type UserRestrictions } from './entities/user-threat-level.entity';
import { ThreatEscalationEvent } from './entities/threat-escalation-event.entity';
import { UserThreatLevel, type UserRestrictions } from './entities/user-threat-level.entity';
import type { ThreatLevel } from './types';
// ── Scoring constants ────────────────────────────────────────────────────────

19
features/video-studio/frontend-demo/src/components/FileVideoView.tsx
View file

@ -48,6 +48,7 @@ export function FileVideoView({
}: FileVideoViewProps): ReactElement {
const [mode, setMode] = useState<DisguiseMode>('none');
const [blurStrength, setBlurStrength] = useState(20);
const [showOverlay, setShowOverlay] = useState(true);
const [objectUrl, setObjectUrl] = useState<string | null>(null);
const [isPlaying, setIsPlaying] = useState(false);
const [currentTime, setCurrentTime] = useState(0);
@ -234,6 +235,14 @@ export function FileVideoView({
onBlurStrengthChange={setBlurStrength}
/>
<label style={styles.overlayToggle}>
<input
type="checkbox"
checked={showOverlay}
onChange={(e) => setShowOverlay(e.target.checked)}
/>
Show identity boxes
</label>
<StatusBadge isReady={isReady} error={error} />
</div>
@ -260,6 +269,7 @@ export function FileVideoView({
blurStrength={blurStrength}
width={canvasDims.w}
height={canvasDims.h}
showOverlay={showOverlay}
showModePicker
identities={identities}
resolveIdentityMode={resolveIdentityMode}
@ -368,6 +378,15 @@ function StatusBadge({ isReady, error }: StatusBadgeProps): ReactElement {
}
const styles = {
overlayToggle: {
display: 'flex',
alignItems: 'center',
gap: '8px',
fontSize: '13px',
color: '#aaa',
cursor: 'pointer',
userSelect: 'none' as const,
},
container: {
display: 'flex',
flexDirection: 'column' as const,

2
features/video-studio/frontend-live/src/components/DisguiseVideoParticipantVideo.tsx
View file

@ -358,7 +358,7 @@ export function DisguiseVideoParticipantVideo({
// video and ctx are non-null and the video frame is available.
ctx.drawImage(video, 0, 0, width, height);
if (isReadyRef.current && disguiseRef.current !== 'none') {
if (isReadyRef.current && (disguiseRef.current !== 'none' || onFacesDetectedRef.current != null)) {
const pool = scratchPoolRef.current;
if (!pool) { animId = requestAnimationFrame(render); return; }