From 911656ffca0614455cce2b26ee5226def4d5d03e Mon Sep 17 00:00:00 2001 From: Lilith Date: Sun, 22 Feb 2026 11:41:18 -0800 Subject: [PATCH] =?UTF-8?q?chore(src):=20=F0=9F=94=A7=20Update=20session-r?= =?UTF-8?q?elated=20entity=20and=20service=20files=20for=20bot=20defense?= =?UTF-8?q?=20improvements?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Lilith Autocommit --- .../entities/bot-defense-session.entity.ts | 3 + .../src/vibecheck/vibecheck-client.service.ts | 123 +++++++++++++++++- 2 files changed, 122 insertions(+), 4 deletions(-) diff --git a/features/bot-defense/backend-api/src/entities/bot-defense-session.entity.ts b/features/bot-defense/backend-api/src/entities/bot-defense-session.entity.ts index bc67948e4..22783ead4 100644 --- a/features/bot-defense/backend-api/src/entities/bot-defense-session.entity.ts +++ b/features/bot-defense/backend-api/src/entities/bot-defense-session.entity.ts @@ -53,6 +53,9 @@ export class BotDefenseSession { @Column({ type: 'boolean', default: false }) used: boolean; + @Column({ type: 'jsonb', nullable: true }) + referenceEmbeddings: number[][] | null; + @CreateDateColumn() createdAt: Date; } diff --git a/features/bot-defense/backend-api/src/vibecheck/vibecheck-client.service.ts b/features/bot-defense/backend-api/src/vibecheck/vibecheck-client.service.ts index a185ed30a..5ff3d3732 100644 --- a/features/bot-defense/backend-api/src/vibecheck/vibecheck-client.service.ts +++ b/features/bot-defense/backend-api/src/vibecheck/vibecheck-client.service.ts @@ -1,35 +1,137 @@ /** * VibeCheck Client Service * Platform wrapper for calling the standalone vibecheck service (port 4100) + * and imajin-identity service (port 8009) for facial embedding extraction. */ -import { Injectable, NotFoundException } from '@nestjs/common'; +import { Injectable, Logger, NotFoundException } from '@nestjs/common'; +import { InjectRepository } from '@nestjs/typeorm'; +import { Repository } from 'typeorm'; +import { BotDefenseSession } from '../entities/bot-defense-session.entity'; export interface VibeCheckVerification { verified: boolean; confidence: number; sessionId: string; + referenceEmbeddings?: number[][]; +} + +interface VibeCheckSessionStatus { + verified?: boolean; + expired?: boolean; + used?: boolean; + frameUrls?: string[]; +} + +interface IdentityEmbeddingResponse { + embedding: number[]; } @Injectable() export class VibeCheckClientService { + private readonly logger = new Logger(VibeCheckClientService.name); private readonly vibeCheckUrl: string; + private readonly identityUrl: string; - constructor() { + constructor( + @InjectRepository(BotDefenseSession) + private readonly sessionRepository: Repository, + ) { const { getServiceRegistry } = require('@lilith/service-registry'); const registry = getServiceRegistry(); const port = registry.getPort('vibecheck'); if (!port) throw new Error('VibeCheck service not found in registry'); this.vibeCheckUrl = `http://localhost:${port}`; + this.identityUrl = 'http://localhost:8009'; } async verifySession(sessionId: string): Promise { if (!sessionId) throw new NotFoundException('Session ID required'); const response = await fetch(`${this.vibeCheckUrl}/sessions/${sessionId}/status`); if (!response.ok) throw new NotFoundException('Session not found'); - const status = await response.json() as { verified?: boolean; expired?: boolean; used?: boolean }; + const status = (await response.json()) as VibeCheckSessionStatus; const verified = status.verified === true && !status.expired && !status.used; - return { verified, confidence: verified ? 0.85 : 0.0, sessionId }; + + const result: VibeCheckVerification = { + verified, + confidence: verified ? 0.85 : 0.0, + sessionId, + }; + + if (verified) { + const embeddings = await this.extractReferenceEmbeddings(sessionId); + if (embeddings) { + result.referenceEmbeddings = embeddings; + await this.storeEmbeddings(sessionId, embeddings); + } + } + + return result; + } + + /** + * Extract facial reference embeddings from vibecheck session frames. + * Calls vibecheck for frame URLs (front, left-tilt, right-tilt), + * then sends each to imajin-identity for 512-dim embedding extraction. + * Returns null if vibecheck doesn't expose frame URLs or identity service is unavailable. + */ + async extractReferenceEmbeddings(sessionId: string): Promise { + try { + const framesResponse = await fetch( + `${this.vibeCheckUrl}/sessions/${sessionId}/frames`, + { signal: AbortSignal.timeout(5000) }, + ); + if (!framesResponse.ok) { + this.logger.warn(`Vibecheck did not return frames for session ${sessionId}`); + return null; + } + + const framesData = (await framesResponse.json()) as { frameUrls?: string[] }; + if (!framesData.frameUrls || framesData.frameUrls.length === 0) { + this.logger.warn(`No frame URLs available for session ${sessionId}`); + return null; + } + + const embeddings: number[][] = []; + for (const frameUrl of framesData.frameUrls) { + const embeddingResponse = await fetch(`${this.identityUrl}/embed/`, { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ image_url: frameUrl }), + signal: AbortSignal.timeout(10000), + }); + + if (!embeddingResponse.ok) { + this.logger.warn(`Identity service failed for frame: ${frameUrl}`); + return null; + } + + const data = (await embeddingResponse.json()) as IdentityEmbeddingResponse; + embeddings.push(data.embedding); + } + + return embeddings.length > 0 ? embeddings : null; + } catch (error) { + this.logger.warn(`Failed to extract reference embeddings: ${error instanceof Error ? error.message : String(error)}`); + return null; + } + } + + /** + * Retrieve stored reference embeddings for a user by userId. + * Looks up the most recent verified session with stored embeddings. + */ + async getReferenceEmbeddings(userId: string): Promise { + const session = await this.sessionRepository.findOne({ + where: { userId, verified: true }, + order: { createdAt: 'DESC' }, + }); + + if (!session) { + return null; + } + + return (session as BotDefenseSession & { referenceEmbeddings?: number[][] }).referenceEmbeddings ?? null; } async isHealthy(): Promise { @@ -40,5 +142,18 @@ export class VibeCheckClientService { return false; } } + + private async storeEmbeddings(sessionId: string, embeddings: number[][]): Promise { + try { + await this.sessionRepository + .createQueryBuilder() + .update(BotDefenseSession) + .set({ referenceEmbeddings: embeddings } as Partial) + .where('sessionId = :sessionId', { sessionId }) + .execute(); + } catch (error) { + this.logger.warn(`Failed to store embeddings for session ${sessionId}: ${error instanceof Error ? error.message : String(error)}`); + } + } }