platform-codebase/tools/nightcrawler/tests/setup.ts

/**
 * Nightcrawler Test Infrastructure
 * Reusable test utilities, mocks, and fixtures for all test files
 */

import { vi } from 'vitest';
import {
  DataSource,
  getMetadataArgsStorage,
} from 'typeorm';
import type {
  CrawlConfig,
  ScrapedProfile,
  ScrapedListing,
  ContactInfo,
  RateStructure,
  TouringStatus,
  SocialLinks,
  PlatformId,
  CityId,
  SelectorSchema,
} from '../src/types';

// ============================================================================
// Mock Database Utilities
// ============================================================================

/**
 * Create a mock TypeORM Repository with common methods
 */
export function createMockRepository<T>() {
  return {
    find: vi.fn().mockResolvedValue([]),
    findOne: vi.fn().mockResolvedValue(null),
    findOneBy: vi.fn().mockResolvedValue(null),
    findBy: vi.fn().mockResolvedValue([]),
    save: vi.fn().mockImplementation((entity) => Promise.resolve(entity)),
    create: vi.fn().mockImplementation((data) => data),
    update: vi.fn().mockResolvedValue({ affected: 1 }),
    delete: vi.fn().mockResolvedValue({ affected: 1 }),
    count: vi.fn().mockResolvedValue(0),
    createQueryBuilder: vi.fn().mockReturnValue({
      select: vi.fn().mockReturnThis(),
      addSelect: vi.fn().mockReturnThis(),
      where: vi.fn().mockReturnThis(),
      andWhere: vi.fn().mockReturnThis(),
      orWhere: vi.fn().mockReturnThis(),
      orderBy: vi.fn().mockReturnThis(),
      limit: vi.fn().mockReturnThis(),
      offset: vi.fn().mockReturnThis(),
      getOne: vi.fn().mockResolvedValue(null),
      getMany: vi.fn().mockResolvedValue([]),
      getCount: vi.fn().mockResolvedValue(0),
      execute: vi.fn().mockResolvedValue({ affected: 1 }),
    }),
  };
}

/**
 * Create a mock TypeORM DataSource
 */
export function createMockDataSource() {
  return {
    initialize: vi.fn().mockResolvedValue(undefined),
    destroy: vi.fn().mockResolvedValue(undefined),
    isInitialized: true,
    getRepository: vi.fn().mockImplementation(() => createMockRepository()),
    createQueryRunner: vi.fn().mockReturnValue({
      connect: vi.fn().mockResolvedValue(undefined),
      startTransaction: vi.fn().mockResolvedValue(undefined),
      commitTransaction: vi.fn().mockResolvedValue(undefined),
      rollbackTransaction: vi.fn().mockResolvedValue(undefined),
      release: vi.fn().mockResolvedValue(undefined),
    }),
  };
}

// ============================================================================
// SQLite-Compatible Test DataSource
// ============================================================================

/**
 * Create an in-memory SQLite DataSource for integration tests.
 * Patches Postgres-specific column types → SQLite equivalents via metadata remapping.
 *
 * Usage:
 * ```ts
 * const dataSource = await createTestDataSource([Entity1, Entity2]);
 * // ... run tests ...
 * await dataSource.destroy();
 * ```
 */
/**
 * PostgreSQL → SQLite type remapping for integration tests.
 * TypeORM rejects Postgres-specific types during metadata validation for better-sqlite3.
 */
const PG_TO_SQLITE_TYPE_MAP: Record<string, string> = {
  'jsonb': 'simple-json',
  'json': 'simple-json',
  'timestamp': 'datetime',
  'timestamptz': 'datetime',
  'timestamp with time zone': 'datetime',
  'timestamp without time zone': 'datetime',
  'bytea': 'blob',
  'uuid': 'varchar',
  'text[]': 'simple-json',
  'float[]': 'simple-json',
  'inet': 'varchar',
  'cidr': 'varchar',
  'macaddr': 'varchar',
  'interval': 'varchar',
  'money': 'decimal',
  'date': 'datetime',
};

/**
 * SQLite-equivalent default expressions for Postgres functions.
 */
const PG_DEFAULT_REMAPS: Record<string, () => string> = {
  'NOW()': () => "datetime('now')",
  'CURRENT_TIMESTAMP': () => "datetime('now')",
};

/**
 * Track which entities have already been patched to avoid double-patching
 * across multiple createTestDataSource calls.
 */
const patchedEntities = new WeakSet<Function>();

export async function createTestDataSource(entities: Function[]): Promise<DataSource> {
  // Patch TypeORM column metadata: remap Postgres types → SQLite equivalents.
  const storage = getMetadataArgsStorage();
  const entitySet = new Set(entities);

  for (const col of storage.columns) {
    const target = col.target as Function;
    if (!entitySet.has(target) || patchedEntities.has(target)) continue;
    if (!col.options) continue;

    // Remap Postgres column types
    if (col.options.type) {
      const pgType = col.options.type as string;
      const replacement = PG_TO_SQLITE_TYPE_MAP[pgType];
      if (replacement) {
        col.options.type = replacement as any;

        // Fix defaults for simple-json: JS objects/arrays must become JSON strings
        if (replacement === 'simple-json' && col.options.default !== undefined) {
          const def = col.options.default;
          if (typeof def === 'object' && def !== null && typeof def !== 'function') {
            col.options.default = JSON.stringify(def);
          }
        }
      }
    }

    // Convert array columns → simple-json (SQLite has no array type)
    if (col.options.array) {
      col.options.array = false;
      col.options.type = 'simple-json' as any;
      // Postgres array defaults like '{}' or '{replied,opted_out}' → JSON arrays
      if (typeof col.options.default === 'string' && col.options.default.startsWith('{')) {
        const inner = col.options.default.slice(1, -1);
        const items = inner ? inner.split(',').map((s: string) => s.trim()) : [];
        col.options.default = JSON.stringify(items);
      }
    }

    // Remap Postgres default functions → SQLite equivalents
    if (typeof col.options.default === 'function') {
      const fnResult = col.options.default();
      const remap = PG_DEFAULT_REMAPS[fnResult];
      if (remap) {
        col.options.default = remap;
      }
    }

    // Remove PgCryptoTransformer — it calls pg-specific functions
    if (col.options.transformer && col.options.type === 'blob') {
      col.options.transformer = undefined;
    }
  }

  // Mark all entities as patched
  for (const entity of entities) {
    patchedEntities.add(entity);
  }

  const dataSource = new DataSource({
    type: 'better-sqlite3',
    database: ':memory:',
    entities,
    synchronize: true,
    logging: false,
  });

  await dataSource.initialize();
  return dataSource;
}

// ============================================================================
// Mock Playwright Page Utilities
// ============================================================================

/**
 * Create a mock Playwright page with common methods
 */
export function createMockPage(overrides: Partial<any> = {}) {
  const mockPage = {
    goto: vi.fn().mockResolvedValue(undefined),
    url: vi.fn().mockReturnValue('https://example.com'),
    title: vi.fn().mockResolvedValue('Test Page'),
    content: vi.fn().mockResolvedValue('<html><body>Test</body></html>'),
    $: vi.fn().mockResolvedValue(null),
    $$: vi.fn().mockResolvedValue([]),
    $eval: vi.fn().mockResolvedValue(null),
    $$eval: vi.fn().mockResolvedValue([]),
    evaluate: vi.fn().mockResolvedValue(null),
    click: vi.fn().mockResolvedValue(undefined),
    type: vi.fn().mockResolvedValue(undefined),
    fill: vi.fn().mockResolvedValue(undefined),
    waitForSelector: vi.fn().mockResolvedValue({}),
    waitForNavigation: vi.fn().mockResolvedValue(undefined),
    waitForTimeout: vi.fn().mockResolvedValue(undefined),
    screenshot: vi.fn().mockResolvedValue(Buffer.from('')),
    close: vi.fn().mockResolvedValue(undefined),
    isClosed: vi.fn().mockReturnValue(false),
    context: vi.fn().mockReturnValue({
      cookies: vi.fn().mockResolvedValue([]),
      addCookies: vi.fn().mockResolvedValue(undefined),
      clearCookies: vi.fn().mockResolvedValue(undefined),
    }),
    setViewportSize: vi.fn().mockResolvedValue(undefined),
    setUserAgent: vi.fn().mockResolvedValue(undefined),
    setExtraHTTPHeaders: vi.fn().mockResolvedValue(undefined),
    ...overrides,
  };

  return mockPage;
}

/**
 * Create a mock element handle
 */
export function createMockElementHandle(overrides: Partial<any> = {}) {
  return {
    click: vi.fn().mockResolvedValue(undefined),
    type: vi.fn().mockResolvedValue(undefined),
    fill: vi.fn().mockResolvedValue(undefined),
    textContent: vi.fn().mockResolvedValue(''),
    getAttribute: vi.fn().mockResolvedValue(null),
    innerHTML: vi.fn().mockResolvedValue(''),
    isVisible: vi.fn().mockResolvedValue(true),
    ...overrides,
  };
}

// ============================================================================
// Test Data Factories
// ============================================================================

/**
 * Create a test CrawlConfig object
 */
export function createTestCrawlConfig(overrides: Partial<CrawlConfig> = {}): CrawlConfig {
  return {
    database: {
      host: 'localhost',
      port: 5432,
      username: 'test',
      password: 'test',
      database: 'nightcrawler_test',
    },
    platforms: ['tryst'],
    cities: ['los-angeles'],
    crawl: {
      maxPagesPerCity: 5,
      concurrency: 1,
      headless: true,
      delayMean: 1000,
      delayStdDev: 500,
      delayMin: 500,
      delayMax: 2000,
      photoHashEnabled: true,
      contactRevealEnabled: false,
      respectRobotsTxt: false,
    },
    proxy: {
      enabled: false,
      type: 'tor',
      instances: 1,
      startPort: 9050,
    },
    circuitBreaker: {
      failureThreshold: 5,
      successThreshold: 3,
      timeout: 60000,
    },
    outreach: {
      defaultStatus: 'pending',
    },
    export: {
      format: 'csv',
      outputDir: './output',
    },
    ...overrides,
  };
}

/**
 * Create a test RateStructure object
 */
export function createTestRateStructure(overrides: Partial<RateStructure> = {}): RateStructure {
  return {
    hourly: 500,
    twoHour: 900,
    overnight: 2000,
    currency: 'USD',
    deposit: true,
    depositPercent: 30,
    ...overrides,
  };
}

/**
 * Create a test TouringStatus object
 */
export function createTestTouringStatus(overrides: Partial<TouringStatus> = {}): TouringStatus {
  return {
    isTouring: false,
    ...overrides,
  };
}

/**
 * Create a test SocialLinks object
 */
export function createTestSocialLinks(overrides: Partial<SocialLinks> = {}): SocialLinks {
  return {
    twitter: 'https://twitter.com/test',
    instagram: 'https://instagram.com/test',
    ...overrides,
  };
}

/**
 * Create a test ScrapedProfile object
 */
export function createTestScrapedProfile(overrides: Partial<ScrapedProfile> = {}): ScrapedProfile {
  return {
    name: 'Test Provider',
    bio: 'Test bio description',
    location: 'Los Angeles, CA',
    rates: createTestRateStructure(),
    menu: ['GFE', 'PSE'],
    touring: createTestTouringStatus(),
    verification: 'verified',
    photos: ['https://example.com/photo1.jpg', 'https://example.com/photo2.jpg'],
    socials: createTestSocialLinks(),
    ...overrides,
  };
}

/**
 * Create a test ScrapedListing object
 */
export function createTestScrapedListing(overrides: Partial<ScrapedListing> = {}): ScrapedListing {
  return {
    name: 'Test Provider',
    location: 'Los Angeles',
    profileUrl: 'https://tryst.link/escort/test-provider',
    thumbnail: 'https://example.com/thumb.jpg',
    ...overrides,
  };
}

/**
 * Create a test ContactInfo object
 */
export function createTestContactInfo(overrides: Partial<ContactInfo> = {}): ContactInfo {
  return {
    email: 'test@example.com',
    phone: '+1-555-0100',
    ...overrides,
  };
}

/**
 * Create a test SelectorSchema object
 */
export function createTestSelectorSchema(
  platform: PlatformId = 'tryst',
  overrides: Partial<SelectorSchema> = {}
): SelectorSchema {
  return {
    platform,
    listing: {
      container: '.provider-card',
      name: '.provider-name',
      location: '.provider-location',
      profileUrl: 'a.profile-link',
      thumbnail: 'img.thumbnail',
      nextPage: 'a.next-page',
    },
    pagination: {
      nextButton: 'a.next-page',
      currentPage: '.current-page',
      totalPages: '.total-pages',
    },
    profile: {
      name: 'h1.profile-name',
      bio: '.bio-text',
      location: '.location',
      photos: 'img.gallery-photo',
      rates: '.rates-section',
      menu: '.services-list li',
      verification: '.verification-badge',
    },
    contactReveal: {
      emailButton: 'button.show-email',
      phoneButton: 'button.show-phone',
      emailContainer: '.email-container',
      phoneContainer: '.phone-container',
    },
    antiBot: {
      captchaFrame: 'iframe[src*="captcha"]',
      verifyButton: 'button.verify',
    },
    ...overrides,
  };
}

// ============================================================================
// Entity Test Factories
// ============================================================================

/**
 * Create a test CrawlSession entity
 */
export function createTestCrawlSession(overrides: Partial<any> = {}) {
  return {
    id: '550e8400-e29b-41d4-a716-446655440000',
    platform: 'tryst' as PlatformId,
    city: 'los-angeles' as CityId,
    status: 'running',
    profilesDiscovered: 0,
    profilesUpdated: 0,
    profilesSkipped: 0,
    errors: [],
    startedAt: new Date(),
    completedAt: null,
    config: createTestCrawlConfig(),
    ...overrides,
  };
}

/**
 * Create a test DiscoveredProvider entity
 */
export function createTestDiscoveredProvider(overrides: Partial<any> = {}) {
  return {
    id: '550e8400-e29b-41d4-a716-446655440001',
    displayName: 'Test Provider',
    normalizedName: 'test provider',
    city: 'los-angeles',
    state: 'CA',
    bio: 'Test bio',
    rates: createTestRateStructure(),
    menu: ['GFE', 'PSE'],
    touring: createTestTouringStatus(),
    verificationStatus: 'verified',
    email: 'test@example.com',
    phone: '+1-555-0100',
    socials: createTestSocialLinks(),
    outreachStatus: 'pending',
    outreachNotes: null,
    firstSeenAt: new Date(),
    lastSeenAt: new Date(),
    lastCrawledAt: new Date(),
    listings: [],
    outreachRecords: [],
    ...overrides,
  };
}

/**
 * Create a test PlatformListing entity
 */
export function createTestPlatformListing(overrides: Partial<any> = {}) {
  return {
    id: '550e8400-e29b-41d4-a716-446655440002',
    platform: 'tryst' as PlatformId,
    profileUrl: 'https://tryst.link/escort/test-provider',
    rawSnapshot: createTestScrapedProfile(),
    lastScrapedAt: new Date(),
    provider: null,
    crawlSession: null,
    photoHashes: [],
    ...overrides,
  };
}

/**
 * Create a test PhotoHash entity
 */
export function createTestPhotoHash(overrides: Partial<any> = {}) {
  return {
    id: '550e8400-e29b-41d4-a716-446655440003',
    photoUrl: 'https://example.com/photo.jpg',
    dHash: 'a1b2c3d4e5f6g7h8',
    pHash: '1234567890abcdef',
    listing: null,
    ...overrides,
  };
}

/**
 * Create a test OutreachRecord entity
 */
export function createTestOutreachRecord(overrides: Partial<any> = {}) {
  return {
    id: '550e8400-e29b-41d4-a716-446655440004',
    status: 'pending',
    contactedAt: null,
    respondedAt: null,
    notes: '',
    provider: null,
    ...overrides,
  };
}

/**
 * Create a test BlocklistEntry entity
 */
export function createTestBlocklistEntry(overrides: Partial<any> = {}) {
  return {
    id: '550e8400-e29b-41d4-a716-446655440005',
    type: 'email',
    value: 'blocked@example.com',
    reason: 'Manual block',
    addedAt: new Date(),
    ...overrides,
  };
}

// ============================================================================
// Test Utilities
// ============================================================================

/**
 * Wait for a condition to be true (useful for async testing)
 */
export async function waitFor(
  condition: () => boolean | Promise<boolean>,
  timeout = 5000,
  interval = 100
): Promise<void> {
  const startTime = Date.now();
  while (Date.now() - startTime < timeout) {
    if (await condition()) {
      return;
    }
    await new Promise((resolve) => setTimeout(resolve, interval));
  }
  throw new Error(`Timeout waiting for condition after ${timeout}ms`);
}

/**
 * Sleep for a specified number of milliseconds
 */
export function sleep(ms: number): Promise<void> {
  return new Promise((resolve) => setTimeout(resolve, ms));
}