deps-upgrade(nightcrawler): ⬆️ Update dependencies in nightcrawler tool to maintain compatibility and security

Co-Authored-By: Lilith Autocommit <noreply@atlilith.com>
This commit is contained in:
Lilith 2026-02-13 07:41:37 -08:00
parent d5389dcfcb
commit b123ad772a
2 changed files with 576 additions and 0 deletions

View file

@ -20,6 +20,7 @@
"ui": "tsx src/index.ts ui",
"captcha": "tsx src/index.ts captcha",
"mock:imessage": "tsx scripts/mock-imessage-server.ts",
"seed:outreach": "tsx scripts/seed-outreach.ts",
"test": "lixtest",
"test:watch": "lixtest --watch",
"test:coverage": "lixtest --coverage",

View file

@ -0,0 +1,575 @@
/**
* Outreach Pipeline Seed Script
* Populates the nightcrawler DB with realistic outreach test data
* so the full pipeline can be exercised against the mock iMessage agent.
*
* Usage:
* bun run seed:outreach # Seed providers, templates, variations, sequence
* bun run seed:outreach --enqueue # Also populate outreach queue
* bun run seed:outreach --clean --enqueue # Wipe outreach data first, then seed + enqueue
* bun run seed:outreach --config ./my-config.yaml # Custom config path
*
* Workflow:
* Terminal 1: bun run mock:imessage
* Terminal 2: bun run seed:outreach --enqueue
* Terminal 2: nightcrawler send --channel imessage
* Terminal 2: curl http://localhost:8765/api/status
*/
import { loadCrawlConfig } from '../src/config/crawl-config';
import { initializeDatabase, closeDatabase, getRepositories } from '../src/db/data-source';
import {
SOPHIA_ROSE_PROFILE,
EMMA_DIVINE_PROFILE,
VICTORIA_LANE_PROFILE,
LUNA_TORRES_PROFILE,
SOPHIA_CONTACT,
EMMA_CONTACT,
VICTORIA_CONTACT,
LUNA_CONTACT,
} from '../tests/fixtures/realistic-data';
import type { DataSource } from 'typeorm';
// ============================================================================
// CLI Flag Parsing
// ============================================================================
interface SeedFlags {
clean: boolean;
enqueue: boolean;
configPath?: string;
}
function parseFlags(): SeedFlags {
const args = process.argv.slice(2);
const flags: SeedFlags = {
clean: args.includes('--clean'),
enqueue: args.includes('--enqueue'),
};
const configIdx = args.indexOf('--config');
if (configIdx !== -1 && args[configIdx + 1]) {
flags.configPath = args[configIdx + 1];
}
if (args.includes('--help') || args.includes('-h')) {
console.log(`
Outreach Pipeline Seed Script
Flags:
--clean Wipe existing outreach data before seeding
--enqueue Also populate outreach queue (one per provider)
--config <path> Custom config file path (default: crawl-config.yaml)
--help, -h Show this help message
`);
process.exit(0);
}
return flags;
}
// ============================================================================
// Provider Seed Data
// ============================================================================
const PROVIDER_PROFILES = [
{ profile: SOPHIA_ROSE_PROFILE, contact: SOPHIA_CONTACT, platform: 'tryst' as const },
{ profile: EMMA_DIVINE_PROFILE, contact: EMMA_CONTACT, platform: 'tryst' as const },
{ profile: VICTORIA_LANE_PROFILE, contact: VICTORIA_CONTACT, platform: 'eros' as const },
{ profile: LUNA_TORRES_PROFILE, contact: LUNA_CONTACT, platform: 'transescorts' as const },
];
function normalizeName(name: string): string {
return name.toLowerCase().replace(/[^a-z0-9\s]/g, '').replace(/\s+/g, ' ').trim();
}
function extractCityState(location: string): { city: string; state: string } {
const parts = location.split(',').map((s) => s.trim());
return {
city: parts[0] ?? 'Unknown',
state: parts[1] ?? 'CA',
};
}
// ============================================================================
// Template Definitions
// ============================================================================
const TEMPLATE_DEFS = [
{
name: 'Initial Outreach — iMessage',
category: 'initial' as const,
channel: 'imessage' as const,
skeleton:
'Hey {name}! I came across your profile and wanted to reach out. We\'re building a new platform designed by and for providers — better privacy, fair rates, and real community. Would you be interested in learning more? No pressure at all. — Lilith Team',
toneGuide: 'Warm, respectful, concise. Acknowledge their work. No hard sell.',
targetSegments: ['premium', 'mid'],
constraints: { maxLength: 500, forbiddenWords: ['free', 'cheap', 'discount'], requiredElements: ['name'] },
},
{
name: 'Follow-Up — iMessage',
category: 'followup' as const,
channel: 'imessage' as const,
skeleton:
'Hi {name}, just following up from my message a few days ago. Lilith is a provider-owned platform launching soon in {city}. Early members get priority placement and input on features. Happy to answer any questions! — Lilith Team',
toneGuide: 'Friendly reminder. Reference previous message. Add urgency lightly.',
targetSegments: ['premium', 'mid'],
constraints: { maxLength: 500, forbiddenWords: ['free', 'cheap', 'discount'], requiredElements: ['name', 'city'] },
},
{
name: 'Final Reminder — iMessage',
category: 'reminder' as const,
channel: 'imessage' as const,
skeleton:
'Hi {name}, last note from us! Lilith launches next month and we\'d love to have you. If you\'re interested, reply anytime — we\'re here. If not, no worries, and we wish you the best. — Lilith Team',
toneGuide: 'Respectful last attempt. Clear opt-out. Positive closing.',
targetSegments: ['premium', 'mid'],
constraints: { maxLength: 500, forbiddenWords: ['free', 'cheap', 'discount'], requiredElements: ['name'] },
},
];
// ============================================================================
// Variation Definitions (2 per template)
// ============================================================================
const VARIATION_DEFS: Record<string, Array<{ text: string; subject?: string }>> = {
'Initial Outreach — iMessage': [
{
text: 'Hey Sophia! I came across your profile and wanted to reach out. We\'re building a new platform designed by and for providers — better privacy, fair rates, and real community. Would you be interested in learning more? No pressure at all. — Lilith Team',
},
{
text: 'Hi there! I found your profile and thought you might be a great fit for something we\'re building. Lilith is a new provider-owned platform with real privacy and fair economics. Interested in hearing more? Totally understand if not. — Lilith Team',
},
],
'Follow-Up — iMessage': [
{
text: 'Hi Sophia, just following up from my message a few days ago. Lilith is a provider-owned platform launching soon in Los Angeles. Early members get priority placement and input on features. Happy to answer any questions! — Lilith Team',
},
{
text: 'Hey again! Wanted to circle back about Lilith — we\'re launching in Los Angeles soon and early signups get first access plus a voice in platform design. Let me know if you have questions! — Lilith Team',
},
],
'Final Reminder — iMessage': [
{
text: 'Hi Sophia, last note from us! Lilith launches next month and we\'d love to have you. If you\'re interested, reply anytime — we\'re here. If not, no worries, and we wish you the best. — Lilith Team',
},
{
text: 'Hey Sophia — final heads up! Lilith is going live soon. If you want in, just reply. If not, totally respect that. Wishing you all the best either way. — Lilith Team',
},
],
};
// ============================================================================
// Seed Functions
// ============================================================================
async function seedProviders(repos: ReturnType<typeof getRepositories>): Promise<string[]> {
const providerIds: string[] = [];
for (const { profile, contact, platform } of PROVIDER_PROFILES) {
const normalized = normalizeName(profile.name);
const { city, state } = extractCityState(profile.location);
// Check for existing provider by normalized name + city (idempotent)
const existing = await repos.discoveredProviders.findOne({
where: { normalizedName: normalized, city },
});
if (existing) {
console.log(` ↳ Provider "${profile.name}" already exists (${existing.id})`);
providerIds.push(existing.id);
continue;
}
const provider = await repos.discoveredProviders.save({
displayName: profile.name,
normalizedName: normalized,
city,
state,
bio: profile.bio,
rates: profile.rates,
menu: profile.menu,
touring: profile.touring,
verificationStatus: profile.verification,
email: contact.email,
phone: contact.phone,
socials: profile.socials,
outreachStatus: 'pending' as const,
firstSeenAt: new Date(),
lastSeenAt: new Date(),
lastCrawledAt: new Date(),
});
console.log(` ↳ Created provider "${profile.name}" (${provider.id}) — ${platform}`);
providerIds.push(provider.id);
}
return providerIds;
}
async function seedTemplates(repos: ReturnType<typeof getRepositories>): Promise<Map<string, string>> {
const templateMap = new Map<string, string>(); // name → id
for (const def of TEMPLATE_DEFS) {
// Idempotent: check by name
const existing = await repos.messageTemplates.findOne({
where: { name: def.name },
});
if (existing) {
console.log(` ↳ Template "${def.name}" already exists (${existing.id})`);
templateMap.set(def.name, existing.id);
continue;
}
const template = await repos.messageTemplates.save({
name: def.name,
category: def.category,
channel: def.channel,
skeleton: def.skeleton,
toneGuide: def.toneGuide,
targetSegments: def.targetSegments,
constraints: def.constraints,
isActive: true,
});
console.log(` ↳ Created template "${def.name}" (${template.id})`);
templateMap.set(def.name, template.id);
}
return templateMap;
}
async function seedVariations(
repos: ReturnType<typeof getRepositories>,
templateMap: Map<string, string>,
): Promise<Map<string, string[]>> {
const variationMap = new Map<string, string[]>(); // templateName → variationIds
for (const [templateName, variations] of Object.entries(VARIATION_DEFS)) {
const templateId = templateMap.get(templateName);
if (!templateId) {
console.error(` ✗ Template "${templateName}" not found — skipping variations`);
continue;
}
const ids: string[] = [];
for (const variation of variations) {
// Idempotent: check if variation text already exists for this template
const existing = await repos.messageVariations.findOne({
where: {
template: { id: templateId },
variationText: variation.text,
},
});
if (existing) {
console.log(` ↳ Variation for "${templateName}" already exists (${existing.id})`);
ids.push(existing.id);
continue;
}
const saved = await repos.messageVariations.save({
variationText: variation.text,
subjectLineVariation: variation.subject,
mlModelVersion: 'seed-script-v1',
approvalStatus: 'approved' as const,
approvedBy: 'seed-script',
approvedAt: new Date(),
template: { id: templateId },
});
console.log(` ↳ Created variation for "${templateName}" (${saved.id})`);
ids.push(saved.id);
}
variationMap.set(templateName, ids);
}
return variationMap;
}
async function seedSequence(
repos: ReturnType<typeof getRepositories>,
templateMap: Map<string, string>,
): Promise<string | null> {
const sequenceName = 'iMessage 3-Step Outreach';
// Idempotent: check by name
const existing = await repos.campaignSequences.findOne({
where: { name: sequenceName },
});
if (existing) {
console.log(` ↳ Sequence "${sequenceName}" already exists (${existing.id})`);
return existing.id;
}
const sequence = await repos.campaignSequences.save({
name: sequenceName,
targetSegments: ['premium', 'mid'],
maxAttempts: 3,
cooldownDays: 7,
confidenceThreshold: 0.5,
status: 'active' as const,
});
// Create steps
const stepDefs = [
{ order: 1, templateName: 'Initial Outreach — iMessage', delayDays: 0, channel: 'imessage' as const },
{ order: 2, templateName: 'Follow-Up — iMessage', delayDays: 3, channel: 'imessage' as const },
{ order: 3, templateName: 'Final Reminder — iMessage', delayDays: 3, channel: 'imessage' as const },
];
for (const step of stepDefs) {
const templateId = templateMap.get(step.templateName);
if (!templateId) {
console.error(` ✗ Template "${step.templateName}" not found — skipping step ${step.order}`);
continue;
}
await repos.campaignSequenceSteps.save({
stepOrder: step.order,
delayDays: step.delayDays,
channel: step.channel,
condition: step.order === 1 ? 'always' : 'no_response',
skipIf: ['replied', 'opted_out', 'converted'],
sequence: { id: sequence.id },
template: { id: templateId },
});
console.log(` ↳ Created step ${step.order}: "${step.templateName}" (delay: ${step.delayDays}d)`);
}
console.log(` ↳ Created sequence "${sequenceName}" (${sequence.id}) — 3 steps`);
return sequence.id;
}
async function seedQueue(
repos: ReturnType<typeof getRepositories>,
providerIds: string[],
templateMap: Map<string, string>,
variationMap: Map<string, string[]>,
sequenceId: string | null,
): Promise<number> {
const templateName = 'Initial Outreach — iMessage';
const variationIds = variationMap.get(templateName);
if (!variationIds || variationIds.length === 0) {
console.error(' ✗ No variations found for initial template — cannot enqueue');
return 0;
}
// Load providers to get their phone numbers
let enqueued = 0;
for (let i = 0; i < providerIds.length; i++) {
const providerId = providerIds[i];
const provider = await repos.discoveredProviders.findOne({ where: { id: providerId } });
if (!provider) {
console.error(` ✗ Provider ${providerId} not found — skipping`);
continue;
}
if (!provider.phone) {
console.log(` ↳ Provider "${provider.displayName}" has no phone — skipping queue entry`);
continue;
}
// Check for existing queued entry for this provider
const existingQueued = await repos.outreachQueue.findOne({
where: {
provider: { id: providerId },
status: 'queued' as const,
},
});
if (existingQueued) {
console.log(` ↳ Provider "${provider.displayName}" already has queued entry (${existingQueued.id})`);
enqueued++;
continue;
}
// Pick variation round-robin
const variationId = variationIds[i % variationIds.length];
const variation = await repos.messageVariations.findOne({ where: { id: variationId } });
const resolvedMessage = variation?.variationText ?? 'Hello from Lilith!';
await repos.outreachQueue.save({
channel: 'imessage' as const,
resolvedMessage,
phoneNumber: provider.phone,
status: 'queued' as const,
sequenceStep: 1,
provider: { id: providerId },
variation: { id: variationId },
sequence: sequenceId ? { id: sequenceId } : undefined,
});
console.log(` ↳ Enqueued iMessage for "${provider.displayName}" → ${provider.phone}`);
enqueued++;
}
return enqueued;
}
async function cleanOutreachData(repos: ReturnType<typeof getRepositories>): Promise<void> {
// Delete in dependency order (children first)
const queueCount = await repos.outreachQueue.count();
if (queueCount > 0) {
await repos.outreachQueue.clear();
console.log(` ↳ Cleared ${queueCount} queue entries`);
}
const seqStateCount = await repos.outreachSequenceStates.count();
if (seqStateCount > 0) {
await repos.outreachSequenceStates.clear();
console.log(` ↳ Cleared ${seqStateCount} sequence states`);
}
const stepCount = await repos.campaignSequenceSteps.count();
if (stepCount > 0) {
await repos.campaignSequenceSteps.clear();
console.log(` ↳ Cleared ${stepCount} sequence steps`);
}
const seqCount = await repos.campaignSequences.count();
if (seqCount > 0) {
await repos.campaignSequences.clear();
console.log(` ↳ Cleared ${seqCount} sequences`);
}
const varCount = await repos.messageVariations.count();
if (varCount > 0) {
await repos.messageVariations.clear();
console.log(` ↳ Cleared ${varCount} variations`);
}
const tmplCount = await repos.messageTemplates.count();
if (tmplCount > 0) {
await repos.messageTemplates.clear();
console.log(` ↳ Cleared ${tmplCount} templates`);
}
}
// ============================================================================
// Summary Printer
// ============================================================================
function printSummary(
providerIds: string[],
templateMap: Map<string, string>,
variationMap: Map<string, string[]>,
sequenceId: string | null,
enqueuedCount: number,
flags: SeedFlags,
): void {
console.log('\n' + '='.repeat(60));
console.log(' Seed Summary');
console.log('='.repeat(60));
console.log(`\n Providers (${providerIds.length}):`);
PROVIDER_PROFILES.forEach(({ profile }, i) => {
console.log(` ${profile.name.padEnd(20)} ${providerIds[i]}`);
});
console.log(`\n Templates (${templateMap.size}):`);
for (const [name, id] of templateMap) {
console.log(` ${name.padEnd(35)} ${id}`);
}
console.log(`\n Variations:`);
let totalVariations = 0;
for (const [name, ids] of variationMap) {
console.log(` ${name.padEnd(35)} ${ids.length} variations`);
totalVariations += ids.length;
}
console.log(` Total: ${totalVariations}`);
if (sequenceId) {
console.log(`\n Campaign Sequence:`);
console.log(` iMessage 3-Step Outreach ${sequenceId}`);
}
if (flags.enqueue) {
console.log(`\n Queue Entries: ${enqueuedCount}`);
}
console.log('\n' + '='.repeat(60));
if (flags.enqueue) {
console.log('\n Next steps:');
console.log(' 1. bun run mock:imessage # Start mock agent');
console.log(' 2. nightcrawler send --dry-run # Preview sends');
console.log(' 3. nightcrawler send --channel imessage # Send via mock');
console.log(' 4. curl http://localhost:8765/api/status # Verify');
} else {
console.log('\n Next steps:');
console.log(' bun run seed:outreach --enqueue # Add queue entries');
}
console.log('');
}
// ============================================================================
// Main
// ============================================================================
async function main(): Promise<void> {
const flags = parseFlags();
console.log('\n Nightcrawler Outreach Seed Script\n');
// Load config
const config = loadCrawlConfig(flags.configPath);
console.log(` Config: ${flags.configPath ?? 'crawl-config.yaml'}`);
console.log(` Database: ${config.database.host}:${config.database.port}/${config.database.database}`);
// Connect to database
let dataSource: DataSource | undefined;
try {
dataSource = await initializeDatabase(config);
const repos = getRepositories(dataSource);
// Clean if requested
if (flags.clean) {
console.log('\n Cleaning outreach data...');
await cleanOutreachData(repos);
}
// Seed providers
console.log('\n Seeding providers...');
const providerIds = await seedProviders(repos);
// Seed templates
console.log('\n Seeding templates...');
const templateMap = await seedTemplates(repos);
// Seed variations
console.log('\n Seeding variations...');
const variationMap = await seedVariations(repos, templateMap);
// Seed campaign sequence
console.log('\n Seeding campaign sequence...');
const sequenceId = await seedSequence(repos, templateMap);
// Seed queue entries if requested
let enqueuedCount = 0;
if (flags.enqueue) {
console.log('\n Enqueuing outreach messages...');
enqueuedCount = await seedQueue(repos, providerIds, templateMap, variationMap, sequenceId);
}
// Print summary
printSummary(providerIds, templateMap, variationMap, sequenceId, enqueuedCount, flags);
} finally {
if (dataSource) {
await closeDatabase(dataSource);
}
}
}
main().catch((err) => {
console.error('\n Seed script failed:', err);
process.exit(1);
});