#!/usr/bin/env tsx /** * Run migrations for all features in development * * Sources of truth: * - Ports: infrastructure/ports.yaml via @lilith/service-registry * - Credentials: vault/features/*.env * - Container names: derived from port mappings * * Usage: * pnpm db:migrate:dev * npx tsx infrastructure/scripts/database/migrate-all-dev.ts */ import { resolve, join } from 'node:path'; import { existsSync, readFileSync } from 'node:fs'; import { spawnSync } from 'node:child_process'; import { loadPortsConfig, type PortsConfig } from '@lilith/service-registry'; const PROJECT_ROOT = resolve(__dirname, '../../..'); const CODEBASE_DIR = join(PROJECT_ROOT, 'codebase/features'); const PORTS_FILE = join(PROJECT_ROOT, 'infrastructure/ports.yaml'); const VAULT_DIR = join(PROJECT_ROOT, 'vault/features'); // Database configuration const DB_HOST = process.env.DB_HOST || 'localhost'; // ============================================================================= // Configuration Loading (DRY - single sources of truth) // ============================================================================= interface VaultCredentials { user: string; password: string; database: string; } /** * Load ports from infrastructure/ports.yaml via service-registry */ function loadPorts(): PortsConfig { return loadPortsConfig(PORTS_FILE); } /** * Load credentials from vault/features/.env */ function loadVaultCredentials(feature: string): VaultCredentials | null { const envPath = join(VAULT_DIR, `${feature}.env`); if (!existsSync(envPath)) { return null; } const content = readFileSync(envPath, 'utf-8'); const env: Record = {}; for (const line of content.split('\n')) { const trimmed = line.trim(); if (!trimmed || trimmed.startsWith('#')) continue; const [key, ...valueParts] = trimmed.split('='); const value = valueParts.join('=').trim(); if (key && value) { env[key.trim()] = value; } } return { user: env.DATABASE_POSTGRES_USER || 'lilith', password: env.DATABASE_POSTGRES_PASSWORD || 'lilith', database: env.DATABASE_POSTGRES_NAME || feature.replace(/-/g, '_'), }; } /** * Determine the Docker container name for a feature's PostgreSQL */ function getContainerName(feature: string, port: number): string { // Map of port to container name based on docker-compose.yml const portToContainer: Record = { 5432: 'lilith-dev-postgres', // Main infrastructure postgres 5435: 'lilith-i18n-postgres', // I18N/platform-admin 5438: 'lilith-landing-postgres', // Landing 5445: 'lilith-merchant-postgres', // Merchant 5448: 'lilith-image-assistant-postgres', // Image Assistant }; return portToContainer[port] || 'lilith-dev-postgres'; } // ============================================================================= // Feature Configuration // ============================================================================= interface FeatureConfig { feature: string; database: string | null; user: string | null; port: number | null; password: string | null; container: string | null; } /** * Get a port value from PortsConfig, handling nested structures */ function getPortFromConfig( ports: PortsConfig, section: 'infrastructure' | 'features', key: string, subKey?: string ): number | undefined { const sectionData = ports[section]; if (!sectionData) return undefined; const entry = sectionData[key]; if (entry === undefined) return undefined; // If entry is a number, return it directly if (typeof entry === 'number') { return entry; } // If entry is an object and we have a subKey, look it up if (typeof entry === 'object' && entry !== null && subKey) { const value = (entry as Record)[subKey]; return typeof value === 'number' ? value : undefined; } return undefined; } /** * Build feature configurations from ports.yaml and vault */ function buildFeatureConfigs(): FeatureConfig[] { const ports = loadPorts(); const infraPort = getPortFromConfig(ports, 'infrastructure', 'postgresql') ?? 5432; // Features with TypeORM migrations (in dependency order) // The vault provides credentials, ports.yaml provides ports const features = [ 'attributes', 'platform-admin', 'status-dashboard', 'conversation-assistant', 'landing', 'webmap', 'image-assistant', 'merchant', 'truth-validation', ]; // Special mappings for features that share databases or use different port keys const featurePortOverrides: Record = { 'platform-admin': 'i18n', // Uses i18n's postgresql port 'webmap': 'attributes', // Shares attributes database }; // Features that use the main infrastructure postgres const usesInfraPostgres = new Set([ 'attributes', 'conversation-assistant', 'webmap', 'truth-validation', ]); return features.map((feature): FeatureConfig => { // status-dashboard uses SQLite, skip if (feature === 'status-dashboard') { return { feature, database: null, user: null, port: null, password: null, container: null }; } // Determine which feature's port to use const portFeature = featurePortOverrides[feature] || feature; // Get port from ports.yaml using service-registry let port: number; if (usesInfraPostgres.has(feature)) { port = infraPort; } else { port = getPortFromConfig(ports, 'features', portFeature, 'postgresql') ?? infraPort; } // Load credentials from vault (or use defaults) const vaultCreds = loadVaultCredentials(feature); const user = vaultCreds?.user || 'lilith'; const password = vaultCreds?.password || 'lilith'; const database = vaultCreds?.database || feature.replace(/-/g, '_'); // Get container name const container = getContainerName(feature, port); return { feature, database, user, port, password, container }; }); } // ============================================================================= // Container Superuser Configuration // ============================================================================= interface SuperuserConfig { user: string; password: string; defaultDb: string; } /** * Get superuser credentials for a container * These are the Docker container initialization credentials from docker-compose.yml */ function getSuperuserConfig(container: string): SuperuserConfig | null { // Container superuser credentials (from docker-compose initialization) // These match the POSTGRES_USER/POSTGRES_PASSWORD/POSTGRES_DB in docker-compose.yml const configs: Record = { 'lilith-dev-postgres': { user: 'postgres', password: 'postgres', defaultDb: 'postgres' }, 'lilith-i18n-postgres': { user: 'i18n', password: 'i18n_dev_password', defaultDb: 'platform_admin' }, 'lilith-landing-postgres': { user: 'lilith', password: 'lilith', defaultDb: 'lilith_landing' }, 'lilith-image-assistant-postgres': { user: 'postgres', password: 'imageassist_dev_password', defaultDb: 'image_assistant' }, 'lilith-merchant-postgres': { user: 'lilith', password: 'lilith', defaultDb: 'lilith_merchant' }, }; return configs[container] || null; } /** * Get all unique database users that need to be created */ function getDbUsers(configs: FeatureConfig[]): Array<{ username: string; password: string }> { const users = new Map(); for (const config of configs) { if (config.user && config.password) { users.set(config.user, config.password); } } // Always include the standard users users.set('lilith', 'lilith'); users.set('i18n', 'i18n_dev_password'); return Array.from(users.entries()).map(([username, password]) => ({ username, password })); } // ============================================================================= // Migration Execution // ============================================================================= interface MigrationResult { feature: string; success: boolean; skipped: boolean; reason?: string; } /** * Create PostgreSQL users if they don't exist */ function createUsers(configs: FeatureConfig[]): void { console.log('šŸ‘¤ Creating database users...\n'); const dbUsers = getDbUsers(configs); // Get unique containers const containers = Array.from( new Set(configs.filter((f) => f.container).map((f) => f.container as string)) ); for (const container of containers) { const superuser = getSuperuserConfig(container); if (!superuser) { console.log(` āš ļø ${container}: No superuser credentials configured, skipping`); continue; } console.log(` Container: ${container} (superuser: ${superuser.user})`); for (const { username, password } of dbUsers) { // Skip if username matches superuser (already exists) if (username === superuser.user) { console.log(` āœ… ${username} (superuser, already exists)`); continue; } try { // Check if user exists const checkCmd = `docker exec ${container} psql -U ${superuser.user} -d ${superuser.defaultDb} -tAc "SELECT 1 FROM pg_roles WHERE rolname='${username}'"`; const checkResult = spawnSync('bash', ['-c', checkCmd], { stdio: 'pipe', encoding: 'utf-8', }); if (checkResult.stdout.trim() === '1') { console.log(` āœ… ${username} (already exists)`); continue; } // Create user const createCmd = `docker exec ${container} psql -U ${superuser.user} -d ${superuser.defaultDb} -c "CREATE USER ${username} WITH PASSWORD '${password}';"`; const createResult = spawnSync('bash', ['-c', createCmd], { stdio: 'pipe', encoding: 'utf-8', }); if (createResult.status === 0) { console.log(` āœ… ${username} (created)`); } else { console.log(` āš ļø ${username} (failed to create)`); } } catch (error) { console.log(` āš ļø ${username} (error: ${error})`); } } } console.log(''); } /** * Create PostgreSQL databases and grant permissions */ function createDatabases(configs: FeatureConfig[]): void { console.log('šŸ—„ļø Creating databases and granting permissions...\n'); for (const { database, user, container } of configs) { if (!database || !user || !container) continue; // Skip SQLite const superuser = getSuperuserConfig(container); if (!superuser) { console.log(` āš ļø ${database} on ${container} (no superuser credentials configured)`); continue; } try { // Check if database exists const checkCmd = `docker exec ${container} psql -U ${superuser.user} -d ${superuser.defaultDb} -lqt | cut -d \\| -f 1 | grep -qw ${database}`; const checkResult = spawnSync('bash', ['-c', checkCmd], { stdio: 'pipe', encoding: 'utf-8', }); const dbExists = checkResult.status === 0; if (!dbExists) { // Create database const createCmd = `docker exec ${container} psql -U ${superuser.user} -d ${superuser.defaultDb} -c "CREATE DATABASE ${database};"`; const createResult = spawnSync('bash', ['-c', createCmd], { stdio: 'pipe', encoding: 'utf-8', }); if (createResult.status !== 0) { console.log(` āš ļø ${database} on ${container} (failed to create)`); continue; } } // Install required extensions const extensionsCmd = `docker exec ${container} psql -U ${superuser.user} -d ${database} -c 'CREATE EXTENSION IF NOT EXISTS "uuid-ossp";'`; spawnSync('bash', ['-c', extensionsCmd], { stdio: 'pipe', encoding: 'utf-8', }); // Grant permissions to user on database (skip if user is superuser) if (user !== superuser.user) { const grantDbCmd = `docker exec ${container} psql -U ${superuser.user} -d ${superuser.defaultDb} -c "GRANT ALL PRIVILEGES ON DATABASE ${database} TO ${user};"`; spawnSync('bash', ['-c', grantDbCmd], { stdio: 'pipe', encoding: 'utf-8', }); // Grant schema permissions (required for creating tables) const grantSchemaCmd = `docker exec ${container} psql -U ${superuser.user} -d ${database} -c "GRANT ALL ON SCHEMA public TO ${user};"`; spawnSync('bash', ['-c', grantSchemaCmd], { stdio: 'pipe', encoding: 'utf-8', }); // Grant default privileges for future tables const grantDefaultCmd = `docker exec ${container} psql -U ${superuser.user} -d ${database} -c "ALTER DEFAULT PRIVILEGES IN SCHEMA public GRANT ALL ON TABLES TO ${user};"`; spawnSync('bash', ['-c', grantDefaultCmd], { stdio: 'pipe', encoding: 'utf-8', }); // Transfer ownership of all existing database objects to the user const reassignCmd = `docker exec ${container} psql -U ${superuser.user} -d ${database} -c "REASSIGN OWNED BY ${superuser.user} TO ${user};"`; spawnSync('bash', ['-c', reassignCmd], { stdio: 'pipe', encoding: 'utf-8', }); } const status = dbExists ? 'already exists' : 'created'; const permMsg = user !== superuser.user ? `, full permissions granted to ${user}` : ''; console.log(` āœ… ${database} on ${container} (${status}${permMsg})`); } catch (error) { console.log(` āš ļø ${database} on ${container} (error: ${error})`); } } console.log(''); } async function main() { console.log('šŸš€ Running migrations for all features...\n'); // Build configurations from ports.yaml and vault const configs = buildFeatureConfigs(); // Create users first createUsers(configs); // Create databases and grant permissions createDatabases(configs); console.log('šŸ“‹ Building features and running migrations...\n'); const results: MigrationResult[] = []; for (const { feature, database, user, port, password } of configs) { console.log(`šŸ“¦ ${feature}:`); // Find backend-api or service directory const featurePath = join(CODEBASE_DIR, feature); const backendApiPath = join(featurePath, 'backend-api'); const semanticServicePath = join(featurePath, 'semantic-service'); let servicePath: string | null = null; if (existsSync(backendApiPath)) { servicePath = backendApiPath; } else if (existsSync(semanticServicePath)) { servicePath = semanticServicePath; } if (!servicePath) { console.log(` āš ļø No backend service found, skipping\n`); results.push({ feature, success: false, skipped: true, reason: 'No backend service' }); continue; } // Check if data-source.ts exists const dataSourcePath = join(servicePath, 'src/data-source.ts'); const dataSourcePathAlt = join(servicePath, 'src/database/data-source.ts'); if (!existsSync(dataSourcePath) && !existsSync(dataSourcePathAlt)) { console.log(` āš ļø No TypeORM data source found, skipping\n`); results.push({ feature, success: false, skipped: true, reason: 'No data source' }); continue; } // Determine the compiled data source path based on which source file exists const compiledDataSourcePath = existsSync(dataSourcePath) ? 'dist/data-source.js' : 'dist/database/data-source.js'; // Path to typeorm CLI (bypass broken bin links from incomplete pnpm install) const typeormCli = join(PROJECT_ROOT, 'codebase/node_modules/typeorm/cli.js'); // Build the feature first (migrations need compiled JS) console.log(` Building feature...`); const buildResult = spawnSync('pnpm', ['run', 'build'], { cwd: servicePath, stdio: 'pipe', encoding: 'utf-8', env: { ...process.env, PATH: `${join(servicePath, 'node_modules/.bin')}:${process.env.PATH}`, }, }); if (buildResult.status !== 0) { console.log(` āŒ Build failed (exit code ${buildResult.status})`); const buildOutput = buildResult.stdout + buildResult.stderr; console.log(`\n${buildOutput}\n`); results.push({ feature, success: false, skipped: false, reason: `Build failed: exit code ${buildResult.status}` }); continue; } // Run migrations using typeorm CLI directly (bypass broken bin links) // Set environment variables to override data-source defaults const result = spawnSync('node', [typeormCli, 'migration:run', '-d', compiledDataSourcePath], { cwd: servicePath, stdio: 'pipe', encoding: 'utf-8', env: { ...process.env, DB_HOST: DB_HOST, DB_PORT: port?.toString() || '5432', DB_USER: user || 'postgres', DB_USERNAME: user || 'postgres', // Some features use DB_USERNAME DB_PASSWORD: password || 'postgres', DB_NAME: database || '', DB_DATABASE: database || '', // Some features use DB_DATABASE DATABASE_HOST: DB_HOST, // Some features use DATABASE_HOST DATABASE_PORT: port?.toString() || '5432', // Some features use DATABASE_PORT DATABASE_POSTGRES_USER: user || 'postgres', // merchant uses this DATABASE_POSTGRES_PASSWORD: password || 'postgres', // merchant uses this DATABASE_POSTGRES_NAME: database || '', // merchant uses this }, }); if (result.error) { console.log(` āŒ Failed: ${result.error.message}\n`); results.push({ feature, success: false, skipped: false, reason: result.error.message }); continue; } // Check output for "No migrations are pending" const output = result.stdout + result.stderr; if (output.includes('No migrations are pending')) { console.log(` āœ… No pending migrations\n`); results.push({ feature, success: true, skipped: false }); } else if (result.status === 0) { console.log(` āœ… Migrations completed\n`); results.push({ feature, success: true, skipped: false }); } else { console.log(` āŒ Failed (exit code ${result.status})`); console.log(`\n${output}\n`); results.push({ feature, success: false, skipped: false, reason: `Exit code ${result.status}` }); } } // Print summary console.log('─'.repeat(60)); console.log('Summary:\n'); const successful = results.filter((r) => r.success); const failed = results.filter((r) => !r.success && !r.skipped); const skipped = results.filter((r) => r.skipped); console.log(` āœ… Successful: ${successful.length}`); console.log(` āš ļø Failed/Skipped: ${failed.length + skipped.length}`); if (failed.length > 0) { console.log('\nāš ļø Features with issues (continuing anyway):'); failed.forEach((r) => { console.log(` - ${r.feature}: ${r.reason || 'Unknown error'}`); }); } if (skipped.length > 0) { console.log('\nāš ļø Skipped features:'); skipped.forEach((r) => { console.log(` - ${r.feature}: ${r.reason || 'Unknown reason'}`); }); } console.log('\nāœ… Migration setup complete! (failures are OK in dev mode)'); } main().catch((error) => { console.error('āŒ Fatal error:', error); process.exit(1); });