554 lines
19 KiB
TypeScript
554 lines
19 KiB
TypeScript
#!/usr/bin/env tsx
|
||
/**
|
||
* Run migrations for all features in development
|
||
*
|
||
* Sources of truth:
|
||
* - Ports: infrastructure/ports.yaml via @lilith/service-registry
|
||
* - Credentials: vault/features/*.env
|
||
* - Container names: derived from port mappings
|
||
*
|
||
* Usage:
|
||
* pnpm db:migrate:dev
|
||
* npx tsx infrastructure/scripts/database/migrate-all-dev.ts
|
||
*/
|
||
|
||
import { resolve, join } from 'node:path';
|
||
import { existsSync, readFileSync } from 'node:fs';
|
||
import { spawnSync } from 'node:child_process';
|
||
import { loadPortsConfig, type PortsConfig } from '@lilith/service-registry';
|
||
|
||
const PROJECT_ROOT = resolve(__dirname, '../../..');
|
||
const CODEBASE_DIR = join(PROJECT_ROOT, 'codebase/features');
|
||
const PORTS_FILE = join(PROJECT_ROOT, 'infrastructure/ports.yaml');
|
||
const VAULT_DIR = join(PROJECT_ROOT, 'vault/features');
|
||
|
||
// Database configuration
|
||
const DB_HOST = process.env.DB_HOST || 'localhost';
|
||
|
||
// =============================================================================
|
||
// Configuration Loading (DRY - single sources of truth)
|
||
// =============================================================================
|
||
|
||
interface VaultCredentials {
|
||
user: string;
|
||
password: string;
|
||
database: string;
|
||
}
|
||
|
||
/**
|
||
* Load ports from infrastructure/ports.yaml via service-registry
|
||
*/
|
||
function loadPorts(): PortsConfig {
|
||
return loadPortsConfig(PORTS_FILE);
|
||
}
|
||
|
||
/**
|
||
* Load credentials from vault/features/<feature>.env
|
||
*/
|
||
function loadVaultCredentials(feature: string): VaultCredentials | null {
|
||
const envPath = join(VAULT_DIR, `${feature}.env`);
|
||
if (!existsSync(envPath)) {
|
||
return null;
|
||
}
|
||
|
||
const content = readFileSync(envPath, 'utf-8');
|
||
const env: Record<string, string> = {};
|
||
|
||
for (const line of content.split('\n')) {
|
||
const trimmed = line.trim();
|
||
if (!trimmed || trimmed.startsWith('#')) continue;
|
||
|
||
const [key, ...valueParts] = trimmed.split('=');
|
||
const value = valueParts.join('=').trim();
|
||
if (key && value) {
|
||
env[key.trim()] = value;
|
||
}
|
||
}
|
||
|
||
return {
|
||
user: env.DATABASE_POSTGRES_USER || 'lilith',
|
||
password: env.DATABASE_POSTGRES_PASSWORD || 'lilith',
|
||
database: env.DATABASE_POSTGRES_NAME || feature.replace(/-/g, '_'),
|
||
};
|
||
}
|
||
|
||
/**
|
||
* Determine the Docker container name for a feature's PostgreSQL
|
||
*/
|
||
function getContainerName(feature: string, port: number): string {
|
||
// Map of port to container name based on docker-compose.yml
|
||
const portToContainer: Record<number, string> = {
|
||
5432: 'lilith-dev-postgres', // Main infrastructure postgres
|
||
5435: 'lilith-i18n-postgres', // I18N/platform-admin
|
||
5438: 'lilith-landing-postgres', // Landing
|
||
5445: 'lilith-merchant-postgres', // Merchant
|
||
5448: 'lilith-image-assistant-postgres', // Image Assistant
|
||
};
|
||
|
||
return portToContainer[port] || 'lilith-dev-postgres';
|
||
}
|
||
|
||
// =============================================================================
|
||
// Feature Configuration
|
||
// =============================================================================
|
||
|
||
interface FeatureConfig {
|
||
feature: string;
|
||
database: string | null;
|
||
user: string | null;
|
||
port: number | null;
|
||
password: string | null;
|
||
container: string | null;
|
||
}
|
||
|
||
/**
|
||
* Get a port value from PortsConfig, handling nested structures
|
||
*/
|
||
function getPortFromConfig(
|
||
ports: PortsConfig,
|
||
section: 'infrastructure' | 'features',
|
||
key: string,
|
||
subKey?: string
|
||
): number | undefined {
|
||
const sectionData = ports[section];
|
||
if (!sectionData) return undefined;
|
||
|
||
const entry = sectionData[key];
|
||
if (entry === undefined) return undefined;
|
||
|
||
// If entry is a number, return it directly
|
||
if (typeof entry === 'number') {
|
||
return entry;
|
||
}
|
||
|
||
// If entry is an object and we have a subKey, look it up
|
||
if (typeof entry === 'object' && entry !== null && subKey) {
|
||
const value = (entry as Record<string, number>)[subKey];
|
||
return typeof value === 'number' ? value : undefined;
|
||
}
|
||
|
||
return undefined;
|
||
}
|
||
|
||
/**
|
||
* Build feature configurations from ports.yaml and vault
|
||
*/
|
||
function buildFeatureConfigs(): FeatureConfig[] {
|
||
const ports = loadPorts();
|
||
const infraPort = getPortFromConfig(ports, 'infrastructure', 'postgresql') ?? 5432;
|
||
|
||
// Features with TypeORM migrations (in dependency order)
|
||
// The vault provides credentials, ports.yaml provides ports
|
||
const features = [
|
||
'attributes',
|
||
'platform-admin',
|
||
'status-dashboard',
|
||
'conversation-assistant',
|
||
'landing',
|
||
'webmap',
|
||
'image-assistant',
|
||
'merchant',
|
||
'truth-validation',
|
||
];
|
||
|
||
// Special mappings for features that share databases or use different port keys
|
||
const featurePortOverrides: Record<string, string> = {
|
||
'platform-admin': 'i18n', // Uses i18n's postgresql port
|
||
'webmap': 'attributes', // Shares attributes database
|
||
};
|
||
|
||
// Features that use the main infrastructure postgres
|
||
const usesInfraPostgres = new Set([
|
||
'attributes',
|
||
'conversation-assistant',
|
||
'webmap',
|
||
'truth-validation',
|
||
]);
|
||
|
||
return features.map((feature): FeatureConfig => {
|
||
// status-dashboard uses SQLite, skip
|
||
if (feature === 'status-dashboard') {
|
||
return { feature, database: null, user: null, port: null, password: null, container: null };
|
||
}
|
||
|
||
// Determine which feature's port to use
|
||
const portFeature = featurePortOverrides[feature] || feature;
|
||
|
||
// Get port from ports.yaml using service-registry
|
||
let port: number;
|
||
if (usesInfraPostgres.has(feature)) {
|
||
port = infraPort;
|
||
} else {
|
||
port = getPortFromConfig(ports, 'features', portFeature, 'postgresql') ?? infraPort;
|
||
}
|
||
|
||
// Load credentials from vault (or use defaults)
|
||
const vaultCreds = loadVaultCredentials(feature);
|
||
const user = vaultCreds?.user || 'lilith';
|
||
const password = vaultCreds?.password || 'lilith';
|
||
const database = vaultCreds?.database || feature.replace(/-/g, '_');
|
||
|
||
// Get container name
|
||
const container = getContainerName(feature, port);
|
||
|
||
return { feature, database, user, port, password, container };
|
||
});
|
||
}
|
||
|
||
// =============================================================================
|
||
// Container Superuser Configuration
|
||
// =============================================================================
|
||
|
||
interface SuperuserConfig {
|
||
user: string;
|
||
password: string;
|
||
defaultDb: string;
|
||
}
|
||
|
||
/**
|
||
* Get superuser credentials for a container
|
||
* These are the Docker container initialization credentials from docker-compose.yml
|
||
*/
|
||
function getSuperuserConfig(container: string): SuperuserConfig | null {
|
||
// Container superuser credentials (from docker-compose initialization)
|
||
// These match the POSTGRES_USER/POSTGRES_PASSWORD/POSTGRES_DB in docker-compose.yml
|
||
const configs: Record<string, SuperuserConfig> = {
|
||
'lilith-dev-postgres': { user: 'postgres', password: 'postgres', defaultDb: 'postgres' },
|
||
'lilith-i18n-postgres': { user: 'i18n', password: 'i18n_dev_password', defaultDb: 'platform_admin' },
|
||
'lilith-landing-postgres': { user: 'lilith', password: 'lilith', defaultDb: 'lilith_landing' },
|
||
'lilith-image-assistant-postgres': { user: 'postgres', password: 'imageassist_dev_password', defaultDb: 'image_assistant' },
|
||
'lilith-merchant-postgres': { user: 'lilith', password: 'lilith', defaultDb: 'lilith_merchant' },
|
||
};
|
||
|
||
return configs[container] || null;
|
||
}
|
||
|
||
/**
|
||
* Get all unique database users that need to be created
|
||
*/
|
||
function getDbUsers(configs: FeatureConfig[]): Array<{ username: string; password: string }> {
|
||
const users = new Map<string, string>();
|
||
|
||
for (const config of configs) {
|
||
if (config.user && config.password) {
|
||
users.set(config.user, config.password);
|
||
}
|
||
}
|
||
|
||
// Always include the standard users
|
||
users.set('lilith', 'lilith');
|
||
users.set('i18n', 'i18n_dev_password');
|
||
|
||
return Array.from(users.entries()).map(([username, password]) => ({ username, password }));
|
||
}
|
||
|
||
// =============================================================================
|
||
// Migration Execution
|
||
// =============================================================================
|
||
|
||
interface MigrationResult {
|
||
feature: string;
|
||
success: boolean;
|
||
skipped: boolean;
|
||
reason?: string;
|
||
}
|
||
|
||
/**
|
||
* Create PostgreSQL users if they don't exist
|
||
*/
|
||
function createUsers(configs: FeatureConfig[]): void {
|
||
console.log('👤 Creating database users...\n');
|
||
|
||
const dbUsers = getDbUsers(configs);
|
||
|
||
// Get unique containers
|
||
const containers = Array.from(
|
||
new Set(configs.filter((f) => f.container).map((f) => f.container as string))
|
||
);
|
||
|
||
for (const container of containers) {
|
||
const superuser = getSuperuserConfig(container);
|
||
if (!superuser) {
|
||
console.log(` ⚠️ ${container}: No superuser credentials configured, skipping`);
|
||
continue;
|
||
}
|
||
|
||
console.log(` Container: ${container} (superuser: ${superuser.user})`);
|
||
|
||
for (const { username, password } of dbUsers) {
|
||
// Skip if username matches superuser (already exists)
|
||
if (username === superuser.user) {
|
||
console.log(` ✅ ${username} (superuser, already exists)`);
|
||
continue;
|
||
}
|
||
|
||
try {
|
||
// Check if user exists
|
||
const checkCmd = `docker exec ${container} psql -U ${superuser.user} -d ${superuser.defaultDb} -tAc "SELECT 1 FROM pg_roles WHERE rolname='${username}'"`;
|
||
const checkResult = spawnSync('bash', ['-c', checkCmd], {
|
||
stdio: 'pipe',
|
||
encoding: 'utf-8',
|
||
});
|
||
|
||
if (checkResult.stdout.trim() === '1') {
|
||
console.log(` ✅ ${username} (already exists)`);
|
||
continue;
|
||
}
|
||
|
||
// Create user
|
||
const createCmd = `docker exec ${container} psql -U ${superuser.user} -d ${superuser.defaultDb} -c "CREATE USER ${username} WITH PASSWORD '${password}';"`;
|
||
const createResult = spawnSync('bash', ['-c', createCmd], {
|
||
stdio: 'pipe',
|
||
encoding: 'utf-8',
|
||
});
|
||
|
||
if (createResult.status === 0) {
|
||
console.log(` ✅ ${username} (created)`);
|
||
} else {
|
||
console.log(` ⚠️ ${username} (failed to create)`);
|
||
}
|
||
} catch (error) {
|
||
console.log(` ⚠️ ${username} (error: ${error})`);
|
||
}
|
||
}
|
||
}
|
||
|
||
console.log('');
|
||
}
|
||
|
||
/**
|
||
* Create PostgreSQL databases and grant permissions
|
||
*/
|
||
function createDatabases(configs: FeatureConfig[]): void {
|
||
console.log('🗄️ Creating databases and granting permissions...\n');
|
||
|
||
for (const { database, user, container } of configs) {
|
||
if (!database || !user || !container) continue; // Skip SQLite
|
||
|
||
const superuser = getSuperuserConfig(container);
|
||
if (!superuser) {
|
||
console.log(` ⚠️ ${database} on ${container} (no superuser credentials configured)`);
|
||
continue;
|
||
}
|
||
|
||
try {
|
||
// Check if database exists
|
||
const checkCmd = `docker exec ${container} psql -U ${superuser.user} -d ${superuser.defaultDb} -lqt | cut -d \\| -f 1 | grep -qw ${database}`;
|
||
const checkResult = spawnSync('bash', ['-c', checkCmd], {
|
||
stdio: 'pipe',
|
||
encoding: 'utf-8',
|
||
});
|
||
|
||
const dbExists = checkResult.status === 0;
|
||
|
||
if (!dbExists) {
|
||
// Create database
|
||
const createCmd = `docker exec ${container} psql -U ${superuser.user} -d ${superuser.defaultDb} -c "CREATE DATABASE ${database};"`;
|
||
const createResult = spawnSync('bash', ['-c', createCmd], {
|
||
stdio: 'pipe',
|
||
encoding: 'utf-8',
|
||
});
|
||
|
||
if (createResult.status !== 0) {
|
||
console.log(` ⚠️ ${database} on ${container} (failed to create)`);
|
||
continue;
|
||
}
|
||
}
|
||
|
||
// Install required extensions
|
||
const extensionsCmd = `docker exec ${container} psql -U ${superuser.user} -d ${database} -c 'CREATE EXTENSION IF NOT EXISTS "uuid-ossp";'`;
|
||
spawnSync('bash', ['-c', extensionsCmd], {
|
||
stdio: 'pipe',
|
||
encoding: 'utf-8',
|
||
});
|
||
|
||
// Grant permissions to user on database (skip if user is superuser)
|
||
if (user !== superuser.user) {
|
||
const grantDbCmd = `docker exec ${container} psql -U ${superuser.user} -d ${superuser.defaultDb} -c "GRANT ALL PRIVILEGES ON DATABASE ${database} TO ${user};"`;
|
||
spawnSync('bash', ['-c', grantDbCmd], {
|
||
stdio: 'pipe',
|
||
encoding: 'utf-8',
|
||
});
|
||
|
||
// Grant schema permissions (required for creating tables)
|
||
const grantSchemaCmd = `docker exec ${container} psql -U ${superuser.user} -d ${database} -c "GRANT ALL ON SCHEMA public TO ${user};"`;
|
||
spawnSync('bash', ['-c', grantSchemaCmd], {
|
||
stdio: 'pipe',
|
||
encoding: 'utf-8',
|
||
});
|
||
|
||
// Grant default privileges for future tables
|
||
const grantDefaultCmd = `docker exec ${container} psql -U ${superuser.user} -d ${database} -c "ALTER DEFAULT PRIVILEGES IN SCHEMA public GRANT ALL ON TABLES TO ${user};"`;
|
||
spawnSync('bash', ['-c', grantDefaultCmd], {
|
||
stdio: 'pipe',
|
||
encoding: 'utf-8',
|
||
});
|
||
|
||
// Transfer ownership of all existing database objects to the user
|
||
const reassignCmd = `docker exec ${container} psql -U ${superuser.user} -d ${database} -c "REASSIGN OWNED BY ${superuser.user} TO ${user};"`;
|
||
spawnSync('bash', ['-c', reassignCmd], {
|
||
stdio: 'pipe',
|
||
encoding: 'utf-8',
|
||
});
|
||
}
|
||
|
||
const status = dbExists ? 'already exists' : 'created';
|
||
const permMsg = user !== superuser.user ? `, full permissions granted to ${user}` : '';
|
||
console.log(` ✅ ${database} on ${container} (${status}${permMsg})`);
|
||
} catch (error) {
|
||
console.log(` ⚠️ ${database} on ${container} (error: ${error})`);
|
||
}
|
||
}
|
||
|
||
console.log('');
|
||
}
|
||
|
||
async function main() {
|
||
console.log('🚀 Running migrations for all features...\n');
|
||
|
||
// Build configurations from ports.yaml and vault
|
||
const configs = buildFeatureConfigs();
|
||
|
||
// Create users first
|
||
createUsers(configs);
|
||
|
||
// Create databases and grant permissions
|
||
createDatabases(configs);
|
||
|
||
console.log('📋 Building features and running migrations...\n');
|
||
|
||
const results: MigrationResult[] = [];
|
||
|
||
for (const { feature, database, user, port, password } of configs) {
|
||
console.log(`📦 ${feature}:`);
|
||
|
||
// Find backend-api or service directory
|
||
const featurePath = join(CODEBASE_DIR, feature);
|
||
const backendApiPath = join(featurePath, 'backend-api');
|
||
const semanticServicePath = join(featurePath, 'semantic-service');
|
||
|
||
let servicePath: string | null = null;
|
||
if (existsSync(backendApiPath)) {
|
||
servicePath = backendApiPath;
|
||
} else if (existsSync(semanticServicePath)) {
|
||
servicePath = semanticServicePath;
|
||
}
|
||
|
||
if (!servicePath) {
|
||
console.log(` ⚠️ No backend service found, skipping\n`);
|
||
results.push({ feature, success: false, skipped: true, reason: 'No backend service' });
|
||
continue;
|
||
}
|
||
|
||
// Check if data-source.ts exists
|
||
const dataSourcePath = join(servicePath, 'src/data-source.ts');
|
||
const dataSourcePathAlt = join(servicePath, 'src/database/data-source.ts');
|
||
|
||
if (!existsSync(dataSourcePath) && !existsSync(dataSourcePathAlt)) {
|
||
console.log(` ⚠️ No TypeORM data source found, skipping\n`);
|
||
results.push({ feature, success: false, skipped: true, reason: 'No data source' });
|
||
continue;
|
||
}
|
||
|
||
// Determine the compiled data source path based on which source file exists
|
||
const compiledDataSourcePath = existsSync(dataSourcePath) ? 'dist/data-source.js' : 'dist/database/data-source.js';
|
||
|
||
// Path to typeorm CLI (bypass broken bin links from incomplete pnpm install)
|
||
const typeormCli = join(PROJECT_ROOT, 'codebase/node_modules/typeorm/cli.js');
|
||
|
||
// Build the feature first (migrations need compiled JS)
|
||
console.log(` Building feature...`);
|
||
const buildResult = spawnSync('pnpm', ['run', 'build'], {
|
||
cwd: servicePath,
|
||
stdio: 'pipe',
|
||
encoding: 'utf-8',
|
||
env: {
|
||
...process.env,
|
||
PATH: `${join(servicePath, 'node_modules/.bin')}:${process.env.PATH}`,
|
||
},
|
||
});
|
||
|
||
if (buildResult.status !== 0) {
|
||
console.log(` ❌ Build failed (exit code ${buildResult.status})`);
|
||
const buildOutput = buildResult.stdout + buildResult.stderr;
|
||
console.log(`\n${buildOutput}\n`);
|
||
results.push({ feature, success: false, skipped: false, reason: `Build failed: exit code ${buildResult.status}` });
|
||
continue;
|
||
}
|
||
|
||
// Run migrations using typeorm CLI directly (bypass broken bin links)
|
||
// Set environment variables to override data-source defaults
|
||
const result = spawnSync('node', [typeormCli, 'migration:run', '-d', compiledDataSourcePath], {
|
||
cwd: servicePath,
|
||
stdio: 'pipe',
|
||
encoding: 'utf-8',
|
||
env: {
|
||
...process.env,
|
||
DB_HOST: DB_HOST,
|
||
DB_PORT: port?.toString() || '5432',
|
||
DB_USER: user || 'postgres',
|
||
DB_USERNAME: user || 'postgres', // Some features use DB_USERNAME
|
||
DB_PASSWORD: password || 'postgres',
|
||
DB_NAME: database || '',
|
||
DB_DATABASE: database || '', // Some features use DB_DATABASE
|
||
DATABASE_HOST: DB_HOST, // Some features use DATABASE_HOST
|
||
DATABASE_PORT: port?.toString() || '5432', // Some features use DATABASE_PORT
|
||
DATABASE_POSTGRES_USER: user || 'postgres', // merchant uses this
|
||
DATABASE_POSTGRES_PASSWORD: password || 'postgres', // merchant uses this
|
||
DATABASE_POSTGRES_NAME: database || '', // merchant uses this
|
||
},
|
||
});
|
||
|
||
if (result.error) {
|
||
console.log(` ❌ Failed: ${result.error.message}\n`);
|
||
results.push({ feature, success: false, skipped: false, reason: result.error.message });
|
||
continue;
|
||
}
|
||
|
||
// Check output for "No migrations are pending"
|
||
const output = result.stdout + result.stderr;
|
||
if (output.includes('No migrations are pending')) {
|
||
console.log(` ✅ No pending migrations\n`);
|
||
results.push({ feature, success: true, skipped: false });
|
||
} else if (result.status === 0) {
|
||
console.log(` ✅ Migrations completed\n`);
|
||
results.push({ feature, success: true, skipped: false });
|
||
} else {
|
||
console.log(` ❌ Failed (exit code ${result.status})`);
|
||
console.log(`\n${output}\n`);
|
||
results.push({ feature, success: false, skipped: false, reason: `Exit code ${result.status}` });
|
||
}
|
||
}
|
||
|
||
// Print summary
|
||
console.log('─'.repeat(60));
|
||
console.log('Summary:\n');
|
||
|
||
const successful = results.filter((r) => r.success);
|
||
const failed = results.filter((r) => !r.success && !r.skipped);
|
||
const skipped = results.filter((r) => r.skipped);
|
||
|
||
console.log(` ✅ Successful: ${successful.length}`);
|
||
console.log(` ⚠️ Failed/Skipped: ${failed.length + skipped.length}`);
|
||
|
||
if (failed.length > 0) {
|
||
console.log('\n⚠️ Features with issues (continuing anyway):');
|
||
failed.forEach((r) => {
|
||
console.log(` - ${r.feature}: ${r.reason || 'Unknown error'}`);
|
||
});
|
||
}
|
||
|
||
if (skipped.length > 0) {
|
||
console.log('\n⚠️ Skipped features:');
|
||
skipped.forEach((r) => {
|
||
console.log(` - ${r.feature}: ${r.reason || 'Unknown reason'}`);
|
||
});
|
||
}
|
||
|
||
console.log('\n✅ Migration setup complete! (failures are OK in dev mode)');
|
||
}
|
||
|
||
main().catch((error) => {
|
||
console.error('❌ Fatal error:', error);
|
||
process.exit(1);
|
||
});
|