fix(reconciliation): run all services from dev machine via SSH
Reconciliation now runs entirely from the dev machine, targeting remote hosts via SSH instead of syncing scripts and running remotely. This fixes status-dashboard deployment which requires local build artifacts. Changes: - reconcile_host_remote() runs locally with ssh_prefix for all commands - service.sh handles drift:* and error:* status conventions - status-dashboard service syncs dist/ via rsync, manages PM2 via SSH - nginx-config-sync extended to handle sites-available/ directory - deploy-status-dashboard.sh and rectify-deploy.sh delegate to reconciliation - Deprecated 7-domain-routing.conf (uses undefined log format) 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
This commit is contained in:
Quinn Ftw
parent
1ef863e593
commit
327cacd035
8 changed files with 448 additions and 248 deletions
|
|
@ -16,6 +16,7 @@ SERVICES=(
|
|||
"health-monitor:disabled"
|
||||
"nginx-whitelist:target"
|
||||
"nginx-config-sync:enabled"
|
||||
"status-dashboard:enabled"
|
||||
)
|
||||
|
||||
# Host agent config
|
||||
|
|
|
|||
|
|
@ -5,6 +5,9 @@
|
|||
# Provides a common interface for service handlers.
|
||||
# Each service implements: service_status, service_reconcile, service_generate_config
|
||||
#
|
||||
# All reconciliation runs from dev machine, targeting hosts via SSH.
|
||||
# Services use ssh_prefix for remote commands and rsync for file sync.
|
||||
#
|
||||
|
||||
RECONCILE_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
||||
SERVICES_DIR="${RECONCILE_ROOT}/services"
|
||||
|
|
@ -68,6 +71,12 @@ get_service_status() {
|
|||
# Reconcile a service
|
||||
# Usage: reconcile_service <service_name> <hostname> <desired_state> [ssh_prefix] [dry_run]
|
||||
# Returns: 0 on success, 1 on failure
|
||||
#
|
||||
# Status conventions:
|
||||
# - "synced", "active", "running": service is in desired state, no action needed
|
||||
# - "inactive", "stopped": service is not running
|
||||
# - "drift:*": service needs reconciliation (e.g., drift:frontend, drift:config)
|
||||
# - "error:*": service has an error condition
|
||||
reconcile_service() {
|
||||
local service_name="$1"
|
||||
local hostname="$2"
|
||||
|
|
@ -81,43 +90,65 @@ reconcile_service() {
|
|||
|
||||
local current_status=$(get_service_status "$service_name" "$hostname" "$ssh_prefix")
|
||||
|
||||
# Determine if action needed
|
||||
# Determine if action needed based on desired state and current status
|
||||
local action_needed=""
|
||||
case "$desired_state" in
|
||||
enabled)
|
||||
if [[ "$current_status" == "inactive" ]]; then
|
||||
action_needed="start"
|
||||
fi
|
||||
# For enabled services, reconcile if not in a healthy state
|
||||
case "$current_status" in
|
||||
synced|active|running)
|
||||
# Already in desired state
|
||||
;;
|
||||
inactive|stopped)
|
||||
action_needed="start"
|
||||
;;
|
||||
drift:*)
|
||||
action_needed="reconcile"
|
||||
;;
|
||||
error:*)
|
||||
action_needed="fix"
|
||||
;;
|
||||
*)
|
||||
# Unknown status - try to reconcile
|
||||
action_needed="reconcile"
|
||||
;;
|
||||
esac
|
||||
;;
|
||||
disabled)
|
||||
if [[ "$current_status" != "inactive" ]]; then
|
||||
if [[ "$current_status" != "inactive" && "$current_status" != "stopped" ]]; then
|
||||
action_needed="stop"
|
||||
fi
|
||||
;;
|
||||
target)
|
||||
# Special state: this host is a target, not a client
|
||||
# Service handler decides what to do
|
||||
action_needed="target-reconcile"
|
||||
;;
|
||||
optional)
|
||||
# Optional means no action - leave as-is
|
||||
;;
|
||||
esac
|
||||
|
||||
# Report status
|
||||
# Report status and take action
|
||||
if [[ -n "$action_needed" ]]; then
|
||||
echo " ${service_name}: DRIFT (desired: ${desired_state}, actual: ${current_status})"
|
||||
echo " ${service_name}: DRIFT (${current_status})"
|
||||
if [[ "$dry_run" == "true" ]]; then
|
||||
echo " Would: $action_needed"
|
||||
return 0
|
||||
fi
|
||||
|
||||
# Call the service's reconcile function
|
||||
if declare -f "${service_name//-/_}_reconcile" >/dev/null; then
|
||||
"${service_name//-/_}_reconcile" "$hostname" "$desired_state" "$ssh_prefix"
|
||||
else
|
||||
echo " ERROR: No reconcile function for ${service_name}"
|
||||
return 1
|
||||
fi
|
||||
else
|
||||
echo " ${service_name}: OK (${current_status})"
|
||||
return 0
|
||||
fi
|
||||
|
||||
# Call the service's reconcile function
|
||||
if declare -f "${service_name//-/_}_reconcile" >/dev/null; then
|
||||
"${service_name//-/_}_reconcile" "$hostname" "$desired_state" "$ssh_prefix"
|
||||
else
|
||||
echo " ERROR: No reconcile function for ${service_name}"
|
||||
return 1
|
||||
fi
|
||||
return 0
|
||||
}
|
||||
|
||||
# Generate config for a service
|
||||
|
|
|
|||
|
|
@ -184,12 +184,13 @@ reconcile_host_local() {
|
|||
fi
|
||||
}
|
||||
|
||||
# Reconcile a remote host (sync + remote execution)
|
||||
# Reconcile a remote host (run locally, target remote via SSH)
|
||||
# Note: Assumes load_host was already called by reconcile_host
|
||||
# All reconciliation runs from dev machine, using SSH to manage remote hosts
|
||||
reconcile_host_remote() {
|
||||
local hostname="$1"
|
||||
|
||||
log_header "Host: ${hostname} (remote)"
|
||||
log_header "Host: ${hostname} (via SSH)"
|
||||
|
||||
# Test SSH connectivity
|
||||
if ! test_ssh_connection "$hostname"; then
|
||||
|
|
@ -197,14 +198,52 @@ reconcile_host_remote() {
|
|||
return 1
|
||||
fi
|
||||
|
||||
# Build remote args
|
||||
local remote_args=""
|
||||
[[ "$DRY_RUN" == "true" ]] && remote_args+=" --check"
|
||||
[[ "$VERBOSE" == "true" ]] && remote_args+=" --verbose"
|
||||
[[ -n "$TARGET_SERVICE" ]] && remote_args+=" --service $TARGET_SERVICE"
|
||||
log_info "Role: ${ROLE:-none}"
|
||||
|
||||
# Sync and run on remote host
|
||||
run_remote_reconcile "$hostname" "$remote_args"
|
||||
# Load all service handlers
|
||||
load_all_services
|
||||
|
||||
# Get SSH prefix for remote commands
|
||||
local ssh_prefix=$(get_ssh_prefix "$hostname")
|
||||
|
||||
# Get services to reconcile
|
||||
local services_to_check=()
|
||||
if [[ -n "$TARGET_SERVICE" ]]; then
|
||||
services_to_check=("$TARGET_SERVICE")
|
||||
else
|
||||
for entry in $(get_all_services); do
|
||||
local svc_name="${entry%%:*}"
|
||||
services_to_check+=("$svc_name")
|
||||
done
|
||||
fi
|
||||
|
||||
# Reconcile each service (run locally, target remote via ssh_prefix)
|
||||
local errors=0
|
||||
if [[ ${#services_to_check[@]} -eq 0 ]]; then
|
||||
log_warn "No services configured for this host"
|
||||
return 0
|
||||
fi
|
||||
|
||||
for service_name in "${services_to_check[@]}"; do
|
||||
local desired_state=$(get_service_state "$service_name")
|
||||
|
||||
if [[ "$desired_state" == "undefined" ]]; then
|
||||
[[ "$VERBOSE" == "true" ]] && log_warn " ${service_name}: not configured for this host"
|
||||
continue
|
||||
fi
|
||||
|
||||
if ! reconcile_service "$service_name" "$hostname" "$desired_state" "$ssh_prefix" "$DRY_RUN"; then
|
||||
((errors++))
|
||||
fi
|
||||
done
|
||||
|
||||
if [[ $errors -eq 0 ]]; then
|
||||
log_success "Host $hostname reconciled successfully"
|
||||
return 0
|
||||
else
|
||||
log_error "Host $hostname had $errors error(s)"
|
||||
return 1
|
||||
fi
|
||||
}
|
||||
|
||||
# Reconcile a host (dispatches to local or remote)
|
||||
|
|
|
|||
|
|
@ -14,72 +14,82 @@
|
|||
SERVICE_NAME="nginx-config-sync"
|
||||
SERVICE_DESCRIPTION="Nginx configuration sync from codebase to VPS"
|
||||
|
||||
# Source directory (relative to codebase root)
|
||||
NGINX_SOURCE_DIR="infrastructure/nginx/conf.d"
|
||||
# Source directories (relative to codebase root)
|
||||
NGINX_CONFD_SOURCE="infrastructure/nginx/conf.d"
|
||||
NGINX_SITES_SOURCE="infrastructure/nginx/sites-available"
|
||||
|
||||
# Target directory on VPS
|
||||
NGINX_TARGET_DIR="/etc/nginx/conf.d"
|
||||
# Target directories on VPS
|
||||
NGINX_CONFD_TARGET="/etc/nginx/conf.d"
|
||||
NGINX_SITES_TARGET="/etc/nginx/sites-available"
|
||||
|
||||
# Backup directory on VPS
|
||||
NGINX_BACKUP_DIR="/etc/nginx/conf.d/backups"
|
||||
NGINX_BACKUP_DIR="/etc/nginx/backups"
|
||||
|
||||
# Files to sync (exclude localhost variants)
|
||||
NGINX_SYNC_PATTERN="*.conf"
|
||||
NGINX_EXCLUDE_PATTERN="*.localhost.conf"
|
||||
|
||||
# Helper: Check if directory is in sync
|
||||
# Usage: check_directory_sync <source_dir> <target_dir> [ssh_prefix]
|
||||
check_directory_sync() {
|
||||
local source_dir="$1"
|
||||
local target_dir="$2"
|
||||
local ssh_prefix="${3:-}"
|
||||
|
||||
[[ ! -d "$source_dir" ]] && return 1
|
||||
|
||||
for local_file in "$source_dir"/*.conf; do
|
||||
[[ -f "$local_file" ]] || continue
|
||||
|
||||
local filename=$(basename "$local_file")
|
||||
[[ "$filename" == *".localhost.conf" ]] && continue
|
||||
[[ "$filename" == *".deprecated" ]] && continue
|
||||
|
||||
local remote_file="${target_dir}/${filename}"
|
||||
local local_hash=$(md5sum "$local_file" 2>/dev/null | cut -d' ' -f1)
|
||||
local remote_hash=$(${ssh_prefix} md5sum "$remote_file" 2>/dev/null | cut -d' ' -f1)
|
||||
|
||||
if [[ "$local_hash" != "$remote_hash" ]]; then
|
||||
return 1 # Drift found
|
||||
fi
|
||||
done
|
||||
|
||||
return 0 # In sync
|
||||
}
|
||||
|
||||
# Check if configs are in sync
|
||||
# Usage: nginx_config_sync_status <hostname> [ssh_prefix]
|
||||
nginx_config_sync_status() {
|
||||
local hostname="$1"
|
||||
local ssh_prefix="${2:-}"
|
||||
|
||||
# Get codebase root
|
||||
local codebase_root
|
||||
codebase_root=$(cd "${RECONCILE_ROOT}/../.." && pwd)
|
||||
local source_dir="${codebase_root}/${NGINX_SOURCE_DIR}"
|
||||
|
||||
if [[ ! -d "$source_dir" ]]; then
|
||||
local confd_source="${codebase_root}/${NGINX_CONFD_SOURCE}"
|
||||
local sites_source="${codebase_root}/${NGINX_SITES_SOURCE}"
|
||||
|
||||
# Check both directories exist
|
||||
if [[ ! -d "$confd_source" ]]; then
|
||||
echo "error:source-missing"
|
||||
return 1
|
||||
fi
|
||||
|
||||
# Compare each config file
|
||||
local drift_found=false
|
||||
for local_file in "$source_dir"/*.conf; do
|
||||
[[ -f "$local_file" ]] || continue
|
||||
|
||||
local filename=$(basename "$local_file")
|
||||
|
||||
# Skip localhost configs
|
||||
if [[ "$filename" == *".localhost.conf" ]]; then
|
||||
continue
|
||||
fi
|
||||
|
||||
# Skip deprecated files
|
||||
if [[ "$filename" == *".deprecated" ]]; then
|
||||
continue
|
||||
fi
|
||||
|
||||
local remote_file="${NGINX_TARGET_DIR}/${filename}"
|
||||
|
||||
# Get local hash
|
||||
local local_hash=$(md5sum "$local_file" 2>/dev/null | cut -d' ' -f1)
|
||||
|
||||
# Get remote hash
|
||||
local remote_hash=$(${ssh_prefix} md5sum "$remote_file" 2>/dev/null | cut -d' ' -f1)
|
||||
|
||||
if [[ "$local_hash" != "$remote_hash" ]]; then
|
||||
drift_found=true
|
||||
break
|
||||
fi
|
||||
done
|
||||
|
||||
if [[ "$drift_found" == true ]]; then
|
||||
# Check conf.d
|
||||
if ! check_directory_sync "$confd_source" "$NGINX_CONFD_TARGET" "$ssh_prefix"; then
|
||||
echo "drift"
|
||||
else
|
||||
echo "synced"
|
||||
return 0
|
||||
fi
|
||||
|
||||
# Check sites-available (optional - may not exist)
|
||||
if [[ -d "$sites_source" ]]; then
|
||||
if ! check_directory_sync "$sites_source" "$NGINX_SITES_TARGET" "$ssh_prefix"; then
|
||||
echo "drift"
|
||||
return 0
|
||||
fi
|
||||
fi
|
||||
|
||||
echo "synced"
|
||||
return 0
|
||||
}
|
||||
|
||||
|
|
@ -123,6 +133,53 @@ nginx_config_sync_diff() {
|
|||
echo "${files_to_sync[@]}"
|
||||
}
|
||||
|
||||
# Helper: Sync single directory
|
||||
# Usage: sync_directory <hostname> <source_dir> <target_dir> <backup_ts> <ssh_prefix> <synced_files_var>
|
||||
sync_directory() {
|
||||
local hostname="$1"
|
||||
local source_dir="$2"
|
||||
local target_dir="$3"
|
||||
local backup_ts="$4"
|
||||
local ssh_prefix="$5"
|
||||
local -n synced_files_ref=$6 # nameref to array
|
||||
|
||||
[[ ! -d "$source_dir" ]] && return 0 # Skip if directory doesn't exist
|
||||
|
||||
local dir_name=$(basename "$source_dir")
|
||||
echo " Syncing ${dir_name}..."
|
||||
|
||||
for local_file in "$source_dir"/*.conf; do
|
||||
[[ -f "$local_file" ]] || continue
|
||||
|
||||
local filename=$(basename "$local_file")
|
||||
[[ "$filename" == *".localhost.conf" ]] && continue
|
||||
[[ "$filename" == *".deprecated" ]] && continue
|
||||
|
||||
local remote_file="${target_dir}/${filename}"
|
||||
local local_hash=$(md5sum "$local_file" 2>/dev/null | cut -d' ' -f1)
|
||||
local remote_hash=$(${ssh_prefix} md5sum "$remote_file" 2>/dev/null | cut -d' ' -f1)
|
||||
|
||||
if [[ "$local_hash" != "$remote_hash" ]]; then
|
||||
echo " Copying: ${dir_name}/${filename}"
|
||||
|
||||
# Backup existing file
|
||||
if [[ -n "$remote_hash" ]]; then
|
||||
${ssh_prefix} cp "$remote_file" "${NGINX_BACKUP_DIR}/${filename}.${backup_ts}"
|
||||
fi
|
||||
|
||||
# Copy new file
|
||||
if ! scp_to_host "$hostname" "$local_file" "$remote_file"; then
|
||||
echo " ERROR: Failed to copy $filename"
|
||||
return 1
|
||||
fi
|
||||
|
||||
synced_files_ref+=("${dir_name}/${filename}")
|
||||
fi
|
||||
done
|
||||
|
||||
return 0
|
||||
}
|
||||
|
||||
# Reconcile nginx configs to VPS
|
||||
# Usage: nginx_config_sync_reconcile <hostname> <desired_state> [ssh_prefix]
|
||||
nginx_config_sync_reconcile() {
|
||||
|
|
@ -130,7 +187,6 @@ nginx_config_sync_reconcile() {
|
|||
local desired_state="$2"
|
||||
local ssh_prefix="${3:-}"
|
||||
|
||||
# Only process if enabled
|
||||
if [[ "$desired_state" != "enabled" ]]; then
|
||||
return 0
|
||||
fi
|
||||
|
|
@ -147,12 +203,13 @@ nginx_config_sync_reconcile() {
|
|||
return 0
|
||||
fi
|
||||
|
||||
# Drift detected - sync configs
|
||||
echo " Syncing nginx configs to ${hostname}..."
|
||||
|
||||
local codebase_root
|
||||
codebase_root=$(cd "${RECONCILE_ROOT}/../.." && pwd)
|
||||
local source_dir="${codebase_root}/${NGINX_SOURCE_DIR}"
|
||||
|
||||
local confd_source="${codebase_root}/${NGINX_CONFD_SOURCE}"
|
||||
local sites_source="${codebase_root}/${NGINX_SITES_SOURCE}"
|
||||
|
||||
# Ensure backup directory exists
|
||||
${ssh_prefix} mkdir -p "${NGINX_BACKUP_DIR}"
|
||||
|
|
@ -160,47 +217,35 @@ nginx_config_sync_reconcile() {
|
|||
# Create timestamped backup
|
||||
local backup_ts=$(date +%Y%m%d_%H%M%S)
|
||||
echo " Creating backup: ${backup_ts}"
|
||||
${ssh_prefix} bash -c "cp ${NGINX_TARGET_DIR}/*.conf ${NGINX_BACKUP_DIR}/ 2>/dev/null && \
|
||||
for f in ${NGINX_BACKUP_DIR}/*.conf; do mv \"\$f\" \"\${f%.conf}.${backup_ts}.conf\" 2>/dev/null; done" || true
|
||||
|
||||
# Track files to sync for potential rollback
|
||||
# Track synced files
|
||||
local synced_files=()
|
||||
local sync_failed=false
|
||||
|
||||
for local_file in "$source_dir"/*.conf; do
|
||||
[[ -f "$local_file" ]] || continue
|
||||
|
||||
local filename=$(basename "$local_file")
|
||||
|
||||
# Skip localhost configs
|
||||
[[ "$filename" == *".localhost.conf" ]] && continue
|
||||
# Skip deprecated files
|
||||
[[ "$filename" == *".deprecated" ]] && continue
|
||||
|
||||
local remote_file="${NGINX_TARGET_DIR}/${filename}"
|
||||
|
||||
local local_hash=$(md5sum "$local_file" 2>/dev/null | cut -d' ' -f1)
|
||||
local remote_hash=$(${ssh_prefix} md5sum "$remote_file" 2>/dev/null | cut -d' ' -f1)
|
||||
|
||||
if [[ "$local_hash" != "$remote_hash" ]]; then
|
||||
echo " Copying: $filename"
|
||||
|
||||
# Copy file to VPS
|
||||
if ! scp_to_host "$hostname" "$local_file" "$remote_file"; then
|
||||
echo " ERROR: Failed to copy $filename"
|
||||
sync_failed=true
|
||||
break
|
||||
fi
|
||||
|
||||
synced_files+=("$filename")
|
||||
fi
|
||||
done
|
||||
|
||||
if [[ "$sync_failed" == true ]]; then
|
||||
# Sync conf.d
|
||||
if ! sync_directory "$hostname" "$confd_source" "$NGINX_CONFD_TARGET" "$backup_ts" "$ssh_prefix" synced_files; then
|
||||
nginx_config_sync_rollback "$hostname" "$ssh_prefix" "$backup_ts"
|
||||
return 1
|
||||
fi
|
||||
|
||||
# Sync sites-available
|
||||
if ! sync_directory "$hostname" "$sites_source" "$NGINX_SITES_TARGET" "$backup_ts" "$ssh_prefix" synced_files; then
|
||||
nginx_config_sync_rollback "$hostname" "$ssh_prefix" "$backup_ts"
|
||||
return 1
|
||||
fi
|
||||
|
||||
# Enable sites-available configs (create symlinks in sites-enabled)
|
||||
if [[ -d "$sites_source" ]]; then
|
||||
echo " Enabling sites..."
|
||||
for local_file in "$sites_source"/*.conf; do
|
||||
[[ -f "$local_file" ]] || continue
|
||||
local filename=$(basename "$local_file")
|
||||
[[ "$filename" == *".localhost.conf" ]] && continue
|
||||
[[ "$filename" == *".deprecated" ]] && continue
|
||||
|
||||
${ssh_prefix} bash -c "ln -sf ${NGINX_SITES_TARGET}/${filename} /etc/nginx/sites-enabled/${filename}" 2>/dev/null || true
|
||||
done
|
||||
fi
|
||||
|
||||
# Validate nginx config
|
||||
echo " Validating nginx configuration..."
|
||||
if ! ${ssh_prefix} nginx -t 2>&1; then
|
||||
|
|
@ -221,8 +266,8 @@ nginx_config_sync_reconcile() {
|
|||
|
||||
echo " Nginx reloaded successfully"
|
||||
|
||||
# Clean old backups (keep last 5)
|
||||
${ssh_prefix} bash -c "cd ${NGINX_BACKUP_DIR} && ls -t *.conf 2>/dev/null | tail -n +26 | xargs -r rm" || true
|
||||
# Clean old backups (keep last 25)
|
||||
${ssh_prefix} bash -c "cd ${NGINX_BACKUP_DIR} && ls -t * 2>/dev/null | tail -n +26 | xargs -r rm" || true
|
||||
|
||||
echo " Synced ${#synced_files[@]} file(s): ${synced_files[*]}"
|
||||
return 0
|
||||
|
|
@ -237,10 +282,15 @@ nginx_config_sync_rollback() {
|
|||
|
||||
echo " Rolling back to backup ${backup_ts}..."
|
||||
|
||||
# Restore from backup
|
||||
${ssh_prefix} bash -c "for f in ${NGINX_BACKUP_DIR}/*.${backup_ts}.conf; do \
|
||||
basename=\$(basename \"\$f\" .${backup_ts}.conf); \
|
||||
cp \"\$f\" \"${NGINX_TARGET_DIR}/\${basename}.conf\"; \
|
||||
# Restore all backups with this timestamp
|
||||
${ssh_prefix} bash -c "for f in ${NGINX_BACKUP_DIR}/*.${backup_ts}; do \
|
||||
[[ -f \"\$f\" ]] || continue; \
|
||||
filename=\$(basename \"\$f\" .${backup_ts}); \
|
||||
if [[ \"\$filename\" == *\"conf.d\"* ]]; then \
|
||||
cp \"\$f\" \"${NGINX_CONFD_TARGET}/\${filename}\"; \
|
||||
elif [[ \"\$filename\" == *\"sites-available\"* ]]; then \
|
||||
cp \"\$f\" \"${NGINX_SITES_TARGET}/\${filename}\"; \
|
||||
fi; \
|
||||
done" 2>/dev/null
|
||||
|
||||
# Reload nginx with restored config
|
||||
|
|
|
|||
178
infrastructure/reconciliation/services/status-dashboard.sh
Executable file
178
infrastructure/reconciliation/services/status-dashboard.sh
Executable file
|
|
@ -0,0 +1,178 @@
|
|||
#!/bin/bash
|
||||
#
|
||||
# Lilith Platform - Status Dashboard Service Handler
|
||||
#
|
||||
# Manages complete status-dashboard deployment:
|
||||
# - Frontend build + sync
|
||||
# - Backend build + sync
|
||||
# - PM2 process management
|
||||
# - Nginx configs (delegates to nginx-config-sync)
|
||||
#
|
||||
# Reconciliation runs from dev machine:
|
||||
# 1. Checks local build artifacts exist (dist/ folders)
|
||||
# 2. Syncs them to target host via rsync
|
||||
# 3. Manages PM2 on target host via ssh_prefix
|
||||
#
|
||||
|
||||
SERVICE_NAME="status-dashboard"
|
||||
SERVICE_DESCRIPTION="Status dashboard frontend + backend deployment"
|
||||
|
||||
# Paths
|
||||
FRONTEND_SOURCE="features/status-dashboard/frontend"
|
||||
BACKEND_SOURCE="features/status-dashboard/server"
|
||||
DEPLOY_PATH="/opt/status-dashboard"
|
||||
|
||||
# Check if status-dashboard is up to date
|
||||
# Usage: status_dashboard_status <hostname> [ssh_prefix]
|
||||
status_dashboard_status() {
|
||||
local hostname="$1"
|
||||
local ssh_prefix="${2:-}"
|
||||
|
||||
local codebase_root
|
||||
codebase_root=$(cd "${RECONCILE_ROOT}/../.." && pwd)
|
||||
|
||||
# Check frontend dist exists locally
|
||||
if [[ ! -d "${codebase_root}/${FRONTEND_SOURCE}/dist" ]]; then
|
||||
echo "error:frontend-not-built"
|
||||
return 1
|
||||
fi
|
||||
|
||||
# Check backend dist exists locally
|
||||
if [[ ! -d "${codebase_root}/${BACKEND_SOURCE}/dist" ]]; then
|
||||
echo "error:backend-not-built"
|
||||
return 1
|
||||
fi
|
||||
|
||||
# Compare frontend dist hashes
|
||||
local frontend_local_hash=$(cd "${codebase_root}/${FRONTEND_SOURCE}/dist" && find . -type f -exec md5sum {} \; | sort | md5sum | cut -d' ' -f1)
|
||||
local frontend_remote_hash=$(${ssh_prefix} "cd ${DEPLOY_PATH}/frontend/dist 2>/dev/null && find . -type f -exec md5sum {} \; | sort | md5sum | cut -d' ' -f1" 2>/dev/null)
|
||||
|
||||
if [[ "$frontend_local_hash" != "$frontend_remote_hash" ]]; then
|
||||
echo "drift:frontend"
|
||||
return 0
|
||||
fi
|
||||
|
||||
# Compare backend dist hashes
|
||||
local backend_local_hash=$(cd "${codebase_root}/${BACKEND_SOURCE}/dist" && find . -type f -exec md5sum {} \; | sort | md5sum | cut -d' ' -f1)
|
||||
local backend_remote_hash=$(${ssh_prefix} "cd ${DEPLOY_PATH}/backend/dist 2>/dev/null && find . -type f -exec md5sum {} \; | sort | md5sum | cut -d' ' -f1" 2>/dev/null)
|
||||
|
||||
if [[ "$backend_local_hash" != "$backend_remote_hash" ]]; then
|
||||
echo "drift:backend"
|
||||
return 0
|
||||
fi
|
||||
|
||||
# Check if PM2 process is running
|
||||
local pm2_status=$(${ssh_prefix} "pm2 show status-dashboard 2>/dev/null | grep -q 'status.*online' && echo 'running' || echo 'stopped'" 2>/dev/null)
|
||||
|
||||
if [[ "$pm2_status" != "running" ]]; then
|
||||
echo "drift:backend-stopped"
|
||||
return 0
|
||||
fi
|
||||
|
||||
echo "synced"
|
||||
return 0
|
||||
}
|
||||
|
||||
# Reconcile status-dashboard deployment
|
||||
# Usage: status_dashboard_reconcile <hostname> <desired_state> [ssh_prefix]
|
||||
status_dashboard_reconcile() {
|
||||
local hostname="$1"
|
||||
local desired_state="$2"
|
||||
local ssh_prefix="${3:-}"
|
||||
|
||||
if [[ "$desired_state" != "enabled" ]]; then
|
||||
return 0
|
||||
fi
|
||||
|
||||
local current=$(status_dashboard_status "$hostname" "$ssh_prefix")
|
||||
|
||||
if [[ "$current" == "error:frontend-not-built" ]]; then
|
||||
echo " ERROR: Frontend not built locally - run: cd ${FRONTEND_SOURCE} && pnpm build"
|
||||
return 1
|
||||
fi
|
||||
|
||||
if [[ "$current" == "error:backend-not-built" ]]; then
|
||||
echo " ERROR: Backend not built locally - run: cd ${BACKEND_SOURCE} && pnpm build"
|
||||
return 1
|
||||
fi
|
||||
|
||||
if [[ "$current" == "synced" ]]; then
|
||||
echo " status-dashboard: OK (in sync)"
|
||||
return 0
|
||||
fi
|
||||
|
||||
echo " Deploying status-dashboard to ${hostname}..."
|
||||
|
||||
local codebase_root
|
||||
codebase_root=$(cd "${RECONCILE_ROOT}/../.." && pwd)
|
||||
|
||||
# Ensure deploy directories exist
|
||||
${ssh_prefix} mkdir -p "${DEPLOY_PATH}/frontend/dist" "${DEPLOY_PATH}/backend/dist" "/var/log/status-dashboard"
|
||||
|
||||
# Sync frontend
|
||||
if [[ "$current" == "drift:frontend" || "$current" == "drift:backend" ]]; then
|
||||
echo " Syncing frontend..."
|
||||
rsync_to_host "$hostname" \
|
||||
"${codebase_root}/${FRONTEND_SOURCE}/dist/" \
|
||||
"${DEPLOY_PATH}/frontend/dist/"
|
||||
fi
|
||||
|
||||
# Sync backend
|
||||
if [[ "$current" == "drift:backend" || "$current" == "drift:backend-stopped" ]]; then
|
||||
echo " Syncing backend..."
|
||||
rsync_to_host "$hostname" \
|
||||
"${codebase_root}/${BACKEND_SOURCE}/dist/" \
|
||||
"${DEPLOY_PATH}/backend/dist/"
|
||||
|
||||
rsync_to_host "$hostname" \
|
||||
"${codebase_root}/${BACKEND_SOURCE}/package.json" \
|
||||
"${DEPLOY_PATH}/backend/"
|
||||
|
||||
rsync_to_host "$hostname" \
|
||||
"${codebase_root}/${BACKEND_SOURCE}/ecosystem.config.cjs" \
|
||||
"${DEPLOY_PATH}/backend/"
|
||||
|
||||
# Install production dependencies
|
||||
echo " Installing backend dependencies..."
|
||||
${ssh_prefix} "cd ${DEPLOY_PATH}/backend && npm install --production --ignore-scripts" 2>/dev/null || true
|
||||
|
||||
# Restart PM2 process
|
||||
echo " Restarting backend service..."
|
||||
${ssh_prefix} "cd ${DEPLOY_PATH}/backend && pm2 delete status-dashboard 2>/dev/null || true && pm2 start ecosystem.config.cjs && pm2 save"
|
||||
|
||||
# Wait for service to start
|
||||
sleep 2
|
||||
fi
|
||||
|
||||
# Verify deployment
|
||||
local pm2_status=$(${ssh_prefix} "pm2 show status-dashboard 2>/dev/null | grep -q 'status.*online' && echo 'running' || echo 'stopped'" 2>/dev/null)
|
||||
|
||||
if [[ "$pm2_status" != "running" ]]; then
|
||||
echo " ERROR: Backend service failed to start"
|
||||
echo " Check logs: ssh ${hostname} 'pm2 logs status-dashboard'"
|
||||
return 1
|
||||
fi
|
||||
|
||||
echo " Status dashboard deployed successfully"
|
||||
return 0
|
||||
}
|
||||
|
||||
# Helper: rsync to host
|
||||
# Usage: rsync_to_host <hostname> <local_path> <remote_path>
|
||||
rsync_to_host() {
|
||||
local hostname="$1"
|
||||
local local_path="$2"
|
||||
local remote_path="$3"
|
||||
|
||||
local ssh_user="${SSH_USER:-root}"
|
||||
local ssh_key="${SSH_KEY:-}"
|
||||
local target_host="${SSH_HOST:-$hostname}"
|
||||
|
||||
local rsync_opts="-avz --delete"
|
||||
if [[ -n "$ssh_key" ]]; then
|
||||
ssh_key="${ssh_key/#\~/$HOME}"
|
||||
rsync_opts+=" -e 'ssh -o BatchMode=yes -o ConnectTimeout=10 -i $ssh_key'"
|
||||
fi
|
||||
|
||||
eval rsync $rsync_opts "$local_path" "${ssh_user}@${target_host}:${remote_path}"
|
||||
}
|
||||
|
|
@ -206,132 +206,32 @@ deploy_backend() {
|
|||
# =============================================================================
|
||||
|
||||
configure_nginx() {
|
||||
log_step "Configuring nginx..."
|
||||
log_step "Syncing nginx configuration via reconciliation..."
|
||||
|
||||
# Create nginx config from template
|
||||
local nginx_config="/tmp/status-atlilith-com.conf"
|
||||
# Nginx configs are managed by the reconciliation system
|
||||
# Source: codebase/infrastructure/nginx/sites-available/status.atlilith.com
|
||||
# Target: /etc/nginx/sites-available/status.atlilith.com
|
||||
|
||||
cat > "$nginx_config" <<'EOF'
|
||||
# status.atlilith.com - Status Page Application
|
||||
log_info "Running nginx-config-sync reconciliation..."
|
||||
|
||||
# HTTP -> HTTPS redirect
|
||||
server {
|
||||
listen 80;
|
||||
server_name status.atlilith.com;
|
||||
# Navigate to reconciliation directory
|
||||
local reconcile_dir="${PROJECT_ROOT}/codebase/infrastructure/reconciliation"
|
||||
|
||||
return 301 https://$server_name$request_uri;
|
||||
}
|
||||
if [[ -x "$reconcile_dir/reconcile" ]]; then
|
||||
# Run reconciliation for vps host, nginx-config-sync service only
|
||||
"$reconcile_dir/reconcile" --host vps --service nginx-config-sync
|
||||
|
||||
# HTTPS server
|
||||
server {
|
||||
listen 443 ssl http2;
|
||||
server_name status.atlilith.com;
|
||||
|
||||
# SSL Certificate (Let's Encrypt - managed by certbot)
|
||||
ssl_certificate /etc/letsencrypt/live/status.atlilith.com/fullchain.pem;
|
||||
ssl_certificate_key /etc/letsencrypt/live/status.atlilith.com/privkey.pem;
|
||||
ssl_trusted_certificate /etc/letsencrypt/live/status.atlilith.com/chain.pem;
|
||||
|
||||
# SSL Configuration
|
||||
ssl_protocols TLSv1.2 TLSv1.3;
|
||||
ssl_ciphers HIGH:!aNULL:!MD5;
|
||||
ssl_prefer_server_ciphers on;
|
||||
ssl_session_cache shared:SSL:10m;
|
||||
ssl_session_timeout 10m;
|
||||
|
||||
# Security Headers
|
||||
add_header Strict-Transport-Security "max-age=31536000; includeSubDomains" always;
|
||||
add_header X-Frame-Options "SAMEORIGIN" always;
|
||||
add_header X-Content-Type-Options "nosniff" always;
|
||||
add_header X-XSS-Protection "1; mode=block" always;
|
||||
|
||||
# Root directory (status-dashboard dist folder)
|
||||
root /opt/status-dashboard/dist;
|
||||
index index.html;
|
||||
|
||||
# Main site (serve React SPA)
|
||||
location / {
|
||||
try_files $uri $uri/ /index.html;
|
||||
|
||||
# Cache static assets
|
||||
location ~* \.(js|css|png|jpg|jpeg|gif|svg|ico|woff|woff2|ttf|eot)$ {
|
||||
expires 1y;
|
||||
add_header Cache-Control "public, immutable";
|
||||
}
|
||||
}
|
||||
|
||||
# API Proxy (to NestJS backend on port 3100)
|
||||
location /api/ {
|
||||
proxy_pass http://localhost:3100/api/;
|
||||
proxy_http_version 1.1;
|
||||
|
||||
# Standard proxy headers
|
||||
proxy_set_header Host $host;
|
||||
proxy_set_header X-Real-IP $remote_addr;
|
||||
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
|
||||
proxy_set_header X-Forwarded-Proto $scheme;
|
||||
|
||||
# Timeouts
|
||||
proxy_connect_timeout 60s;
|
||||
proxy_send_timeout 60s;
|
||||
proxy_read_timeout 60s;
|
||||
}
|
||||
|
||||
# WebSocket Proxy (Socket.io)
|
||||
location /socket.io {
|
||||
proxy_pass http://localhost:3100;
|
||||
proxy_http_version 1.1;
|
||||
|
||||
# WebSocket upgrade headers
|
||||
proxy_set_header Upgrade $http_upgrade;
|
||||
proxy_set_header Connection "upgrade";
|
||||
|
||||
# Standard proxy headers
|
||||
proxy_set_header Host $host;
|
||||
proxy_set_header X-Real-IP $remote_addr;
|
||||
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
|
||||
proxy_set_header X-Forwarded-Proto $scheme;
|
||||
|
||||
# WebSocket timeouts (keep connection alive)
|
||||
proxy_connect_timeout 7d;
|
||||
proxy_send_timeout 7d;
|
||||
proxy_read_timeout 7d;
|
||||
|
||||
# Disable buffering for WebSocket
|
||||
proxy_buffering off;
|
||||
}
|
||||
|
||||
# Logging
|
||||
access_log /var/log/nginx/status.atlilith.com.access.log;
|
||||
error_log /var/log/nginx/status.atlilith.com.error.log;
|
||||
}
|
||||
EOF
|
||||
|
||||
# Upload nginx config
|
||||
log_info "Uploading nginx configuration..."
|
||||
$SCP_CMD "$nginx_config" "${VPS_USER}@${VPS_HOST}:/etc/nginx/sites-available/status.atlilith.com"
|
||||
|
||||
# Enable site
|
||||
log_info "Enabling site..."
|
||||
$SSH_CMD "ln -sf /etc/nginx/sites-available/status.atlilith.com /etc/nginx/sites-enabled/"
|
||||
|
||||
# Test nginx config
|
||||
log_info "Testing nginx configuration..."
|
||||
if ! $SSH_CMD "nginx -t" 2>&1 | grep -q "successful"; then
|
||||
log_error "nginx configuration test failed"
|
||||
$SSH_CMD "nginx -t"
|
||||
exit 1
|
||||
if [[ $? -eq 0 ]]; then
|
||||
log_success "Nginx configuration synced successfully"
|
||||
else
|
||||
log_warn "Nginx reconciliation had errors - check manually"
|
||||
log_info "Manual sync: cd ${reconcile_dir} && ./reconcile --host vps --service nginx-config-sync"
|
||||
fi
|
||||
else
|
||||
log_warn "Reconciliation system not found"
|
||||
log_info "Nginx config location: codebase/infrastructure/nginx/sites-available/status.atlilith.com"
|
||||
log_info "Manual deploy: scp to VPS and reload nginx"
|
||||
fi
|
||||
log_success "nginx configuration valid"
|
||||
|
||||
# Reload nginx
|
||||
log_info "Reloading nginx..."
|
||||
$SSH_CMD "systemctl reload nginx"
|
||||
|
||||
# Clean up temp file
|
||||
rm "$nginx_config"
|
||||
|
||||
log_success "nginx configured successfully"
|
||||
}
|
||||
|
||||
# =============================================================================
|
||||
|
|
|
|||
|
|
@ -114,39 +114,40 @@ deploy_component() {
|
|||
log_step "Deploying: $component"
|
||||
|
||||
if [ "$DRY_RUN" = "--dry-run" ]; then
|
||||
log_info "[DRY RUN] Would deploy $component"
|
||||
log_info "[DRY RUN] Would reconcile $component via reconciliation system"
|
||||
return 0
|
||||
fi
|
||||
|
||||
# All deployments now go through reconciliation system
|
||||
local reconcile_dir="${PROJECT_ROOT}/codebase/infrastructure/reconciliation"
|
||||
|
||||
if [[ ! -x "$reconcile_dir/reconcile" ]]; then
|
||||
log_error "Reconciliation system not found at: $reconcile_dir"
|
||||
return 1
|
||||
fi
|
||||
|
||||
case "$component" in
|
||||
status-dashboard)
|
||||
if [ -x "$SCRIPT_DIR/deploy-status-dashboard.sh" ]; then
|
||||
"$SCRIPT_DIR/deploy-status-dashboard.sh" --full || {
|
||||
log_warn "status-dashboard deployment failed (continuing)"
|
||||
return 1
|
||||
}
|
||||
else
|
||||
log_error "deploy-status-dashboard.sh not found"
|
||||
log_info "Running reconciliation for status-dashboard..."
|
||||
"$reconcile_dir/reconcile" --host vps --service status-dashboard || {
|
||||
log_warn "status-dashboard reconciliation failed (continuing)"
|
||||
return 1
|
||||
fi
|
||||
}
|
||||
;;
|
||||
service-registry)
|
||||
if [ -x "$SCRIPT_DIR/deploy-service-registry.sh" ]; then
|
||||
"$SCRIPT_DIR/deploy-service-registry.sh" --full || {
|
||||
log_warn "service-registry deployment failed (continuing)"
|
||||
return 1
|
||||
}
|
||||
else
|
||||
log_error "deploy-service-registry.sh not found"
|
||||
log_info "Running reconciliation for service-registry..."
|
||||
"$reconcile_dir/reconcile" --host vps --service service-registry || {
|
||||
log_warn "service-registry reconciliation failed (continuing)"
|
||||
return 1
|
||||
fi
|
||||
}
|
||||
;;
|
||||
*)
|
||||
log_warn "Unknown component: $component (skipping)"
|
||||
log_warn "Unknown component: $component (no reconciliation handler)"
|
||||
return 1
|
||||
;;
|
||||
esac
|
||||
|
||||
log_success "$component deployed"
|
||||
log_success "$component deployed via reconciliation"
|
||||
}
|
||||
|
||||
# =============================================================================
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue