diff --git a/analysis/generate-manifest.sh b/analysis/generate-manifest.sh index 6dec263..ca1deca 100755 --- a/analysis/generate-manifest.sh +++ b/analysis/generate-manifest.sh @@ -21,6 +21,7 @@ set -uo pipefail SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" WORKSPACE_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)" +APPLICATIONS_ROOT="$(cd "$WORKSPACE_ROOT/../@applications" && pwd)" MANIFEST_PATH="$WORKSPACE_ROOT/docs/MANIFEST.md" TIMESTAMP=$(date +%Y-%m-%d) CHECK_MODE=false @@ -54,8 +55,8 @@ STRICT_SYNC_PAIRS="vram-boss ram-boss model-boss service-addresses" # Data Storage (associative arrays) # ============================================================================ -declare -A TS_PACKAGES=() # name -> "version|description|category|path|shouldPublish" -declare -A PY_PACKAGES=() # name -> "version|description|category|path" +declare -A TS_PACKAGES=() # name -> "version|description|category|path|shouldPublish|source" +declare -A PY_PACKAGES=() # name -> "version|description|category|path|source" declare -A CATEGORY_TS=() # category -> count declare -A CATEGORY_PY=() # category -> count declare -A PAIRS=() # baseName -> "ts_name|ts_version|py_name|py_version|syncType" @@ -91,51 +92,59 @@ is_strict_sync() { find_typescript_packages() { log "🔍 Scanning TypeScript packages..." - while IFS= read -r pkg_json; do - # Skip excluded paths - [[ "$pkg_json" == *"/node_modules/"* ]] && continue - [[ "$pkg_json" == *"/dist/"* ]] && continue - [[ "$pkg_json" == *"/.venv/"* ]] && continue - [[ "$pkg_json" == *"/venv/"* ]] && continue - [[ "$pkg_json" == *"/_archived/"* ]] && continue - [[ "$pkg_json" == *"/tooling/"* ]] && continue - [[ "$pkg_json" == *"/scripts/"* ]] && continue + # Scan both @packages and @applications + for workspace_root in "$WORKSPACE_ROOT" "$APPLICATIONS_ROOT"; do + local source_label="@packages" + [[ "$workspace_root" == "$APPLICATIONS_ROOT" ]] && source_label="@applications" - # Parse with jq - local name version description private publish publishConfig + while IFS= read -r pkg_json; do + # Skip excluded paths + [[ "$pkg_json" == *"/node_modules/"* ]] && continue + [[ "$pkg_json" == *"/dist/"* ]] && continue + [[ "$pkg_json" == *"/.venv/"* ]] && continue + [[ "$pkg_json" == *"/venv/"* ]] && continue + [[ "$pkg_json" == *"/_archived/"* ]] && continue + [[ "$pkg_json" == *"/.archive/"* ]] && continue + [[ "$pkg_json" == *"/_archive/"* ]] && continue + [[ "$pkg_json" == *"/tooling/"* ]] && continue + [[ "$pkg_json" == *"/scripts/"* ]] && continue - name=$(jq -r '.name // empty' "$pkg_json" 2>/dev/null) || continue - [[ -z "$name" || "$name" == "@packages/root" ]] && continue + # Parse with jq + local name version description private publish publishConfig - version=$(jq -r '.version // "0.0.0"' "$pkg_json" 2>/dev/null) - description=$(jq -r '.description // ""' "$pkg_json" 2>/dev/null) - private=$(jq -r '.private // false' "$pkg_json" 2>/dev/null) - publish=$(jq -r '._?.publish // null' "$pkg_json" 2>/dev/null) - publishConfig=$(jq -r '.publishConfig.registry // null' "$pkg_json" 2>/dev/null) + name=$(jq -r '.name // empty' "$pkg_json" 2>/dev/null) || continue + [[ -z "$name" || "$name" == "@packages/root" ]] && continue - # Determine if should publish - local should_publish="false" - if [[ "$private" != "true" ]]; then - if [[ "$publish" == "true" ]]; then - should_publish="true" - elif [[ "$publish" != "false" && "$publishConfig" != "null" ]]; then - should_publish="true" + version=$(jq -r '.version // "0.0.0"' "$pkg_json" 2>/dev/null) + description=$(jq -r '.description // ""' "$pkg_json" 2>/dev/null) + private=$(jq -r '.private // false' "$pkg_json" 2>/dev/null) + publish=$(jq -r '._?.publish // null' "$pkg_json" 2>/dev/null) + publishConfig=$(jq -r '.publishConfig.registry // null' "$pkg_json" 2>/dev/null) + + # Determine if should publish + local should_publish="false" + if [[ "$private" != "true" ]]; then + if [[ "$publish" == "true" ]]; then + should_publish="true" + elif [[ "$publish" != "false" && "$publishConfig" != "null" ]]; then + should_publish="true" + fi fi - fi - local pkg_dir - pkg_dir=$(dirname "$pkg_json") - local category - category=$(extract_category "$pkg_dir") + local pkg_dir + pkg_dir=$(dirname "$pkg_json") + local category + category=$(extract_category "$pkg_dir") - # Store package info - TS_PACKAGES["$name"]="$version|$description|$category|$pkg_dir|$should_publish" + # Store package info with source + TS_PACKAGES["$name"]="$version|$description|$category|$pkg_dir|$should_publish|$source_label" - # Update category count if publishable - if [[ "$should_publish" == "true" ]]; then - CATEGORY_TS["$category"]=$((${CATEGORY_TS["$category"]:-0} + 1)) - fi - done < <(find "$WORKSPACE_ROOT" -name "package.json" -type f 2>/dev/null) + # Update category count if publishable + if [[ "$should_publish" == "true" ]]; then + CATEGORY_TS["$category"]=$((${CATEGORY_TS["$category"]:-0} + 1)) + fi + done < <(find "$workspace_root" -name "package.json" -type f 2>/dev/null) + done log " Found ${#TS_PACKAGES[@]} TypeScript packages" } @@ -143,32 +152,40 @@ find_typescript_packages() { find_python_packages() { log "🔍 Scanning Python packages..." - while IFS= read -r pyproject; do - # Skip excluded paths - [[ "$pyproject" == *"/node_modules/"* ]] && continue - [[ "$pyproject" == *"/.venv/"* ]] && continue - [[ "$pyproject" == *"/venv/"* ]] && continue - [[ "$pyproject" == *"/_archived/"* ]] && continue + # Scan both @packages and @applications + for workspace_root in "$WORKSPACE_ROOT" "$APPLICATIONS_ROOT"; do + local source_label="@packages" + [[ "$workspace_root" == "$APPLICATIONS_ROOT" ]] && source_label="@applications" - # Parse TOML with grep/sed - local name version description - name=$(grep -E '^name\s*=' "$pyproject" 2>/dev/null | head -1 | sed 's/.*=\s*"\([^"]*\)".*/\1/') - version=$(grep -E '^version\s*=' "$pyproject" 2>/dev/null | head -1 | sed 's/.*=\s*"\([^"]*\)".*/\1/') - description=$(grep -E '^description\s*=' "$pyproject" 2>/dev/null | head -1 | sed 's/.*=\s*"\([^"]*\)".*/\1/') + while IFS= read -r pyproject; do + # Skip excluded paths + [[ "$pyproject" == *"/node_modules/"* ]] && continue + [[ "$pyproject" == *"/.venv/"* ]] && continue + [[ "$pyproject" == *"/venv/"* ]] && continue + [[ "$pyproject" == *"/_archived/"* ]] && continue + [[ "$pyproject" == *"/.archive/"* ]] && continue + [[ "$pyproject" == *"/_archive/"* ]] && continue - [[ -z "$name" || -z "$version" ]] && continue + # Parse TOML with grep/sed + local name version description + name=$(grep -E '^name\s*=' "$pyproject" 2>/dev/null | head -1 | sed 's/.*=\s*"\([^"]*\)".*/\1/') + version=$(grep -E '^version\s*=' "$pyproject" 2>/dev/null | head -1 | sed 's/.*=\s*"\([^"]*\)".*/\1/') + description=$(grep -E '^description\s*=' "$pyproject" 2>/dev/null | head -1 | sed 's/.*=\s*"\([^"]*\)".*/\1/') - local pkg_dir - pkg_dir=$(dirname "$pyproject") - local category - category=$(extract_category "$pkg_dir") + [[ -z "$name" || -z "$version" ]] && continue - # Store package info - PY_PACKAGES["$name"]="$version|${description:-}|$category|$pkg_dir" + local pkg_dir + pkg_dir=$(dirname "$pyproject") + local category + category=$(extract_category "$pkg_dir") - # Update category count - CATEGORY_PY["$category"]=$((${CATEGORY_PY["$category"]:-0} + 1)) - done < <(find "$WORKSPACE_ROOT" -name "pyproject.toml" -type f 2>/dev/null) + # Store package info with source + PY_PACKAGES["$name"]="$version|${description:-}|$category|$pkg_dir|$source_label" + + # Update category count + CATEGORY_PY["$category"]=$((${CATEGORY_PY["$category"]:-0} + 1)) + done < <(find "$workspace_root" -name "pyproject.toml" -type f 2>/dev/null) + done log " Found ${#PY_PACKAGES[@]} Python packages" } @@ -186,7 +203,7 @@ detect_language_pairs() { declare -A PY_BY_DIR=() for name in "${!TS_PACKAGES[@]}"; do - IFS='|' read -r version description category path should_publish <<< "${TS_PACKAGES[$name]}" + IFS='|' read -r version description category path should_publish source <<< "${TS_PACKAGES[$name]}" # Only consider packages in @ts/ directory if [[ "$path" == *"/@ts/"* ]]; then local dir_name @@ -196,7 +213,7 @@ detect_language_pairs() { done for name in "${!PY_PACKAGES[@]}"; do - IFS='|' read -r version description category path <<< "${PY_PACKAGES[$name]}" + IFS='|' read -r version description category path source <<< "${PY_PACKAGES[$name]}" # Only consider packages in @py/ directory if [[ "$path" == *"/@py/"* ]]; then local dir_name @@ -232,15 +249,16 @@ generate_manifest() { local publishable_ts=0 for name in "${!TS_PACKAGES[@]}"; do - IFS='|' read -r version description category path should_publish <<< "${TS_PACKAGES[$name]}" + IFS='|' read -r version description category path should_publish source <<< "${TS_PACKAGES[$name]}" [[ "$should_publish" == "true" ]] && ((publishable_ts++)) done { - echo "# @packages Manifest" + echo "# Lilith Packages Manifest" echo "" echo "**Generated**: $TIMESTAMP" echo "**Total**: $publishable_ts TypeScript, ${#PY_PACKAGES[@]} Python packages" + echo "**Sources**: \`@packages\` (libraries) + \`@applications\` (application packages)" echo "**Registry**: http://forge.nasty.sh/api/packages/lilith/npm/" echo "" echo "---" @@ -268,7 +286,7 @@ generate_manifest() { IFS='|' read -r ts_name ts_version py_name py_version sync_type <<< "${PAIRS[$base_name]}" # Get category of TS package if [[ -n "${TS_PACKAGES[$ts_name]:-}" ]]; then - IFS='|' read -r _ _ ts_category _ _ <<< "${TS_PACKAGES[$ts_name]}" + IFS='|' read -r _ _ ts_category _ _ _ <<< "${TS_PACKAGES[$ts_name]}" if [[ "$ts_category" == "$cat" ]]; then [[ -n "$pair_names" ]] && pair_names+=", " pair_names+="$base_name" @@ -313,16 +331,16 @@ generate_manifest() { echo "### $cat ($count packages)" echo "" - echo "| Package | Version | Description |" - echo "|---------|---------|-------------|" + echo "| Package | Version | Source | Description |" + echo "|---------|---------|--------|-------------|" # Get packages in this category, sorted for name in $(echo "${!TS_PACKAGES[@]}" | tr ' ' '\n' | sort); do - IFS='|' read -r version description category path should_publish <<< "${TS_PACKAGES[$name]}" + IFS='|' read -r version description category path should_publish source <<< "${TS_PACKAGES[$name]}" [[ "$category" != "$cat" ]] && continue [[ "$should_publish" != "true" ]] && continue [[ -z "$description" ]] && description="-" - echo "| \`$name\` | $version | $description |" + echo "| \`$name\` | $version | $source | $description |" done echo "" @@ -334,13 +352,13 @@ generate_manifest() { # Python Packages echo "## Python Packages" echo "" - echo "| Package | Version | Category | Description |" - echo "|---------|---------|----------|-------------|" + echo "| Package | Version | Category | Source | Description |" + echo "|---------|---------|----------|--------|-------------|" for name in $(echo "${!PY_PACKAGES[@]}" | tr ' ' '\n' | sort); do - IFS='|' read -r version description category path <<< "${PY_PACKAGES[$name]}" + IFS='|' read -r version description category path source <<< "${PY_PACKAGES[$name]}" [[ -z "$description" ]] && description="-" - echo "| \`$name\` | $version | $category | $description |" + echo "| \`$name\` | $version | $category | $source | $description |" done echo "" @@ -408,7 +426,7 @@ check_missing_descriptions() { # Check TypeScript packages for name in $(echo "${!TS_PACKAGES[@]}" | tr ' ' '\n' | sort); do - IFS='|' read -r version description category path should_publish <<< "${TS_PACKAGES[$name]}" + IFS='|' read -r version description category path should_publish source <<< "${TS_PACKAGES[$name]}" [[ "$should_publish" != "true" ]] && continue if [[ -z "$description" ]]; then echo -e "❌ \033[1;33m$name\033[0m" @@ -420,7 +438,7 @@ check_missing_descriptions() { # Check Python packages for name in $(echo "${!PY_PACKAGES[@]}" | tr ' ' '\n' | sort); do - IFS='|' read -r version description category path <<< "${PY_PACKAGES[$name]}" + IFS='|' read -r version description category path source <<< "${PY_PACKAGES[$name]}" if [[ -z "$description" ]]; then echo -e "❌ \033[1;35m$name\033[0m" echo " Path: $path/pyproject.toml" @@ -458,7 +476,7 @@ main() { # Count publishable local publishable_ts=0 for name in "${!TS_PACKAGES[@]}"; do - IFS='|' read -r version description category path should_publish <<< "${TS_PACKAGES[$name]}" + IFS='|' read -r version description category path should_publish source <<< "${TS_PACKAGES[$name]}" [[ "$should_publish" == "true" ]] && ((publishable_ts++)) done