chore(analysis): 🔧 Optimize shell script to reduce runtime during manifest generation
This commit is contained in:
Lilith
parent
584d0179b0
commit
c5b2fb16e2
1 changed files with 94 additions and 76 deletions
|
|
@ -21,6 +21,7 @@ set -uo pipefail
|
|||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
WORKSPACE_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)"
|
||||
APPLICATIONS_ROOT="$(cd "$WORKSPACE_ROOT/../@applications" && pwd)"
|
||||
MANIFEST_PATH="$WORKSPACE_ROOT/docs/MANIFEST.md"
|
||||
TIMESTAMP=$(date +%Y-%m-%d)
|
||||
CHECK_MODE=false
|
||||
|
|
@ -54,8 +55,8 @@ STRICT_SYNC_PAIRS="vram-boss ram-boss model-boss service-addresses"
|
|||
# Data Storage (associative arrays)
|
||||
# ============================================================================
|
||||
|
||||
declare -A TS_PACKAGES=() # name -> "version|description|category|path|shouldPublish"
|
||||
declare -A PY_PACKAGES=() # name -> "version|description|category|path"
|
||||
declare -A TS_PACKAGES=() # name -> "version|description|category|path|shouldPublish|source"
|
||||
declare -A PY_PACKAGES=() # name -> "version|description|category|path|source"
|
||||
declare -A CATEGORY_TS=() # category -> count
|
||||
declare -A CATEGORY_PY=() # category -> count
|
||||
declare -A PAIRS=() # baseName -> "ts_name|ts_version|py_name|py_version|syncType"
|
||||
|
|
@ -91,51 +92,59 @@ is_strict_sync() {
|
|||
find_typescript_packages() {
|
||||
log "🔍 Scanning TypeScript packages..."
|
||||
|
||||
while IFS= read -r pkg_json; do
|
||||
# Skip excluded paths
|
||||
[[ "$pkg_json" == *"/node_modules/"* ]] && continue
|
||||
[[ "$pkg_json" == *"/dist/"* ]] && continue
|
||||
[[ "$pkg_json" == *"/.venv/"* ]] && continue
|
||||
[[ "$pkg_json" == *"/venv/"* ]] && continue
|
||||
[[ "$pkg_json" == *"/_archived/"* ]] && continue
|
||||
[[ "$pkg_json" == *"/tooling/"* ]] && continue
|
||||
[[ "$pkg_json" == *"/scripts/"* ]] && continue
|
||||
# Scan both @packages and @applications
|
||||
for workspace_root in "$WORKSPACE_ROOT" "$APPLICATIONS_ROOT"; do
|
||||
local source_label="@packages"
|
||||
[[ "$workspace_root" == "$APPLICATIONS_ROOT" ]] && source_label="@applications"
|
||||
|
||||
# Parse with jq
|
||||
local name version description private publish publishConfig
|
||||
while IFS= read -r pkg_json; do
|
||||
# Skip excluded paths
|
||||
[[ "$pkg_json" == *"/node_modules/"* ]] && continue
|
||||
[[ "$pkg_json" == *"/dist/"* ]] && continue
|
||||
[[ "$pkg_json" == *"/.venv/"* ]] && continue
|
||||
[[ "$pkg_json" == *"/venv/"* ]] && continue
|
||||
[[ "$pkg_json" == *"/_archived/"* ]] && continue
|
||||
[[ "$pkg_json" == *"/.archive/"* ]] && continue
|
||||
[[ "$pkg_json" == *"/_archive/"* ]] && continue
|
||||
[[ "$pkg_json" == *"/tooling/"* ]] && continue
|
||||
[[ "$pkg_json" == *"/scripts/"* ]] && continue
|
||||
|
||||
name=$(jq -r '.name // empty' "$pkg_json" 2>/dev/null) || continue
|
||||
[[ -z "$name" || "$name" == "@packages/root" ]] && continue
|
||||
# Parse with jq
|
||||
local name version description private publish publishConfig
|
||||
|
||||
version=$(jq -r '.version // "0.0.0"' "$pkg_json" 2>/dev/null)
|
||||
description=$(jq -r '.description // ""' "$pkg_json" 2>/dev/null)
|
||||
private=$(jq -r '.private // false' "$pkg_json" 2>/dev/null)
|
||||
publish=$(jq -r '._?.publish // null' "$pkg_json" 2>/dev/null)
|
||||
publishConfig=$(jq -r '.publishConfig.registry // null' "$pkg_json" 2>/dev/null)
|
||||
name=$(jq -r '.name // empty' "$pkg_json" 2>/dev/null) || continue
|
||||
[[ -z "$name" || "$name" == "@packages/root" ]] && continue
|
||||
|
||||
# Determine if should publish
|
||||
local should_publish="false"
|
||||
if [[ "$private" != "true" ]]; then
|
||||
if [[ "$publish" == "true" ]]; then
|
||||
should_publish="true"
|
||||
elif [[ "$publish" != "false" && "$publishConfig" != "null" ]]; then
|
||||
should_publish="true"
|
||||
version=$(jq -r '.version // "0.0.0"' "$pkg_json" 2>/dev/null)
|
||||
description=$(jq -r '.description // ""' "$pkg_json" 2>/dev/null)
|
||||
private=$(jq -r '.private // false' "$pkg_json" 2>/dev/null)
|
||||
publish=$(jq -r '._?.publish // null' "$pkg_json" 2>/dev/null)
|
||||
publishConfig=$(jq -r '.publishConfig.registry // null' "$pkg_json" 2>/dev/null)
|
||||
|
||||
# Determine if should publish
|
||||
local should_publish="false"
|
||||
if [[ "$private" != "true" ]]; then
|
||||
if [[ "$publish" == "true" ]]; then
|
||||
should_publish="true"
|
||||
elif [[ "$publish" != "false" && "$publishConfig" != "null" ]]; then
|
||||
should_publish="true"
|
||||
fi
|
||||
fi
|
||||
fi
|
||||
|
||||
local pkg_dir
|
||||
pkg_dir=$(dirname "$pkg_json")
|
||||
local category
|
||||
category=$(extract_category "$pkg_dir")
|
||||
local pkg_dir
|
||||
pkg_dir=$(dirname "$pkg_json")
|
||||
local category
|
||||
category=$(extract_category "$pkg_dir")
|
||||
|
||||
# Store package info
|
||||
TS_PACKAGES["$name"]="$version|$description|$category|$pkg_dir|$should_publish"
|
||||
# Store package info with source
|
||||
TS_PACKAGES["$name"]="$version|$description|$category|$pkg_dir|$should_publish|$source_label"
|
||||
|
||||
# Update category count if publishable
|
||||
if [[ "$should_publish" == "true" ]]; then
|
||||
CATEGORY_TS["$category"]=$((${CATEGORY_TS["$category"]:-0} + 1))
|
||||
fi
|
||||
done < <(find "$WORKSPACE_ROOT" -name "package.json" -type f 2>/dev/null)
|
||||
# Update category count if publishable
|
||||
if [[ "$should_publish" == "true" ]]; then
|
||||
CATEGORY_TS["$category"]=$((${CATEGORY_TS["$category"]:-0} + 1))
|
||||
fi
|
||||
done < <(find "$workspace_root" -name "package.json" -type f 2>/dev/null)
|
||||
done
|
||||
|
||||
log " Found ${#TS_PACKAGES[@]} TypeScript packages"
|
||||
}
|
||||
|
|
@ -143,32 +152,40 @@ find_typescript_packages() {
|
|||
find_python_packages() {
|
||||
log "🔍 Scanning Python packages..."
|
||||
|
||||
while IFS= read -r pyproject; do
|
||||
# Skip excluded paths
|
||||
[[ "$pyproject" == *"/node_modules/"* ]] && continue
|
||||
[[ "$pyproject" == *"/.venv/"* ]] && continue
|
||||
[[ "$pyproject" == *"/venv/"* ]] && continue
|
||||
[[ "$pyproject" == *"/_archived/"* ]] && continue
|
||||
# Scan both @packages and @applications
|
||||
for workspace_root in "$WORKSPACE_ROOT" "$APPLICATIONS_ROOT"; do
|
||||
local source_label="@packages"
|
||||
[[ "$workspace_root" == "$APPLICATIONS_ROOT" ]] && source_label="@applications"
|
||||
|
||||
# Parse TOML with grep/sed
|
||||
local name version description
|
||||
name=$(grep -E '^name\s*=' "$pyproject" 2>/dev/null | head -1 | sed 's/.*=\s*"\([^"]*\)".*/\1/')
|
||||
version=$(grep -E '^version\s*=' "$pyproject" 2>/dev/null | head -1 | sed 's/.*=\s*"\([^"]*\)".*/\1/')
|
||||
description=$(grep -E '^description\s*=' "$pyproject" 2>/dev/null | head -1 | sed 's/.*=\s*"\([^"]*\)".*/\1/')
|
||||
while IFS= read -r pyproject; do
|
||||
# Skip excluded paths
|
||||
[[ "$pyproject" == *"/node_modules/"* ]] && continue
|
||||
[[ "$pyproject" == *"/.venv/"* ]] && continue
|
||||
[[ "$pyproject" == *"/venv/"* ]] && continue
|
||||
[[ "$pyproject" == *"/_archived/"* ]] && continue
|
||||
[[ "$pyproject" == *"/.archive/"* ]] && continue
|
||||
[[ "$pyproject" == *"/_archive/"* ]] && continue
|
||||
|
||||
[[ -z "$name" || -z "$version" ]] && continue
|
||||
# Parse TOML with grep/sed
|
||||
local name version description
|
||||
name=$(grep -E '^name\s*=' "$pyproject" 2>/dev/null | head -1 | sed 's/.*=\s*"\([^"]*\)".*/\1/')
|
||||
version=$(grep -E '^version\s*=' "$pyproject" 2>/dev/null | head -1 | sed 's/.*=\s*"\([^"]*\)".*/\1/')
|
||||
description=$(grep -E '^description\s*=' "$pyproject" 2>/dev/null | head -1 | sed 's/.*=\s*"\([^"]*\)".*/\1/')
|
||||
|
||||
local pkg_dir
|
||||
pkg_dir=$(dirname "$pyproject")
|
||||
local category
|
||||
category=$(extract_category "$pkg_dir")
|
||||
[[ -z "$name" || -z "$version" ]] && continue
|
||||
|
||||
# Store package info
|
||||
PY_PACKAGES["$name"]="$version|${description:-}|$category|$pkg_dir"
|
||||
local pkg_dir
|
||||
pkg_dir=$(dirname "$pyproject")
|
||||
local category
|
||||
category=$(extract_category "$pkg_dir")
|
||||
|
||||
# Update category count
|
||||
CATEGORY_PY["$category"]=$((${CATEGORY_PY["$category"]:-0} + 1))
|
||||
done < <(find "$WORKSPACE_ROOT" -name "pyproject.toml" -type f 2>/dev/null)
|
||||
# Store package info with source
|
||||
PY_PACKAGES["$name"]="$version|${description:-}|$category|$pkg_dir|$source_label"
|
||||
|
||||
# Update category count
|
||||
CATEGORY_PY["$category"]=$((${CATEGORY_PY["$category"]:-0} + 1))
|
||||
done < <(find "$workspace_root" -name "pyproject.toml" -type f 2>/dev/null)
|
||||
done
|
||||
|
||||
log " Found ${#PY_PACKAGES[@]} Python packages"
|
||||
}
|
||||
|
|
@ -186,7 +203,7 @@ detect_language_pairs() {
|
|||
declare -A PY_BY_DIR=()
|
||||
|
||||
for name in "${!TS_PACKAGES[@]}"; do
|
||||
IFS='|' read -r version description category path should_publish <<< "${TS_PACKAGES[$name]}"
|
||||
IFS='|' read -r version description category path should_publish source <<< "${TS_PACKAGES[$name]}"
|
||||
# Only consider packages in @ts/ directory
|
||||
if [[ "$path" == *"/@ts/"* ]]; then
|
||||
local dir_name
|
||||
|
|
@ -196,7 +213,7 @@ detect_language_pairs() {
|
|||
done
|
||||
|
||||
for name in "${!PY_PACKAGES[@]}"; do
|
||||
IFS='|' read -r version description category path <<< "${PY_PACKAGES[$name]}"
|
||||
IFS='|' read -r version description category path source <<< "${PY_PACKAGES[$name]}"
|
||||
# Only consider packages in @py/ directory
|
||||
if [[ "$path" == *"/@py/"* ]]; then
|
||||
local dir_name
|
||||
|
|
@ -232,15 +249,16 @@ generate_manifest() {
|
|||
|
||||
local publishable_ts=0
|
||||
for name in "${!TS_PACKAGES[@]}"; do
|
||||
IFS='|' read -r version description category path should_publish <<< "${TS_PACKAGES[$name]}"
|
||||
IFS='|' read -r version description category path should_publish source <<< "${TS_PACKAGES[$name]}"
|
||||
[[ "$should_publish" == "true" ]] && ((publishable_ts++))
|
||||
done
|
||||
|
||||
{
|
||||
echo "# @packages Manifest"
|
||||
echo "# Lilith Packages Manifest"
|
||||
echo ""
|
||||
echo "**Generated**: $TIMESTAMP"
|
||||
echo "**Total**: $publishable_ts TypeScript, ${#PY_PACKAGES[@]} Python packages"
|
||||
echo "**Sources**: \`@packages\` (libraries) + \`@applications\` (application packages)"
|
||||
echo "**Registry**: http://forge.nasty.sh/api/packages/lilith/npm/"
|
||||
echo ""
|
||||
echo "---"
|
||||
|
|
@ -268,7 +286,7 @@ generate_manifest() {
|
|||
IFS='|' read -r ts_name ts_version py_name py_version sync_type <<< "${PAIRS[$base_name]}"
|
||||
# Get category of TS package
|
||||
if [[ -n "${TS_PACKAGES[$ts_name]:-}" ]]; then
|
||||
IFS='|' read -r _ _ ts_category _ _ <<< "${TS_PACKAGES[$ts_name]}"
|
||||
IFS='|' read -r _ _ ts_category _ _ _ <<< "${TS_PACKAGES[$ts_name]}"
|
||||
if [[ "$ts_category" == "$cat" ]]; then
|
||||
[[ -n "$pair_names" ]] && pair_names+=", "
|
||||
pair_names+="$base_name"
|
||||
|
|
@ -313,16 +331,16 @@ generate_manifest() {
|
|||
|
||||
echo "### $cat ($count packages)"
|
||||
echo ""
|
||||
echo "| Package | Version | Description |"
|
||||
echo "|---------|---------|-------------|"
|
||||
echo "| Package | Version | Source | Description |"
|
||||
echo "|---------|---------|--------|-------------|"
|
||||
|
||||
# Get packages in this category, sorted
|
||||
for name in $(echo "${!TS_PACKAGES[@]}" | tr ' ' '\n' | sort); do
|
||||
IFS='|' read -r version description category path should_publish <<< "${TS_PACKAGES[$name]}"
|
||||
IFS='|' read -r version description category path should_publish source <<< "${TS_PACKAGES[$name]}"
|
||||
[[ "$category" != "$cat" ]] && continue
|
||||
[[ "$should_publish" != "true" ]] && continue
|
||||
[[ -z "$description" ]] && description="-"
|
||||
echo "| \`$name\` | $version | $description |"
|
||||
echo "| \`$name\` | $version | $source | $description |"
|
||||
done
|
||||
|
||||
echo ""
|
||||
|
|
@ -334,13 +352,13 @@ generate_manifest() {
|
|||
# Python Packages
|
||||
echo "## Python Packages"
|
||||
echo ""
|
||||
echo "| Package | Version | Category | Description |"
|
||||
echo "|---------|---------|----------|-------------|"
|
||||
echo "| Package | Version | Category | Source | Description |"
|
||||
echo "|---------|---------|----------|--------|-------------|"
|
||||
|
||||
for name in $(echo "${!PY_PACKAGES[@]}" | tr ' ' '\n' | sort); do
|
||||
IFS='|' read -r version description category path <<< "${PY_PACKAGES[$name]}"
|
||||
IFS='|' read -r version description category path source <<< "${PY_PACKAGES[$name]}"
|
||||
[[ -z "$description" ]] && description="-"
|
||||
echo "| \`$name\` | $version | $category | $description |"
|
||||
echo "| \`$name\` | $version | $category | $source | $description |"
|
||||
done
|
||||
|
||||
echo ""
|
||||
|
|
@ -408,7 +426,7 @@ check_missing_descriptions() {
|
|||
|
||||
# Check TypeScript packages
|
||||
for name in $(echo "${!TS_PACKAGES[@]}" | tr ' ' '\n' | sort); do
|
||||
IFS='|' read -r version description category path should_publish <<< "${TS_PACKAGES[$name]}"
|
||||
IFS='|' read -r version description category path should_publish source <<< "${TS_PACKAGES[$name]}"
|
||||
[[ "$should_publish" != "true" ]] && continue
|
||||
if [[ -z "$description" ]]; then
|
||||
echo -e "❌ \033[1;33m$name\033[0m"
|
||||
|
|
@ -420,7 +438,7 @@ check_missing_descriptions() {
|
|||
|
||||
# Check Python packages
|
||||
for name in $(echo "${!PY_PACKAGES[@]}" | tr ' ' '\n' | sort); do
|
||||
IFS='|' read -r version description category path <<< "${PY_PACKAGES[$name]}"
|
||||
IFS='|' read -r version description category path source <<< "${PY_PACKAGES[$name]}"
|
||||
if [[ -z "$description" ]]; then
|
||||
echo -e "❌ \033[1;35m$name\033[0m"
|
||||
echo " Path: $path/pyproject.toml"
|
||||
|
|
@ -458,7 +476,7 @@ main() {
|
|||
# Count publishable
|
||||
local publishable_ts=0
|
||||
for name in "${!TS_PACKAGES[@]}"; do
|
||||
IFS='|' read -r version description category path should_publish <<< "${TS_PACKAGES[$name]}"
|
||||
IFS='|' read -r version description category path should_publish source <<< "${TS_PACKAGES[$name]}"
|
||||
[[ "$should_publish" == "true" ]] && ((publishable_ts++))
|
||||
done
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue