From 2a182e9f7663865999e435c56813da43deaeaf79 Mon Sep 17 00:00:00 2001 From: "Trez.One" Date: Sat, 29 Nov 2025 09:03:17 -0500 Subject: [PATCH] Accomodating OVOS changes. --- .gitea/workflows/compose-services-readme.yml | 52 +++++- .gitea/workflows/pr-docker-deploy.yml | 177 +++++++++++-------- 2 files changed, 154 insertions(+), 75 deletions(-) diff --git a/.gitea/workflows/compose-services-readme.yml b/.gitea/workflows/compose-services-readme.yml index 0343957..119a6a7 100644 --- a/.gitea/workflows/compose-services-readme.yml +++ b/.gitea/workflows/compose-services-readme.yml @@ -5,6 +5,9 @@ on: - cron: 30 */2 * * * workflow_dispatch: +env: + HC_VAULT_VERSION: "1.21.1" + jobs: readme-services: name: Generate Services List @@ -14,19 +17,62 @@ jobs: uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 with: ref: 'main' + submodules: true - name: Install yq uses: dcarbone/install-yq-action@4075b4dca348d74bd83f2bf82d30f25d7c54539b # v1.3.1 - - name: Generate service list + - name: Generate .env from Hashicorp Vault + uses: https://git.trez.wtf/Trez/hc-vault-env@main + with: + HC_VAULT_VERSION: ${{ env.HC_VAULT_VERSION }} + HC_VAULT_ADDR: ${{ secrets.TREZ_VAULT_ADDR }} + HC_VAULT_AUTH: token + HC_VAULT_TOKEN: ${{ secrets.VAULT_GITEA_TOKEN }} + HC_VAULT_SECRETS_PATH: rinoa-docker/env + + - name: Find docker-compose YAML files + id: find-compose-files run: | - yq 'explode(.) | .services | to_entries | map({"service": .key, "image": (.value.image | sub("@sha256:.*$"; "")), "description": (.value.labels."homepage.description" // "")})' docker-compose.yml > services.yml + COMPOSE_FILES=$(find . -type f -name "docker-compose*.yml" \ + -a ! -name "*windows*" \ + -a ! -name "*gui*" \ + -a ! -name "*macos*" \ + -a ! -name "*hivemind*" \ + -a ! -name "*server*" \ + | sort) + + echo "compose_files<> "$GITHUB_OUTPUT" + echo "$COMPOSE_FILES" >> "$GITHUB_OUTPUT" + echo "EOF" >> "$GITHUB_OUTPUT" + + echo "Found Compose files:" + echo "$COMPOSE_FILES" + + - name: Generate combined service list + run: | + echo "Processing combined Compose files:" + echo "${{ steps.find-compose-files.outputs.compose_files }}" + + docker_compose_args="" + while IFS= read -r f; do + [ -z "$f" ] && continue + docker_compose_args="$docker_compose_args -f $f" + done <<< "${{ steps.find-compose-files.outputs.compose_files }}" + + echo "Running: docker compose --env-file .env $docker_compose_args config" + docker compose --env-file .env $docker_compose_args config > combined_compose.yml + + echo "Extracting services..." + yq eval -o=json '.services | to_entries | map({"service": .key, "image": (.value.image | sub("@sha256:.*$"; "") // "N/A")})' combined_compose.yml > temp_services.json + + jq -s add temp_services.json | jq 'unique_by(.service)' > services.json - name: Generate Markdown Table uses: gazab/create-markdown-table@6686233d7008e8d8b9d4bbdbfd1fb1ae510019f0 # v1.0.7 id: service-table with: - file: ./services.yml + file: ./services.json - name: Regenerate README run: | diff --git a/.gitea/workflows/pr-docker-deploy.yml b/.gitea/workflows/pr-docker-deploy.yml index 86b5edc..39bfbd2 100644 --- a/.gitea/workflows/pr-docker-deploy.yml +++ b/.gitea/workflows/pr-docker-deploy.yml @@ -1,13 +1,12 @@ -name: Docker Compose Deployment +name: Gitea Branch PR & Docker Deployment on: workflow_dispatch: - # push: - # branches-ignore: - # - "main" - # - "renovate/**" - # paths: - # - "**/docker-compose.yml" + push: + branches-ignore: + - "main" + paths: + - "**/docker-compose*.yml" env: HC_VAULT_VERSION: "1.21.1" @@ -29,7 +28,7 @@ jobs: with: gotify_api_base: "${{ secrets.RUNNER_GOTIFY_URL }}" gotify_app_token: "${{ secrets.RUNNER_GOTIFY_TOKEN }}" - notification_title: "GITEA: PR Check @ Ultima" + notification_title: "GITEA: PR Check @ Benedikta" notification_message: "Checking for existing PR... 🔍" - name: PR Check/Creation @@ -45,7 +44,7 @@ jobs: with: gotify_api_base: "${{ secrets.RUNNER_GOTIFY_URL }}" gotify_app_token: "${{ secrets.RUNNER_GOTIFY_TOKEN }}" - notification_title: "GITEA: PR Check @ Ultima" + notification_title: "GITEA: PR Check @ Benedikta" notification_message: "PR Check done 🎟️" generate-service-list: @@ -70,36 +69,74 @@ jobs: notification_title: "GITEA: Services TBD" notification_message: "Generating list of services to deploy..." - - name: Save both versions of docker-compose.yml - run: | - git show origin/main:docker-compose.yml > docker-compose-main.yml || touch docker-compose-main.yml - cp docker-compose.yml docker-compose-head.yml - - name: Detect added, deleted, and modified services id: detect_services run: | echo "Getting services from main and ${{ github.ref_name }}" - yq '.services | keys | .[]' docker-compose-main.yml | sort > services_main.txt - yq '.services | keys | .[]' docker-compose-head.yml | sort > services_head.txt + + # Dynamically find all docker-compose YAML files (root + compose folder) + COMPOSE_FILES=($(find . -maxdepth 2 -type f -name 'docker-compose*.yml' | sort)) + + echo "Found Compose files:" + printf '%s\n' "${COMPOSE_FILES[@]}" + + # Temp files to store all services + touch services_main_all.txt services_head_all.txt + + for f in "${COMPOSE_FILES[@]}"; do + echo "Processing $f" + + # Create a safe filename by replacing slashes with underscores + safe_f=$(echo "$f" | sed 's|[./]|_|g') + + # Fetch main version + git show origin/main:"$f" > "main_${safe_f}" 2>/dev/null || touch "main_${safe_f}" + cp "$f" "head_${safe_f}" + + # Extract services and append to global list + yq '.services | keys | .[]' "main_${safe_f}" >> services_main_all.txt 2>/dev/null || true + yq '.services | keys | .[]' "head_${safe_f}" >> services_head_all.txt 2>/dev/null || true + done + + # Sort and deduplicate + sort -u services_main_all.txt -o services_main_all.txt + sort -u services_head_all.txt -o services_head_all.txt echo "Creating list of modified services..." touch service_changes.txt - comm -13 services_main.txt services_head.txt | while read service; do + # Added services + comm -13 services_main_all.txt services_head_all.txt | while read service; do echo "$service: added" >> service_changes.txt done - comm -12 services_main.txt services_head.txt | while read service; do - yq ".services[\"$service\"]" docker-compose-main.yml > tmp_main.yml - yq ".services[\"$service\"]" docker-compose-head.yml > tmp_head.yml - if ! diff -q tmp_main.yml tmp_head.yml > /dev/null; then + # Modified services (parallelized) + comm -12 services_main_all.txt services_head_all.txt | xargs -n1 -P4 -I{} bash -c ' + service="{}" + modified=0 + for f in "${COMPOSE_FILES[@]}"; do + safe_f=$(echo "$f" | sed "s|[./]|_|g") + yq ".services[\"$service\"]" "main_${safe_f}" > tmp_main.yml 2>/dev/null || continue + yq ".services[\"$service\"]" "head_${safe_f}" > tmp_head.yml 2>/dev/null || continue + if ! diff -q tmp_main.yml tmp_head.yml > /dev/null; then + modified=1 + break + fi + done + if [[ $modified -eq 1 ]]; then echo "$service: modified" >> service_changes.txt fi - done + ' echo "Detected service changes:" cat service_changes.txt + if [[ -z $(cat service_changes.txt) ]]; then + echo "watchtower" > service_changes.txt + echo "Placeholder:" + cat service_changes.txt + fi + mod_svcs=$(cut -d':' -f1 service_changes.txt | sort | uniq | tr '\n' ' ' | sed 's/ *$//') echo "docker_svc_list<> "$GITHUB_OUTPUT" echo "$mod_svcs" >> "$GITHUB_OUTPUT" @@ -113,12 +150,15 @@ jobs: name: Docker Compose Dry Run needs: [generate-service-list] runs-on: ubuntu-latest + outputs: + compose_file_list: ${{ steps.compose_file_list.outputs.compose_list }} env: VAULT_ADDR: ${{ secrets.TREZ_VAULT_ADDR }} VAULT_TOKEN: ${{ secrets.VAULT_GITEA_TOKEN }} VAULT_NAMESPACE: "" - RIKKU_REGISTRY_PASSWORD: ${{ secrets.BOT_GITEA_PASSWORD }} + REGISTRY_PASSWORD: ${{ secrets.BOT_GITEA_PASSWORD }} DOCKER_SVC_LIST: ${{ needs.generate-service-list.outputs.svc_deploy_list }} + steps: - name: Checkout uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 @@ -126,7 +166,7 @@ jobs: - name: Login to Docker Hub uses: docker/login-action@5e57cd118135c172c3672efd75eb46360885c0ef # v3.6.0 env: - DOCKER_HOST: tcp://192.168.1.249:2375 + DOCKER_HOST: tcp://192.168.1.250:2375 with: username: ${{ secrets.DOCKERHUB_USER }} password: ${{ secrets.DOCKERHUB_PASSWORD }} @@ -134,7 +174,7 @@ jobs: - name: Login to GitHub Container Registry uses: docker/login-action@5e57cd118135c172c3672efd75eb46360885c0ef # v3.6.0 env: - DOCKER_HOST: tcp://192.168.1.249:2375 + DOCKER_HOST: tcp://192.168.1.250:2375 with: registry: ghcr.io username: ${{ secrets.GHCR_USER }} @@ -143,7 +183,7 @@ jobs: - name: Login to Private Gitea Registry uses: docker/login-action@5e57cd118135c172c3672efd75eb46360885c0ef # v3.6.0 env: - DOCKER_HOST: tcp://192.168.1.249:2375 + DOCKER_HOST: tcp://192.168.1.250:2375 with: registry: git.trez.wtf username: ${{ secrets.BOT_GITEA_USER }} @@ -154,9 +194,23 @@ jobs: with: gotify_api_base: "${{ secrets.RUNNER_GOTIFY_URL }}" gotify_app_token: "${{ secrets.RUNNER_GOTIFY_TOKEN }}" - notification_title: "GITEA: Docker Compose Dry Run @ Ultima" + notification_title: "GITEA: Docker Compose Dry Run @ Benedikta" notification_message: "Starting Docker Compose dry run..." + - name: Get list of Compose files + id: compose_file_list + run: | + compose_list=$(find . -type f -name "docker-compose*.yml" \ + -a ! -name "*windows*" \ + -a ! -name "*gui*" \ + -a ! -name "*macos*" \ + -a ! -name "*hivemind*" \ + -a ! -name "*server*" \ + | sed -e ':a;N;$!ba;s/[\r\n]/ /g') + + echo "compose_list=$compose_list" >> "$GITHUB_OUTPUT" + echo "Compose files: $compose_list" + - name: Generate .env from Hashicorp Vault uses: https://git.trez.wtf/Trez/hc-vault-env@main with: @@ -164,26 +218,22 @@ jobs: HC_VAULT_ADDR: ${{ secrets.TREZ_VAULT_ADDR }} HC_VAULT_AUTH: token HC_VAULT_TOKEN: ${{ secrets.VAULT_GITEA_TOKEN }} - HC_VAULT_SECRETS_PATH: ultima-docker/env - - - name: Pre-pull/build service images in parallel - uses: https://git.trez.wtf/Trez/docker-select-image-pull@main - env: - DOCKER_HOST: tcp://192.168.1.249:2375 - with: - services: ${{ env.DOCKER_SVC_LIST }} + HC_VAULT_SECRETS_PATH: rinoa-docker/env - name: Docker Compose Dry Run uses: cssnr/stack-deploy-action@d58b92bcd776afc57ef12f55bafff71200fd218e # v1.4.0 with: mode: compose - file: docker-compose.yml - name: "ultima" - host: 192.168.1.249 - user: pi + file: ${{ steps.compose_file_list.outputs.compose_list }} + name: "ovosmisc" + host: 192.168.1.250 + user: ovos ssh_key: ${{ secrets.RUNNER_SSH_PRIVATE_KEY }} - args: --remove-orphans -d --dry-run ${{ needs.generate-service-list.outputs.svc_deploy_list }} + args: --remove-orphans --dry-run ${{ needs.generate-service-list.outputs.svc_deploy_list }} env_file: ".env" + registry_host: "docker.io" + registry_user: ${{ secrets.DOCKERHUB_USER }} + registry_pass: ${{ secrets.DOCKERHUB_PASSWORD }} summary: true - name: Gotify Notification @@ -191,17 +241,12 @@ jobs: with: gotify_api_base: "${{ secrets.RUNNER_GOTIFY_URL }}" gotify_app_token: "${{ secrets.RUNNER_GOTIFY_TOKEN }}" - notification_title: "GITEA: Docker Compose Dry Run @ Ultima" + notification_title: "GITEA: Docker Compose Dry Run @ Benedikta" notification_message: "Docker Compose dry run completed successfully." pr-merge: name: PR Merge - needs: - [ - generate-service-list, - docker-compose-dry-run, - regenerate-readme-modified-services, - ] + needs: [generate-service-list, docker-compose-dry-run] runs-on: ubuntu-latest steps: - name: Checkout @@ -242,12 +287,11 @@ jobs: docker-compose-deploy: name: Docker Compose Deployment runs-on: ubuntu-latest - needs: [pr-merge] + needs: [generate-service-list, docker-compose-dry-run, pr-merge] env: VAULT_ADDR: ${{ secrets.TREZ_VAULT_ADDR }} VAULT_TOKEN: ${{ secrets.VAULT_GITEA_TOKEN }} - RIKKU_REGISTRY_PASSWORD: ${{ secrets.BOT_GITEA_PASSWORD }} - DOCKER_SVC_LIST: ${{ needs.generate-service-list.outputs.svc_deploy_list }} + REGISTRY_PASSWORD: ${{ secrets.BOT_GITEA_PASSWORD }} steps: - name: Checkout uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 @@ -257,7 +301,7 @@ jobs: - name: Login to Docker Hub uses: docker/login-action@5e57cd118135c172c3672efd75eb46360885c0ef # v3.6.0 env: - DOCKER_HOST: tcp://192.168.1.249:2375 + DOCKER_HOST: tcp://192.168.1.250:2375 with: username: ${{ secrets.DOCKERHUB_USER }} password: ${{ secrets.DOCKERHUB_PASSWORD }} @@ -265,7 +309,7 @@ jobs: - name: Login to GitHub Container Registry uses: docker/login-action@5e57cd118135c172c3672efd75eb46360885c0ef # v3.6.0 env: - DOCKER_HOST: tcp://192.168.1.249:2375 + DOCKER_HOST: tcp://192.168.1.250:2375 with: registry: ghcr.io username: ${{ secrets.GHCR_USER }} @@ -274,7 +318,7 @@ jobs: - name: Login to Private Gitea Registry uses: docker/login-action@5e57cd118135c172c3672efd75eb46360885c0ef # v3.6.0 env: - DOCKER_HOST: tcp://192.168.1.249:2375 + DOCKER_HOST: tcp://192.168.1.250:2375 with: registry: git.trez.wtf username: ${{ secrets.BOT_GITEA_USER }} @@ -285,7 +329,7 @@ jobs: with: gotify_api_base: "${{ secrets.RUNNER_GOTIFY_URL }}" gotify_app_token: "${{ secrets.RUNNER_GOTIFY_TOKEN }}" - notification_title: "GITEA: Docker Compose Deployment @ Ultima" + notification_title: "GITEA: Docker Compose Deployment @ Benedikta" notification_message: "Starting Docker Compose run..." - name: Generate .env from Hashicorp Vault @@ -295,36 +339,25 @@ jobs: HC_VAULT_ADDR: ${{ secrets.TREZ_VAULT_ADDR }} HC_VAULT_AUTH: token HC_VAULT_TOKEN: ${{ secrets.VAULT_GITEA_TOKEN }} - HC_VAULT_SECRETS_PATH: ultima-docker/env + HC_VAULT_SECRETS_PATH: rinoa-docker/env - name: Docker Compose Deployment uses: cssnr/stack-deploy-action@d58b92bcd776afc57ef12f55bafff71200fd218e # v1.4.0 with: mode: compose - file: docker-compose.yml - name: "ultima" - host: 192.168.1.249 - user: pi + file: ${{ needs.docker-compose-dry-run.outputs.compose_file_list }} + name: "ovosmisc" + host: 192.168.1.250 + user: ovos ssh_key: ${{ secrets.RUNNER_SSH_PRIVATE_KEY }} - args: --remove-orphans -d ${{ needs.generate-service-list.outputs.svc_deploy_list }} + args: --remove-orphans ${{ needs.generate-service-list.outputs.svc_deploy_list }} env_file: ".env" summary: true - - name: Docker Compose Healthcheck - uses: jaracogmbh/docker-compose-health-check-action@973fbdccf7c8e396b652d3501984c8e530a9fa80 # v1.0.0 - env: - DOCKER_HOST: tcp://192.168.1.249:2375 - with: - max-retries: 30 - retry-interval: 10 - compose-file: "docker-compose.yml" - skip-exited: "true" - skip-no-healthcheck: "true" - - name: Gotify Notification uses: eikendev/gotify-action@ca0339b85ee8db9fda9c0718aaa7f95e17b3c617 # 0.0.4 with: gotify_api_base: "${{ secrets.RUNNER_GOTIFY_URL }}" gotify_app_token: "${{ secrets.RUNNER_GOTIFY_TOKEN }}" - notification_title: "GITEA: Docker Compose Deployment @ Ultima" + notification_title: "GITEA: Docker Compose Deployment @ Benedikta" notification_message: "Deployment completed successfully."