Accomodating OVOS changes.
List of Services README Generation / Generate Services List (push) Failing after 2m9s
Renovate / renovate (push) Successful in 4m13s

This commit is contained in:
2025-11-29 09:03:17 -05:00
parent 92913f0d88
commit 2a182e9f76
2 changed files with 154 additions and 75 deletions
+49 -3
View File
@@ -5,6 +5,9 @@ on:
- cron: 30 */2 * * * - cron: 30 */2 * * *
workflow_dispatch: workflow_dispatch:
env:
HC_VAULT_VERSION: "1.21.1"
jobs: jobs:
readme-services: readme-services:
name: Generate Services List name: Generate Services List
@@ -14,19 +17,62 @@ jobs:
uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1
with: with:
ref: 'main' ref: 'main'
submodules: true
- name: Install yq - name: Install yq
uses: dcarbone/install-yq-action@4075b4dca348d74bd83f2bf82d30f25d7c54539b # v1.3.1 uses: dcarbone/install-yq-action@4075b4dca348d74bd83f2bf82d30f25d7c54539b # v1.3.1
- name: Generate service list - name: Generate .env from Hashicorp Vault
uses: https://git.trez.wtf/Trez/hc-vault-env@main
with:
HC_VAULT_VERSION: ${{ env.HC_VAULT_VERSION }}
HC_VAULT_ADDR: ${{ secrets.TREZ_VAULT_ADDR }}
HC_VAULT_AUTH: token
HC_VAULT_TOKEN: ${{ secrets.VAULT_GITEA_TOKEN }}
HC_VAULT_SECRETS_PATH: rinoa-docker/env
- name: Find docker-compose YAML files
id: find-compose-files
run: | run: |
yq 'explode(.) | .services | to_entries | map({"service": .key, "image": (.value.image | sub("@sha256:.*$"; "")), "description": (.value.labels."homepage.description" // "")})' docker-compose.yml > services.yml COMPOSE_FILES=$(find . -type f -name "docker-compose*.yml" \
-a ! -name "*windows*" \
-a ! -name "*gui*" \
-a ! -name "*macos*" \
-a ! -name "*hivemind*" \
-a ! -name "*server*" \
| sort)
echo "compose_files<<EOF" >> "$GITHUB_OUTPUT"
echo "$COMPOSE_FILES" >> "$GITHUB_OUTPUT"
echo "EOF" >> "$GITHUB_OUTPUT"
echo "Found Compose files:"
echo "$COMPOSE_FILES"
- name: Generate combined service list
run: |
echo "Processing combined Compose files:"
echo "${{ steps.find-compose-files.outputs.compose_files }}"
docker_compose_args=""
while IFS= read -r f; do
[ -z "$f" ] && continue
docker_compose_args="$docker_compose_args -f $f"
done <<< "${{ steps.find-compose-files.outputs.compose_files }}"
echo "Running: docker compose --env-file .env $docker_compose_args config"
docker compose --env-file .env $docker_compose_args config > combined_compose.yml
echo "Extracting services..."
yq eval -o=json '.services | to_entries | map({"service": .key, "image": (.value.image | sub("@sha256:.*$"; "") // "N/A")})' combined_compose.yml > temp_services.json
jq -s add temp_services.json | jq 'unique_by(.service)' > services.json
- name: Generate Markdown Table - name: Generate Markdown Table
uses: gazab/create-markdown-table@6686233d7008e8d8b9d4bbdbfd1fb1ae510019f0 # v1.0.7 uses: gazab/create-markdown-table@6686233d7008e8d8b9d4bbdbfd1fb1ae510019f0 # v1.0.7
id: service-table id: service-table
with: with:
file: ./services.yml file: ./services.json
- name: Regenerate README - name: Regenerate README
run: | run: |
+105 -72
View File
@@ -1,13 +1,12 @@
name: Docker Compose Deployment name: Gitea Branch PR & Docker Deployment
on: on:
workflow_dispatch: workflow_dispatch:
# push: push:
# branches-ignore: branches-ignore:
# - "main" - "main"
# - "renovate/**" paths:
# paths: - "**/docker-compose*.yml"
# - "**/docker-compose.yml"
env: env:
HC_VAULT_VERSION: "1.21.1" HC_VAULT_VERSION: "1.21.1"
@@ -29,7 +28,7 @@ jobs:
with: with:
gotify_api_base: "${{ secrets.RUNNER_GOTIFY_URL }}" gotify_api_base: "${{ secrets.RUNNER_GOTIFY_URL }}"
gotify_app_token: "${{ secrets.RUNNER_GOTIFY_TOKEN }}" gotify_app_token: "${{ secrets.RUNNER_GOTIFY_TOKEN }}"
notification_title: "GITEA: PR Check @ Ultima" notification_title: "GITEA: PR Check @ Benedikta"
notification_message: "Checking for existing PR... 🔍" notification_message: "Checking for existing PR... 🔍"
- name: PR Check/Creation - name: PR Check/Creation
@@ -45,7 +44,7 @@ jobs:
with: with:
gotify_api_base: "${{ secrets.RUNNER_GOTIFY_URL }}" gotify_api_base: "${{ secrets.RUNNER_GOTIFY_URL }}"
gotify_app_token: "${{ secrets.RUNNER_GOTIFY_TOKEN }}" gotify_app_token: "${{ secrets.RUNNER_GOTIFY_TOKEN }}"
notification_title: "GITEA: PR Check @ Ultima" notification_title: "GITEA: PR Check @ Benedikta"
notification_message: "PR Check done 🎟️" notification_message: "PR Check done 🎟️"
generate-service-list: generate-service-list:
@@ -70,36 +69,74 @@ jobs:
notification_title: "GITEA: Services TBD" notification_title: "GITEA: Services TBD"
notification_message: "Generating list of services to deploy..." notification_message: "Generating list of services to deploy..."
- name: Save both versions of docker-compose.yml
run: |
git show origin/main:docker-compose.yml > docker-compose-main.yml || touch docker-compose-main.yml
cp docker-compose.yml docker-compose-head.yml
- name: Detect added, deleted, and modified services - name: Detect added, deleted, and modified services
id: detect_services id: detect_services
run: | run: |
echo "Getting services from main and ${{ github.ref_name }}" echo "Getting services from main and ${{ github.ref_name }}"
yq '.services | keys | .[]' docker-compose-main.yml | sort > services_main.txt
yq '.services | keys | .[]' docker-compose-head.yml | sort > services_head.txt # Dynamically find all docker-compose YAML files (root + compose folder)
COMPOSE_FILES=($(find . -maxdepth 2 -type f -name 'docker-compose*.yml' | sort))
echo "Found Compose files:"
printf '%s\n' "${COMPOSE_FILES[@]}"
# Temp files to store all services
touch services_main_all.txt services_head_all.txt
for f in "${COMPOSE_FILES[@]}"; do
echo "Processing $f"
# Create a safe filename by replacing slashes with underscores
safe_f=$(echo "$f" | sed 's|[./]|_|g')
# Fetch main version
git show origin/main:"$f" > "main_${safe_f}" 2>/dev/null || touch "main_${safe_f}"
cp "$f" "head_${safe_f}"
# Extract services and append to global list
yq '.services | keys | .[]' "main_${safe_f}" >> services_main_all.txt 2>/dev/null || true
yq '.services | keys | .[]' "head_${safe_f}" >> services_head_all.txt 2>/dev/null || true
done
# Sort and deduplicate
sort -u services_main_all.txt -o services_main_all.txt
sort -u services_head_all.txt -o services_head_all.txt
echo "Creating list of modified services..." echo "Creating list of modified services..."
touch service_changes.txt touch service_changes.txt
comm -13 services_main.txt services_head.txt | while read service; do # Added services
comm -13 services_main_all.txt services_head_all.txt | while read service; do
echo "$service: added" >> service_changes.txt echo "$service: added" >> service_changes.txt
done done
comm -12 services_main.txt services_head.txt | while read service; do # Modified services (parallelized)
yq ".services[\"$service\"]" docker-compose-main.yml > tmp_main.yml comm -12 services_main_all.txt services_head_all.txt | xargs -n1 -P4 -I{} bash -c '
yq ".services[\"$service\"]" docker-compose-head.yml > tmp_head.yml service="{}"
if ! diff -q tmp_main.yml tmp_head.yml > /dev/null; then modified=0
for f in "${COMPOSE_FILES[@]}"; do
safe_f=$(echo "$f" | sed "s|[./]|_|g")
yq ".services[\"$service\"]" "main_${safe_f}" > tmp_main.yml 2>/dev/null || continue
yq ".services[\"$service\"]" "head_${safe_f}" > tmp_head.yml 2>/dev/null || continue
if ! diff -q tmp_main.yml tmp_head.yml > /dev/null; then
modified=1
break
fi
done
if [[ $modified -eq 1 ]]; then
echo "$service: modified" >> service_changes.txt echo "$service: modified" >> service_changes.txt
fi fi
done '
echo "Detected service changes:" echo "Detected service changes:"
cat service_changes.txt cat service_changes.txt
if [[ -z $(cat service_changes.txt) ]]; then
echo "watchtower" > service_changes.txt
echo "Placeholder:"
cat service_changes.txt
fi
mod_svcs=$(cut -d':' -f1 service_changes.txt | sort | uniq | tr '\n' ' ' | sed 's/ *$//') mod_svcs=$(cut -d':' -f1 service_changes.txt | sort | uniq | tr '\n' ' ' | sed 's/ *$//')
echo "docker_svc_list<<EOF" >> "$GITHUB_OUTPUT" echo "docker_svc_list<<EOF" >> "$GITHUB_OUTPUT"
echo "$mod_svcs" >> "$GITHUB_OUTPUT" echo "$mod_svcs" >> "$GITHUB_OUTPUT"
@@ -113,12 +150,15 @@ jobs:
name: Docker Compose Dry Run name: Docker Compose Dry Run
needs: [generate-service-list] needs: [generate-service-list]
runs-on: ubuntu-latest runs-on: ubuntu-latest
outputs:
compose_file_list: ${{ steps.compose_file_list.outputs.compose_list }}
env: env:
VAULT_ADDR: ${{ secrets.TREZ_VAULT_ADDR }} VAULT_ADDR: ${{ secrets.TREZ_VAULT_ADDR }}
VAULT_TOKEN: ${{ secrets.VAULT_GITEA_TOKEN }} VAULT_TOKEN: ${{ secrets.VAULT_GITEA_TOKEN }}
VAULT_NAMESPACE: "" VAULT_NAMESPACE: ""
RIKKU_REGISTRY_PASSWORD: ${{ secrets.BOT_GITEA_PASSWORD }} REGISTRY_PASSWORD: ${{ secrets.BOT_GITEA_PASSWORD }}
DOCKER_SVC_LIST: ${{ needs.generate-service-list.outputs.svc_deploy_list }} DOCKER_SVC_LIST: ${{ needs.generate-service-list.outputs.svc_deploy_list }}
steps: steps:
- name: Checkout - name: Checkout
uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1
@@ -126,7 +166,7 @@ jobs:
- name: Login to Docker Hub - name: Login to Docker Hub
uses: docker/login-action@5e57cd118135c172c3672efd75eb46360885c0ef # v3.6.0 uses: docker/login-action@5e57cd118135c172c3672efd75eb46360885c0ef # v3.6.0
env: env:
DOCKER_HOST: tcp://192.168.1.249:2375 DOCKER_HOST: tcp://192.168.1.250:2375
with: with:
username: ${{ secrets.DOCKERHUB_USER }} username: ${{ secrets.DOCKERHUB_USER }}
password: ${{ secrets.DOCKERHUB_PASSWORD }} password: ${{ secrets.DOCKERHUB_PASSWORD }}
@@ -134,7 +174,7 @@ jobs:
- name: Login to GitHub Container Registry - name: Login to GitHub Container Registry
uses: docker/login-action@5e57cd118135c172c3672efd75eb46360885c0ef # v3.6.0 uses: docker/login-action@5e57cd118135c172c3672efd75eb46360885c0ef # v3.6.0
env: env:
DOCKER_HOST: tcp://192.168.1.249:2375 DOCKER_HOST: tcp://192.168.1.250:2375
with: with:
registry: ghcr.io registry: ghcr.io
username: ${{ secrets.GHCR_USER }} username: ${{ secrets.GHCR_USER }}
@@ -143,7 +183,7 @@ jobs:
- name: Login to Private Gitea Registry - name: Login to Private Gitea Registry
uses: docker/login-action@5e57cd118135c172c3672efd75eb46360885c0ef # v3.6.0 uses: docker/login-action@5e57cd118135c172c3672efd75eb46360885c0ef # v3.6.0
env: env:
DOCKER_HOST: tcp://192.168.1.249:2375 DOCKER_HOST: tcp://192.168.1.250:2375
with: with:
registry: git.trez.wtf registry: git.trez.wtf
username: ${{ secrets.BOT_GITEA_USER }} username: ${{ secrets.BOT_GITEA_USER }}
@@ -154,9 +194,23 @@ jobs:
with: with:
gotify_api_base: "${{ secrets.RUNNER_GOTIFY_URL }}" gotify_api_base: "${{ secrets.RUNNER_GOTIFY_URL }}"
gotify_app_token: "${{ secrets.RUNNER_GOTIFY_TOKEN }}" gotify_app_token: "${{ secrets.RUNNER_GOTIFY_TOKEN }}"
notification_title: "GITEA: Docker Compose Dry Run @ Ultima" notification_title: "GITEA: Docker Compose Dry Run @ Benedikta"
notification_message: "Starting Docker Compose dry run..." notification_message: "Starting Docker Compose dry run..."
- name: Get list of Compose files
id: compose_file_list
run: |
compose_list=$(find . -type f -name "docker-compose*.yml" \
-a ! -name "*windows*" \
-a ! -name "*gui*" \
-a ! -name "*macos*" \
-a ! -name "*hivemind*" \
-a ! -name "*server*" \
| sed -e ':a;N;$!ba;s/[\r\n]/ /g')
echo "compose_list=$compose_list" >> "$GITHUB_OUTPUT"
echo "Compose files: $compose_list"
- name: Generate .env from Hashicorp Vault - name: Generate .env from Hashicorp Vault
uses: https://git.trez.wtf/Trez/hc-vault-env@main uses: https://git.trez.wtf/Trez/hc-vault-env@main
with: with:
@@ -164,26 +218,22 @@ jobs:
HC_VAULT_ADDR: ${{ secrets.TREZ_VAULT_ADDR }} HC_VAULT_ADDR: ${{ secrets.TREZ_VAULT_ADDR }}
HC_VAULT_AUTH: token HC_VAULT_AUTH: token
HC_VAULT_TOKEN: ${{ secrets.VAULT_GITEA_TOKEN }} HC_VAULT_TOKEN: ${{ secrets.VAULT_GITEA_TOKEN }}
HC_VAULT_SECRETS_PATH: ultima-docker/env HC_VAULT_SECRETS_PATH: rinoa-docker/env
- name: Pre-pull/build service images in parallel
uses: https://git.trez.wtf/Trez/docker-select-image-pull@main
env:
DOCKER_HOST: tcp://192.168.1.249:2375
with:
services: ${{ env.DOCKER_SVC_LIST }}
- name: Docker Compose Dry Run - name: Docker Compose Dry Run
uses: cssnr/stack-deploy-action@d58b92bcd776afc57ef12f55bafff71200fd218e # v1.4.0 uses: cssnr/stack-deploy-action@d58b92bcd776afc57ef12f55bafff71200fd218e # v1.4.0
with: with:
mode: compose mode: compose
file: docker-compose.yml file: ${{ steps.compose_file_list.outputs.compose_list }}
name: "ultima" name: "ovosmisc"
host: 192.168.1.249 host: 192.168.1.250
user: pi user: ovos
ssh_key: ${{ secrets.RUNNER_SSH_PRIVATE_KEY }} ssh_key: ${{ secrets.RUNNER_SSH_PRIVATE_KEY }}
args: --remove-orphans -d --dry-run ${{ needs.generate-service-list.outputs.svc_deploy_list }} args: --remove-orphans --dry-run ${{ needs.generate-service-list.outputs.svc_deploy_list }}
env_file: ".env" env_file: ".env"
registry_host: "docker.io"
registry_user: ${{ secrets.DOCKERHUB_USER }}
registry_pass: ${{ secrets.DOCKERHUB_PASSWORD }}
summary: true summary: true
- name: Gotify Notification - name: Gotify Notification
@@ -191,17 +241,12 @@ jobs:
with: with:
gotify_api_base: "${{ secrets.RUNNER_GOTIFY_URL }}" gotify_api_base: "${{ secrets.RUNNER_GOTIFY_URL }}"
gotify_app_token: "${{ secrets.RUNNER_GOTIFY_TOKEN }}" gotify_app_token: "${{ secrets.RUNNER_GOTIFY_TOKEN }}"
notification_title: "GITEA: Docker Compose Dry Run @ Ultima" notification_title: "GITEA: Docker Compose Dry Run @ Benedikta"
notification_message: "Docker Compose dry run completed successfully." notification_message: "Docker Compose dry run completed successfully."
pr-merge: pr-merge:
name: PR Merge name: PR Merge
needs: needs: [generate-service-list, docker-compose-dry-run]
[
generate-service-list,
docker-compose-dry-run,
regenerate-readme-modified-services,
]
runs-on: ubuntu-latest runs-on: ubuntu-latest
steps: steps:
- name: Checkout - name: Checkout
@@ -242,12 +287,11 @@ jobs:
docker-compose-deploy: docker-compose-deploy:
name: Docker Compose Deployment name: Docker Compose Deployment
runs-on: ubuntu-latest runs-on: ubuntu-latest
needs: [pr-merge] needs: [generate-service-list, docker-compose-dry-run, pr-merge]
env: env:
VAULT_ADDR: ${{ secrets.TREZ_VAULT_ADDR }} VAULT_ADDR: ${{ secrets.TREZ_VAULT_ADDR }}
VAULT_TOKEN: ${{ secrets.VAULT_GITEA_TOKEN }} VAULT_TOKEN: ${{ secrets.VAULT_GITEA_TOKEN }}
RIKKU_REGISTRY_PASSWORD: ${{ secrets.BOT_GITEA_PASSWORD }} REGISTRY_PASSWORD: ${{ secrets.BOT_GITEA_PASSWORD }}
DOCKER_SVC_LIST: ${{ needs.generate-service-list.outputs.svc_deploy_list }}
steps: steps:
- name: Checkout - name: Checkout
uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1
@@ -257,7 +301,7 @@ jobs:
- name: Login to Docker Hub - name: Login to Docker Hub
uses: docker/login-action@5e57cd118135c172c3672efd75eb46360885c0ef # v3.6.0 uses: docker/login-action@5e57cd118135c172c3672efd75eb46360885c0ef # v3.6.0
env: env:
DOCKER_HOST: tcp://192.168.1.249:2375 DOCKER_HOST: tcp://192.168.1.250:2375
with: with:
username: ${{ secrets.DOCKERHUB_USER }} username: ${{ secrets.DOCKERHUB_USER }}
password: ${{ secrets.DOCKERHUB_PASSWORD }} password: ${{ secrets.DOCKERHUB_PASSWORD }}
@@ -265,7 +309,7 @@ jobs:
- name: Login to GitHub Container Registry - name: Login to GitHub Container Registry
uses: docker/login-action@5e57cd118135c172c3672efd75eb46360885c0ef # v3.6.0 uses: docker/login-action@5e57cd118135c172c3672efd75eb46360885c0ef # v3.6.0
env: env:
DOCKER_HOST: tcp://192.168.1.249:2375 DOCKER_HOST: tcp://192.168.1.250:2375
with: with:
registry: ghcr.io registry: ghcr.io
username: ${{ secrets.GHCR_USER }} username: ${{ secrets.GHCR_USER }}
@@ -274,7 +318,7 @@ jobs:
- name: Login to Private Gitea Registry - name: Login to Private Gitea Registry
uses: docker/login-action@5e57cd118135c172c3672efd75eb46360885c0ef # v3.6.0 uses: docker/login-action@5e57cd118135c172c3672efd75eb46360885c0ef # v3.6.0
env: env:
DOCKER_HOST: tcp://192.168.1.249:2375 DOCKER_HOST: tcp://192.168.1.250:2375
with: with:
registry: git.trez.wtf registry: git.trez.wtf
username: ${{ secrets.BOT_GITEA_USER }} username: ${{ secrets.BOT_GITEA_USER }}
@@ -285,7 +329,7 @@ jobs:
with: with:
gotify_api_base: "${{ secrets.RUNNER_GOTIFY_URL }}" gotify_api_base: "${{ secrets.RUNNER_GOTIFY_URL }}"
gotify_app_token: "${{ secrets.RUNNER_GOTIFY_TOKEN }}" gotify_app_token: "${{ secrets.RUNNER_GOTIFY_TOKEN }}"
notification_title: "GITEA: Docker Compose Deployment @ Ultima" notification_title: "GITEA: Docker Compose Deployment @ Benedikta"
notification_message: "Starting Docker Compose run..." notification_message: "Starting Docker Compose run..."
- name: Generate .env from Hashicorp Vault - name: Generate .env from Hashicorp Vault
@@ -295,36 +339,25 @@ jobs:
HC_VAULT_ADDR: ${{ secrets.TREZ_VAULT_ADDR }} HC_VAULT_ADDR: ${{ secrets.TREZ_VAULT_ADDR }}
HC_VAULT_AUTH: token HC_VAULT_AUTH: token
HC_VAULT_TOKEN: ${{ secrets.VAULT_GITEA_TOKEN }} HC_VAULT_TOKEN: ${{ secrets.VAULT_GITEA_TOKEN }}
HC_VAULT_SECRETS_PATH: ultima-docker/env HC_VAULT_SECRETS_PATH: rinoa-docker/env
- name: Docker Compose Deployment - name: Docker Compose Deployment
uses: cssnr/stack-deploy-action@d58b92bcd776afc57ef12f55bafff71200fd218e # v1.4.0 uses: cssnr/stack-deploy-action@d58b92bcd776afc57ef12f55bafff71200fd218e # v1.4.0
with: with:
mode: compose mode: compose
file: docker-compose.yml file: ${{ needs.docker-compose-dry-run.outputs.compose_file_list }}
name: "ultima" name: "ovosmisc"
host: 192.168.1.249 host: 192.168.1.250
user: pi user: ovos
ssh_key: ${{ secrets.RUNNER_SSH_PRIVATE_KEY }} ssh_key: ${{ secrets.RUNNER_SSH_PRIVATE_KEY }}
args: --remove-orphans -d ${{ needs.generate-service-list.outputs.svc_deploy_list }} args: --remove-orphans ${{ needs.generate-service-list.outputs.svc_deploy_list }}
env_file: ".env" env_file: ".env"
summary: true summary: true
- name: Docker Compose Healthcheck
uses: jaracogmbh/docker-compose-health-check-action@973fbdccf7c8e396b652d3501984c8e530a9fa80 # v1.0.0
env:
DOCKER_HOST: tcp://192.168.1.249:2375
with:
max-retries: 30
retry-interval: 10
compose-file: "docker-compose.yml"
skip-exited: "true"
skip-no-healthcheck: "true"
- name: Gotify Notification - name: Gotify Notification
uses: eikendev/gotify-action@ca0339b85ee8db9fda9c0718aaa7f95e17b3c617 # 0.0.4 uses: eikendev/gotify-action@ca0339b85ee8db9fda9c0718aaa7f95e17b3c617 # 0.0.4
with: with:
gotify_api_base: "${{ secrets.RUNNER_GOTIFY_URL }}" gotify_api_base: "${{ secrets.RUNNER_GOTIFY_URL }}"
gotify_app_token: "${{ secrets.RUNNER_GOTIFY_TOKEN }}" gotify_app_token: "${{ secrets.RUNNER_GOTIFY_TOKEN }}"
notification_title: "GITEA: Docker Compose Deployment @ Ultima" notification_title: "GITEA: Docker Compose Deployment @ Benedikta"
notification_message: "Deployment completed successfully." notification_message: "Deployment completed successfully."