diff --git a/.gitea/workflows/dag-config-check.yml b/.gitea/workflows/dag-config-check.yml index 7c7f991..c4a4c1c 100644 --- a/.gitea/workflows/dag-config-check.yml +++ b/.gitea/workflows/dag-config-check.yml @@ -3,35 +3,75 @@ name: Validate DAGs on: workflow_dispatch: push: - paths: ['./app-configs/rinoa/dagu/dags/**'] + paths: + - 'app-configs/rinoa/dagu/dags/**' + - '**/dag-config-check.yml' branches-ignore: - main env: - DAGS_PATH: "./app-configs/rinoa/dagu/dags" + DAGS_PATH: "app-configs/rinoa/dagu/dags" VAULT_ADDR: ${{ secrets.VAULT_ADDR }} - VAULT_TOKEN: ${{ secrets.VAULT_TOKEN }} + VAULT_TOKEN: ${{ secrets.VAULT_GITEA_TOKEN }} jobs: - validate: + validate-dags: + name: DAGU DAG(s) Validation runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 + - name: Checkout Repo + uses: actions/checkout@v4 + + - name: Gotify Notification + uses: eikendev/gotify-action@master + with: + gotify_api_base: '${{ secrets.GOTIFY_URL }}' + gotify_app_token: '${{ secrets.RUNNER_GOTIFY_TOKEN }}' + notification_title: 'GITEA: Dagu Validation' + notification_message: 'Setting up Ansible and Vault... 🏗️' - name: Install Ansible uses: alex-oleshkevich/setup-ansible@v1.0.1 with: version: "11.4.0" - - name: Set up Vault CLI - uses: hashicorp/setup-vault@v2 + - name: Cache Ansible Galaxy Collections + uses: actions/cache@v3 with: - version: 1.18.0 + path: collections + key: ${{ runner.os }}-ansible-${{ hashFiles('./collections/requirements.yml') }} + restore-keys: | + ${{ runner.os }}-ansible- + + - name: Set up Vault CLI + uses: eLco/setup-vault@v1 + with: + vault_version: 1.18.0 + + - name: Cache pip + uses: actions/cache@v4 + with: + path: ~/.cache/pip + key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements*.txt') }} + restore-keys: | + ${{ runner.os }}-pip- - name: Install hvac (Vault Python SDK) run: pip install hvac - - name: Render .yaml.j2 templates + - name: Gotify Notification + uses: eikendev/gotify-action@master + with: + gotify_api_base: '${{ secrets.GOTIFY_URL }}' + gotify_app_token: '${{ secrets.RUNNER_GOTIFY_TOKEN }}' + notification_title: 'GITEA: Dagu Validation' + notification_message: 'Ansible and Vault setups completed, starting Dagu validation... 🔬' + + - name: Install dagu + run: | + curl -L https://raw.githubusercontent.com/dagu-org/dagu/main/scripts/installer.sh | bash + + - name: Render DAG Jinja templates uses: dawidd6/action-ansible-playbook@v2 with: directory: . @@ -39,16 +79,37 @@ jobs: requirements: collections/requirements.yml playbook: playbooks/rinoa-render-dags.yml - - name: Install dagu - uses: jaxxstorm/action-install-gh-release@v1 - with: - repo: dagu-org/dagu - platform: linux - arch: amd64 - - name: Validate DAGs run: | for dag in $(find ${DAGS_PATH} -type f -name "*.yaml" -a ! -name "*example*"); do - echo "Validating $dag" - dagu dry "$dag" + echo -e "\n\n===========Validating ${dag}===========\n" + dagu dry "${dag}" done + echo "exit_code=$status" >> $GITHUB_OUTPUT + + - name: Gotify Notification + if: steps.validate-dags.outputs.exit_code == '0' + uses: eikendev/gotify-action@master + with: + gotify_api_base: '${{ secrets.GOTIFY_URL }}' + gotify_app_token: '${{ secrets.RUNNER_GOTIFY_TOKEN }}' + notification_title: 'GITEA: Dagu Validation' + notification_message: 'Dagu DAG validation passed! 🏁' + + - name: Gotify Notification + if: steps.validate-dags.outputs.exit_code != '0' + uses: eikendev/gotify-action@master + with: + gotify_api_base: '${{ secrets.GOTIFY_URL }}' + gotify_app_token: '${{ secrets.RUNNER_GOTIFY_TOKEN }}' + notification_title: 'GITEA: Dagu Validation' + notification_message: 'Dagu DAG validation failed! 🚫' + + - name: Trigger downstream workflow + run: | + BRANCH="${GITHUB_REF_NAME}" + curl -X POST \ + -H "Authorization: token ${{ secrets.BOT_GITEA_TOKEN }}" \ + -H "Content-Type: application/json" \ + "https://git.trez.wtf/api/v1/repos/Trez/tar-valon-ansible/actions/workflows/gitea_tar-valon_ansible_deploy.yml/dispatches" \ + -d "{\"ref\":\"${BRANCH}\"}" \ No newline at end of file diff --git a/.gitea/workflows/gitea_tar-valon_ansible_deploy.yml b/.gitea/workflows/gitea_tar-valon_ansible_deploy.yml index e574e12..4eb262f 100644 --- a/.gitea/workflows/gitea_tar-valon_ansible_deploy.yml +++ b/.gitea/workflows/gitea_tar-valon_ansible_deploy.yml @@ -2,20 +2,26 @@ name: Gitea Branch PR & Ansible Deployment on: workflow_dispatch: - workflow_run: - workflows: [Home Assistant Config Check, Validate DAGs] - types: - - completed - branches: main + # workflow_run will be supported in Gitea 1.25.x + # workflow_run: + # workflows: [Home Assistant Config Check, Validate DAGs] + # types: + # - completed + # branches: main push: branches-ignore: - 'main' paths: + - 'app-configs/**' + - 'collections/**' + - 'group_vars/**' + - 'inventory/**' + - 'playbooks/**' - '!app-configs/rikku/homeassistant/**' - '!app-configs/rinoa/dagu/dags/**' - - 'app-configs/**' env: + TEA_VERSION: '0.10.1' VAULT_ADDR: ${{ secrets.VAULT_ADDR }} VAULT_TOKEN: ${{ secrets.VAULT_GITEA_TOKEN }} @@ -33,9 +39,9 @@ jobs: - name: Install tea CLI uses: supplypike/setup-bin@v4 with: - uri: 'https://gitea.com/gitea/tea/releases/download/v0.9.2/tea-0.9.2-linux-amd64' + uri: 'https://gitea.com/gitea/tea/releases/download/v${{ env.TEA_VERSION }}/tea-${{ env.TEA_VERSION }}-linux-amd64' name: 'tea' - version: '0.9.2' + version: ${{ env.TEA_VERSION }} - name: Gotify Notification uses: eikendev/gotify-action@master @@ -96,13 +102,21 @@ jobs: with: version: "11.4.0" + - name: Set up Vault CLI + uses: eLco/setup-vault@v1 + with: + vault_version: 1.18.0 + + - name: Cache pip + uses: actions/cache@v4 + with: + path: ~/.cache/pip + key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements*.txt') }} + restore-keys: | + ${{ runner.os }}-pip- + - name: Install Vault & hvac - run: | - sudo apt-get update && sudo apt-get install -y unzip - curl -fsSL https://releases.hashicorp.com/vault/1.18.0/vault_1.18.0_linux_amd64.zip -o vault.zip - unzip vault.zip - sudo mv vault /usr/local/bin/ - pip install hvac + run: pip install hvac - name: Gotify Notification uses: eikendev/gotify-action@master @@ -144,9 +158,9 @@ jobs: - name: Install tea uses: supplypike/setup-bin@v4 with: - uri: 'https://gitea.com/gitea/tea/releases/download/v0.9.2/tea-0.9.2-linux-amd64' + uri: 'https://gitea.com/gitea/tea/releases/download/v${{ env.TEA_VERSION }}/tea-${{ env.TEA_VERSION }}-linux-amd64' name: 'tea' - version: '0.9.2' + version: ${{ env.TEA_VERSION }} - name: PR Merge id: pr_merge @@ -181,18 +195,34 @@ jobs: with: ref: main + - name: Cache Ansible Galaxy Collections + uses: actions/cache@v3 + with: + path: collections + key: ${{ runner.os }}-ansible-${{ hashFiles('./collections/requirements.yml') }} + restore-keys: | + ${{ runner.os }}-ansible- + - name: Install Ansible uses: alex-oleshkevich/setup-ansible@v1.0.1 with: version: "11.4.0" - - name: Install Vault & hvac - run: | - sudo apt-get update && sudo apt-get install -y unzip - curl -fsSL https://releases.hashicorp.com/vault/1.18.0/vault_1.18.0_linux_amd64.zip -o vault.zip - unzip vault.zip - sudo mv vault /usr/local/bin/ - pip install hvac + - name: Set up Vault CLI + uses: eLco/setup-vault@v1 + with: + vault_version: 1.18.0 + + - name: Cache pip + uses: actions/cache@v4 + with: + path: ~/.cache/pip + key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements*.txt') }} + restore-keys: | + ${{ runner.os }}-pip- + + - name: Install hvac + run: pip install hvac - name: Gotify Notification uses: eikendev/gotify-action@master diff --git a/app-configs/rinoa/dagu/dags/mariadb-backups.yaml.j2 b/app-configs/rinoa/dagu/dags/mariadb-backups.yaml.j2 index 87c5666..0437c46 100644 --- a/app-configs/rinoa/dagu/dags/mariadb-backups.yaml.j2 +++ b/app-configs/rinoa/dagu/dags/mariadb-backups.yaml.j2 @@ -1,3 +1,6 @@ +{% set vault_addr = 'https://vault.trez.wtf' %} +{% set secrets_path = 'rinoa-docker/env' %} + name: mariadb-backup description: "Backup of all databases from MariaDB container" schedule: "30 23 * * *" @@ -21,7 +24,7 @@ steps: script: | for mdatabase in $(echo ${RINOA_MADB_LIST}) ; do mkdir -p ${mdatabase} - mariadb_dump --user=root --password"${MARIADB_ROOT_PASSWORD}" --databases ${madb} > ${mdatabase}/${mdatabase}_$(date +%Y-%m-%dT%H-%M-%S).sql + mariadb_dump --u root --p"${MARIADB_ROOT_PASSWORD}" --databases ${madb} > ${mdatabase}/dump-$(date +%Y%m%d)-$(cat /proc/sys/kernel/random/uuid).sql done - name: db-backup-cleanup diff --git a/group_vars/all.yml b/group_vars/all.yml index 51355e7..14d76ef 100644 --- a/group_vars/all.yml +++ b/group_vars/all.yml @@ -5,12 +5,12 @@ template_base_path: "{{ playbook_dir }}/app-configs" vault_addr: "https://vault.trez.wtf" vault_token: !vault | $ANSIBLE_VAULT;1.1;AES256 - 37656166373362653966353330313633313864646236643832616134646335613737383561383433 - 6565666166326532396662646365663136393339666336300a346435363038303638333462356464 - 30643538643165643765366334383662666133376466323436306633623939383531363630383836 - 6239396633666636640a666363383662323562663639386436363937376435626332656161393662 - 37373434303365333261346537373062633437323062373139613633333336316537633930303965 - 37373832646530303734323939616562653431316534313164616132636337313565643230323862 - 32643834363666353038643261663932623331646462643765336566346461643566326533343631 - 32396235343534623132383962383765393063626536383631323138316432646636656563353035 - 6461 \ No newline at end of file + 65333865316461653235633230653362613830633362636662643632323430376135396333386138 + 3365623536336433653431326164376638633138396334610a326230373862626164373965373266 + 38306434346665323237336130316161346662313366643461666134366239323732653632653135 + 6230666338653038310a326239316132366530333461653364326133303432363364353162326438 + 39376665306133393631356561316433363035333134633232316535643434353763306638643965 + 35373638326531333461623766616530363366623137303535623366336539643331643934376332 + 35636435666438373936376366623566346536336638653635643431653336613138383239323564 + 36666332303661623133306133373965383731396566323834353133363363326133376134313534 + 6139 diff --git a/rinoa-render-dags.yml b/rinoa-render-dags.yml index 82fdb32..66e3cfe 100644 --- a/rinoa-render-dags.yml +++ b/rinoa-render-dags.yml @@ -5,6 +5,7 @@ gather_facts: false vars: + # Hardcoded fallback for DAGs path dags_path: "{{ lookup('env', 'DAGS_PATH') }}" vault_addr: "{{ lookup('env', 'VAULT_ADDR') }}" vault_token: "{{ lookup('env', 'VAULT_TOKEN') }}" @@ -12,26 +13,11 @@ tasks: - name: Build list of DAG template files ansible.builtin.set_fact: - dag_templates: "{{ lookup('fileglob', dags_path ~ '/*.yaml.j2', wantlist=True) }}" - - - name: Pre-check Vault secrets in templates - when: dag_templates | length > 0 - block: - - name: Find all Vault lookup expressions in templates - ansible.builtin.set_fact: - vault_keys: >- - {{ - dag_templates - | map('lookup', 'file', wantlist=True) - | map('regex_findall', - "lookup\\('community.hashi_vault.vault_kv2_get',\\s*'[^']+',\\s*engine_mount_point='[^']+',\\s*url=[^,]+,\\s*token=[^\\)]+\\)\\['secret'\\]\\['([^']+)'\\]") - | sum(start=[]) - }} - - - name: Warn if any Vault keys might be missing - loop: "{{ vault_keys }}" - ansible.builtin.debug: - msg: "Vault key '{{ item }}' will be required by templates" + dag_templates: >- + {{ + lookup('ansible.builtin.fileglob', dags_path ~ '/*.yaml.j2', wantlist=True) + | default([]) + }} - name: Render DAG templates in-place (guarded) when: dag_templates | length > 0