From 6b37ad8ea4663a57b4725fe88a89b0c57cb1398e Mon Sep 17 00:00:00 2001 From: "Trez.One" Date: Tue, 20 May 2025 09:06:49 -0400 Subject: [PATCH 01/10] Ansible DRY fixes for Vault lookups. --- ansible/app-configs/apprise_apprise.yml.j2 | 4 ++-- .../app-configs/authelia_configuration.yml.j2 | 20 ++++++++-------- .../crowdsec_local-api-credentials.yaml.j2 | 2 +- .../ghost_config.production.json.j2 | 6 ++--- ansible/app-configs/gitea_app.ini.j2 | 10 ++++---- .../gitea_gitea-sonarqube-bot_config.yaml.j2 | 8 +++---- .../grafana_pyroscope_config.yaml.j2 | 4 ++-- .../app-configs/grafana_tempo_tempo.yaml.j2 | 4 ++-- ansible/app-configs/homepage_services.yaml.j2 | 2 +- ansible/app-configs/homepage_settings.yaml.j2 | 2 +- ansible/app-configs/invidious_config.yml.j2 | 8 +++---- .../invoice-ninja_invoice-ninja.env.j2 | 8 +++---- .../app-configs/librechat_librechat.env.j2 | 22 ++++++++--------- ansible/app-configs/lidarr_config.xml.j2 | 2 +- .../lidify_settings_config.json.j2 | 12 +++++----- ansible/app-configs/loggifly_config.yaml.j2 | 2 +- .../multi-scrobbler_config.json.j2 | 22 ++++++++--------- .../app-configs/netbird_management.json.j2 | 24 +++++++++---------- .../app-configs/netbird_turnserver.conf.j2 | 2 +- ansible/app-configs/postal_postal.yml.j2 | 8 +++---- ansible/app-configs/prowlarr_config.xml.j2 | 2 +- .../radarec_settings_config.json.j2 | 4 ++-- ansible/app-configs/radarr_config.xml.j2 | 2 +- ansible/app-configs/readarr_config.xml.j2 | 2 +- ansible/app-configs/sabnzbdvpn_sabnzbd.ini.j2 | 8 +++---- ansible/app-configs/searxng_settings.yml.j2 | 2 +- ansible/app-configs/sonarr_config.xml.j2 | 2 +- .../sonashow_settings_config.json.j2 | 4 ++-- ansible/app-configs/soularr_config.ini.j2 | 4 ++-- ansible/app-configs/soulseek_slskd.yml.j2 | 6 ++--- ansible/app-configs/sourcebot_config.json.j2 | 2 +- ansible/app-configs/traccar_traccar.xml.j2 | 2 +- ansible/app-configs/vector_vector.yaml.j2 | 2 +- ansible/app-configs/wazuh_wazuh.yml.j2 | 2 +- ansible/app-configs/zitadel_config.yaml.j2 | 4 ++-- ansible/app-configs/zitadel_secrets.yaml.j2 | 4 ++-- ansible/docker_config_deploy.yml | 6 +++++ 37 files changed, 118 insertions(+), 112 deletions(-) diff --git a/ansible/app-configs/apprise_apprise.yml.j2 b/ansible/app-configs/apprise_apprise.yml.j2 index dfaeb255..f840500a 100644 --- a/ansible/app-configs/apprise_apprise.yml.j2 +++ b/ansible/app-configs/apprise_apprise.yml.j2 @@ -2,5 +2,5 @@ {% set secrets_path = 'rinoa-docker/env' %} urls: - - gotify://gotify/{{ lookup('community.hashi_vault.vault_kv2_get', 'env', engine_mount_point='rinoa-docker', url=vault_addr, token=vault_token_cleaned)['secret']['APPRISE_GOTIFY_TOKEN'] }} - - mailtos://{{ lookup('community.hashi_vault.vault_kv2_get', 'env', engine_mount_point='rinoa-docker', url=vault_addr, token=vault_token_cleaned)['secret']['POSTAL_SMTP_AUTH_USER'] }}:{{ lookup('community.hashi_vault.vault_kv2_get', 'env', engine_mount_point='rinoa-docker', url=vault_addr, token=vault_token_cleaned)['secret']['POSTAL_SMTP_AUTH_PASSWORD'] }}@trez.wtf25?smtp=postal-smtp&from=noreply@trez.wtf \ No newline at end of file + - gotify://gotify/{{ vault_secrets['APPRISE_GOTIFY_TOKEN'] }} + - mailtos://{{ vault_secrets['POSTAL_SMTP_AUTH_USER'] }}:{{ vault_secrets['POSTAL_SMTP_AUTH_PASSWORD'] }}@trez.wtf25?smtp=postal-smtp&from=noreply@trez.wtf \ No newline at end of file diff --git a/ansible/app-configs/authelia_configuration.yml.j2 b/ansible/app-configs/authelia_configuration.yml.j2 index 48764283..d7f93949 100644 --- a/ansible/app-configs/authelia_configuration.yml.j2 +++ b/ansible/app-configs/authelia_configuration.yml.j2 @@ -64,11 +64,11 @@ authentication_backend: mail: mail display_name: displayName user: uid=authelia,ou=people,dc=trez,dc=wtf - password: '{{ lookup('community.hashi_vault.vault_kv2_get', 'env', engine_mount_point='rinoa-docker', url=vault_addr, token=vault_token_cleaned)['secret']['AUTHELIA_AUTH_BIND_LDAP_PASSWORD'] }}' + password: '{{ vault_secrets['AUTHELIA_AUTH_BIND_LDAP_PASSWORD'] }}' refresh_interval: 5m identity_validation: reset_password: - jwt_secret: '{{ lookup('community.hashi_vault.vault_kv2_get', 'env', engine_mount_point='rinoa-docker', url=vault_addr, token=vault_token_cleaned)['secret']['AUTHELIA_JWT_SECRET'] }}' + jwt_secret: '{{ vault_secrets['AUTHELIA_JWT_SECRET'] }}' password_policy: standard: enabled: true @@ -104,7 +104,7 @@ access_control: - ['user:the.trezured.one'] session: name: authelia_session - secret: '{{ lookup('community.hashi_vault.vault_kv2_get', 'env', engine_mount_point='rinoa-docker', url=vault_addr, token=vault_token_cleaned)['secret']['AUTHELIA_SESSION_SECRET'] }}' + secret: '{{ vault_secrets['AUTHELIA_SESSION_SECRET'] }}' expiration: 1h inactivity: 5m remember_me: 1M @@ -115,12 +115,12 @@ session: host: redis port: 6379 storage: - encryption_key: '{{ lookup('community.hashi_vault.vault_kv2_get', 'env', engine_mount_point='rinoa-docker', url=vault_addr, token=vault_token_cleaned)['secret']['AUTHELIA_STORAGE_ENCRYPTION_KEY'] }}' + encryption_key: '{{ vault_secrets['AUTHELIA_STORAGE_ENCRYPTION_KEY'] }}' postgres: address: 'tcp://authelia-pg:5432' database: authelia username: authelia - password: '{{ lookup('community.hashi_vault.vault_kv2_get', 'env', engine_mount_point='rinoa-docker', url=vault_addr, token=vault_token_cleaned)['secret']['AUTHELIA_STORAGE_POSTGRES_PASSWORD'] }}' + password: '{{ vault_secrets['AUTHELIA_STORAGE_POSTGRES_PASSWORD'] }}' timeout: '5s' regulation: max_retries: 3 @@ -131,8 +131,8 @@ notifier: smtp: address: 'smtp://postal-smtp:25' timeout: '5s' - username: '{{ lookup('community.hashi_vault.vault_kv2_get', 'env', engine_mount_point='rinoa-docker', url=vault_addr, token=vault_token_cleaned)['secret']['POSTAL_SMTP_AUTH_USER'] }}' - password: '{{ lookup('community.hashi_vault.vault_kv2_get', 'env', engine_mount_point='rinoa-docker', url=vault_addr, token=vault_token_cleaned)['secret']['POSTAL_SMTP_AUTH_PASSWORD'] }}' + username: '{{ vault_secrets['POSTAL_SMTP_AUTH_USER'] }}' + password: '{{ vault_secrets['POSTAL_SMTP_AUTH_PASSWORD'] }}' sender: "Authelia " identifier: 'localhost' subject: "[Authelia] {title}" @@ -142,10 +142,10 @@ notifier: disable_html_emails: false identity_providers: oidc: - hmac_secret: '{{ lookup('community.hashi_vault.vault_kv2_get', 'env', engine_mount_point='rinoa-docker', url=vault_addr, token=vault_token_cleaned)['secret']['AUTHELIA_OIDC_HMAC_SECRET'] }}' + hmac_secret: '{{ vault_secrets['AUTHELIA_OIDC_HMAC_SECRET'] }}' jwks: - key: | - {{ lookup("community.hashi_vault.vault_kv2_get", "env", engine_mount_point="rinoa-docker", url=vault_addr, token=vault_token_cleaned)["secret"]["AUTHELIA_OIDC_JWKS_KEY"] | replace("\\n", "\n") | indent(10) }} + {{ vault_secrets["AUTHELIA_OIDC_JWKS_KEY"] | replace("\\n", "\n") | indent(10) }} cors: allowed_origins_from_client_redirect_uris: true endpoints: @@ -157,7 +157,7 @@ identity_providers: clients: - client_id: 'netbird' client_name: 'NetBird' - client_secret: '{{ lookup('community.hashi_vault.vault_kv2_get', 'env', engine_mount_point='rinoa-docker', url=vault_addr, token=vault_token_cleaned)['secret']['AUTHELIA_NETBIRD_CLIENT_SECRET'] }}' + client_secret: '{{ vault_secrets['AUTHELIA_NETBIRD_CLIENT_SECRET'] }}' public: false authorization_policy: 'two_factor' redirect_uris: diff --git a/ansible/app-configs/crowdsec_local-api-credentials.yaml.j2 b/ansible/app-configs/crowdsec_local-api-credentials.yaml.j2 index 5335296f..84c339d4 100644 --- a/ansible/app-configs/crowdsec_local-api-credentials.yaml.j2 +++ b/ansible/app-configs/crowdsec_local-api-credentials.yaml.j2 @@ -3,4 +3,4 @@ url: http://0.0.0.0:8080 login: localhost -password: {{ lookup('community.hashi_vault.vault_kv2_get', 'env', engine_mount_point='rinoa-docker', url=vault_addr, token=vault_token_cleaned)['secret']['CROWDSEC_LOCAL_API_KEY'] }} \ No newline at end of file +password: {{ vault_secrets['CROWDSEC_LOCAL_API_KEY'] }} \ No newline at end of file diff --git a/ansible/app-configs/ghost_config.production.json.j2 b/ansible/app-configs/ghost_config.production.json.j2 index 40e0dd14..6e878074 100644 --- a/ansible/app-configs/ghost_config.production.json.j2 +++ b/ansible/app-configs/ghost_config.production.json.j2 @@ -9,7 +9,7 @@ "host" : "mariadb", "port" : 3306, "user" : "ghost", - "password" : "{{ lookup('community.hashi_vault.vault_kv2_get', 'env', engine_mount_point='rinoa-docker', url=vault_addr, token=vault_token_cleaned)['secret']['GHOST_DB_PASSWORD'] }}", + "password" : "{{ vault_secrets['GHOST_DB_PASSWORD'] }}", "database" : "ghost_db" } }, @@ -21,8 +21,8 @@ "port": 25, "secure": false, "auth": { - "user": "{{ lookup('community.hashi_vault.vault_kv2_get', 'env', engine_mount_point='rinoa-docker', url=vault_addr, token=vault_token_cleaned)['secret']['POSTAL_SMTP_AUTH_USER'] }}", - "pass": "{{ lookup('community.hashi_vault.vault_kv2_get', 'env', engine_mount_point='rinoa-docker', url=vault_addr, token=vault_token_cleaned)['secret']['POSTAL_SMTP_AUTH_PASSWORD'] }}" + "user": "{{ vault_secrets['POSTAL_SMTP_AUTH_USER'] }}", + "pass": "{{ vault_secrets['POSTAL_SMTP_AUTH_PASSWORD'] }}" } } }, diff --git a/ansible/app-configs/gitea_app.ini.j2 b/ansible/app-configs/gitea_app.ini.j2 index bc4e810b..5764285f 100644 --- a/ansible/app-configs/gitea_app.ini.j2 +++ b/ansible/app-configs/gitea_app.ini.j2 @@ -27,7 +27,7 @@ DISABLE_SSH = false SSH_PORT = 22 SSH_LISTEN_PORT = 22 LFS_START_SERVER = true -LFS_JWT_SECRET = {{ lookup('community.hashi_vault.vault_kv2_get', 'env', engine_mount_point='rinoa-docker', url=vault_addr, token=vault_token_cleaned)['secret']['GITEA_LFS_JWT_SECRET'] }} +LFS_JWT_SECRET = {{ vault_secrets['GITEA_LFS_JWT_SECRET'] }} OFFLINE_MODE = true [database] @@ -36,7 +36,7 @@ DB_TYPE = postgres HOST = gitea-db:5432 NAME = gitea USER = gitea -PASSWD = {{ lookup('community.hashi_vault.vault_kv2_get', 'env', engine_mount_point='rinoa-docker', url=vault_addr, token=vault_token_cleaned)['secret']['GITEA_PG_DB_PASSWORD'] }} +PASSWD = {{ vault_secrets['GITEA_PG_DB_PASSWORD'] }} LOG_SQL = false SCHEMA = SSL_MODE = disable @@ -70,7 +70,7 @@ INSTALL_LOCK = true SECRET_KEY = REVERSE_PROXY_LIMIT = 1 REVERSE_PROXY_TRUSTED_PROXIES = * -INTERNAL_TOKEN = {{ lookup('community.hashi_vault.vault_kv2_get', 'env', engine_mount_point='rinoa-docker', url=vault_addr, token=vault_token_cleaned)['secret']['GITEA_INTERNAL_TOKEN'] }} +INTERNAL_TOKEN = {{ vault_secrets['GITEA_INTERNAL_TOKEN'] }} PASSWORD_HASH_ALGO = pbkdf2 [service] @@ -89,7 +89,7 @@ NO_REPLY_ADDRESS = noreply@trez.wtf PATH = /data/git/lfs [mailer] -PASSWD = {{ lookup('community.hashi_vault.vault_kv2_get', 'env', engine_mount_point='rinoa-docker', url=vault_addr, token=vault_token_cleaned)['secret']['POSTAL_SMTP_AUTH_PASSWORD'] }} +PASSWD = {{ vault_secrets['POSTAL_SMTP_AUTH_PASSWORD'] }} PROTOCOL = smtp ENABLED = true FROM = '"Gitea" ' @@ -112,7 +112,7 @@ DEFAULT_MERGE_STYLE = merge DEFAULT_TRUST_MODEL = committer [oauth2] -JWT_SECRET = {{ lookup('community.hashi_vault.vault_kv2_get', 'env', engine_mount_point='rinoa-docker', url=vault_addr, token=vault_token_cleaned)['secret']['GITEA_OAUTH2_JWT_SECRET'] }} +JWT_SECRET = {{ vault_secrets['GITEA_OAUTH2_JWT_SECRET'] }} [ui] THEMES = diff --git a/ansible/app-configs/gitea_gitea-sonarqube-bot_config.yaml.j2 b/ansible/app-configs/gitea_gitea-sonarqube-bot_config.yaml.j2 index 90b9fb69..62f5f53f 100644 --- a/ansible/app-configs/gitea_gitea-sonarqube-bot_config.yaml.j2 +++ b/ansible/app-configs/gitea_gitea-sonarqube-bot_config.yaml.j2 @@ -9,7 +9,7 @@ gitea: # Created access token for the user that shall be used as bot account. # User needs "Read project" permissions with access to "Pull Requests" token: - value: "{{ lookup('community.hashi_vault.vault_kv2_get', 'env', engine_mount_point='rinoa-docker', url=vault_addr, token=vault_token_cleaned)['secret']['GITEA_SONARQUBE_BOT_GITEA_TOKEN'] }}" + value: "{{ vault_secrets['GITEA_SONARQUBE_BOT_GITEA_TOKEN'] }}" # # or path to file containing the plain text secret # file: /path/to/gitea/token @@ -18,7 +18,7 @@ gitea: # The bot looks for `X-Gitea-Signature` header containing the sha256 hmac hash of the plain text secret. If the header # exists and no webhookSecret is defined here, the bot will ignore the request, because it cannot be validated. webhook: - secret: "{{ lookup('community.hashi_vault.vault_kv2_get', 'env', engine_mount_point='rinoa-docker', url=vault_addr, token=vault_token_cleaned)['secret']['GITEA_SONARQUBE_BOT_GITEA_WEBHOOK_SECRET'] }}" + secret: "{{ vault_secrets['GITEA_SONARQUBE_BOT_GITEA_WEBHOOK_SECRET'] }}" # # or path to file containing the plain text secret # secretFile: /path/to/gitea/webhook/secret @@ -35,7 +35,7 @@ sonarqube: # Created access token for the user that shall be used as bot account. # User needs "Browse on project" permissions token: - value: "{{ lookup('community.hashi_vault.vault_kv2_get', 'env', engine_mount_point='rinoa-docker', url=vault_addr, token=vault_token_cleaned)['secret']['GITEA_SONARQUBE_BOT_SQUBE_TOKEN'] }}" + value: "{{ vault_secrets['GITEA_SONARQUBE_BOT_SQUBE_TOKEN'] }}" # # or path to file containing the plain text secret # file: /path/to/sonarqube/token @@ -45,7 +45,7 @@ sonarqube: # If the header exists and no webhookSecret is defined here, the bot will ignore the request, because it cannot be # validated. webhook: - secret: "{{ lookup('community.hashi_vault.vault_kv2_get', 'env', engine_mount_point='rinoa-docker', url=vault_addr, token=vault_token_cleaned)['secret']['GITEA_SONARQUBE_BOT_SQUBE_WEBHOOK_SECRET'] }}" + secret: "{{ vault_secrets['GITEA_SONARQUBE_BOT_SQUBE_WEBHOOK_SECRET'] }}" # # or path to file containing the plain text secret # secretFile: /path/to/sonarqube/webhook/secret diff --git a/ansible/app-configs/grafana_pyroscope_config.yaml.j2 b/ansible/app-configs/grafana_pyroscope_config.yaml.j2 index fe8066be..739222d9 100644 --- a/ansible/app-configs/grafana_pyroscope_config.yaml.j2 +++ b/ansible/app-configs/grafana_pyroscope_config.yaml.j2 @@ -4,8 +4,8 @@ storage: bucket_name: pyroscope endpoint: minio:9000 region: us-east-fh-pln - access_key_id: {{ lookup('community.hashi_vault.vault_kv2_get', 'env', engine_mount_point='rinoa-docker', url=vault_addr, token=vault_token_cleaned)['secret']['MINIO_PYROSCOPE_STORAGE_ACCESS_KEY'] }} - secret_access_key: {{ lookup('community.hashi_vault.vault_kv2_get', 'env', engine_mount_point='rinoa-docker', url=vault_addr, token=vault_token_cleaned)['secret']['MINIO_PYROSCOPE_STORAGE_SECRET_KEY'] }} + access_key_id: {{ vault_secrets['MINIO_PYROSCOPE_STORAGE_ACCESS_KEY'] }} + secret_access_key: {{ vault_secrets['MINIO_PYROSCOPE_STORAGE_SECRET_KEY'] }} insecure: true analytics: diff --git a/ansible/app-configs/grafana_tempo_tempo.yaml.j2 b/ansible/app-configs/grafana_tempo_tempo.yaml.j2 index fbfd0050..07bfcff7 100644 --- a/ansible/app-configs/grafana_tempo_tempo.yaml.j2 +++ b/ansible/app-configs/grafana_tempo_tempo.yaml.j2 @@ -46,8 +46,8 @@ storage: s3: bucket: tempo # how to store data in s3 endpoint: minio:9000 - access_key: {{ lookup('community.hashi_vault.vault_kv2_get', 'env', engine_mount_point='rinoa-docker', url=vault_addr, token=vault_token_cleaned)['secret']['MINIO_TEMPO_STORAGE_ACCESS_KEY'] }} - secret_key: {{ lookup('community.hashi_vault.vault_kv2_get', 'env', engine_mount_point='rinoa-docker', url=vault_addr, token=vault_token_cleaned)['secret']['MINIO_TEMPO_STORAGE_SECRET_KEY'] }} + access_key: {{ vault_secrets['MINIO_TEMPO_STORAGE_ACCESS_KEY'] }} + secret_key: {{ vault_secrets['MINIO_TEMPO_STORAGE_SECRET_KEY'] }} insecure: true usage_report: diff --git a/ansible/app-configs/homepage_services.yaml.j2 b/ansible/app-configs/homepage_services.yaml.j2 index 7a610bbc..6161e3a0 100644 --- a/ansible/app-configs/homepage_services.yaml.j2 +++ b/ansible/app-configs/homepage_services.yaml.j2 @@ -29,5 +29,5 @@ widget: type: homeassistant url: http://192.168.1.252:8123 - key: {{ lookup('community.hashi_vault.vault_kv2_get', 'env', engine_mount_point='rinoa-docker', url=vault_addr, token=vault_token_cleaned)['secret']['HOMEPAGE_HOME_ASSISTANT_API_KEY'] }} + key: {{ vault_secrets['HOMEPAGE_HOME_ASSISTANT_API_KEY'] }} diff --git a/ansible/app-configs/homepage_settings.yaml.j2 b/ansible/app-configs/homepage_settings.yaml.j2 index 53bc3b69..de216146 100644 --- a/ansible/app-configs/homepage_settings.yaml.j2 +++ b/ansible/app-configs/homepage_settings.yaml.j2 @@ -6,7 +6,7 @@ # https://gethomepage.dev/en/configs/settings providers: - openweathermap: {{ lookup('community.hashi_vault.vault_kv2_get', 'env', engine_mount_point='rinoa-docker', url=vault_addr, token=vault_token_cleaned)['secret']['HOMEPAGE_OPENWEATHERMAP_API_KEY'] }} + openweathermap: {{ vault_secrets['HOMEPAGE_OPENWEATHERMAP_API_KEY'] }} # weatherapi: weatherapiapikey title: Rinoa Dashboard (trez.WTF) headerStyle: underlined diff --git a/ansible/app-configs/invidious_config.yml.j2 b/ansible/app-configs/invidious_config.yml.j2 index 7aff109b..1a63ebd4 100644 --- a/ansible/app-configs/invidious_config.yml.j2 +++ b/ansible/app-configs/invidious_config.yml.j2 @@ -16,7 +16,7 @@ db: host: invidious-db port: 5432 dbname: invidious - password: {{ lookup('community.hashi_vault.vault_kv2_get', 'env', engine_mount_point='rinoa-docker', url=vault_addr, token=vault_token_cleaned)['secret']['INVID_PG_DB_PASSWORD'] }} + password: {{ vault_secrets['INVID_PG_DB_PASSWORD'] }} ## ## Database configuration using a single URI. This is an @@ -210,8 +210,8 @@ https_only: false ## Accepted values: String ## Default: ## -po_token: {{ lookup('community.hashi_vault.vault_kv2_get', 'env', engine_mount_point='rinoa-docker', url=vault_addr, token=vault_token_cleaned)['secret']['INVID_PO_TOKEN'] }} -visitor_data: {{ lookup('community.hashi_vault.vault_kv2_get', 'env', engine_mount_point='rinoa-docker', url=vault_addr, token=vault_token_cleaned)['secret']['INVID_VISITOR_DATA'] }} +po_token: {{ vault_secrets['INVID_PO_TOKEN'] }} +visitor_data: {{ vault_secrets['INVID_VISITOR_DATA'] }} # ----------------------------- # Logging @@ -471,7 +471,7 @@ jobs: ## Accepted values: a string ## Default: ## -hmac_key: {{ lookup('community.hashi_vault.vault_kv2_get', 'env', engine_mount_point='rinoa-docker', url=vault_addr, token=vault_token_cleaned)['secret']['INVID_HMAC_KEY'] }} +hmac_key: {{ vault_secrets['INVID_HMAC_KEY'] }} ## ## List of video IDs where the "download" widget must be diff --git a/ansible/app-configs/invoice-ninja_invoice-ninja.env.j2 b/ansible/app-configs/invoice-ninja_invoice-ninja.env.j2 index 2c8dc97a..331932de 100644 --- a/ansible/app-configs/invoice-ninja_invoice-ninja.env.j2 +++ b/ansible/app-configs/invoice-ninja_invoice-ninja.env.j2 @@ -3,7 +3,7 @@ # IN application vars IN_APP_URL=https://biz.trez.wtf -IN_APP_KEY={{ lookup('community.hashi_vault.vault_kv2_get', 'env', engine_mount_point='rinoa-docker', url=vault_addr, token=vault_token_cleaned)['secret']['IN_APP_KEY'] }} +IN_APP_KEY={{ vault_secrets['IN_APP_KEY'] }} IN_APP_DEBUG=true IN_REQUIRE_HTTPS=false IN_PHANTOMJS_PDF_GENERATION=false @@ -18,7 +18,7 @@ IN_DB_HOST=mariadb IN_DB_PORT=3306 IN_DB_DATABASE=invoice_ninja IN_DB_USERNAME=ininja -IN_DB_PASSWORD={{ lookup('community.hashi_vault.vault_kv2_get', 'env', engine_mount_point='rinoa-docker', url=vault_addr, token=vault_token_cleaned)['secret']['IN_MYSQL_PASSWORD'] }} +IN_DB_PASSWORD={{ vault_secrets['IN_MYSQL_PASSWORD'] }} # Create initial user # Default to these values if empty @@ -31,8 +31,8 @@ IN_PASSWORD= IN_MAIL_MAILER=log IN_MAIL_HOST=postal-smtp IN_MAIL_PORT=25 -IN_MAIL_USERNAME={{ lookup('community.hashi_vault.vault_kv2_get', 'env', engine_mount_point='rinoa-docker', url=vault_addr, token=vault_token_cleaned)['secret']['POSTAL_SMTP_AUTH_USER'] }} -IN_MAIL_PASSWORD={{ lookup('community.hashi_vault.vault_kv2_get', 'env', engine_mount_point='rinoa-docker', url=vault_addr, token=vault_token_cleaned)['secret']['POSTAL_SMTP_AUTH_PASSWORD'] }} +IN_MAIL_USERNAME={{ vault_secrets['POSTAL_SMTP_AUTH_USER'] }} +IN_MAIL_PASSWORD={{ vault_secrets['POSTAL_SMTP_AUTH_PASSWORD'] }} IN_MAIL_ENCRYPTION=null IN_MAIL_FROM_ADDRESS='noreply@trez.wtf' IN_MAIL_FROM_NAME='Treasured IT' diff --git a/ansible/app-configs/librechat_librechat.env.j2 b/ansible/app-configs/librechat_librechat.env.j2 index 456f9bbc..b8bcc7a9 100644 --- a/ansible/app-configs/librechat_librechat.env.j2 +++ b/ansible/app-configs/librechat_librechat.env.j2 @@ -17,7 +17,7 @@ HOST=localhost PORT=3080 -MONGO_URI=mongodb://librechat:{{ lookup('community.hashi_vault.vault_kv2_get', 'env', engine_mount_point='rinoa-docker', url=vault_addr, token=vault_token_cleaned)['secret']['LIBRECHAT_MONGODB_PASSWORD'] }}@mongodb:27017/librechat?replicaSet=rinoa +MONGO_URI=mongodb://librechat:{{ vault_secrets['LIBRECHAT_MONGODB_PASSWORD'] }}@mongodb:27017/librechat?replicaSet=rinoa DOMAIN_CLIENT=https://ai.trez.wtf DOMAIN_SERVER=https://ai.trez.wtf @@ -73,12 +73,12 @@ PROXY= # ANYSCALE_API_KEY= # APIPIE_API_KEY= # COHERE_API_KEY= -DEEPSEEK_API_KEY={{ lookup('community.hashi_vault.vault_kv2_get', 'env', engine_mount_point='rinoa-docker', url=vault_addr, token=vault_token_cleaned)['secret']['LIBRECHAT_DEEPSEEK_API_KEY'] }} +DEEPSEEK_API_KEY={{ vault_secrets['LIBRECHAT_DEEPSEEK_API_KEY'] }} # DATABRICKS_API_KEY= # FIREWORKS_API_KEY= # GROQ_API_KEY= # HUGGINGFACE_TOKEN= -MISTRAL_API_KEY={{ lookup('community.hashi_vault.vault_kv2_get', 'env', engine_mount_point='rinoa-docker', url=vault_addr, token=vault_token_cleaned)['secret']['LIBRECHAT_MISTRAL_API_KEY'] }} +MISTRAL_API_KEY={{ vault_secrets['LIBRECHAT_MISTRAL_API_KEY'] }} # OPENROUTER_KEY= # PERPLEXITY_API_KEY= # SHUTTLEAI_API_KEY= @@ -90,7 +90,7 @@ MISTRAL_API_KEY={{ lookup('community.hashi_vault.vault_kv2_get', 'env', engine_m # Anthropic # #============# -ANTHROPIC_API_KEY={{ lookup('community.hashi_vault.vault_kv2_get', 'env', engine_mount_point='rinoa-docker', url=vault_addr, token=vault_token_cleaned)['secret']['LIBRECHAT_ANTHROPIC_API_KEY'] }} +ANTHROPIC_API_KEY={{ vault_secrets['LIBRECHAT_ANTHROPIC_API_KEY'] }} ANTHROPIC_MODELS=claude-3-7-sonnet-latest,claude-3-7-sonnet-20250219,claude-3-5-haiku-20241022,claude-3-5-sonnet-20241022,claude-3-5-sonnet-latest,claude-3-5-sonnet-20240620,claude-3-opus-20240229,claude-3-sonnet-20240229,claude-3-haiku-20240307,claude-2.1,claude-2,claude-1.2,claude-1,claude-1-100k,claude-instant-1,claude-instant-1-100k # ANTHROPIC_REVERSE_PROXY= @@ -177,7 +177,7 @@ ANTHROPIC_MODELS=claude-3-7-sonnet-latest,claude-3-7-sonnet-20250219,claude-3-5- # OpenAI # #============# -OPENAI_API_KEY={{ lookup('community.hashi_vault.vault_kv2_get', 'env', engine_mount_point='rinoa-docker', url=vault_addr, token=vault_token_cleaned)['secret']['LIBRECHAT_OPENAI_API_KEY'] }} +OPENAI_API_KEY={{ vault_secrets['LIBRECHAT_OPENAI_API_KEY'] }} OPENAI_MODELS=o1,o1-mini,o1-preview,gpt-4o,chatgpt-4o-latest,gpt-4o-mini,gpt-3.5-turbo-0125,gpt-3.5-turbo-0301,gpt-3.5-turbo,gpt-4,gpt-4-0613,gpt-4-vision-preview,gpt-3.5-turbo-0613,gpt-3.5-turbo-16k-0613,gpt-4-0125-preview,gpt-4-turbo-preview,gpt-4-1106-preview,gpt-3.5-turbo-1106,gpt-3.5-turbo-instruct,gpt-3.5-turbo-instruct-0914,gpt-3.5-turbo-16k DEBUG_OPENAI=false @@ -226,8 +226,8 @@ DEBUG_OPENAI=false # DEBUG_PLUGINS= -CREDS_KEY={{ lookup('community.hashi_vault.vault_kv2_get', 'env', engine_mount_point='rinoa-docker', url=vault_addr, token=vault_token_cleaned)['secret']['LIBRECHAT_CREDS_KEY'] }} -CREDS_IV={{ lookup('community.hashi_vault.vault_kv2_get', 'env', engine_mount_point='rinoa-docker', url=vault_addr, token=vault_token_cleaned)['secret']['LIBRECHAT_CREDS_IV'] }} +CREDS_KEY={{ vault_secrets['LIBRECHAT_CREDS_KEY'] }} +CREDS_IV={{ vault_secrets['LIBRECHAT_CREDS_IV'] }} # Azure AI Search #----------------- @@ -298,7 +298,7 @@ ZAPIER_NLA_API_KEY= SEARCH=true MEILI_NO_ANALYTICS=true MEILI_HOST=http://meilisearch:7700 -MEILI_MASTER_KEY={{ lookup('community.hashi_vault.vault_kv2_get', 'env', engine_mount_point='rinoa-docker', url=vault_addr, token=vault_token_cleaned)['secret']['MEILISEARCH_MASTER_KEY'] }} +MEILI_MASTER_KEY={{ vault_secrets['MEILISEARCH_MASTER_KEY'] }} # Optional: Disable indexing, useful in a multi-node setup # where only one instance should perform an index sync. @@ -384,8 +384,8 @@ ALLOW_UNVERIFIED_EMAIL_LOGIN=true SESSION_EXPIRY=1000 * 60 * 15 REFRESH_TOKEN_EXPIRY=(1000 * 60 * 60 * 24) * 7 -JWT_SECRET={{ lookup('community.hashi_vault.vault_kv2_get', 'env', engine_mount_point='rinoa-docker', url=vault_addr, token=vault_token_cleaned)['secret']['LIBRECHAT_JWT_SECRET'] }} -JWT_REFRESH_SECRET={{ lookup('community.hashi_vault.vault_kv2_get', 'env', engine_mount_point='rinoa-docker', url=vault_addr, token=vault_token_cleaned)['secret']['LIBRECHAT_JWT_REFRESH_SECRET'] }} +JWT_SECRET={{ vault_secrets['LIBRECHAT_JWT_SECRET'] }} +JWT_REFRESH_SECRET={{ vault_secrets['LIBRECHAT_JWT_REFRESH_SECRET'] }} # Discord @@ -547,4 +547,4 @@ USE_REDIS=true #=====================================================# # OpenWeather # #=====================================================# -OPENWEATHER_API_KEY={{ lookup('community.hashi_vault.vault_kv2_get', 'env', engine_mount_point='rinoa-docker', url=vault_addr, token=vault_token_cleaned)['secret']['HOMEPAGE_OPENWEATHERMAP_API_KEY'] }} \ No newline at end of file +OPENWEATHER_API_KEY={{ vault_secrets['HOMEPAGE_OPENWEATHERMAP_API_KEY'] }} \ No newline at end of file diff --git a/ansible/app-configs/lidarr_config.xml.j2 b/ansible/app-configs/lidarr_config.xml.j2 index 7ff4318b..cf112524 100644 --- a/ansible/app-configs/lidarr_config.xml.j2 +++ b/ansible/app-configs/lidarr_config.xml.j2 @@ -7,7 +7,7 @@ 6868 False True - {{ lookup('community.hashi_vault.vault_kv2_get', 'env', engine_mount_point='rinoa-docker', url=vault_addr, token=vault_token_cleaned)['secret']['LIDARR_API_KEY'] }} + {{ vault_secrets['LIDARR_API_KEY'] }} Forms master trace diff --git a/ansible/app-configs/lidify_settings_config.json.j2 b/ansible/app-configs/lidify_settings_config.json.j2 index cc87a3b6..f6de799a 100644 --- a/ansible/app-configs/lidify_settings_config.json.j2 +++ b/ansible/app-configs/lidify_settings_config.json.j2 @@ -3,11 +3,11 @@ { "lidarr_address": "http://lidarr:8686", - "lidarr_api_key": "{{ lookup('community.hashi_vault.vault_kv2_get', 'env', engine_mount_point='rinoa-docker', url=vault_addr, token=vault_token_cleaned)['secret']['LIDARR_API_KEY'] }}", - "spotify_client_secret": "{{ lookup('community.hashi_vault.vault_kv2_get', 'env', engine_mount_point='rinoa-docker', url=vault_addr, token=vault_token_cleaned)['secret']['YOUR_SPOTIFY_SECRET'] }}", + "lidarr_api_key": "{{ vault_secrets['LIDARR_API_KEY'] }}", + "spotify_client_secret": "{{ vault_secrets['YOUR_SPOTIFY_SECRET'] }}", "root_folder_path": "/data/media/music", - "spotify_client_id": "{{ lookup('community.hashi_vault.vault_kv2_get', 'env', engine_mount_point='rinoa-docker', url=vault_addr, token=vault_token_cleaned)['secret']['YOUR_SPOTIFY_ID'] }}", - "spotify_client_secret": "{{ lookup('community.hashi_vault.vault_kv2_get', 'env', engine_mount_point='rinoa-docker', url=vault_addr, token=vault_token_cleaned)['secret']['YOUR_SPOTIFY_SECRET'] }}", + "spotify_client_id": "{{ vault_secrets['YOUR_SPOTIFY_ID'] }}", + "spotify_client_secret": "{{ vault_secrets['YOUR_SPOTIFY_SECRET'] }}", "fallback_to_top_result": false, "lidarr_api_timeout": 120.0, "quality_profile_id": 1, @@ -17,8 +17,8 @@ "app_name": "lidify", "app_rev": "0.09", "app_url": "lidify.trez.wtf", - "last_fm_api_key": "{{ lookup('community.hashi_vault.vault_kv2_get', 'env', engine_mount_point='rinoa-docker', url=vault_addr, token=vault_token_cleaned)['secret']['LASTFM_API_KEY'] }}", - "last_fm_api_secret": "{{ lookup('community.hashi_vault.vault_kv2_get', 'env', engine_mount_point='rinoa-docker', url=vault_addr, token=vault_token_cleaned)['secret']['LASTFM_API_SECRET'] }}", + "last_fm_api_key": "{{ vault_secrets['LASTFM_API_KEY'] }}", + "last_fm_api_secret": "{{ vault_secrets['LASTFM_API_SECRET'] }}", "mode": "LastFM", "auto_start": false, "auto_start_delay": 60 diff --git a/ansible/app-configs/loggifly_config.yaml.j2 b/ansible/app-configs/loggifly_config.yaml.j2 index d567b316..08156290 100644 --- a/ansible/app-configs/loggifly_config.yaml.j2 +++ b/ansible/app-configs/loggifly_config.yaml.j2 @@ -20,7 +20,7 @@ global_keywords: - fatal notifications: apprise: - url: gotify://gotify/{{ lookup('community.hashi_vault.vault_kv2_get', 'env', engine_mount_point='rinoa-docker', url=vault_addr, token=vault_token_cleaned)['secret']['APPRISE_GOTIFY_TOKEN'] }} # Any Apprise-compatible URL (https://github.com/caronc/apprise/wiki) + url: gotify://gotify/{{ vault_secrets['APPRISE_GOTIFY_TOKEN'] }} # Any Apprise-compatible URL (https://github.com/caronc/apprise/wiki) # settings are optional because they all have default values settings: log_level: INFO # DEBUG, INFO, WARNING, ERROR diff --git a/ansible/app-configs/multi-scrobbler_config.json.j2 b/ansible/app-configs/multi-scrobbler_config.json.j2 index 697a0cd5..691fa538 100644 --- a/ansible/app-configs/multi-scrobbler_config.json.j2 +++ b/ansible/app-configs/multi-scrobbler_config.json.j2 @@ -27,8 +27,8 @@ "clients": [], "name": "spotify", "data": { - "clientId": "{{ lookup('community.hashi_vault.vault_kv2_get', 'env', engine_mount_point='rinoa-docker', url=vault_addr, token=vault_token_cleaned)['secret']['YOUR_SPOTIFY_ID'] }}", - "clientSecret": "{{ lookup('community.hashi_vault.vault_kv2_get', 'env', engine_mount_point='rinoa-docker', url=vault_addr, token=vault_token_cleaned)['secret']['YOUR_SPOTIFY_SECRET'] }}", + "clientId": "{{ vault_secrets['YOUR_SPOTIFY_ID'] }}", + "clientSecret": "{{ vault_secrets['YOUR_SPOTIFY_SECRET'] }}", "redirectUri": "http://localhost:9078/callback" } }, @@ -38,8 +38,8 @@ "clients": [], "name": "lastfm", "data": { - "apiKey": "{{ lookup('community.hashi_vault.vault_kv2_get', 'env', engine_mount_point='rinoa-docker', url=vault_addr, token=vault_token_cleaned)['secret']['LASTFM_API_KEY'] }}", - "secret": "{{ lookup('community.hashi_vault.vault_kv2_get', 'env', engine_mount_point='rinoa-docker', url=vault_addr, token=vault_token_cleaned)['secret']['LASTFM_API_SECRET'] }}", + "apiKey": "{{ vault_secrets['LASTFM_API_KEY'] }}", + "secret": "{{ vault_secrets['LASTFM_API_SECRET'] }}", "redirectUri": "http://localhost:9078/lastfm/callback" } }, @@ -49,7 +49,7 @@ "clients": [], "name": "listenBrainz", "data": { - "token": "{{ lookup('community.hashi_vault.vault_kv2_get', 'env', engine_mount_point='rinoa-docker', url=vault_addr, token=vault_token_cleaned)['secret']['MALOJA_LISTENBRAINZ_TOKEN'] }}", + "token": "{{ vault_secrets['MALOJA_LISTENBRAINZ_TOKEN'] }}", "username": "Trez.One" } }, @@ -61,7 +61,7 @@ "data": { "url": "http://navidrome:4533", "user": "admin", - "password": "{{ lookup('community.hashi_vault.vault_kv2_get', 'env', engine_mount_point='rinoa-docker', url=vault_addr, token=vault_token_cleaned)['secret']['NAVIDROME_PASSWORD'] }}" + "password": "{{ vault_secrets['NAVIDROME_PASSWORD'] }}" } } ], @@ -71,8 +71,8 @@ "enable": true, "name": "lastFmClient", "data": { - "apiKey": "{{ lookup('community.hashi_vault.vault_kv2_get', 'env', engine_mount_point='rinoa-docker', url=vault_addr, token=vault_token_cleaned)['secret']['LASTFM_API_KEY'] }}", - "secret": "{{ lookup('community.hashi_vault.vault_kv2_get', 'env', engine_mount_point='rinoa-docker', url=vault_addr, token=vault_token_cleaned)['secret']['LASTFM_API_SECRET'] }}", + "apiKey": "{{ vault_secrets['LASTFM_API_KEY'] }}", + "secret": "{{ vault_secrets['LASTFM_API_SECRET'] }}", "redirectUri": "http://localhost:9078/lastfm/callback" } }, @@ -81,7 +81,7 @@ "enable": true, "name": "ListenBrainzClient", "data": { - "token": "{{ lookup('community.hashi_vault.vault_kv2_get', 'env', engine_mount_point='rinoa-docker', url=vault_addr, token=vault_token_cleaned)['secret']['MALOJA_LISTENBRAINZ_TOKEN'] }}", + "token": "{{ vault_secrets['MALOJA_LISTENBRAINZ_TOKEN'] }}", "username": "Trez.One" } }, @@ -91,7 +91,7 @@ "name": "maloja", "data": { "url": "http://maloja:42010", - "apiKey": "{{ lookup('community.hashi_vault.vault_kv2_get', 'env', engine_mount_point='rinoa-docker', url=vault_addr, token=vault_token_cleaned)['secret']['MALOJA_API_KEY'] }}" + "apiKey": "{{ vault_secrets['MALOJA_API_KEY'] }}" } } ], @@ -100,7 +100,7 @@ "name": "Gotify", "type": "gotify", "url": "http://gotify", - "token": "{{ lookup('community.hashi_vault.vault_kv2_get', 'env', engine_mount_point='rinoa-docker', url=vault_addr, token=vault_token_cleaned)['secret']['MULTI_SCROBBLER_GOTIFY_TOKEN'] }}", + "token": "{{ vault_secrets['MULTI_SCROBBLER_GOTIFY_TOKEN'] }}", "priorities": { "info": 5, "warn": 7, diff --git a/ansible/app-configs/netbird_management.json.j2 b/ansible/app-configs/netbird_management.json.j2 index 80d2bb29..30cf49ff 100644 --- a/ansible/app-configs/netbird_management.json.j2 +++ b/ansible/app-configs/netbird_management.json.j2 @@ -2,7 +2,7 @@ "Stuns": [ { "Proto": "udp", - "URI": "stun:netbird.{{ lookup('community.hashi_vault.vault_kv2_get', 'env', engine_mount_point='rinoa-docker', url=vault_addr, token=vault_token_cleaned)['secret']['MY_TLD'] }}:3478", + "URI": "stun:netbird.{{ vault_secrets['MY_TLD'] }}:3478", "Username": "", "Password": null } @@ -11,9 +11,9 @@ "Turns": [ { "Proto": "udp", - "URI": "turn:netbird.{{ lookup('community.hashi_vault.vault_kv2_get', 'env', engine_mount_point='rinoa-docker', url=vault_addr, token=vault_token_cleaned)['secret']['MY_TLD'] }}:3478", + "URI": "turn:netbird.{{ vault_secrets['MY_TLD'] }}:3478", "Username": "self", - "Password": "{{ lookup('community.hashi_vault.vault_kv2_get', 'env', engine_mount_point='rinoa-docker', url=vault_addr, token=vault_token_cleaned)['secret']['NETBIRD_TURN_PASSWORD'] }}" + "Password": "{{ vault_secrets['NETBIRD_TURN_PASSWORD'] }}" } ], "CredentialsTTL": "12h", @@ -22,14 +22,14 @@ }, "Relay": { "Addresses": [ - "rel://netbird.{{ lookup('community.hashi_vault.vault_kv2_get', 'env', engine_mount_point='rinoa-docker', url=vault_addr, token=vault_token_cleaned)['secret']['MY_TLD'] }}:33080" + "rel://netbird.{{ vault_secrets['MY_TLD'] }}:33080" ], "CredentialsTTL": "24h", - "Secret": "{{ lookup('community.hashi_vault.vault_kv2_get', 'env', engine_mount_point='rinoa-docker', url=vault_addr, token=vault_token_cleaned)['secret']['NETBIRD_RELAY_AUTH_SECRET'] }}" + "Secret": "{{ vault_secrets['NETBIRD_RELAY_AUTH_SECRET'] }}" }, "Signal": { "Proto": "https", - "URI": "netbird.{{ lookup('community.hashi_vault.vault_kv2_get', 'env', engine_mount_point='rinoa-docker', url=vault_addr, token=vault_token_cleaned)['secret']['MY_TLD'] }}:10001", + "URI": "netbird.{{ vault_secrets['MY_TLD'] }}:10001", "Username": "", "Password": null }, @@ -47,14 +47,14 @@ }, "HttpConfig": { "Address": "0.0.0.0:33073", - "AuthIssuer": "https://auth.{{ lookup('community.hashi_vault.vault_kv2_get', 'env', engine_mount_point='rinoa-docker', url=vault_addr, token=vault_token_cleaned)['secret']['MY_TLD'] }}", + "AuthIssuer": "https://auth.{{ vault_secrets['MY_TLD'] }}", "AuthAudience": "netbird", - "AuthKeysLocation": "https://auth.{{ lookup('community.hashi_vault.vault_kv2_get', 'env', engine_mount_point='rinoa-docker', url=vault_addr, token=vault_token_cleaned)['secret']['MY_TLD'] }}/jwks.json", + "AuthKeysLocation": "https://auth.{{ vault_secrets['MY_TLD'] }}/jwks.json", "AuthUserIDClaim": "", "CertFile": "", "CertKey": "", "IdpSignKeyRefreshEnabled": true, - "OIDCConfigEndpoint": "https://auth.{{ lookup('community.hashi_vault.vault_kv2_get', 'env', engine_mount_point='rinoa-docker', url=vault_addr, token=vault_token_cleaned)['secret']['MY_TLD'] }}/.well-known/openid-configuration" + "OIDCConfigEndpoint": "https://auth.{{ vault_secrets['MY_TLD'] }}/.well-known/openid-configuration" }, "IdpManagerConfig": {}, "DeviceAuthorizationFlow": {}, @@ -62,10 +62,10 @@ "ProviderConfig": { "Audience": "netbird", "ClientID": "netbird", - "ClientSecret": "{{ lookup('community.hashi_vault.vault_kv2_get', 'env', engine_mount_point='rinoa-docker', url=vault_addr, token=vault_token_cleaned)['secret']['AUTHELIA_NETBIRD_CLIENT_SECRET'] }}", + "ClientSecret": "{{ vault_secrets['AUTHELIA_NETBIRD_CLIENT_SECRET'] }}", "Domain": "", - "AuthorizationEndpoint": "https://auth.{{ lookup('community.hashi_vault.vault_kv2_get', 'env', engine_mount_point='rinoa-docker', url=vault_addr, token=vault_token_cleaned)['secret']['MY_TLD'] }}/api/oidc/authorization", - "TokenEndpoint": "https://auth.{{ lookup('community.hashi_vault.vault_kv2_get', 'env', engine_mount_point='rinoa-docker', url=vault_addr, token=vault_token_cleaned)['secret']['MY_TLD'] }}/api/oidc/token", + "AuthorizationEndpoint": "https://auth.{{ vault_secrets['MY_TLD'] }}/api/oidc/authorization", + "TokenEndpoint": "https://auth.{{ vault_secrets['MY_TLD'] }}/api/oidc/token", "Scope": "openid profile email offline_access api", "RedirectURLs": [ "http://localhost:53000" diff --git a/ansible/app-configs/netbird_turnserver.conf.j2 b/ansible/app-configs/netbird_turnserver.conf.j2 index 97030f7a..c8ec6c4f 100644 --- a/ansible/app-configs/netbird_turnserver.conf.j2 +++ b/ansible/app-configs/netbird_turnserver.conf.j2 @@ -250,7 +250,7 @@ lt-cred-mech #user=username1:key1 #user=username2:key2 # OR: -user=self:{{ lookup('community.hashi_vault.vault_kv2_get', 'env', engine_mount_point='rinoa-docker', url=vault_addr, token=vault_token_cleaned)['secret']['NETBIRD_TURN_PASSWORD'] }} +user=self:{{ vault_secrets['NETBIRD_TURN_PASSWORD'] }} #user=username2:password2 # # Keys must be generated by turnadmin utility. The key value depends diff --git a/ansible/app-configs/postal_postal.yml.j2 b/ansible/app-configs/postal_postal.yml.j2 index b365f101..661d71b0 100644 --- a/ansible/app-configs/postal_postal.yml.j2 +++ b/ansible/app-configs/postal_postal.yml.j2 @@ -18,13 +18,13 @@ web_server: main_db: host: mariadb username: postal - password: {{ lookup('community.hashi_vault.vault_kv2_get', 'env', engine_mount_point='rinoa-docker', url=vault_addr, token=vault_token_cleaned)['secret']['POSTAL_MYSQL_PASSWORD'] }} + password: {{ vault_secrets['POSTAL_MYSQL_PASSWORD'] }} database: postal message_db: host: mariadb username: postal - password: {{ lookup('community.hashi_vault.vault_kv2_get', 'env', engine_mount_point='rinoa-docker', url=vault_addr, token=vault_token_cleaned)['secret']['POSTAL_MYSQL_PASSWORD'] }} + password: {{ vault_secrets['POSTAL_MYSQL_PASSWORD'] }} prefix: postal smtp_server: @@ -52,11 +52,11 @@ smtp: host: postal-smtp port: 25 username: rinoa/postal-smtp - password: "{{ lookup('community.hashi_vault.vault_kv2_get', 'env', engine_mount_point='rinoa-docker', url=vault_addr, token=vault_token_cleaned)['secret']['POSTAL_SMTP_AUTH_PASSWORD'] }}" + password: "{{ vault_secrets['POSTAL_SMTP_AUTH_PASSWORD'] }}" from_name: Postal @ Rinoa from_address: noreply@trez.wtf rails: # This is generated automatically by the config initialization. It should be a random # string unique to your installation. - secret_key: "{{ lookup('community.hashi_vault.vault_kv2_get', 'env', engine_mount_point='rinoa-docker', url=vault_addr, token=vault_token_cleaned)['secret']['POSTAL_RAILS_SECRET_KEY'] }}" + secret_key: "{{ vault_secrets['POSTAL_RAILS_SECRET_KEY'] }}" diff --git a/ansible/app-configs/prowlarr_config.xml.j2 b/ansible/app-configs/prowlarr_config.xml.j2 index f45b8eea..d7ab871f 100644 --- a/ansible/app-configs/prowlarr_config.xml.j2 +++ b/ansible/app-configs/prowlarr_config.xml.j2 @@ -7,7 +7,7 @@ 6969 False True - {{ lookup('community.hashi_vault.vault_kv2_get', 'env', engine_mount_point='rinoa-docker', url=vault_addr, token=vault_token_cleaned)['secret']['PROWLARR_API_KEY'] }} + {{ vault_secrets['PROWLARR_API_KEY'] }} Forms Enabled master diff --git a/ansible/app-configs/radarec_settings_config.json.j2 b/ansible/app-configs/radarec_settings_config.json.j2 index a35180cc..9bbd522d 100644 --- a/ansible/app-configs/radarec_settings_config.json.j2 +++ b/ansible/app-configs/radarec_settings_config.json.j2 @@ -3,9 +3,9 @@ { "radarr_address": "http://radarr:7878", - "radarr_api_key": "{{ lookup('community.hashi_vault.vault_kv2_get', 'env', engine_mount_point='rinoa-docker', url=vault_addr, token=vault_token_cleaned)['secret']['RADARR_API_KEY'] }}", + "radarr_api_key": "{{ vault_secrets['RADARR_API_KEY'] }}", "root_folder_path": "/data/media/movies", - "tmdb_api_key": "{{ lookup('community.hashi_vault.vault_kv2_get', 'env', engine_mount_point='rinoa-docker', url=vault_addr, token=vault_token_cleaned)['secret']['TMDB_API_KEY'] }}", + "tmdb_api_key": "{{ vault_secrets['TMDB_API_KEY'] }}", "fallback_to_top_result": false, "radarr_api_timeout": 120.0, "quality_profile_id": 1, diff --git a/ansible/app-configs/radarr_config.xml.j2 b/ansible/app-configs/radarr_config.xml.j2 index e9a9baaa..2e4fafda 100644 --- a/ansible/app-configs/radarr_config.xml.j2 +++ b/ansible/app-configs/radarr_config.xml.j2 @@ -8,7 +8,7 @@ 7878 - {{ lookup('community.hashi_vault.vault_kv2_get', 'env', engine_mount_point='rinoa-docker', url=vault_addr, token=vault_token_cleaned)['secret']['RADARR_API_KEY'] }} + {{ vault_secrets['RADARR_API_KEY'] }} Forms Docker 9898 diff --git a/ansible/app-configs/readarr_config.xml.j2 b/ansible/app-configs/readarr_config.xml.j2 index 5eec003e..96fbcc5c 100644 --- a/ansible/app-configs/readarr_config.xml.j2 +++ b/ansible/app-configs/readarr_config.xml.j2 @@ -7,7 +7,7 @@ 6868 False True - {{ lookup('community.hashi_vault.vault_kv2_get', 'env', engine_mount_point='rinoa-docker', url=vault_addr, token=vault_token_cleaned)['secret']['READARR_API_KEY'] }} + {{ vault_secrets['READARR_API_KEY'] }} Forms develop info diff --git a/ansible/app-configs/sabnzbdvpn_sabnzbd.ini.j2 b/ansible/app-configs/sabnzbdvpn_sabnzbd.ini.j2 index f9e2f6e7..2dd410ad 100644 --- a/ansible/app-configs/sabnzbdvpn_sabnzbd.ini.j2 +++ b/ansible/app-configs/sabnzbdvpn_sabnzbd.ini.j2 @@ -22,7 +22,7 @@ host = 0.0.0.0 port = 8080 https_port = 8090 username = thetrezuredone -password = {{ lookup('community.hashi_vault.vault_kv2_get', 'env', engine_mount_point='rinoa-docker', url=vault_addr, token=vault_token_cleaned)['secret']['SABNZBDVPN_PASSWORD'] }} +password = {{ vault_secrets['SABNZBDVPN_PASSWORD'] }} bandwidth_max = 1000M cache_limit = 1G web_dir = Glitter @@ -33,7 +33,7 @@ https_chain = "" enable_https = 1 inet_exposure = 0 local_ranges = , -api_key = {{ lookup('community.hashi_vault.vault_kv2_get', 'env', engine_mount_point='rinoa-docker', url=vault_addr, token=vault_token_cleaned)['secret']['SABNZBDVPN_API_KEY'] }} +api_key = {{ vault_secrets['SABNZBDVPN_API_KEY'] }} nzb_key = 3c0fa874bb2748b58c1bd7512e649946 permissions = 775 download_dir = /storage/downloads/incomplete @@ -342,7 +342,7 @@ host = news.newshosting.com port = 563 timeout = 60 username = thetrezuredone -password = {{ lookup('community.hashi_vault.vault_kv2_get', 'env', engine_mount_point='rinoa-docker', url=vault_addr, token=vault_token_cleaned)['secret']['SLSK_USER_PASSWORD'] }} +password = {{ vault_secrets['SLSK_USER_PASSWORD'] }} connections = 8 ssl = 1 ssl_verify = 3 @@ -363,7 +363,7 @@ host = news.easynews.com port = 443 timeout = 60 username = TrezOne -password = {{ lookup('community.hashi_vault.vault_kv2_get', 'env', engine_mount_point='rinoa-docker', url=vault_addr, token=vault_token_cleaned)['secret']['SABNZBDVPN_EASYNEWS_PASSWORD'] }} +password = {{ vault_secrets['SABNZBDVPN_EASYNEWS_PASSWORD'] }} connections = 60 ssl = 0 ssl_verify = 3 diff --git a/ansible/app-configs/searxng_settings.yml.j2 b/ansible/app-configs/searxng_settings.yml.j2 index a9c081c8..01427409 100644 --- a/ansible/app-configs/searxng_settings.yml.j2 +++ b/ansible/app-configs/searxng_settings.yml.j2 @@ -82,7 +82,7 @@ server: # If your instance owns a /etc/searxng/settings.yml file, then set the following # values there. - secret_key: {{ lookup('community.hashi_vault.vault_kv2_get', 'env', engine_mount_point='rinoa-docker', url=vault_addr, token=vault_token_cleaned)['secret']['SEARXNG_SECRET_KEY'] }} # Is overwritten by ${SEARXNG_SECRET} + secret_key: {{ vault_secrets['SEARXNG_SECRET_KEY'] }} # Is overwritten by ${SEARXNG_SECRET} # Proxying image results through searx image_proxy: true # 1.0 and 1.1 are supported diff --git a/ansible/app-configs/sonarr_config.xml.j2 b/ansible/app-configs/sonarr_config.xml.j2 index cb4f0f35..cfc93815 100644 --- a/ansible/app-configs/sonarr_config.xml.j2 +++ b/ansible/app-configs/sonarr_config.xml.j2 @@ -8,7 +8,7 @@ 9898 * - {{ lookup('community.hashi_vault.vault_kv2_get', 'env', engine_mount_point='rinoa-docker', url=vault_addr, token=vault_token_cleaned)['secret']['SONARR_API_KEY'] }} + {{ vault_secrets['SONARR_API_KEY'] }} Forms Docker True diff --git a/ansible/app-configs/sonashow_settings_config.json.j2 b/ansible/app-configs/sonashow_settings_config.json.j2 index 5441e156..340d0ca6 100644 --- a/ansible/app-configs/sonashow_settings_config.json.j2 +++ b/ansible/app-configs/sonashow_settings_config.json.j2 @@ -3,10 +3,10 @@ { "sonarr_address": "http://192.168.1.2:8989", - "sonarr_api_key": "{{ lookup('community.hashi_vault.vault_kv2_get', 'env', engine_mount_point='rinoa-docker', url=vault_addr, token=vault_token_cleaned)['secret']['SONARR_API_KEY'] }}", + "sonarr_api_key": "{{ vault_secrets['SONARR_API_KEY'] }}", "root_folder_path": "/data/media/shows", "tvdb_api_key": "", - "tmdb_api_key": "{{ lookup('community.hashi_vault.vault_kv2_get', 'env', engine_mount_point='rinoa-docker', url=vault_addr, token=vault_token_cleaned)['secret']['TMDB_API_KEY'] }}", + "tmdb_api_key": "{{ vault_secrets['TMDB_API_KEY'] }}", "fallback_to_top_result": false, "sonarr_api_timeout": 120.0, "quality_profile_id": 1, diff --git a/ansible/app-configs/soularr_config.ini.j2 b/ansible/app-configs/soularr_config.ini.j2 index 2de58ec5..5e2c3ffc 100644 --- a/ansible/app-configs/soularr_config.ini.j2 +++ b/ansible/app-configs/soularr_config.ini.j2 @@ -2,7 +2,7 @@ {% set secrets_path = 'rinoa-docker/env' %} [Lidarr] -api_key = {{ lookup('community.hashi_vault.vault_kv2_get', 'env', engine_mount_point='rinoa-docker', url=vault_addr, token=vault_token_cleaned)['secret']['LIDARR_API_KEY'] }} +api_key = {{ vault_secrets['LIDARR_API_KEY'] }} host_url = http://lidarr:8686 #This should be the path mounted in lidarr that points to your slskd download directory. #If Lidarr is not running in Docker then this may just be the same dir as Slskd is using below. @@ -10,7 +10,7 @@ download_dir = /storage [Slskd] #Api key from Slskd. Need to set this up manually. See link to Slskd docs above. -api_key = {{ lookup('community.hashi_vault.vault_kv2_get', 'env', engine_mount_point='rinoa-docker', url=vault_addr, token=vault_token_cleaned)['secret']['SLSKD_API_KEY'] }} +api_key = {{ vault_secrets['SLSKD_API_KEY'] }} host_url = http://gluetun:5030 #Slskd download directory. Should have set it up when installing Slskd. download_dir = /app/downloads diff --git a/ansible/app-configs/soulseek_slskd.yml.j2 b/ansible/app-configs/soulseek_slskd.yml.j2 index 2890f689..0898820f 100644 --- a/ansible/app-configs/soulseek_slskd.yml.j2 +++ b/ansible/app-configs/soulseek_slskd.yml.j2 @@ -198,15 +198,15 @@ rooms: web: authentication: username: slskd - password: {{ lookup('community.hashi_vault.vault_kv2_get', 'env', engine_mount_point='rinoa-docker', url=vault_addr, token=vault_token_cleaned)['secret']['SLSKD_WEB_PASSSWORD'] }} + password: {{ vault_secrets['SLSKD_WEB_PASSSWORD'] }} api_keys: my_api_key: - key: {{ lookup('community.hashi_vault.vault_kv2_get', 'env', engine_mount_point='rinoa-docker', url=vault_addr, token=vault_token_cleaned)['secret']['SLSKD_API_KEY'] }} + key: {{ vault_secrets['SLSKD_API_KEY'] }} role: readwrite cidr: 0.0.0.0/0,::/0 soulseek: address: vps.slsknet.org port: 2271 username: Trez.One - password: {{ lookup('community.hashi_vault.vault_kv2_get', 'env', engine_mount_point='rinoa-docker', url=vault_addr, token=vault_token_cleaned)['secret']['SLSK_USER_PASSWORD'] }} + password: {{ vault_secrets['SLSK_USER_PASSWORD'] }} diagnostic_level: Info diff --git a/ansible/app-configs/sourcebot_config.json.j2 b/ansible/app-configs/sourcebot_config.json.j2 index 5a522c03..3c7bf845 100644 --- a/ansible/app-configs/sourcebot_config.json.j2 +++ b/ansible/app-configs/sourcebot_config.json.j2 @@ -6,7 +6,7 @@ "repos": [ { "type": "gitea", - "token": "{{ lookup('community.hashi_vault.vault_kv2_get', 'env', engine_mount_point='rinoa-docker', url=vault_addr, token=vault_token_cleaned)['secret']['GITEA_SONARQUBE_BOT_GITEA_TOKEN'] }}", + "token": "{{ vault_secrets['GITEA_SONARQUBE_BOT_GITEA_TOKEN'] }}", "url": "https://git.trez.wtf", "revisions": { "branches": [ diff --git a/ansible/app-configs/traccar_traccar.xml.j2 b/ansible/app-configs/traccar_traccar.xml.j2 index 8d1f9bc5..f58bb252 100644 --- a/ansible/app-configs/traccar_traccar.xml.j2 +++ b/ansible/app-configs/traccar_traccar.xml.j2 @@ -24,6 +24,6 @@ org.postgresql.Driver jdbc:postgresql://traccar-pg:5432/traccar-db traccar - {{ lookup('community.hashi_vault.vault_kv2_get', 'env', engine_mount_point='rinoa-docker', url=vault_addr, token=vault_token_cleaned)['secret']['WAZUH_API_PASSWORD'] }} + {{ vault_secrets['WAZUH_API_PASSWORD'] }} diff --git a/ansible/app-configs/vector_vector.yaml.j2 b/ansible/app-configs/vector_vector.yaml.j2 index 45610c91..de9d1e12 100644 --- a/ansible/app-configs/vector_vector.yaml.j2 +++ b/ansible/app-configs/vector_vector.yaml.j2 @@ -21,7 +21,7 @@ auth: strategy: basic user: admin - password: {{ lookup('community.hashi_vault.vault_kv2_get', 'env', engine_mount_point='rinoa-docker', url=vault_addr, token=vault_token_cleaned)['secret']['PARSEABLE_PASSWORD'] }} + password: {{ vault_secrets['PARSEABLE_PASSWORD'] }} request: headers: X-P-Stream: rinoa-docker-logs diff --git a/ansible/app-configs/wazuh_wazuh.yml.j2 b/ansible/app-configs/wazuh_wazuh.yml.j2 index bb1995c8..38e8b586 100644 --- a/ansible/app-configs/wazuh_wazuh.yml.j2 +++ b/ansible/app-configs/wazuh_wazuh.yml.j2 @@ -6,5 +6,5 @@ hosts: url: "https://wazuh.manager" port: 55000 username: wazuh-wui - password: {{ lookup('community.hashi_vault.vault_kv2_get', 'env', engine_mount_point='rinoa-docker', url=vault_addr, token=vault_token_cleaned)['secret']['WAZUH_API_PASSWORD'] }} + password: {{ vault_secrets['WAZUH_API_PASSWORD'] }} run_as: false diff --git a/ansible/app-configs/zitadel_config.yaml.j2 b/ansible/app-configs/zitadel_config.yaml.j2 index 708a5a64..ebb462ea 100644 --- a/ansible/app-configs/zitadel_config.yaml.j2 +++ b/ansible/app-configs/zitadel_config.yaml.j2 @@ -37,7 +37,7 @@ SMTPConfiguration: SMTP: # must include the port, like smtp.mailtrap.io:2525. IPv6 is also supported, like [2001:db8::1]:2525 Host: 'postal-smtp:25' - User: {{ lookup('community.hashi_vault.vault_kv2_get', 'env', engine_mount_point='rinoa-docker', url=vault_addr, token=vault_token_cleaned)['secret']['POSTAL_SMTP_AUTH_USER'] }} - Password: {{ lookup('community.hashi_vault.vault_kv2_get', 'env', engine_mount_point='rinoa-docker', url=vault_addr, token=vault_token_cleaned)['secret']['POSTAL_SMTP_AUTH_PASSWORD'] }} + User: {{ vault_secrets['POSTAL_SMTP_AUTH_USER'] }} + Password: {{ vault_secrets['POSTAL_SMTP_AUTH_PASSWORD'] }} From: 'noreply@trez.wtf' FromName: 'Zitadel @ Rinoa' \ No newline at end of file diff --git a/ansible/app-configs/zitadel_secrets.yaml.j2 b/ansible/app-configs/zitadel_secrets.yaml.j2 index 201034c8..0f509e43 100644 --- a/ansible/app-configs/zitadel_secrets.yaml.j2 +++ b/ansible/app-configs/zitadel_secrets.yaml.j2 @@ -7,7 +7,7 @@ Database: User: # If the user doesn't exist already, it is created Username: 'zitadel' - Password: {{ lookup('community.hashi_vault.vault_kv2_get', 'env', engine_mount_point='rinoa-docker', url=vault_addr, token=vault_token_cleaned)['secret']['ZITADEL_DB_PASSWORD'] }} + Password: {{ vault_secrets['ZITADEL_DB_PASSWORD'] }} Admin: Username: 'root' - Password: {{ lookup('community.hashi_vault.vault_kv2_get', 'env', engine_mount_point='rinoa-docker', url=vault_addr, token=vault_token_cleaned)['secret']['ZITADEL_DB_ADMIN_PASSWORD'] }} \ No newline at end of file + Password: {{ vault_secrets['ZITADEL_DB_ADMIN_PASSWORD'] }} \ No newline at end of file diff --git a/ansible/docker_config_deploy.yml b/ansible/docker_config_deploy.yml index 798016c4..a1f7acb5 100644 --- a/ansible/docker_config_deploy.yml +++ b/ansible/docker_config_deploy.yml @@ -5,6 +5,12 @@ appdata_base_path: "~/.docker/config/appdata" tasks: + - name: Fetch Vault secrets once + ansible.builtin.set_fact: + vault_secrets: "{{ lookup('community.hashi_vault.vault_kv2_get', 'env', + engine_mount_point='rinoa-docker', url=vault_addr, + token=vault_token_cleaned)['secret'] }}" + - name: Ensure target directories exist ansible.builtin.file: path: "{{ appdata_base_path }}/{{ (item | basename | regex_replace('\\.j2$', '') | regex_replace('_', '/') | regex_replace('/[^/]+$', '')) }}" From d08563b985f62e1c8d0c1f0843f4d3ff768e2e88 Mon Sep 17 00:00:00 2001 From: "Trez.One" Date: Tue, 20 May 2025 09:13:33 -0400 Subject: [PATCH 02/10] Small fixes. --- .gitea/workflows/pr-ansible-config-deployment.yaml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.gitea/workflows/pr-ansible-config-deployment.yaml b/.gitea/workflows/pr-ansible-config-deployment.yaml index 25f4e2ac..854fa098 100644 --- a/.gitea/workflows/pr-ansible-config-deployment.yaml +++ b/.gitea/workflows/pr-ansible-config-deployment.yaml @@ -57,7 +57,7 @@ jobs: notification_title: 'GITEA: PR Check' notification_message: 'PR Created 🎟️' ansible-linting: - name: Docker Compose & Ansible Lints + name: Ansible Linting needs: [check-and-create-pr] runs-on: ubuntu-latest env: @@ -112,7 +112,7 @@ jobs: notification_message: 'Docker Compose dry run completed successfully.' pr-merge: name: PR Merge - needs: [regenerate-readme-modified-services] + needs: [ansible-linting] runs-on: ubuntu-latest steps: - name: Checkout @@ -139,8 +139,8 @@ jobs: gotify_app_token: '${{ secrets.RINOA_RUNNER_GOTIFY_TOKEN }}' notification_title: 'GITEA: PR Merge Successful' notification_message: 'PR #${{ steps.pr_merge.outputs.pr_index }} merged.' - ansible-config-docker-compose-deploy: - name: Ansible Configs & Docker Compose Deployment + ansible-config-deploy: + name: Ansible Playbook Run runs-on: ubuntu-latest needs: [pr-merge] env: From 124d4a86ba71fe774e309f24d7d4ea3927333a2a Mon Sep 17 00:00:00 2001 From: "Trez.One" Date: Tue, 20 May 2025 09:16:00 -0400 Subject: [PATCH 03/10] Small fixes. --- .../pr-ansible-config-deployment.yaml | 192 ------------------ 1 file changed, 192 deletions(-) delete mode 100644 .gitea/workflows/pr-ansible-config-deployment.yaml diff --git a/.gitea/workflows/pr-ansible-config-deployment.yaml b/.gitea/workflows/pr-ansible-config-deployment.yaml deleted file mode 100644 index 854fa098..00000000 --- a/.gitea/workflows/pr-ansible-config-deployment.yaml +++ /dev/null @@ -1,192 +0,0 @@ -name: Gitea Branch PR & Ansible Deployment -on: - push: - branches-ignore: - - 'main' - paths: - - '**.j2' - - 'ansible/**.yml' -jobs: - check-and-create-pr: - if: github.ref != 'refs/heads/main' - name: Check and Create PR - runs-on: ubuntu-latest - steps: - - name: Checkout Code - uses: actions/checkout@v4 - with: - fetch-depth: 1 - - name: Cache tea CLI - id: cache-tea - uses: actions/cache@v4 - with: - path: /opt/hostedtoolcache/tea/0.9.2/x64 - key: tea-${{ runner.os }}-0.9.2 - - name: Install tea - uses: supplypike/setup-bin@v4 - with: - uri: 'https://gitea.com/gitea/tea/releases/download/v0.9.2/tea-0.9.2-linux-amd64' - name: 'tea' - version: '0.9.2' - - name: Gotify Notification - uses: eikendev/gotify-action@master - with: - gotify_api_base: '${{ secrets.RINOA_GOTIFY_URL }}' - gotify_app_token: '${{ secrets.RINOA_RUNNER_GOTIFY_TOKEN }}' - notification_title: 'GITEA: PR Check' - notification_message: 'Checking for existing PR... 🔍' - - name: Check if open PR exists - id: check-opened-pr-step - continue-on-error: true - run: | - tea login add --name gitea-rinoa --url "${{ secrets.RINOA_GITEA_URL }}" --user gitea-sonarqube-bot --password "${{ secrets.BOT_GITEA_PASSWORD }}" --token ${{ secrets.BOT_GITEA_TOKEN }} - pr_exists=$(tea pr list --repo ${{ github.repository }} --state open --fields index,title,head | egrep ${{ github.ref_name }} | tail -1 | wc -l) - echo "exists=$pr_exists" >> $GITHUB_OUTPUT - - name: Create PR - if: ${{ steps.check-opened-pr-step.outputs.exists == '0' }} - run: | - tea login default gitea-rinoa - pr_index_old=$(tea pr ls --repo ${{ github.repository }} --state all --fields index,title,head --output csv | sed -e 's|"||g' | egrep '^[0-9]' | head -1 | awk -F"," '{print $1}') - pr_index_new=$(expr ${pr_index_old} + 1) - tea pr c -r ${{ github.repository }} -t "Automated PR for ${{ github.ref_name }} - #${pr_index_new}" -d "Automatically created PR for branch: ${{ github.ref_name }}" -a ${{ github.actor }} -L "Docker Compose, Ansible Configs.j2" - - name: Gotify Notification - uses: eikendev/gotify-action@master - with: - gotify_api_base: '${{ secrets.RINOA_GOTIFY_URL }}' - gotify_app_token: '${{ secrets.RINOA_RUNNER_GOTIFY_TOKEN }}' - notification_title: 'GITEA: PR Check' - notification_message: 'PR Created 🎟️' - ansible-linting: - name: Ansible Linting - needs: [check-and-create-pr] - runs-on: ubuntu-latest - env: - VAULT_ADDR: ${{ secrets.RINOA_VAULT_ADDR }} - VAULT_TOKEN: ${{ secrets.VAULT_GITEA_TOKEN }} - VAULT_NAMESPACE: "" - steps: - - name: Checkout - uses: actions/checkout@v4 - - name: Fetch base branch - run: | - git fetch origin ${{ github.event.pull_request.base.ref }} - - name: Cache Ansible Galaxy Collections - uses: actions/cache@v3 - with: - path: ansible/collections - key: ${{ runner.os }}-ansible-${{ hashFiles('./ansible/collections/requirements.yml') }} - restore-keys: | - ${{ runner.os }}-ansible- - - name: Install Ansible - uses: alex-oleshkevich/setup-ansible@v1.0.1 - with: - version: "11.0.0" - - name: Install Vault - uses: cpanato/vault-installer@main - - name: Install hvac - run: pip install hvac - - name: Gotify Notification - uses: eikendev/gotify-action@master - with: - gotify_api_base: '${{ secrets.RINOA_GOTIFY_URL }}' - gotify_app_token: '${{ secrets.RINOA_RUNNER_GOTIFY_TOKEN }}' - notification_title: 'GITEA: Ansible Config Dry Run @ Rinoa' - notification_message: 'Starting Ansible dry run...' - - name: Ansible Playbook Dry Run - uses: arillso/action.playbook@0.1.0 - with: - check: true - galaxy_collections_path: ansible/collections - galaxy_requirements_file: ansible/collections/requirements.yml - inventory: ansible/inventory/hosts.yml - playbook: ansible/docker_config_deploy.yml - private_key: ${{ secrets.RINOA_ANSIBLE_PRIVATE_KEY }} - vault_password: ${{ secrets.ANSIBLE_VAULT_PASSWORD }} - verbose: 0 - - name: Gotify Notification - uses: eikendev/gotify-action@master - with: - gotify_api_base: '${{ secrets.RINOA_GOTIFY_URL }}' - gotify_app_token: '${{ secrets.RINOA_RUNNER_GOTIFY_TOKEN }}' - notification_title: 'GITEA: Docker Compose Dry Run @ Rinoa' - notification_message: 'Docker Compose dry run completed successfully.' - pr-merge: - name: PR Merge - needs: [ansible-linting] - runs-on: ubuntu-latest - steps: - - name: Checkout - uses: actions/checkout@v4 - - name: Install tea - uses: supplypike/setup-bin@v4 - with: - uri: 'https://gitea.com/gitea/tea/releases/download/v0.9.2/tea-0.9.2-linux-amd64' - name: 'tea' - version: '0.9.2' - - name: PR Merge - id: pr_merge - run: | - tea login add --name gitea-rinoa --url ${{ secrets.RINOA_GITEA_URL }} --user gitea-sonarqube-bot --password "${{ secrets.BOT_GITEA_PASSWORD }}" --token ${{ secrets.BOT_GITEA_TOKEN }} - tea login default gitea-rinoa - echo "Merging PR..." - pr_index=$(tea pr ls --repo ${{ github.repository }} --state open --fields index,title,head,state --output csv | egrep ${{ github.ref_name }} | awk -F"," '{print $1}' | sed -e 's|"||g') - tea pr m --repo ${{ github.repository }} --title "Auto Merge of PR ${pr_index} - ${{ github.ref_name }}" --message "Merged by ${{ github.actor }}" ${pr_index} - echo "pr_index=${pr_index}" >> $GITHUB_OUTPUT - - name: Gotify Notification - uses: eikendev/gotify-action@master - with: - gotify_api_base: '${{ secrets.RINOA_GOTIFY_URL }}' - gotify_app_token: '${{ secrets.RINOA_RUNNER_GOTIFY_TOKEN }}' - notification_title: 'GITEA: PR Merge Successful' - notification_message: 'PR #${{ steps.pr_merge.outputs.pr_index }} merged.' - ansible-config-deploy: - name: Ansible Playbook Run - runs-on: ubuntu-latest - needs: [pr-merge] - env: - VAULT_ADDR: ${{ secrets.RINOA_VAULT_ADDR }} - VAULT_TOKEN: ${{ secrets.VAULT_GITEA_TOKEN }} - DOCKER_HOST: tcp://dockerproxy:2375 - steps: - - name: Checkout - uses: actions/checkout@v4 - with: - ref: main - - name: Cache Vault install - id: cache-vault - uses: actions/cache@v4 - with: - path: /opt/hostedtoolcache/vault/1.18.0/x64 - key: vault-${{ runner.os }}-1.18.0 - - name: Install Ansible - uses: alex-oleshkevich/setup-ansible@v1.0.1 - with: - version: "11.0.0" - - name: Install Vault - uses: cpanato/vault-installer@main - - name: Install hvac - run: pip install hvac - - name: Gotify Notification - uses: eikendev/gotify-action@master - with: - gotify_api_base: '${{ secrets.RINOA_GOTIFY_URL }}' - gotify_app_token: '${{ secrets.RINOA_RUNNER_GOTIFY_TOKEN }}' - notification_title: 'GITEA: Ansible Config Deployment @ Rinoa' - notification_message: 'Starting config deployment with Ansible...' - - name: Ansible Playbook Dry Run - uses: arillso/action.playbook@0.1.0 - with: - check: false - galaxy_collections_path: ansible/collections - galaxy_requirements_file: ansible/collections/requirements.yml - inventory: ansible/inventory/hosts.yml - playbook: ansible/docker_config_deploy.yml - private_key: ${{ secrets.RINOA_ANSIBLE_PRIVATE_KEY }} - vault_password: ${{ secrets.ANSIBLE_VAULT_PASSWORD }} - - name: Gotify Notification - uses: eikendev/gotify-action@master - with: - gotify_api_base: '${{ secrets.RINOA_GOTIFY_URL }}' - gotify_app_token: '${{ secrets.RINOA_RUNNER_GOTIFY_TOKEN }}' - notification_title: 'GITEA: Ansible Config Deployment @ Rinoa' - notification_message: 'Deployment completed successfully.' \ No newline at end of file From c443d73f6f1e4114f988ca15c0689ba309abbe6d Mon Sep 17 00:00:00 2001 From: "Trez.One" Date: Tue, 20 May 2025 09:16:16 -0400 Subject: [PATCH 04/10] Small fixes. --- .../pr-ansible-config-deployment.yml | 192 ++++++++++++++++++ 1 file changed, 192 insertions(+) create mode 100644 .gitea/workflows/pr-ansible-config-deployment.yml diff --git a/.gitea/workflows/pr-ansible-config-deployment.yml b/.gitea/workflows/pr-ansible-config-deployment.yml new file mode 100644 index 00000000..854fa098 --- /dev/null +++ b/.gitea/workflows/pr-ansible-config-deployment.yml @@ -0,0 +1,192 @@ +name: Gitea Branch PR & Ansible Deployment +on: + push: + branches-ignore: + - 'main' + paths: + - '**.j2' + - 'ansible/**.yml' +jobs: + check-and-create-pr: + if: github.ref != 'refs/heads/main' + name: Check and Create PR + runs-on: ubuntu-latest + steps: + - name: Checkout Code + uses: actions/checkout@v4 + with: + fetch-depth: 1 + - name: Cache tea CLI + id: cache-tea + uses: actions/cache@v4 + with: + path: /opt/hostedtoolcache/tea/0.9.2/x64 + key: tea-${{ runner.os }}-0.9.2 + - name: Install tea + uses: supplypike/setup-bin@v4 + with: + uri: 'https://gitea.com/gitea/tea/releases/download/v0.9.2/tea-0.9.2-linux-amd64' + name: 'tea' + version: '0.9.2' + - name: Gotify Notification + uses: eikendev/gotify-action@master + with: + gotify_api_base: '${{ secrets.RINOA_GOTIFY_URL }}' + gotify_app_token: '${{ secrets.RINOA_RUNNER_GOTIFY_TOKEN }}' + notification_title: 'GITEA: PR Check' + notification_message: 'Checking for existing PR... 🔍' + - name: Check if open PR exists + id: check-opened-pr-step + continue-on-error: true + run: | + tea login add --name gitea-rinoa --url "${{ secrets.RINOA_GITEA_URL }}" --user gitea-sonarqube-bot --password "${{ secrets.BOT_GITEA_PASSWORD }}" --token ${{ secrets.BOT_GITEA_TOKEN }} + pr_exists=$(tea pr list --repo ${{ github.repository }} --state open --fields index,title,head | egrep ${{ github.ref_name }} | tail -1 | wc -l) + echo "exists=$pr_exists" >> $GITHUB_OUTPUT + - name: Create PR + if: ${{ steps.check-opened-pr-step.outputs.exists == '0' }} + run: | + tea login default gitea-rinoa + pr_index_old=$(tea pr ls --repo ${{ github.repository }} --state all --fields index,title,head --output csv | sed -e 's|"||g' | egrep '^[0-9]' | head -1 | awk -F"," '{print $1}') + pr_index_new=$(expr ${pr_index_old} + 1) + tea pr c -r ${{ github.repository }} -t "Automated PR for ${{ github.ref_name }} - #${pr_index_new}" -d "Automatically created PR for branch: ${{ github.ref_name }}" -a ${{ github.actor }} -L "Docker Compose, Ansible Configs.j2" + - name: Gotify Notification + uses: eikendev/gotify-action@master + with: + gotify_api_base: '${{ secrets.RINOA_GOTIFY_URL }}' + gotify_app_token: '${{ secrets.RINOA_RUNNER_GOTIFY_TOKEN }}' + notification_title: 'GITEA: PR Check' + notification_message: 'PR Created 🎟️' + ansible-linting: + name: Ansible Linting + needs: [check-and-create-pr] + runs-on: ubuntu-latest + env: + VAULT_ADDR: ${{ secrets.RINOA_VAULT_ADDR }} + VAULT_TOKEN: ${{ secrets.VAULT_GITEA_TOKEN }} + VAULT_NAMESPACE: "" + steps: + - name: Checkout + uses: actions/checkout@v4 + - name: Fetch base branch + run: | + git fetch origin ${{ github.event.pull_request.base.ref }} + - name: Cache Ansible Galaxy Collections + uses: actions/cache@v3 + with: + path: ansible/collections + key: ${{ runner.os }}-ansible-${{ hashFiles('./ansible/collections/requirements.yml') }} + restore-keys: | + ${{ runner.os }}-ansible- + - name: Install Ansible + uses: alex-oleshkevich/setup-ansible@v1.0.1 + with: + version: "11.0.0" + - name: Install Vault + uses: cpanato/vault-installer@main + - name: Install hvac + run: pip install hvac + - name: Gotify Notification + uses: eikendev/gotify-action@master + with: + gotify_api_base: '${{ secrets.RINOA_GOTIFY_URL }}' + gotify_app_token: '${{ secrets.RINOA_RUNNER_GOTIFY_TOKEN }}' + notification_title: 'GITEA: Ansible Config Dry Run @ Rinoa' + notification_message: 'Starting Ansible dry run...' + - name: Ansible Playbook Dry Run + uses: arillso/action.playbook@0.1.0 + with: + check: true + galaxy_collections_path: ansible/collections + galaxy_requirements_file: ansible/collections/requirements.yml + inventory: ansible/inventory/hosts.yml + playbook: ansible/docker_config_deploy.yml + private_key: ${{ secrets.RINOA_ANSIBLE_PRIVATE_KEY }} + vault_password: ${{ secrets.ANSIBLE_VAULT_PASSWORD }} + verbose: 0 + - name: Gotify Notification + uses: eikendev/gotify-action@master + with: + gotify_api_base: '${{ secrets.RINOA_GOTIFY_URL }}' + gotify_app_token: '${{ secrets.RINOA_RUNNER_GOTIFY_TOKEN }}' + notification_title: 'GITEA: Docker Compose Dry Run @ Rinoa' + notification_message: 'Docker Compose dry run completed successfully.' + pr-merge: + name: PR Merge + needs: [ansible-linting] + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v4 + - name: Install tea + uses: supplypike/setup-bin@v4 + with: + uri: 'https://gitea.com/gitea/tea/releases/download/v0.9.2/tea-0.9.2-linux-amd64' + name: 'tea' + version: '0.9.2' + - name: PR Merge + id: pr_merge + run: | + tea login add --name gitea-rinoa --url ${{ secrets.RINOA_GITEA_URL }} --user gitea-sonarqube-bot --password "${{ secrets.BOT_GITEA_PASSWORD }}" --token ${{ secrets.BOT_GITEA_TOKEN }} + tea login default gitea-rinoa + echo "Merging PR..." + pr_index=$(tea pr ls --repo ${{ github.repository }} --state open --fields index,title,head,state --output csv | egrep ${{ github.ref_name }} | awk -F"," '{print $1}' | sed -e 's|"||g') + tea pr m --repo ${{ github.repository }} --title "Auto Merge of PR ${pr_index} - ${{ github.ref_name }}" --message "Merged by ${{ github.actor }}" ${pr_index} + echo "pr_index=${pr_index}" >> $GITHUB_OUTPUT + - name: Gotify Notification + uses: eikendev/gotify-action@master + with: + gotify_api_base: '${{ secrets.RINOA_GOTIFY_URL }}' + gotify_app_token: '${{ secrets.RINOA_RUNNER_GOTIFY_TOKEN }}' + notification_title: 'GITEA: PR Merge Successful' + notification_message: 'PR #${{ steps.pr_merge.outputs.pr_index }} merged.' + ansible-config-deploy: + name: Ansible Playbook Run + runs-on: ubuntu-latest + needs: [pr-merge] + env: + VAULT_ADDR: ${{ secrets.RINOA_VAULT_ADDR }} + VAULT_TOKEN: ${{ secrets.VAULT_GITEA_TOKEN }} + DOCKER_HOST: tcp://dockerproxy:2375 + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + ref: main + - name: Cache Vault install + id: cache-vault + uses: actions/cache@v4 + with: + path: /opt/hostedtoolcache/vault/1.18.0/x64 + key: vault-${{ runner.os }}-1.18.0 + - name: Install Ansible + uses: alex-oleshkevich/setup-ansible@v1.0.1 + with: + version: "11.0.0" + - name: Install Vault + uses: cpanato/vault-installer@main + - name: Install hvac + run: pip install hvac + - name: Gotify Notification + uses: eikendev/gotify-action@master + with: + gotify_api_base: '${{ secrets.RINOA_GOTIFY_URL }}' + gotify_app_token: '${{ secrets.RINOA_RUNNER_GOTIFY_TOKEN }}' + notification_title: 'GITEA: Ansible Config Deployment @ Rinoa' + notification_message: 'Starting config deployment with Ansible...' + - name: Ansible Playbook Dry Run + uses: arillso/action.playbook@0.1.0 + with: + check: false + galaxy_collections_path: ansible/collections + galaxy_requirements_file: ansible/collections/requirements.yml + inventory: ansible/inventory/hosts.yml + playbook: ansible/docker_config_deploy.yml + private_key: ${{ secrets.RINOA_ANSIBLE_PRIVATE_KEY }} + vault_password: ${{ secrets.ANSIBLE_VAULT_PASSWORD }} + - name: Gotify Notification + uses: eikendev/gotify-action@master + with: + gotify_api_base: '${{ secrets.RINOA_GOTIFY_URL }}' + gotify_app_token: '${{ secrets.RINOA_RUNNER_GOTIFY_TOKEN }}' + notification_title: 'GITEA: Ansible Config Deployment @ Rinoa' + notification_message: 'Deployment completed successfully.' \ No newline at end of file From 0c314a500015004ee4ee1f23aa43a3ba26b894d6 Mon Sep 17 00:00:00 2001 From: "Trez.One" Date: Tue, 20 May 2025 09:19:04 -0400 Subject: [PATCH 05/10] Small fixes. --- .gitea/workflows/pr-ansible-config-deployment.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitea/workflows/pr-ansible-config-deployment.yml b/.gitea/workflows/pr-ansible-config-deployment.yml index 854fa098..ffab9fa9 100644 --- a/.gitea/workflows/pr-ansible-config-deployment.yml +++ b/.gitea/workflows/pr-ansible-config-deployment.yml @@ -140,7 +140,7 @@ jobs: notification_title: 'GITEA: PR Merge Successful' notification_message: 'PR #${{ steps.pr_merge.outputs.pr_index }} merged.' ansible-config-deploy: - name: Ansible Playbook Run + name: Ansible Playbook Run (Service Configs) runs-on: ubuntu-latest needs: [pr-merge] env: From 6e5eb45ffe1bf0b68128ddbacb34f3c0fc559722 Mon Sep 17 00:00:00 2001 From: "Trez.One" Date: Tue, 20 May 2025 09:22:50 -0400 Subject: [PATCH 06/10] Small fixes. --- ansible/app-configs/crowdsec_acquis.yaml.j2 | 1 - 1 file changed, 1 deletion(-) diff --git a/ansible/app-configs/crowdsec_acquis.yaml.j2 b/ansible/app-configs/crowdsec_acquis.yaml.j2 index 2830970b..4553c94d 100644 --- a/ansible/app-configs/crowdsec_acquis.yaml.j2 +++ b/ansible/app-configs/crowdsec_acquis.yaml.j2 @@ -1,7 +1,6 @@ {% set vault_addr = 'https://vault.trez.wtf' %} {% set secrets_path = 'rinoa-docker/env' %} - source: journalctl journalctl_filter: - "--directory=/var/log/host/" From 48f9659eded880efa7ebfc4abfa0881cf8b25e34 Mon Sep 17 00:00:00 2001 From: "Trez.One" Date: Tue, 20 May 2025 09:58:35 -0400 Subject: [PATCH 07/10] Small fixes. --- .gitea/workflows/pr-ansible-config-deployment.yml | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/.gitea/workflows/pr-ansible-config-deployment.yml b/.gitea/workflows/pr-ansible-config-deployment.yml index ffab9fa9..0a0ef8ce 100644 --- a/.gitea/workflows/pr-ansible-config-deployment.yml +++ b/.gitea/workflows/pr-ansible-config-deployment.yml @@ -6,6 +6,7 @@ on: paths: - '**.j2' - 'ansible/**.yml' + jobs: check-and-create-pr: if: github.ref != 'refs/heads/main' @@ -70,13 +71,13 @@ jobs: - name: Fetch base branch run: | git fetch origin ${{ github.event.pull_request.base.ref }} - - name: Cache Ansible Galaxy Collections - uses: actions/cache@v3 - with: - path: ansible/collections - key: ${{ runner.os }}-ansible-${{ hashFiles('./ansible/collections/requirements.yml') }} - restore-keys: | - ${{ runner.os }}-ansible- + # - name: Cache Ansible Galaxy Collections + # uses: actions/cache@v3 + # with: + # path: ansible/collections + # key: ${{ runner.os }}-ansible-${{ hashFiles('./ansible/collections/requirements.yml') }} + # restore-keys: | + # ${{ runner.os }}-ansible- - name: Install Ansible uses: alex-oleshkevich/setup-ansible@v1.0.1 with: From 6245e88edcf24615f8e5fbee85307dd569ecb1a3 Mon Sep 17 00:00:00 2001 From: "Trez.One" Date: Tue, 20 May 2025 10:13:55 -0400 Subject: [PATCH 08/10] Changing order of tasks in playbook. --- ansible/docker_config_deploy.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/ansible/docker_config_deploy.yml b/ansible/docker_config_deploy.yml index a1f7acb5..6bd3d2fa 100644 --- a/ansible/docker_config_deploy.yml +++ b/ansible/docker_config_deploy.yml @@ -5,12 +5,6 @@ appdata_base_path: "~/.docker/config/appdata" tasks: - - name: Fetch Vault secrets once - ansible.builtin.set_fact: - vault_secrets: "{{ lookup('community.hashi_vault.vault_kv2_get', 'env', - engine_mount_point='rinoa-docker', url=vault_addr, - token=vault_token_cleaned)['secret'] }}" - - name: Ensure target directories exist ansible.builtin.file: path: "{{ appdata_base_path }}/{{ (item | basename | regex_replace('\\.j2$', '') | regex_replace('_', '/') | regex_replace('/[^/]+$', '')) }}" @@ -18,6 +12,12 @@ mode: '0755' loop: "{{ query('fileglob', 'app-configs/*.j2') }}" + - name: Fetch Vault secrets once + ansible.builtin.set_fact: + vault_secrets: "{{ lookup('community.hashi_vault.vault_kv2_get', 'env', + engine_mount_point='rinoa-docker', url=vault_addr, + token=vault_token_cleaned)['secret'] }}" + - name: Deploy configuration templates ansible.builtin.template: src: "{{ item }}" From 77bb59f594059611c5a24e6012fd8684fe24a26f Mon Sep 17 00:00:00 2001 From: "Trez.One" Date: Wed, 21 May 2025 09:16:28 -0400 Subject: [PATCH 09/10] Renaming Grafana alloy config. --- .../app-configs/grafana_alloy_config.alloy.j2 | 404 ++++++++++++++++++ 1 file changed, 404 insertions(+) create mode 100644 ansible/app-configs/grafana_alloy_config.alloy.j2 diff --git a/ansible/app-configs/grafana_alloy_config.alloy.j2 b/ansible/app-configs/grafana_alloy_config.alloy.j2 new file mode 100644 index 00000000..a6441e39 --- /dev/null +++ b/ansible/app-configs/grafana_alloy_config.alloy.j2 @@ -0,0 +1,404 @@ +{% set vault_addr = 'https://vault.trez.wtf' %} +{% set secrets_path = 'rinoa-docker/env' %} + +///////////////////////////////////////////////////////////////////////////////////////////////////////////// +// Agent globals +///////////////////////////////////////////////////////////////////////////////////////////////////////////// +local.file "endpoints" { + // The endpoints file is used to define the endpoints, credentials and options + // for the Agent export to. + filename = "/etc/alloy/endpoints.json" +} + +discovery.docker "rinoadocker" { + host = env("DOCKER_HOST") +} + +tracing { + write_to = [otelcol.exporter.otlp.tempo.input] +} +///////////////////////////////////////////////////////////////////////////////////////////////////////////// +// Metrics +///////////////////////////////////////////////////////////////////////////////////////////////////////////// +prometheus.remote_write "mimir" { + endpoint { + url = json_path(local.file.endpoints.content, ".metrics.url")[0] + basic_auth { + username = json_path(local.file.endpoints.content, ".metrics.basicAuth.username")[0] + password = json_path(local.file.endpoints.content, ".metrics.basicAuth.password")[0] + } + } +} + +prometheus.scrape "prometheus" { + targets = [{ + __address__ = "localhost:12345", + }] + forward_to = [prometheus.remote_write.mimir.receiver] + job_name = "prometheus" +} + +prometheus.exporter.unix "rinoa" { + procfs_path = "/host/proc" + sysfs_path = "/host/sys" + rootfs_path = "/rootfs" +} + +prometheus.scrape "rinoa" { + targets = prometheus.exporter.unix.rinoa.targets + forward_to = [prometheus.remote_write.mimir.receiver] + job_name = "rinoa_host" +} + +prometheus.exporter.cadvisor "docker" { + docker_host = env("DOCKER_HOST") + storage_duration = "5m" +} + +prometheus.scrape "docker" { + targets = prometheus.exporter.cadvisor.docker.targets + forward_to = [prometheus.remote_write.mimir.receiver] + job_name = "docker_stats" +} +///////////////////////////////////////////////////////////////////////////////////////////////////////////// +// Logging +///////////////////////////////////////////////////////////////////////////////////////////////////////////// +loki.write "loki" { + endpoint { + url = json_path(local.file.endpoints.content, ".logs.url")[0] + basic_auth { + username = json_path(local.file.endpoints.content, ".logs.basicAuth.username")[0] + password = json_path(local.file.endpoints.content, ".logs.basicAuth.password")[0] + } + } + external_labels = {} +} + +loki.source.journal "hostjournal" { + forward_to = [loki.write.loki.receiver] + max_age = "24h" + path = "/rootfs/var/log/journal/" + labels = { + job = "host-journal", + } +} + +local.file_match "system" { + path_targets = [{ + __address__ = "localhost", + __path__ = "/rootfs/var/log/*log", + job = "varlogs", + }] +} + +loki.source.file "system" { + targets = local.file_match.system.targets + forward_to = [loki.write.loki.receiver] +} + +loki.source.docker "containers" { + host = env("DOCKER_HOST") + targets = discovery.docker.rinoadocker.targets + forward_to = [loki.write.loki.receiver] + labels = { + job = "containerlogs", + } +} + +loki.process "containers" { + forward_to = [loki.write.loki.receiver] + // stage.docker {} + stage.json { + expressions = { + attrs = "", + output = "log", + stream = "stream", + } + } + + stage.json { + expressions = { + tag = "", + } + source = "attrs" + } + + stage.regex { + expression = "(?P(?:[^|]*[^|])).(?P(?:[^|]*[^|])).(?P(?:[^|]*[^|])).(?P(?:[^|]*[^|]))" + source = "tag" + } + + stage.timestamp { + source = "time" + format = "RFC3339Nano" + } + + stage.labels { + values = { + container_id = null, + container_name = null, + image_id = null, + image_name = null, + stream = null, + tag = null, + } + } + + stage.output { + source = "output" + } +} + +///////////////////////////////////////////////////////////////////////////////////////////////////////////// +// Traces +///////////////////////////////////////////////////////////////////////////////////////////////////////////// +beyla.ebpf "rinoadocker" { + open_port = "80-65535" + routes { + unmatched = "heauristic" + } + output { + traces = [ + otelcol.connector.servicegraph.tracemetrics.input, + otelcol.connector.spanmetrics.tracemetrics.input, + otelcol.processor.batch.default.input, + otelcol.connector.spanlogs.autologging.input, + ] + } +} + +prometheus.scrape "beyla" { + targets = beyla.ebpf.rinoadocker.targets + forward_to = [prometheus.remote_write.mimir.receiver] +} + +otelcol.auth.headers "tempo" { + header { + key = "Authorization" + value = join(["Basic ", json_path(local.file.endpoints.content, ".traces.basicAuthToken")[0]], "") + } +} + +otelcol.processor.batch "default" { + // Wait until we've received 16K of data. + send_batch_size = 16384 + send_batch_max_size = 16384 + // Or until 2 seconds have elapsed. + timeout = "2s" + // When the Agent has enough batched data, send it to the OpenTelemetry exporter named 'tempo'. + output { + traces = [otelcol.exporter.otlp.tempo.input] + } +} + +otelcol.exporter.otlp "tempo" { + // Define the client for exporting. + client { + // Authentication block. + auth = otelcol.auth.headers.tempo.handler + + // Send to the locally running Tempo instance, on port 4317 (OTLP gRPC). + endpoint = json_path(local.file.endpoints.content, ".traces.url")[0] + + // Configure TLS settings for communicating with the endpoint. + tls { + // The connection is insecure. + insecure = json_path(local.file.endpoints.content, ".traces.tls.insecure")[0] + // Do not verify TLS certificates when connecting. + insecure_skip_verify = json_path(local.file.endpoints.content, ".traces.tls.insecureSkipVerify")[0] + } + } +} + +otelcol.connector.spanlogs "autologging" { + // We only want to output a line for each root span (ie. every single trace), and not for every + // process or span (outputting a line for every span would be extremely verbose). + spans = false + roots = true + processes = false + // We want to ensure that the following three span attributes are included in the log line, if + // present. + span_attributes = [ "http.method", "http.target", "http.status_code" ] + + // Overrides the default key in the log line to be `traceId`, which is then used by Grafana to + // identify the trace ID for correlation with the Tempo datasource. + overrides { + trace_id_key = "traceId" + } + // Send to the OpenTelemetry Loki exporter. + output { + logs = [otelcol.exporter.loki.autologging.input] + } +} + +// Simply forwards the incoming OpenTelemetry log format out as a Loki log. +// We need this stage to ensure we can then process the logline as a Loki object. +otelcol.exporter.loki "autologging" { + forward_to = [loki.process.autologging.receiver] +} + +// The Loki processor allows us to accept a correctly formatted Loki log and mutate it into +// a set of fields for output. +loki.process "autologging" { + // The JSON stage simply extracts the `body` (the actual logline) from the Loki log, ignoring + // all other fields. + stage.json { + expressions = { "body" = "" } + } + // The output stage takes the body (the main logline) and uses this as the source for the output + // logline. In this case, it essentially turns it into logfmt. + stage.output { + source = "body" + } + + // Finally send the processed logline onto the Loki exporter. + forward_to = [loki.write.autologging.receiver] +} + +// The Loki writer receives a processed Loki log and then writes it to a Loki instance. +loki.write "autologging" { + // Add the `agent` value to the `job` label, so we can identify it as having been generated + // by Grafana Agent when querying. + external_labels = { + job = "agent", + } + + // Output the Loki log to the local Loki instance. + endpoint { + url = json_path(local.file.endpoints.content, ".logs.url")[0] + + // The basic auth credentials for the Loki instance. + basic_auth { + username = json_path(local.file.endpoints.content, ".logs.basicAuth.username")[0] + password = json_path(local.file.endpoints.content, ".logs.basicAuth.password")[0] + } + } +} + +// The Tail Sampling processor will use a set of policies to determine which received traces to keep +// and send to Tempo. +otelcol.processor.tail_sampling "errors" { + // Total wait time from the start of a trace before making a sampling decision. Note that smaller time + // periods can potentially cause a decision to be made before the end of a trace has occurred. + decision_wait = "30s" + + // The following policies follow a logical OR pattern, meaning that if any of the policies match, + // the trace will be kept. For logical AND, you can use the `and` policy. Every span of a trace is + // examined by each policy in turn. A match will cause a short-circuit. + + // This policy defines that traces that contain errors should be kept. + policy { + // The name of the policy can be used for logging purposes. + name = "sample-erroring-traces" + // The type must match the type of policy to be used, in this case examing the status code + // of every span in the trace. + type = "status_code" + // This block determines the error codes that should match in order to keep the trace, + // in this case the OpenTelemetry 'ERROR' code. + status_code { + status_codes = [ "ERROR" ] + } + } + + // This policy defines that only traces that are longer than 200ms in total should be kept. + policy { + // The name of the policy can be used for logging purposes. + name = "sample-long-traces" + // The type must match the policy to be used, in this case the total latency of the trace. + type = "latency" + // This block determines the total length of the trace in milliseconds. + latency { + threshold_ms = 200 + } + } + + // The output block forwards the kept traces onto the batch processor, which will marshall them + // for exporting to Tempo. + output { + traces = [otelcol.processor.batch.default.input] + } +} + +// The Spanmetrics Connector will generate RED metrics based on the incoming trace span data. +otelcol.connector.spanmetrics "tracemetrics" { + // The namespace explicit adds a prefix to all the generated span metrics names. + // In this case, we'll ensure they match as closely as possible those generated by Tempo. + namespace = "traces.spanmetrics" + + // Each extra dimension (metrics label) to be added to the generated metrics from matching span attributes. These + // need to be defined with a name and optionally a default value (in the following cases, we do not want a default + // value if the span attribute is not present). + dimension { + name = "http.method" + } + dimension { + name = "http.target" + } + dimension { + name = "http.status_code" + } + dimension { + name = "service.version" + } + + // A histogram block must be present, either explicitly defining bucket values or via an exponential block. + // We do the latter here. + histogram { + explicit { + } + } + + // The exemplar block is added to ensure we generate exemplars for traces on relevant metric values. + exemplars { + enabled = true + } + + // Generated metrics data is in OTLP format. We send this data to the OpenTelemetry Prometheus exporter to ensure + // it gets transformed into Prometheus format data. + output { + metrics = [otelcol.exporter.prometheus.tracemetrics.input] + } +} + +// The Servicegraph Connector will generate service graph metrics (edges and nodes) based on incoming trace spans. +otelcol.connector.servicegraph "tracemetrics" { + // Extra dimensions (metrics labels) to be added to the generated metrics from matching span attributes. + // For this component, this is defined as an array. There are no default values and the labels will not be generated + // for missing span attributes. + dimensions = [ + "http.method", + "http.target", + "http.status_code", + "service.version", + ] + + // Generated metrics data is in OTLP format. We send this data to the OpenTelemetry Prometheus exporter to ensure + // it gets transformed into Prometheus format data. + output { + metrics = [otelcol.exporter.prometheus.tracemetrics.input] + } +} + +otelcol.exporter.prometheus "tracemetrics" { + // Forward to our local Prometheus remote writer which will send the metrics to Mimir. + forward_to = [prometheus.remote_write.mimir.receiver] +} +///////////////////////////////////////////////////////////////////////////////////////////////////////////// +// Profiling +///////////////////////////////////////////////////////////////////////////////////////////////////////////// + +pyroscope.write "pyroscope" { + endpoint { + url = json_path(local.file.endpoints.content, ".profiles.url")[0] + basic_auth { + username = json_path(local.file.endpoints.content, ".profiles.basicAuth.username")[0] + password = json_path(local.file.endpoints.content, ".profiles.basicAuth.password")[0] + } + } + external_labels = {} +} + +pyroscope.ebpf "rinoadocker" { + forward_to = [pyroscope.write.pyroscope.receiver] + targets = discovery.docker.rinoadocker.targets +} \ No newline at end of file From 305f81bbdc4932640885c18ab3ccd55056977785 Mon Sep 17 00:00:00 2001 From: "Trez.One" Date: Wed, 21 May 2025 09:17:04 -0400 Subject: [PATCH 10/10] Renaming Grafana alloy config. --- .../app-configs/grafana_alloy_config.alloy | 404 ------------------ 1 file changed, 404 deletions(-) delete mode 100644 ansible/app-configs/grafana_alloy_config.alloy diff --git a/ansible/app-configs/grafana_alloy_config.alloy b/ansible/app-configs/grafana_alloy_config.alloy deleted file mode 100644 index a6441e39..00000000 --- a/ansible/app-configs/grafana_alloy_config.alloy +++ /dev/null @@ -1,404 +0,0 @@ -{% set vault_addr = 'https://vault.trez.wtf' %} -{% set secrets_path = 'rinoa-docker/env' %} - -///////////////////////////////////////////////////////////////////////////////////////////////////////////// -// Agent globals -///////////////////////////////////////////////////////////////////////////////////////////////////////////// -local.file "endpoints" { - // The endpoints file is used to define the endpoints, credentials and options - // for the Agent export to. - filename = "/etc/alloy/endpoints.json" -} - -discovery.docker "rinoadocker" { - host = env("DOCKER_HOST") -} - -tracing { - write_to = [otelcol.exporter.otlp.tempo.input] -} -///////////////////////////////////////////////////////////////////////////////////////////////////////////// -// Metrics -///////////////////////////////////////////////////////////////////////////////////////////////////////////// -prometheus.remote_write "mimir" { - endpoint { - url = json_path(local.file.endpoints.content, ".metrics.url")[0] - basic_auth { - username = json_path(local.file.endpoints.content, ".metrics.basicAuth.username")[0] - password = json_path(local.file.endpoints.content, ".metrics.basicAuth.password")[0] - } - } -} - -prometheus.scrape "prometheus" { - targets = [{ - __address__ = "localhost:12345", - }] - forward_to = [prometheus.remote_write.mimir.receiver] - job_name = "prometheus" -} - -prometheus.exporter.unix "rinoa" { - procfs_path = "/host/proc" - sysfs_path = "/host/sys" - rootfs_path = "/rootfs" -} - -prometheus.scrape "rinoa" { - targets = prometheus.exporter.unix.rinoa.targets - forward_to = [prometheus.remote_write.mimir.receiver] - job_name = "rinoa_host" -} - -prometheus.exporter.cadvisor "docker" { - docker_host = env("DOCKER_HOST") - storage_duration = "5m" -} - -prometheus.scrape "docker" { - targets = prometheus.exporter.cadvisor.docker.targets - forward_to = [prometheus.remote_write.mimir.receiver] - job_name = "docker_stats" -} -///////////////////////////////////////////////////////////////////////////////////////////////////////////// -// Logging -///////////////////////////////////////////////////////////////////////////////////////////////////////////// -loki.write "loki" { - endpoint { - url = json_path(local.file.endpoints.content, ".logs.url")[0] - basic_auth { - username = json_path(local.file.endpoints.content, ".logs.basicAuth.username")[0] - password = json_path(local.file.endpoints.content, ".logs.basicAuth.password")[0] - } - } - external_labels = {} -} - -loki.source.journal "hostjournal" { - forward_to = [loki.write.loki.receiver] - max_age = "24h" - path = "/rootfs/var/log/journal/" - labels = { - job = "host-journal", - } -} - -local.file_match "system" { - path_targets = [{ - __address__ = "localhost", - __path__ = "/rootfs/var/log/*log", - job = "varlogs", - }] -} - -loki.source.file "system" { - targets = local.file_match.system.targets - forward_to = [loki.write.loki.receiver] -} - -loki.source.docker "containers" { - host = env("DOCKER_HOST") - targets = discovery.docker.rinoadocker.targets - forward_to = [loki.write.loki.receiver] - labels = { - job = "containerlogs", - } -} - -loki.process "containers" { - forward_to = [loki.write.loki.receiver] - // stage.docker {} - stage.json { - expressions = { - attrs = "", - output = "log", - stream = "stream", - } - } - - stage.json { - expressions = { - tag = "", - } - source = "attrs" - } - - stage.regex { - expression = "(?P(?:[^|]*[^|])).(?P(?:[^|]*[^|])).(?P(?:[^|]*[^|])).(?P(?:[^|]*[^|]))" - source = "tag" - } - - stage.timestamp { - source = "time" - format = "RFC3339Nano" - } - - stage.labels { - values = { - container_id = null, - container_name = null, - image_id = null, - image_name = null, - stream = null, - tag = null, - } - } - - stage.output { - source = "output" - } -} - -///////////////////////////////////////////////////////////////////////////////////////////////////////////// -// Traces -///////////////////////////////////////////////////////////////////////////////////////////////////////////// -beyla.ebpf "rinoadocker" { - open_port = "80-65535" - routes { - unmatched = "heauristic" - } - output { - traces = [ - otelcol.connector.servicegraph.tracemetrics.input, - otelcol.connector.spanmetrics.tracemetrics.input, - otelcol.processor.batch.default.input, - otelcol.connector.spanlogs.autologging.input, - ] - } -} - -prometheus.scrape "beyla" { - targets = beyla.ebpf.rinoadocker.targets - forward_to = [prometheus.remote_write.mimir.receiver] -} - -otelcol.auth.headers "tempo" { - header { - key = "Authorization" - value = join(["Basic ", json_path(local.file.endpoints.content, ".traces.basicAuthToken")[0]], "") - } -} - -otelcol.processor.batch "default" { - // Wait until we've received 16K of data. - send_batch_size = 16384 - send_batch_max_size = 16384 - // Or until 2 seconds have elapsed. - timeout = "2s" - // When the Agent has enough batched data, send it to the OpenTelemetry exporter named 'tempo'. - output { - traces = [otelcol.exporter.otlp.tempo.input] - } -} - -otelcol.exporter.otlp "tempo" { - // Define the client for exporting. - client { - // Authentication block. - auth = otelcol.auth.headers.tempo.handler - - // Send to the locally running Tempo instance, on port 4317 (OTLP gRPC). - endpoint = json_path(local.file.endpoints.content, ".traces.url")[0] - - // Configure TLS settings for communicating with the endpoint. - tls { - // The connection is insecure. - insecure = json_path(local.file.endpoints.content, ".traces.tls.insecure")[0] - // Do not verify TLS certificates when connecting. - insecure_skip_verify = json_path(local.file.endpoints.content, ".traces.tls.insecureSkipVerify")[0] - } - } -} - -otelcol.connector.spanlogs "autologging" { - // We only want to output a line for each root span (ie. every single trace), and not for every - // process or span (outputting a line for every span would be extremely verbose). - spans = false - roots = true - processes = false - // We want to ensure that the following three span attributes are included in the log line, if - // present. - span_attributes = [ "http.method", "http.target", "http.status_code" ] - - // Overrides the default key in the log line to be `traceId`, which is then used by Grafana to - // identify the trace ID for correlation with the Tempo datasource. - overrides { - trace_id_key = "traceId" - } - // Send to the OpenTelemetry Loki exporter. - output { - logs = [otelcol.exporter.loki.autologging.input] - } -} - -// Simply forwards the incoming OpenTelemetry log format out as a Loki log. -// We need this stage to ensure we can then process the logline as a Loki object. -otelcol.exporter.loki "autologging" { - forward_to = [loki.process.autologging.receiver] -} - -// The Loki processor allows us to accept a correctly formatted Loki log and mutate it into -// a set of fields for output. -loki.process "autologging" { - // The JSON stage simply extracts the `body` (the actual logline) from the Loki log, ignoring - // all other fields. - stage.json { - expressions = { "body" = "" } - } - // The output stage takes the body (the main logline) and uses this as the source for the output - // logline. In this case, it essentially turns it into logfmt. - stage.output { - source = "body" - } - - // Finally send the processed logline onto the Loki exporter. - forward_to = [loki.write.autologging.receiver] -} - -// The Loki writer receives a processed Loki log and then writes it to a Loki instance. -loki.write "autologging" { - // Add the `agent` value to the `job` label, so we can identify it as having been generated - // by Grafana Agent when querying. - external_labels = { - job = "agent", - } - - // Output the Loki log to the local Loki instance. - endpoint { - url = json_path(local.file.endpoints.content, ".logs.url")[0] - - // The basic auth credentials for the Loki instance. - basic_auth { - username = json_path(local.file.endpoints.content, ".logs.basicAuth.username")[0] - password = json_path(local.file.endpoints.content, ".logs.basicAuth.password")[0] - } - } -} - -// The Tail Sampling processor will use a set of policies to determine which received traces to keep -// and send to Tempo. -otelcol.processor.tail_sampling "errors" { - // Total wait time from the start of a trace before making a sampling decision. Note that smaller time - // periods can potentially cause a decision to be made before the end of a trace has occurred. - decision_wait = "30s" - - // The following policies follow a logical OR pattern, meaning that if any of the policies match, - // the trace will be kept. For logical AND, you can use the `and` policy. Every span of a trace is - // examined by each policy in turn. A match will cause a short-circuit. - - // This policy defines that traces that contain errors should be kept. - policy { - // The name of the policy can be used for logging purposes. - name = "sample-erroring-traces" - // The type must match the type of policy to be used, in this case examing the status code - // of every span in the trace. - type = "status_code" - // This block determines the error codes that should match in order to keep the trace, - // in this case the OpenTelemetry 'ERROR' code. - status_code { - status_codes = [ "ERROR" ] - } - } - - // This policy defines that only traces that are longer than 200ms in total should be kept. - policy { - // The name of the policy can be used for logging purposes. - name = "sample-long-traces" - // The type must match the policy to be used, in this case the total latency of the trace. - type = "latency" - // This block determines the total length of the trace in milliseconds. - latency { - threshold_ms = 200 - } - } - - // The output block forwards the kept traces onto the batch processor, which will marshall them - // for exporting to Tempo. - output { - traces = [otelcol.processor.batch.default.input] - } -} - -// The Spanmetrics Connector will generate RED metrics based on the incoming trace span data. -otelcol.connector.spanmetrics "tracemetrics" { - // The namespace explicit adds a prefix to all the generated span metrics names. - // In this case, we'll ensure they match as closely as possible those generated by Tempo. - namespace = "traces.spanmetrics" - - // Each extra dimension (metrics label) to be added to the generated metrics from matching span attributes. These - // need to be defined with a name and optionally a default value (in the following cases, we do not want a default - // value if the span attribute is not present). - dimension { - name = "http.method" - } - dimension { - name = "http.target" - } - dimension { - name = "http.status_code" - } - dimension { - name = "service.version" - } - - // A histogram block must be present, either explicitly defining bucket values or via an exponential block. - // We do the latter here. - histogram { - explicit { - } - } - - // The exemplar block is added to ensure we generate exemplars for traces on relevant metric values. - exemplars { - enabled = true - } - - // Generated metrics data is in OTLP format. We send this data to the OpenTelemetry Prometheus exporter to ensure - // it gets transformed into Prometheus format data. - output { - metrics = [otelcol.exporter.prometheus.tracemetrics.input] - } -} - -// The Servicegraph Connector will generate service graph metrics (edges and nodes) based on incoming trace spans. -otelcol.connector.servicegraph "tracemetrics" { - // Extra dimensions (metrics labels) to be added to the generated metrics from matching span attributes. - // For this component, this is defined as an array. There are no default values and the labels will not be generated - // for missing span attributes. - dimensions = [ - "http.method", - "http.target", - "http.status_code", - "service.version", - ] - - // Generated metrics data is in OTLP format. We send this data to the OpenTelemetry Prometheus exporter to ensure - // it gets transformed into Prometheus format data. - output { - metrics = [otelcol.exporter.prometheus.tracemetrics.input] - } -} - -otelcol.exporter.prometheus "tracemetrics" { - // Forward to our local Prometheus remote writer which will send the metrics to Mimir. - forward_to = [prometheus.remote_write.mimir.receiver] -} -///////////////////////////////////////////////////////////////////////////////////////////////////////////// -// Profiling -///////////////////////////////////////////////////////////////////////////////////////////////////////////// - -pyroscope.write "pyroscope" { - endpoint { - url = json_path(local.file.endpoints.content, ".profiles.url")[0] - basic_auth { - username = json_path(local.file.endpoints.content, ".profiles.basicAuth.username")[0] - password = json_path(local.file.endpoints.content, ".profiles.basicAuth.password")[0] - } - } - external_labels = {} -} - -pyroscope.ebpf "rinoadocker" { - forward_to = [pyroscope.write.pyroscope.receiver] - targets = discovery.docker.rinoadocker.targets -} \ No newline at end of file