Adding Dockflare labels for tunnels; renaming Signoz migrator containers.

Merged by Trez.One

.gitea/workflows/pr-ansible-config-deployment.yaml

@@ -1,10 +1,12 @@
 name: Gitea Branch PR & Ansible Deployment
 on:
+  workflow_dispatch:
   push:
     branches-ignore:
       - 'main'
     paths:
       - '**.j2'
+      - '**/pr-ansible-config-deployment.yaml'
       - 'ansible/**.yml'
 jobs:
   check-and-create-pr:
@@ -40,7 +42,7 @@ jobs:
         continue-on-error: true
         run: |
           tea login add --name gitea-rinoa --url "${{ secrets.RINOA_GITEA_URL }}" --user gitea-sonarqube-bot --password "${{ secrets.BOT_GITEA_PASSWORD }}" --token ${{ secrets.BOT_GITEA_TOKEN }}
-          pr_exists=$(tea pr list --repo ${{ github.repository }} --state open --fields index,title,head | egrep ${{ github.ref_name }} | tail -1 | wc -l)
+          pr_exists=$(tea pr list --repo ${{ github.repository }} --state open --fields index,title,head | egrep '\[ANSIBLE\].*${{ github.ref_name }}' | tail -1 | wc -l)
           echo "exists=$pr_exists" >> $GITHUB_OUTPUT
       - name: Create PR
         if: ${{ steps.check-opened-pr-step.outputs.exists == '0' }}
@@ -48,7 +50,7 @@ jobs:
           tea login default gitea-rinoa
           pr_index_old=$(tea pr ls --repo ${{ github.repository }} --state all --fields index,title,head --output csv | sed -e 's|"||g' | egrep '^[0-9]' | head -1 | awk -F"," '{print $1}')
           pr_index_new=$(expr ${pr_index_old} + 1)
-          tea pr c -r ${{ github.repository }} -t "Automated PR for ${{ github.ref_name }} - #${pr_index_new}" -d "Automatically created PR for branch: ${{ github.ref_name }}" -a ${{ github.actor }} -L "Docker Compose, Ansible Configs.j2"
+          tea pr c -r ${{ github.repository }} -t "[ANSIBLE] Automated PR for ${{ github.ref_name }} - #${pr_index_new}" -d "Automatically created PR for branch: ${{ github.ref_name }}" -a ${{ github.actor }} -L "Ansible Configs.j2"
       - name: Gotify Notification
         uses: eikendev/gotify-action@master
         with:
@@ -57,7 +59,7 @@ jobs:
           notification_title: 'GITEA: PR Check'
           notification_message: 'PR Created 🎟️'
   ansible-linting:
-    name: Docker Compose & Ansible Lints
+    name: Ansible Lint
     needs: [check-and-create-pr]
     runs-on: ubuntu-latest
     env:
@@ -67,9 +69,6 @@ jobs:
     steps:
       - name: Checkout
         uses: actions/checkout@v4
-      - name: Fetch base branch
-        run: |
-          git fetch origin ${{ github.event.pull_request.base.ref }}
       - name: Cache Ansible Galaxy Collections
         uses: actions/cache@v3
         with:
@@ -80,11 +79,12 @@ jobs:
       - name: Install Ansible
         uses: alex-oleshkevich/setup-ansible@v1.0.1
         with:
-          version: "11.0.0"
+          version: "11.4.0"
       - name: Install Vault
         uses: cpanato/vault-installer@main
       - name: Install hvac
-        run: pip install hvac
+        run: |
+          pip install hvac
       - name: Gotify Notification
         uses: eikendev/gotify-action@master
         with:
@@ -93,16 +93,17 @@ jobs:
           notification_title: 'GITEA: Ansible Config Dry Run @ Rinoa'
           notification_message: 'Starting Ansible dry run...'
       - name: Ansible Playbook Dry Run
-        uses: arillso/action.playbook@0.1.0
+        uses: dawidd6/action-ansible-playbook@v3
         with:
-          check: true
-          galaxy_collections_path: ansible/collections
-          galaxy_requirements_file: ansible/collections/requirements.yml
-          inventory: ansible/inventory/hosts.yml
-          playbook: ansible/docker_config_deploy.yml
-          private_key: ${{ secrets.RINOA_ANSIBLE_PRIVATE_KEY }}
+          directory: ansible/
+          playbook: docker_config_deploy.yml
+          key: ${{ secrets.RINOA_ANSIBLE_PRIVATE_KEY }}
           vault_password: ${{ secrets.ANSIBLE_VAULT_PASSWORD }}
-          verbose: 0
+          requirements: collections/requirements.yml
+          options: |
+            --check
+            --inventory inventory/hosts.yml
+            -v
       - name: Gotify Notification
         uses: eikendev/gotify-action@master
         with:
@@ -139,8 +140,8 @@ jobs:
           gotify_app_token: '${{ secrets.RINOA_RUNNER_GOTIFY_TOKEN }}'
           notification_title: 'GITEA: PR Merge Successful'
           notification_message: 'PR #${{ steps.pr_merge.outputs.pr_index }} merged.'
-  ansible-config-docker-compose-deploy:
-    name: Ansible Configs & Docker Compose Deployment
+  ansible-config-deploy:
+    name: Ansible Config Deployment
     runs-on: ubuntu-latest
     needs: [pr-merge]
     env:
@@ -152,6 +153,10 @@ jobs:
         uses: actions/checkout@v4
         with:
           ref: main
+      - name: Set up Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: 3.12
       - name: Cache Vault install
         id: cache-vault
         uses: actions/cache@v4
@@ -161,11 +166,12 @@ jobs:
       - name: Install Ansible
         uses: alex-oleshkevich/setup-ansible@v1.0.1
         with:
-          version: "11.0.0"
+          version: "11.4.0"
       - name: Install Vault
         uses: cpanato/vault-installer@main
       - name: Install hvac
-        run: pip install hvac
+        run: |
+          pip install hvac
       - name: Gotify Notification
         uses: eikendev/gotify-action@master
         with:
@@ -173,16 +179,16 @@ jobs:
           gotify_app_token: '${{ secrets.RINOA_RUNNER_GOTIFY_TOKEN }}'
           notification_title: 'GITEA: Ansible Config Deployment @ Rinoa'
           notification_message: 'Starting config deployment with Ansible...'
-      - name: Ansible Playbook Dry Run
-        uses: arillso/action.playbook@0.1.0
+      - name: Ansible Playbook Config Deploy
+        uses: dawidd6/action-ansible-playbook@v3
         with:
-          check: false
-          galaxy_collections_path: ansible/collections
-          galaxy_requirements_file: ansible/collections/requirements.yml
-          inventory: ansible/inventory/hosts.yml
-          playbook: ansible/docker_config_deploy.yml
-          private_key: ${{ secrets.RINOA_ANSIBLE_PRIVATE_KEY }}
+          directory: ansible/
+          playbook: docker_config_deploy.yml
+          key: ${{ secrets.RINOA_ANSIBLE_PRIVATE_KEY }}
           vault_password: ${{ secrets.ANSIBLE_VAULT_PASSWORD }}
+          requirements: collections/requirements.yml
+          options: |
+            --inventory inventory/hosts.yml
       - name: Gotify Notification
         uses: eikendev/gotify-action@master
         with:

.gitea/workflows/pr-cloudflare-docker-deploy.yml

@@ -1,5 +1,6 @@
 name: Gitea Branch PR, Cloudflare DNS, README generation, & Docker Deployment
 on:
+  workflow_dispatch:
   push:
     branches-ignore:
       - 'main'
@@ -41,7 +42,7 @@ jobs:
         continue-on-error: true
         run: |
           tea login add --name gitea-rinoa --url "${{ secrets.RINOA_GITEA_URL }}" --user gitea-sonarqube-bot --password "${{ secrets.BOT_GITEA_PASSWORD }}" --token ${{ secrets.BOT_GITEA_TOKEN }}
-          pr_exists=$(tea pr list --repo ${{ github.repository }} --state open --fields index,title,head | egrep ${{ github.ref_name }} | tail -1 | wc -l)
+          pr_exists=$(tea pr list --repo ${{ github.repository }} --state open --fields index,title,head | egrep '\[DOCKER\].*${{ github.ref_name }}' | tail -1 | wc -l)
           echo "exists=$pr_exists" >> $GITHUB_OUTPUT
       - name: Create PR
         if: ${{ steps.check-opened-pr-step.outputs.exists == '0' }}
@@ -49,7 +50,7 @@ jobs:
           tea login default gitea-rinoa
           pr_index_old=$(tea pr ls --repo ${{ github.repository }} --state all --fields index,title,head --output csv | sed -e 's|"||g' | egrep '^[0-9]' | head -1 | awk -F"," '{print $1}')
           pr_index_new=$(expr ${pr_index_old} + 1)
-          tea pr c -r ${{ github.repository }} -t "Automated PR for ${{ github.ref_name }} - #${pr_index_new}" -d "Automatically created PR for branch: ${{ github.ref_name }}" -a ${{ github.actor }} -L "Docker Compose, Ansible Configs.j2"
+          tea pr c -r ${{ github.repository }} -t "[DOCKER] Automated PR for ${{ github.ref_name }} - #${pr_index_new}" -d "Automatically created PR for branch: ${{ github.ref_name }}" -a ${{ github.actor }} -L "Docker Compose"
       - name: Gotify Notification
         uses: eikendev/gotify-action@master
         with:
@@ -65,6 +66,7 @@ jobs:
       VAULT_ADDR: ${{ secrets.RINOA_VAULT_ADDR }}
       VAULT_TOKEN: ${{ secrets.VAULT_GITEA_TOKEN }}
       VAULT_NAMESPACE: ""
+      RINOA_REGISTRY_PASSWORD: ${{ secrets.BOT_GITEA_PASSWORD }}
     outputs:
       svc_deploy_list: ${{ steps.modded_svcs.outputs.rinoa_svcs }}
     steps:
@@ -75,7 +77,7 @@ jobs:
           git fetch origin ${{ github.event.pull_request.base.ref }}
       - name: Login to Gitea Container Registry
         run: |
-          docker login -u gitea-sonarqube-bot -p ${{ secrets.BOT_GITEA_PASSWORD }} git.trez.wtf
+          docker login -u gitea-sonarqube-bot -p ${RINOA_REGISTRY_PASSWORD} git.trez.wtf
       - name: Save both versions of docker-compose.yml
         run: |
           git show origin/main:docker-compose.yml > docker-compose-main.yml || touch docker-compose-main.yml
@@ -134,14 +136,17 @@ jobs:
       - name: Docker Compose Dry Run
         timeout-minutes: 360
         continue-on-error: true
-        uses: keatonLiu/docker-compose-remote-action@v1.2
+        uses: chaplyk/docker-compose-remote-action@v1.1
         with:
-          docker_compose_file: docker-compose.yml
-          docker_args: -d --remove-orphans --pull missing ${{ steps.modded_svcs.outputs.rinoa_svcs }}
-          ssh_user: gitea-deploy
           ssh_host: 192.168.1.254
-          ssh_host_public_key: ${{ secrets.RINOA_GITEA_PUBLIC_SSH_KEY }}
-          ssh_private_key: ${{ secrets.RINOA_GITEA_PRIVATE_SSH_KEY }}
+          ssh_port: 22
+          ssh_user: gitea-deploy
+          ssh_key: ${{ secrets.RINOA_GITEA_PRIVATE_SSH_KEY }}
+          service: ${{ steps.modded_svcs.outputs.rinoa_svcs }}
+          compose_file: docker-compose.yml
+          pull: false
+          build: false
+          options: -d --remove-orphans
         env:
           DOCKER_HOST: tcp://dockerproxy:2375
       - name: Gotify Notification
@@ -294,11 +299,13 @@ jobs:
   docker-compose-deploy:
     name: Docker Compose Deployment
     runs-on: ubuntu-latest
-    needs: [pr-merge]
+    needs: [docker-compose-dry-run, pr-merge]
     env:
       VAULT_ADDR: ${{ secrets.RINOA_VAULT_ADDR }}
       VAULT_TOKEN: ${{ secrets.VAULT_GITEA_TOKEN }}
       DOCKER_HOST: tcp://dockerproxy:2375
+      RINOA_REGISTRY_PASSWORD: ${{ secrets.BOT_GITEA_PASSWORD }}
+      DOCKER_SVC_LIST: ${{ needs.docker-compose-dry-run.outputs.svc_deploy_list }}
     steps:
       - name: Checkout
         uses: actions/checkout@v4
@@ -310,15 +317,11 @@ jobs:
         with:
           path: /opt/hostedtoolcache/vault/1.18.0/x64
           key: vault-${{ runner.os }}-1.18.0
-      - name: Install Ansible
-        uses: alex-oleshkevich/setup-ansible@v1.0.1
-        with:
-          version: "11.0.0"
       - name: Install Vault
         uses: cpanato/vault-installer@main
       - name: Login to Gitea Container Registry
         run: |
-          docker login -u gitea-sonarqube-bot -p ${{ secrets.BOT_GITEA_PASSWORD }} git.trez.wtf
+          docker login -u gitea-sonarqube-bot -p ${RINOA_REGISTRY_PASSWORD} git.trez.wtf
       - name: Gotify Notification
         uses: eikendev/gotify-action@master
         with:
@@ -332,14 +335,19 @@ jobs:
       - name: Docker Compose Deployment
         timeout-minutes: 360
         continue-on-error: true
-        uses: keatonLiu/docker-compose-remote-action@v1.2
+        uses: chaplyk/docker-compose-remote-action@v1.1
+        env:
+          DOCKER_HOST: tcp://dockerproxy:2375
         with:
-          docker_compose_file: docker-compose.yml
-          docker_args: -d --remove-orphans --pull missing ${{ docker-compose-dry-run.outputs.svc_deploy_list }}
-          ssh_user: gitea-deploy
           ssh_host: 192.168.1.254
-          ssh_host_public_key: ${{ secrets.RINOA_GITEA_PUBLIC_SSH_KEY }}
-          ssh_private_key: ${{ secrets.RINOA_GITEA_PRIVATE_SSH_KEY }}
+          ssh_port: 22
+          ssh_user: gitea-deploy
+          ssh_key: ${{ secrets.RINOA_GITEA_PRIVATE_SSH_KEY }}
+          service: ${DOCKER_SVC_LIST}
+          compose_file: docker-compose.yml
+          pull: false
+          build: false
+          options: -d --remove-orphans
       - name: Gotify Notification
         uses: eikendev/gotify-action@master
         with:

.gitea/workflows/vault-auto-unseal-flow.yml

@@ -1,7 +1,8 @@
 name: Auto-Unseal for Vault
 on:
+  workflow_dispatch:
   schedule:
-    - cron: "30 2 * * *"
+    - cron: "0 5 * * *"
 jobs:
   auto-unseal:
     name: Unseal Vault

README.md

@@ -31,6 +31,7 @@
 | dawarich-sidekiq | freikin/dawarich:latest |
 | dead-man-hand | ghcr.io/bkupidura/dead-man-hand:latest |
 | docker-socket-proxy | ghcr.io/tecnativa/docker-socket-proxy:latest |
+| dockflare | alplat/dockflare:stable |
 | duplicati | lscr.io/linuxserver/duplicati:latest |
 | excalidraw | excalidraw/excalidraw:latest |
 | explo | ghcr.io/lumepart/explo:latest |
@@ -38,7 +39,7 @@
 | flaresolverr | ghcr.io/flaresolverr/flaresolverr:latest |
 | freescout | tiredofit/freescout:latest |
 | ghost | ghost:latest |
-| gitea | gitea/gitea:1.23.1 |
+| gitea | gitea/gitea:1.24.0 |
 | gitea-db | postgres:14 |
 | gitea-runner | gitea/act_runner:latest |
 | gitea-sonarqube-bot | justusbunsi/gitea-sonarqube-bot:v0.4.0 |
@@ -129,6 +130,13 @@
 | scrutiny | ghcr.io/analogj/scrutiny:master-omnibus |
 | searxng | searxng/searxng:latest |
 | semaphore | semaphoreui/semaphore:v2.12.14 |
+| signoz-init-clickhouse | clickhouse/clickhouse-server:24.1.2-alpine |
+| signoz-zookeeper-1 | bitnami/zookeeper:3.7.1 |
+| signoz-clickhouse | clickhouse/clickhouse-server:24.1.2-alpine |
+| signoz-app | signoz/signoz:v0.86.2 |
+| signoz-otel-collector | signoz/signoz-otel-collector:v0.111.42 |
+| signoz-schema-migrator-sync | signoz/signoz-schema-migrator:v0.111.42 |
+| signoz-schema-migrator-async | signoz/signoz-schema-migrator:v0.111.42 |
 | sonarqube | mc1arke/sonarqube-with-community-branch-plugin:lts |
 | sonarqube-pg-db | postgres:17-alpine |
 | sonarr | lscr.io/linuxserver/sonarr:latest |

ansible/app-configs/adguardhome/AdGuardHome.yaml.j2

@@ -5,7 +5,7 @@ http:
   pprof:
     port: 6060
     enabled: false
-  address: 0.0.0.0:80
+  address: 0.0.0.0:8008
   session_ttl: 720h
 users:
   - name: admin
@@ -91,7 +91,7 @@ tls:
   enabled: true
   server_name: ""
   force_https: false
-  port_https: 443
+  port_https: 446
   port_dns_over_tls: 853
   port_dns_over_quic: 853
   port_dnscrypt: 0

ansible/app-configs/apprise/apprise.yml.j2

@@ -0,0 +1,6 @@
+{% set vault_addr = 'https://vault.trez.wtf' %}
+{% set secrets_path = 'rinoa-docker/env' %}
+
+urls:
+  - gotify://gotify/{{ lookup('community.hashi_vault.vault_kv2_get', 'env', engine_mount_point='rinoa-docker', url=vault_addr, token=vault_token_cleaned)['secret']['APPRISE_GOTIFY_TOKEN'] }}
+  - mailto://{{ lookup('community.hashi_vault.vault_kv2_get', 'env', engine_mount_point='rinoa-docker', url=vault_addr, token=vault_token_cleaned)['secret']['POSTAL_SMTP_AUTH_USER'] }}:{{ lookup('community.hashi_vault.vault_kv2_get', 'env', engine_mount_point='rinoa-docker', url=vault_addr, token=vault_token_cleaned)['secret']['POSTAL_SMTP_AUTH_PASSWORD'] }}@trez.wtf25?smtp=postal-smtp&from=noreply@trez.wtf

ansible/app-configs/apprise_apprise.yml.j2

@@ -1,6 +0,0 @@
-{% set vault_addr = 'https://vault.trez.wtf' %}
-{% set secrets_path = 'rinoa-docker/env' %}
-
-urls:
-  - gotify://gotify/{{ lookup('community.hashi_vault.vault_kv2_get', 'env', engine_mount_point='rinoa-docker', url=vault_addr, token=vault_token_cleaned)['secret']['APPRISE_GOTIFY_TOKEN'] }}
-  - mailtos://{{ lookup('community.hashi_vault.vault_kv2_get', 'env', engine_mount_point='rinoa-docker', url=vault_addr, token=vault_token_cleaned)['secret']['POSTAL_SMTP_AUTH_USER'] }}:{{ lookup('community.hashi_vault.vault_kv2_get', 'env', engine_mount_point='rinoa-docker', url=vault_addr, token=vault_token_cleaned)['secret']['POSTAL_SMTP_AUTH_PASSWORD'] }}@trez.wtf25?smtp=postal-smtp&from=noreply@trez.wtf

ansible/app-configs/crowdsec/acquis.yaml.j2

@@ -0,0 +1,65 @@
+{% set vault_addr = 'https://vault.trez.wtf' %}
+{% set secrets_path = 'rinoa-docker/env' %}
+source: journalctl
+journalctl_filter:
+  - "--directory=/var/log/host/"
+labels:
+  type: syslog
+---
+filenames:
+  - /var/log/swag/*
+labels:
+  type: nginx
+---
+filenames:
+  - /var/log/auth/auth.log
+labels:
+  type: syslog
+---
+filenames:
+  - /var/lib/mysql/log/mysql/*
+  - /var/lib/mysql/databases/*.err
+  - /var/lib/mysql/databases/*.log
+labels:
+  type: mariadb
+---
+source: docker
+container_name:
+  - adguard
+labels:
+  type: adguardhome
+---
+source: docker
+container_name:
+ - mongodb
+labels:
+  type: mongodb
+---
+source: docker
+container_name:
+ - immich-server
+labels:
+  type: immich
+---
+source: docker
+container_name:
+ - uptimekuma
+labels:
+  type: uptime-kuma
+---
+source: docker
+container_name:
+ - jellyfin
+labels:
+  type: jellyfin
+---
+source: docker
+container_name:
+ - navidrome
+labels:
+  type: navidrome
+---
+filenames:
+  - /var/log/audiobookshelf/*.txt
+labels:
+  type: audiobookshelf

ansible/app-configs/crowdsec/config.yaml.j2

@@ -0,0 +1,51 @@
+{% set vault_addr = 'https://vault.trez.wtf' %}
+{% set secrets_path = 'rinoa-docker/env' %}
+common:
+  daemonize: false
+  log_media: stdout
+  log_level: info
+  log_dir: /var/log/
+config_paths:
+  config_dir: /etc/crowdsec/
+  data_dir: /var/lib/crowdsec/data/
+  simulation_path: /etc/crowdsec/simulation.yaml
+  hub_dir: /etc/crowdsec/hub/
+  index_path: /etc/crowdsec/hub/.index.json
+  notification_dir: /etc/crowdsec/notifications/
+  plugin_dir: /usr/local/lib/crowdsec/plugins/
+crowdsec_service:
+  acquisition_path: /etc/crowdsec/acquis.yaml
+  acquisition_dir: /etc/crowdsec/acquis.d
+  parser_routines: 1
+plugin_config:
+  user: nobody
+  group: nobody
+cscli:
+  output: human
+db_config:
+  log_level: info
+  type: sqlite
+  db_path: /var/lib/crowdsec/data/crowdsec.db
+  flush:
+    max_items: 5000
+    max_age: 7d
+  use_wal: false
+api:
+  client:
+    insecure_skip_verify: false
+    credentials_path: /etc/crowdsec/local_api_credentials.yaml
+  server:
+    log_level: info
+    listen_uri: 0.0.0.0:8080
+    profiles_path: /etc/crowdsec/profiles.yaml
+    trusted_ips: # IP ranges, or IPs which can have admin API access
+      - 127.0.0.1
+      - ::1
+    online_client: # Central API credentials (to push signals and receive bad IPs)
+      credentials_path: /etc/crowdsec/online_api_credentials.yaml
+    enable: true
+prometheus:
+  enabled: true
+  level: full
+  listen_addr: 0.0.0.0
+  listen_port: 6060

ansible/app-configs/crowdsec/online-api-credentials.yaml.j2

@@ -0,0 +1,6 @@
+{% set vault_addr = 'https://vault.trez.wtf' %}
+{% set secrets_path = 'rinoa-docker/env' %}
+
+url: https://api.crowdsec.net/
+login: {{ lookup('community.hashi_vault.vault_kv2_get', 'env', engine_mount_point='rinoa-docker', url=vault_addr, token=vault_token_cleaned)['secret']['CROWDSEC_ONLINE_PASSWORD'] }}
+password: {{ lookup('community.hashi_vault.vault_kv2_get', 'env', engine_mount_point='rinoa-docker', url=vault_addr, token=vault_token_cleaned)['secret']['CROWDSEC_ONLINE_PASSWORD'] }}

ansible/app-configs/crowdsec_acquis.yaml.j2

@@ -1,15 +0,0 @@
-{% set vault_addr = 'https://vault.trez.wtf' %}
-{% set secrets_path = 'rinoa-docker/env' %}
-
-
-source: journalctl
-journalctl_filter: 
-  - "--directory=/var/log/host/"
-labels:
-  type: syslog
----
-filenames:
-  - /var/log/swag/*
-labels:
-  type: nginx
----

ansible/app-configs/signoz/clickhouse/cluster.ha.xml.j2

@@ -0,0 +1,75 @@
+<?xml version="1.0"?>
+<clickhouse>
+    <!-- ZooKeeper is used to store metadata about replicas, when using Replicated tables.
+         Optional. If you don't use replicated tables, you could omit that.
+
+         See https://clickhouse.com/docs/en/engines/table-engines/mergetree-family/replication/
+      -->
+    <zookeeper>
+        <node index="1">
+            <host>signoz-zookeeper-1</host>
+            <port>2181</port>
+        </node>
+        <node index="2">
+            <host>zookeeper-2</host>
+            <port>2181</port>
+        </node>
+        <node index="3">
+            <host>zookeeper-3</host>
+            <port>2181</port>
+        </node>
+    </zookeeper>
+
+    <!-- Configuration of clusters that could be used in Distributed tables.
+         https://clickhouse.com/docs/en/operations/table_engines/distributed/
+      -->
+    <remote_servers>
+        <cluster>
+            <!-- Inter-server per-cluster secret for Distributed queries
+                 default: no secret (no authentication will be performed)
+
+                 If set, then Distributed queries will be validated on shards, so at least:
+                 - such cluster should exist on the shard,
+                 - such cluster should have the same secret.
+
+                 And also (and which is more important), the initial_user will
+                 be used as current user for the query.
+
+                 Right now the protocol is pretty simple and it only takes into account:
+                 - cluster name
+                 - query
+
+                 Also it will be nice if the following will be implemented:
+                 - source hostname (see interserver_http_host), but then it will depends from DNS,
+                   it can use IP address instead, but then the you need to get correct on the initiator node.
+                 - target hostname / ip address (same notes as for source hostname)
+                 - time-based security tokens
+            -->
+            <!-- <secret></secret> -->
+            <shard>
+                <!-- Optional. Whether to write data to just one of the replicas. Default: false (write data to all replicas). -->
+                <!-- <internal_replication>false</internal_replication> -->
+                <!-- Optional. Shard weight when writing data. Default: 1. -->
+                <!-- <weight>1</weight> -->
+                <replica>
+                    <host>signoz-clickhouse</host>
+                    <port>9000</port>
+                    <!-- Optional. Priority of the replica for load_balancing. Default: 1 (less value has more priority). -->
+                    <!-- <priority>1</priority> -->
+                </replica>
+            </shard>
+            <shard>
+                <replica>
+                    <host>clickhouse-2</host>
+                    <port>9000</port>
+                </replica>
+            </shard>
+            <shard>
+                <replica>
+                    <host>clickhouse-3</host>
+                    <port>9000</port>
+                </replica>
+            </shard>
+        </cluster>
+    </remote_servers>
+</clickhouse>

ansible/app-configs/signoz/clickhouse/cluster.xml.j2

@@ -0,0 +1,75 @@
+<?xml version="1.0"?>
+<clickhouse>
+    <!-- ZooKeeper is used to store metadata about replicas, when using Replicated tables.
+         Optional. If you don't use replicated tables, you could omit that.
+
+         See https://clickhouse.com/docs/en/engines/table-engines/mergetree-family/replication/
+      -->
+    <zookeeper>
+        <node index="1">
+            <host>signoz-zookeeper-1</host>
+            <port>2181</port>
+        </node>
+        <!-- <node index="2">
+            <host>zookeeper-2</host>
+            <port>2181</port>
+        </node>
+        <node index="3">
+            <host>zookeeper-3</host>
+            <port>2181</port>
+        </node> -->
+    </zookeeper>
+
+    <!-- Configuration of clusters that could be used in Distributed tables.
+         https://clickhouse.com/docs/en/operations/table_engines/distributed/
+      -->
+    <remote_servers>
+        <cluster>
+            <!-- Inter-server per-cluster secret for Distributed queries
+                 default: no secret (no authentication will be performed)
+
+                 If set, then Distributed queries will be validated on shards, so at least:
+                 - such cluster should exist on the shard,
+                 - such cluster should have the same secret.
+
+                 And also (and which is more important), the initial_user will
+                 be used as current user for the query.
+
+                 Right now the protocol is pretty simple and it only takes into account:
+                 - cluster name
+                 - query
+
+                 Also it will be nice if the following will be implemented:
+                 - source hostname (see interserver_http_host), but then it will depends from DNS,
+                   it can use IP address instead, but then the you need to get correct on the initiator node.
+                 - target hostname / ip address (same notes as for source hostname)
+                 - time-based security tokens
+            -->
+            <!-- <secret></secret> -->
+            <shard>
+                <!-- Optional. Whether to write data to just one of the replicas. Default: false (write data to all replicas). -->
+                <!-- <internal_replication>false</internal_replication> -->
+                <!-- Optional. Shard weight when writing data. Default: 1. -->
+                <!-- <weight>1</weight> -->
+                <replica>
+                    <host>signoz-clickhouse</host>
+                    <port>9000</port>
+                    <!-- Optional. Priority of the replica for load_balancing. Default: 1 (less value has more priority). -->
+                    <!-- <priority>1</priority> -->
+                </replica>
+            </shard>
+            <!-- <shard>
+                <replica>
+                    <host>clickhouse-2</host>
+                    <port>9000</port>
+                </replica>
+            </shard>
+            <shard>
+                <replica>
+                    <host>clickhouse-3</host>
+                    <port>9000</port>
+                </replica>
+            </shard> -->
+        </cluster>
+    </remote_servers>
+</clickhouse>

ansible/app-configs/signoz/clickhouse/custom-function.xml.j2

@@ -0,0 +1,21 @@
+<functions>
+    <function>
+        <type>executable</type>
+        <name>histogramQuantile</name>
+        <return_type>Float64</return_type>
+        <argument>
+            <type>Array(Float64)</type>
+            <name>buckets</name>
+        </argument>
+        <argument>
+            <type>Array(Float64)</type>
+            <name>counts</name>
+        </argument>
+        <argument>
+            <type>Float64</type>
+            <name>quantile</name>
+        </argument>
+        <format>CSV</format>
+        <command>./histogramQuantile</command>
+    </function>
+</functions>

ansible/app-configs/signoz/clickhouse/storage.xml.j2

@@ -0,0 +1,41 @@
+<?xml version="1.0"?>
+<clickhouse>
+<storage_configuration>
+    <disks>
+        <default>
+            <keep_free_space_bytes>10485760</keep_free_space_bytes>
+        </default>
+        <s3>
+            <type>s3</type>
+            <!-- For S3 cold storage,
+                    if region is us-east-1, endpoint can be https://<bucket-name>.s3.amazonaws.com
+                    if region is not us-east-1, endpoint should be https://<bucket-name>.s3-<region>.amazonaws.com
+                For GCS cold storage,
+                    endpoint should be https://storage.googleapis.com/<bucket-name>/data/
+                -->
+            <endpoint>https://BUCKET-NAME.s3-REGION-NAME.amazonaws.com/data/</endpoint>
+            <access_key_id>ACCESS-KEY-ID</access_key_id>
+            <secret_access_key>SECRET-ACCESS-KEY</secret_access_key>
+            <!-- In case of S3, uncomment the below configuration in case you want to read
+                AWS credentials from the Environment variables if they exist. -->
+            <!-- <use_environment_credentials>true</use_environment_credentials> -->
+            <!-- In case of GCS, uncomment the below configuration, since GCS does
+                not support batch deletion and result in error messages in logs. -->
+            <!-- <support_batch_delete>false</support_batch_delete> -->
+        </s3>
+   </disks>
+   <policies>
+       <tiered>
+           <volumes>
+                <default>
+                    <disk>default</disk>
+                </default>
+                <s3>
+                    <disk>s3</disk>
+                    <perform_ttl_move_on_insert>0</perform_ttl_move_on_insert>
+                </s3>
+            </volumes>
+        </tiered>
+    </policies>
+</storage_configuration>
+</clickhouse>

ansible/app-configs/signoz/clickhouse/users.xml.j2

@@ -0,0 +1,123 @@
+<?xml version="1.0"?>
+<clickhouse>
+    <!-- See also the files in users.d directory where the settings can be overridden. -->
+
+    <!-- Profiles of settings. -->
+    <profiles>
+        <!-- Default settings. -->
+        <default>
+            <!-- Maximum memory usage for processing single query, in bytes. -->
+            <max_memory_usage>10000000000</max_memory_usage>
+
+            <!-- How to choose between replicas during distributed query processing.
+                 random - choose random replica from set of replicas with minimum number of errors
+                 nearest_hostname - from set of replicas with minimum number of errors, choose replica
+                  with minimum number of different symbols between replica's hostname and local hostname
+                  (Hamming distance).
+                 in_order - first live replica is chosen in specified order.
+                 first_or_random - if first replica one has higher number of errors, pick a random one from replicas with minimum number of errors.
+            -->
+            <load_balancing>random</load_balancing>
+        </default>
+
+        <!-- Profile that allows only read queries. -->
+        <readonly>
+            <readonly>1</readonly>
+        </readonly>
+    </profiles>
+
+    <!-- Users and ACL. -->
+    <users>
+        <!-- If user name was not specified, 'default' user is used. -->
+        <default>
+            <!-- See also the files in users.d directory where the password can be overridden.
+
+                 Password could be specified in plaintext or in SHA256 (in hex format).
+
+                 If you want to specify password in plaintext (not recommended), place it in 'password' element.
+                 Example: <password>qwerty</password>.
+                 Password could be empty.
+
+                 If you want to specify SHA256, place it in 'password_sha256_hex' element.
+                 Example: <password_sha256_hex>65e84be33532fb784c48129675f9eff3a682b27168c0ea744b2cf58ee02337c5</password_sha256_hex>
+                 Restrictions of SHA256: impossibility to connect to ClickHouse using MySQL JS client (as of July 2019).
+
+                 If you want to specify double SHA1, place it in 'password_double_sha1_hex' element.
+                 Example: <password_double_sha1_hex>e395796d6546b1b65db9d665cd43f0e858dd4303</password_double_sha1_hex>
+
+                 If you want to specify a previously defined LDAP server (see 'ldap_servers' in the main config) for authentication,
+                  place its name in 'server' element inside 'ldap' element.
+                 Example: <ldap><server>my_ldap_server</server></ldap>
+
+                 If you want to authenticate the user via Kerberos (assuming Kerberos is enabled, see 'kerberos' in the main config),
+                  place 'kerberos' element instead of 'password' (and similar) elements.
+                 The name part of the canonical principal name of the initiator must match the user name for authentication to succeed.
+                 You can also place 'realm' element inside 'kerberos' element to further restrict authentication to only those requests
+                  whose initiator's realm matches it.
+                 Example: <kerberos />
+                 Example: <kerberos><realm>EXAMPLE.COM</realm></kerberos>
+
+                 How to generate decent password:
+                 Execute: PASSWORD=$(base64 < /dev/urandom | head -c8); echo "$PASSWORD"; echo -n "$PASSWORD" | sha256sum | tr -d '-'
+                 In first line will be password and in second - corresponding SHA256.
+
+                 How to generate double SHA1:
+                 Execute: PASSWORD=$(base64 < /dev/urandom | head -c8); echo "$PASSWORD"; echo -n "$PASSWORD" | sha1sum | tr -d '-' | xxd -r -p | sha1sum | tr -d '-'
+                 In first line will be password and in second - corresponding double SHA1.
+            -->
+            <password></password>
+
+            <!-- List of networks with open access.
+
+                 To open access from everywhere, specify:
+                    <ip>::/0</ip>
+
+                 To open access only from localhost, specify:
+                    <ip>::1</ip>
+                    <ip>127.0.0.1</ip>
+
+                 Each element of list has one of the following forms:
+                 <ip> IP-address or network mask. Examples: 213.180.204.3 or 10.0.0.1/8 or 10.0.0.1/255.255.255.0
+                     2a02:6b8::3 or 2a02:6b8::3/64 or 2a02:6b8::3/ffff:ffff:ffff:ffff::.
+                 <host> Hostname. Example: server01.clickhouse.com.
+                     To check access, DNS query is performed, and all received addresses compared to peer address.
+                 <host_regexp> Regular expression for host names. Example, ^server\d\d-\d\d-\d\.clickhouse\.com$
+                     To check access, DNS PTR query is performed for peer address and then regexp is applied.
+                     Then, for result of PTR query, another DNS query is performed and all received addresses compared to peer address.
+                     Strongly recommended that regexp is ends with $
+                 All results of DNS requests are cached till server restart.
+            -->
+            <networks>
+                <ip>::/0</ip>
+            </networks>
+
+            <!-- Settings profile for user. -->
+            <profile>default</profile>
+
+            <!-- Quota for user. -->
+            <quota>default</quota>
+
+            <!-- User can create other users and grant rights to them. -->
+            <!-- <access_management>1</access_management> -->
+        </default>
+    </users>
+
+    <!-- Quotas. -->
+    <quotas>
+        <!-- Name of quota. -->
+        <default>
+            <!-- Limits for time interval. You could specify many intervals with different limits. -->
+            <interval>
+                <!-- Length of interval. -->
+                <duration>3600</duration>
+
+                <!-- No limits. Just calculate resource usage for time interval. -->
+                <queries>0</queries>
+                <errors>0</errors>
+                <result_rows>0</result_rows>
+                <read_rows>0</read_rows>
+                <execution_time>0</execution_time>
+            </interval>
+        </default>
+    </quotas>
+</clickhouse>

ansible/app-configs/signoz/otel/otel-collector-config.yaml.j2

@@ -0,0 +1,103 @@
+receivers:
+  otlp:
+    protocols:
+      grpc:
+        endpoint: 0.0.0.0:4317
+      http:
+        endpoint: 0.0.0.0:4318
+  prometheus:
+    config:
+      global:
+        scrape_interval: 60s
+      scrape_configs:
+        - job_name: otel-collector
+          static_configs:
+          - targets:
+              - localhost:8888
+            labels:
+              job_name: otel-collector
+processors:
+  batch:
+    send_batch_size: 10000
+    send_batch_max_size: 11000
+    timeout: 10s
+  resourcedetection:
+    # Using OTEL_RESOURCE_ATTRIBUTES envvar, env detector adds custom labels.
+    detectors: [env, system]
+    timeout: 2s
+  signozspanmetrics/delta:
+    metrics_exporter: clickhousemetricswrite, signozclickhousemetrics
+    metrics_flush_interval: 60s
+    latency_histogram_buckets: [100us, 1ms, 2ms, 6ms, 10ms, 50ms, 100ms, 250ms, 500ms, 1000ms, 1400ms, 2000ms, 5s, 10s, 20s, 40s, 60s ]
+    dimensions_cache_size: 100000
+    aggregation_temporality: AGGREGATION_TEMPORALITY_DELTA
+    enable_exp_histogram: true
+    dimensions:
+      - name: service.namespace
+        default: default
+      - name: deployment.environment
+        default: default
+      # This is added to ensure the uniqueness of the timeseries
+      # Otherwise, identical timeseries produced by multiple replicas of
+      # collectors result in incorrect APM metrics
+      - name: signoz.collector.id
+      - name: service.version
+      - name: browser.platform
+      - name: browser.mobile
+      - name: k8s.cluster.name
+      - name: k8s.node.name
+      - name: k8s.namespace.name
+      - name: host.name
+      - name: host.type
+      - name: container.name
+extensions:
+  health_check:
+    endpoint: 0.0.0.0:13133
+  pprof:
+    endpoint: 0.0.0.0:1777
+exporters:
+  clickhousetraces:
+    datasource: tcp://clickhouse:9000/signoz_traces
+    low_cardinal_exception_grouping: ${env:LOW_CARDINAL_EXCEPTION_GROUPING}
+    use_new_schema: true
+  clickhousemetricswrite:
+    endpoint: tcp://clickhouse:9000/signoz_metrics
+    disable_v2: true
+    resource_to_telemetry_conversion:
+      enabled: true
+  clickhousemetricswrite/prometheus:
+    endpoint: tcp://clickhouse:9000/signoz_metrics
+    disable_v2: true
+  signozclickhousemetrics:
+    dsn: tcp://clickhouse:9000/signoz_metrics
+  clickhouselogsexporter:
+    dsn: tcp://clickhouse:9000/signoz_logs
+    timeout: 10s
+    use_new_schema: true
+  # debug: {}
+service:
+  telemetry:
+    logs:
+      encoding: json
+    metrics:
+      address: 0.0.0.0:8888
+  extensions:
+    - health_check
+    - pprof
+  pipelines:
+    traces:
+      receivers: [otlp]
+      processors: [signozspanmetrics/delta, batch]
+      exporters: [clickhousetraces]
+    metrics:
+      receivers: [otlp]
+      processors: [batch]
+      exporters: [clickhousemetricswrite, signozclickhousemetrics]
+    metrics/prometheus:
+      receivers: [prometheus]
+      processors: [batch]
+      exporters: [clickhousemetricswrite/prometheus, signozclickhousemetrics]
+    logs:
+      receivers: [otlp]
+      processors: [batch]
+      exporters: [clickhouselogsexporter]

ansible/app-configs/signoz/otel/otel-collector-opamp-config.yaml.j2

@@ -0,0 +1 @@
+server_endpoint: ws://signoz:4320/v1/opamp

ansible/app-configs/signoz/prometheus.yml.j2

@@ -0,0 +1,25 @@
+# my global config
+global:
+  scrape_interval:     5s # Set the scrape interval to every 15 seconds. Default is every 1 minute.
+  evaluation_interval: 15s # Evaluate rules every 15 seconds. The default is every 1 minute.
+  # scrape_timeout is set to the global default (10s).
+
+# Alertmanager configuration
+alerting:
+  alertmanagers:
+  - static_configs:
+    - targets:
+      - alertmanager:9093
+
+# Load rules once and periodically evaluate them according to the global 'evaluation_interval'.
+rule_files: []
+  # - "first_rules.yml"
+  # - "second_rules.yml"
+  # - 'alerts.yml'
+
+# A scrape configuration containing exactly one endpoint to scrape:
+# Here it's Prometheus itself.
+scrape_configs: []
+
+remote_read:
+  - url: tcp://clickhouse:9000/signoz_metrics

ansible/docker_config_deploy.yml

@@ -1,20 +1,42 @@
----
 - name: Deploy Docker Service Configurations
   hosts: rinoa
   vars:
     appdata_base_path: "~/.docker/config/appdata"
+    template_base_path: "{{ playbook_dir }}/app-configs"
 
   tasks:
+    - name: Recursively collect all Jinja2 templates (*.j2)
+      ansible.builtin.find:
+        paths: "{{ template_base_path }}"
+        patterns: "*.j2"
+        recurse: true
+      register: template_files
+
+    - name: Set relative template path (without .j2) for each file
+      ansible.builtin.set_fact:
+        rel_template_path: >-
+          {{ item.path
+             | regex_replace('^' + (template_base_path | regex_escape) + '/', '')
+             | regex_replace('\\.j2$', '') }}
+      loop: "{{ template_files.files }}"
+      loop_control:
+        loop_var: item
+      register: rel_paths
+
     - name: Ensure target directories exist
       ansible.builtin.file:
-        path: "{{ appdata_base_path }}/{{ (item | basename | regex_replace('\\.j2$', '') | regex_replace('_', '/') | regex_replace('/[^/]+$', '')) }}"
+        path: "{{ appdata_base_path }}/{{ item.ansible_facts.rel_template_path | dirname }}"
         state: directory
         mode: '0755'
-      loop: "{{ query('fileglob', 'app-configs/*.j2') }}"
+      loop: "{{ rel_paths.results }}"
+      loop_control:
+        label: "{{ item.ansible_facts.rel_template_path }}"
 
-    - name: Deploy configuration templates
+    - name: Deploy rendered templates
       ansible.builtin.template:
-        src: "{{ item }}"
-        dest: "{{ appdata_base_path }}/{{ item | basename | regex_replace('\\.j2$', '') | regex_replace('_', '/') }}"
+        src: "{{ item.item.path | regex_replace('^' + (playbook_dir | regex_escape) + '/', '') }}"
+        dest: "{{ appdata_base_path }}/{{ item.ansible_facts.rel_template_path }}"
         mode: '0644'
-      loop: "{{ query('fileglob', 'app-configs/*.j2') }}"
+      loop: "{{ rel_paths.results }}"
+      loop_control:
+        label: "{{ item.ansible_facts.rel_template_path }}"

ansible/group_vars/all.yml

@@ -1,14 +1,14 @@
 vault_addr: "https://vault.trez.wtf"
 vault_token: !vault |
   $ANSIBLE_VAULT;1.1;AES256
-  39306238386563313462666238333237346239326636633731326263653639646235363937386333
-  6138653434613437643134653463363230303038373765380a636162663734393632396638313261
-  39613730633935373063663030616131653731376461333762633131633066366165343536323031
-  3539373461383138310a383734313237313231363539383632323130336536656662313861336261
-  65393033633461363837366462656134386430353236343136616161663364376261623834366466
-  30303765393039376666303937663839663630623063666135313636353432396161333434653435
-  32623634313531343466613966663139333234616137646636636134373264333263343533393331
-  32313530373164653730656662383837626139643364376134376634613237323063343731663734
-  36306335303936633334353564306239663563366435316464343039373965383032
+  62353532343234343230663331623062376533346166343963383464303535646362376233663361
+  3532343530653365663331393339646337653564316337390a646264353561623132366635343032
+  63326535376434353837663334366336613631346161363034646134333439613531376362646161
+  6438316662626566340a346665666234386630633764376336333063363934643162393565386330
+  35333139303939613232303264646236326637613862303339353334623066393966353032333839
+  33323962303635333335376364366336663035303530396262356130373537363134303937353433
+  34393338336666396338616465666466613931373461663761366235643437646136373039353939
+  33643133313264303637646336653537383337336661313765663366356262343064316334313337
+  35306232303132653566356130343366313139336665313737363732613261623439
 vault_token_cleaned: "{{ vault_token | regex_replace('\\n', '') }}"
 secrets_path: "rinoa-docker/env"

docker-compose.yml

@@ -1,13 +1,5 @@
 name: compose
 networks:
-  bitmagnet:
-    driver: bridge
-    ipam:
-      config:
-        - gateway: 192.168.55.1
-          subnet: 192.168.55.0/27
-      driver: default
-    name: compose_bitmagnet
   default:
     name: compose_default
   nextcloud-aio:
@@ -51,6 +43,65 @@ x-maxun: &maxun-env
     CHROMIUM_FLAGS: '--disable-gpu --no-sandbox --headless=new'
     #DEBUG: pw:api
     #PWDEBUG: 1
+x-signoz-common: &signoz-common
+  # networks:
+  #   - signoz-net
+  restart: unless-stopped
+  # logging:
+  #   options:
+  #     max-size: 50m
+  #     max-file: "3"
+x-signoz-clickhouse-defaults: &signoz-clickhouse-defaults
+  <<: *signoz-common
+  # addding non LTS version due to this fix https://github.com/ClickHouse/ClickHouse/commit/32caf8716352f45c1b617274c7508c86b7d1afab
+  image: clickhouse/clickhouse-server:24.1.2-alpine
+  tty: true
+  labels:
+    signoz.io/scrape: "true"
+    signoz.io/port: "9363"
+    signoz.io/path: "/metrics"
+  depends_on:
+    signoz-init-clickhouse:
+      condition: service_completed_successfully
+    signoz-zookeeper-1:
+      condition: service_healthy
+  healthcheck:
+    test:
+      - CMD
+      - wget
+      - --spider
+      - -q
+      - 0.0.0.0:8123/ping
+    interval: 30s
+    timeout: 5s
+    retries: 3
+  ulimits:
+    nproc: 65535
+    nofile:
+      soft: 262144
+      hard: 262144
+x-signoz-zookeeper-defaults: &signoz-zookeeper-defaults
+  <<: *signoz-common
+  image: bitnami/zookeeper:3.7.1
+  user: root
+  labels:
+    signoz.io/scrape: "true"
+    signoz.io/port: "9141"
+    signoz.io/path: "/metrics"
+  healthcheck:
+    test:
+      - CMD-SHELL
+      - curl -s -m 2 http://localhost:8080/commands/ruok | grep error | grep null
+    interval: 30s
+    timeout: 5s
+    retries: 3
+x-signoz-db-depend: &signoz-db-depend
+  <<: *signoz-common
+  depends_on:
+    signoz-clickhouse:
+      condition: service_healthy
+    signoz-schema-migrator-sync:
+      condition: service_completed_successfully
 services:
   actual_server:
     container_name: actualbudget
@@ -79,6 +130,9 @@ services:
     volumes:
       - ${DOCKER_VOLUME_CONFIG}/actual-budget:/data
   adguard:
+    cap_add:
+      - NET_BIND_SERVICE
+      - NET_RAW
     container_name: adguard
     environment:
       TZ: ${TZ}
@@ -86,6 +140,8 @@ services:
     labels:
       swag: enable
       swag_proto: http
+      swag_port: 8008
+      swag_address: 192.168.1.254
       swag_url: adgh.${MY_TLD}
       homepage.group: System Administration
       homepage.name: AdGuard Home
@@ -93,18 +149,19 @@ services:
       homepage.href: https://adgh.${MY_TLD}
       homepage.description: Ad-blocking/DNS
       homepage.widget.type: adguard
-      homepage.widget.url: http://adguard:80
+      homepage.widget.url: http://192.168.1.254:8008
       homepage.widget.username: admin
       homepage.widget.password: ${ADGUARD_PASSWORD}
-    ports:
-      - "192.168.1.254:53:53/udp"
-      - "192.168.1.254:53:53/tcp"
-      - 3001:3000
-      - "192.168.1.254:446:443/tcp"
-      - 8008:80
-      - "192.168.1.254:853:853/tcp"
-      - 67:67
-      - 688:68
+    network_mode: host
+    # ports:
+    #   - "192.168.1.254:53:53/udp"
+    #   - "192.168.1.254:53:53/tcp"
+    #   - 3001:3000
+    #   - "192.168.1.254:446:443/tcp"
+    #   - 8008:80
+    #   - "192.168.1.254:853:853/tcp"
+    #   - 67:67
+    #   - 688:68
     restart: unless-stopped
     user: 1000:1000
     volumes:
@@ -655,7 +712,29 @@ services:
       DOCKER_HOST: tcp://dockerproxy:2375
       GID: 1000
       BOUNCER_KEY_SWAG: ${CROWDSEC_API_KEY}
-      COLLECTIONS: corvese/apache-guacamole crowdsecurity/home-assistant crowdsecurity/http-cve crowdsecurity/iptables crowdsecurity/linux crowdsecurity/mariadb crowdsecurity/nextcloud crowdsecurity/nginx crowdsecurity/whitelist-good-actors Dominic-Wagner/vaultwarden gauth-fr/immich LePresidente/adguardhome LePresidente/authelia LePresidente/gitea LePresidente/jellyfin LePresidente/ombi plague-doctor/audiobookshelf schiz0phr3ne/sonarr sdwilsh/navidrome timokoessler/mongodb timokoessler/uptime-kuma xs539/joplin-server
+      COLLECTIONS: >-
+        corvese/apache-guacamole
+        crowdsecurity/home-assistant
+        crowdsecurity/http-cve
+        crowdsecurity/iptables
+        crowdsecurity/linux
+        crowdsecurity/mariadb
+        crowdsecurity/nextcloud
+        crowdsecurity/nginx
+        crowdsecurity/whitelist-good-actors
+        Dominic-Wagner/vaultwarden
+        gauth-fr/immich
+        LePresidente/adguardhome
+        LePresidente/authelia
+        LePresidente/gitea
+        LePresidente/jellyfin
+        LePresidente/ombi
+        plague-doctor/audiobookshelf
+        schiz0phr3ne/sonarr
+        sdwilsh/navidrome
+        timokoessler/mongodb
+        timokoessler/uptime-kuma
+        xs539/joplin-server
     image: crowdsecurity/crowdsec:latest
     networks:
       default: null
@@ -665,36 +744,16 @@ services:
     security_opt:
       - no-new-privileges=true
     volumes:
-      - source: ${DOCKER_VOLUME_CONFIG}/crowdsec/config.yaml.local
-        target: /etc/crowdsec/config.yaml.local
-        type: bind
-        bind:
-          create_host_path: true
-      - source: ${DOCKER_VOLUME_CONFIG}/crowdsec/local_api_credentials.yaml.local
-        target: /etc/crowdsec/local_api_credentials.yaml.local
-        type: bind
-        bind:
-          create_host_path: true
-      - read_only: true
-        source: ${DOCKER_VOLUME_CONFIG}/swag/log/nginx
-        target: /var/log/swag
-        type: bind
-        bind:
-          create_host_path: true
-      - source: crowdsec-config
-        target: /etc/crowdsec
-        type: volume
-        volume: {}
-      - source: crowdsec-db
-        target: /var/lib/crowdsec/data
-        type: volume
-        volume: {}
-      - bind:
-          create_host_path: true
-        read_only: true
-        source: /var/log/journal
-        target: /var/log/host
-        type: bind
+      - ${DOCKER_VOLUME_CONFIG}/crowdsec/config.yaml.local:/etc/crowdsec/config.yaml
+      - ${DOCKER_VOLUME_CONFIG}/crowdsec/local-api-credentials.yaml:/etc/crowdsec/local_api_credentials.yaml
+      - ${DOCKER_VOLUME_CONFIG}/crowdsec/online-api-credentials.yaml:/etc/crowdsec/online_api_credentials.yaml
+      - ${DOCKER_VOLUME_CONFIG}/swag/log/nginx:/var/log/swag:ro # SWAG
+      - ${DOCKER_VOLUME_CONFIG}/mariadb/:/var/lib/mysql:ro # MariaDB
+      - ${DOCKER_VOLUME_CONFIG}/audiobookshelf/.metadata/logs:/var/log/audiobookself:ro # Audiobookshelf
+      - crowdsec-config:/etc/crowdsec
+      - crowdsec-db:/var/lib/crowdsec/data
+      - /var/log/journal:/var/log/host:ro
+      - /var/log/auth.log:/var/log/host/auth.log:ro
   crowdsec-dashboard:
     container_name: crowdsec-dashboard
     depends_on:
@@ -982,6 +1041,62 @@ services:
         source: /var/run/docker.sock
         target: /var/run/docker.sock
         type: bind
+  dockflare:
+    container_name: dockflare
+    environment:
+      AGENT_STATUS_UPDATE_INTERVAL_SECONDS: 10
+      CF_ACCOUNT_ID: ${CLOUDFLARE_ACCOUNT_ID}
+      CF_API_TOKEN: ${CLOUDFLAREDDNS_ENVIRONMENT_APITOKEN}
+      CF_ZONE_ID: ${CLOUDFLARE_ZONE_ID}
+      CLEANUP_INTERVAL_SECONDS: 300
+      CLOUDFLARED_NETWORK_NAME: compose_default
+      DEFAULT_NO_TLS_VERIFY: false
+      GRACE_PERIOD_SECONDS: 600
+      LABEL_PREFIX: cloudflare.tunnel
+      MAX_CONCURRENT_DNS_OPS: 3
+      RECONCILIATION_BATCH_SIZE: 3
+      SCAN_ALL_NETWORKS: false
+      STATE_FILE_PATH: /app/data/state.json
+      TRUSTED_PROXIES: 192.168.1.0/24,172.18.0.0/16
+      TUNNEL_DNS_SCAN_ZONE_NAMES:
+      TUNNEL_NAME: dockflared-tunnel
+      TZ: ${TZ}
+    image: alplat/dockflare:stable # Or :unstable for the latest features
+    labels:
+      homepage.group: Privacy/Security
+      homepage.name: DockFlare
+      homepage.href: https://cftunn.${MY_TLD}
+      homepage.icon: /icons/dockflare.png
+      homepage.description: Cloudflare Tunnel controller
+      swag: enable
+      swag_auth: authelia
+      swag_proto: http
+      swag_url: cftunn.${MY_TLD}
+      swag.uptime-kuma.enabled: true
+      swag.uptime-kuma.monitor.url: https://cftunn.${MY_TLD}
+      swag.uptime-kuma.monitor.interval: 300
+      ### EXAMPLE CF TUNNEL LABELS ###
+      # Enable DockFlare management for this container
+      # - "cloudflare.tunnel.enable=true" 
+      # The public hostname to expose
+      # - "cloudflare.tunnel.hostname=my-service.example.com"
+      # The internal service address (protocol://container_name_or_ip:port)
+      # Service type (http, https, tcp, ssh, rdp, http_status) is inferred from the prefix.
+      # - "cloudflare.tunnel.service=http://my-service:80" 
+      # Optional: Specify a URL path. Only requests to hostname/path will match.
+      # - "cloudflare.tunnel.path=/app"
+      # Optional: Specify a different Cloudflare Zone for this hostname
+      # - "cloudflare.tunnel.zonename=another.example.com"
+      # Optional: Disable TLS verification if your internal service uses HTTP or a self-signed cert
+      # - "cloudflare.tunnel.no_tls_verify=true"
+      # Optional: Specify Origin Server Name (SNI) for TLS connection to origin
+      # - "cloudflare.tunnel.originsrvname=internal.service.local"
+    ports:
+      - 20756:5000
+    restart: unless-stopped
+    volumes:
+      - /var/run/docker.sock:/var/run/docker.sock:ro
+      - dockflare_data:/app/data
   duplicati:
     container_name: duplicati
     environment:
@@ -1172,13 +1287,14 @@ services:
       TIMEZONE: ${TZ}
     image: tiredofit/freescout:latest
     labels:
-      homepage.group: Lifestyle
+      homepage.group: Personal/Professional Services
       homepage.name: FreeScout
       homepage.icon: sh-freescout.svg
       homepage.href: https://support.${MY_TLD}
       homepage.description: Lightweight help desk and shared inbox
       swag: enable
       swag_proto: http
+      swag_port: 80
       swag_url: support.${MY_TLD}
       swag.uptime-kuma.enabled: true
       swag.uptime-kuma.monitor.url: https://support.${MY_TLD}
@@ -1254,8 +1370,13 @@ services:
       GITEA__mailer__SMTP_PORT: 25
       GITEA__mailer__USER: ${POSTAL_SMTP_AUTH_USER}
       GITEA__mailer__PASSWD: ${POSTAL_SMTP_AUTH_PASSWORD}
-    image: gitea/gitea:1.23.1
+    image: gitea/gitea:1.24.0
     labels:
+      cloudflare.tunnel.enable: true
+      cloudflare.tunnel.hostname: git-ssh.trez.wtf
+      cloudflare.tunnel.service: http://gitea:22
+      cloudflare.tunnel.zonename: trez.wtf
+      cloudflare.tunnel.no_tls_verify: true
       homepage.group: Code/DevOps
       homepage.name: Gitea
       homepage.href: https://git.${MY_TLD}
@@ -1324,7 +1445,7 @@ services:
     environment:
       CONFIG_FILE: /config.yaml
       DOCKER_HOST: tcp://dockerproxy:2375
-      GITEA_INSTANCE_URL: https://git.trez.wtf
+      GITEA_INSTANCE_URL: http://gitea:3000
       GITEA_RUNNER_REGISTRATION_TOKEN: "${GITEA_RUNNER_REGISTRATION_TOKEN}"
       GITEA_RUNNER_NAME: "gitea-runner-1"
     image: gitea/act_runner:latest
@@ -1361,13 +1482,7 @@ services:
       VPN_SERVICE_PROVIDER: private internet access
     expose:
       - 8000
-    extra_hosts:
-      - bitmagnet-pg-db:192.168.55.8
     image: qmcgaw/gluetun:latest
-    networks:
-      bitmagnet:
-        ipv4_address: 192.168.55.7
-      default: null
     ports:
       - 3333:3333
       - 3334:3334
@@ -4320,6 +4435,11 @@ services:
       - "/dev/sdf:/dev/sdf:rwm"
     image: ghcr.io/analogj/scrutiny:master-omnibus
     labels:
+      cloudflare.tunnel.enable: true
+      cloudflare.tunnel.hostname: smartd.trez.wtf
+      cloudflare.tunnel.service: http://scrutiny:8080
+      cloudflare.tunnel.zonename: trez.wtf
+      cloudflare.tunnel.no_tls_verify: true
       homepage.group: Infrastructure/App Performance Monitoring
       homepage.name: Scrutiny
       homepage.href: http://192.168.1.254:8909
@@ -4429,6 +4549,145 @@ services:
       - semaphore_config:/etc/semaphore
       - semaphore_data:/var/lib/semaphore
       - semaphore_tmp:/tmp/semaphore
+  signoz-init-clickhouse:
+    <<: *signoz-common
+    container_name: signoz-init-clickhouse
+    command:
+      - bash
+      - -c
+      - |
+        version="v0.0.1"
+        node_os=$$(uname -s | tr '[:upper:]' '[:lower:]')
+        node_arch=$$(uname -m | sed s/aarch64/arm64/ | sed s/x86_64/amd64/)
+        echo "Fetching histogram-binary for $${node_os}/$${node_arch}"
+        cd /tmp
+        wget -O histogram-quantile.tar.gz "https://github.com/SigNoz/signoz/releases/download/histogram-quantile%2F$${version}/histogram-quantile_$${node_os}_$${node_arch}.tar.gz"
+        tar -xvzf histogram-quantile.tar.gz
+        mv histogram-quantile /var/lib/clickhouse/user_scripts/histogramQuantile
+    image: clickhouse/clickhouse-server:24.1.2-alpine
+    restart: on-failure
+    volumes:
+      - ${DOCKER_VOLUME_CONFIG}/signoz/common/clickhouse/user_scripts/:/var/lib/clickhouse/user_scripts/
+  signoz-zookeeper-1:
+    <<: *signoz-zookeeper-defaults
+    container_name: signoz-zookeeper-1
+    environment:
+      ZOO_SERVER_ID: 1
+      ALLOW_ANONYMOUS_LOGIN: yes
+      ZOO_AUTOPURGE_INTERVAL: 1
+      ZOO_ENABLE_PROMETHEUS_METRICS: yes
+      ZOO_PROMETHEUS_METRICS_PORT_NUMBER: 9141
+    # ports:
+    #   - "2181:2181"
+    #   - "2888:2888"
+    #   - "3888:3888"
+    volumes:
+      - signoz-zookeeper-1:/bitnami/zookeeper
+  signoz-clickhouse:
+    <<: *signoz-clickhouse-defaults
+    container_name: signoz-clickhouse
+    expose:
+      - 9000
+    ports:
+    #   - "9000:9000"
+      - "8123:8123"
+      - "9181:9181"
+    volumes:
+      - ${DOCKER_VOLUME_CONFIG}/signoz/common/clickhouse/config.xml:/etc/clickhouse-server/config.xml
+      - ${DOCKER_VOLUME_CONFIG}/signoz/common/clickhouse/users.xml:/etc/clickhouse-server/users.xml
+      - ${DOCKER_VOLUME_CONFIG}/signoz/common/clickhouse/custom-function.xml:/etc/clickhouse-server/custom-function.xml
+      - ${DOCKER_VOLUME_CONFIG}/signoz/common/clickhouse/user_scripts:/var/lib/clickhouse/user_scripts/
+      - ${DOCKER_VOLUME_CONFIG}/signoz/common/clickhouse/cluster.xml:/etc/clickhouse-server/config.d/cluster.xml
+      - signoz-clickhouse:/var/lib/clickhouse/
+      # - ${DOCKER_VOLUME_CONFIG}/signoz/common/clickhouse/storage.xml:/etc/clickhouse-server/config.d/storage.xml
+  signoz-app:
+    <<: *signoz-db-depend
+    container_name: signoz-app
+    command:
+      - --config=/root/config/prometheus.yml
+    environment:
+       SIGNOZ_ALERTMANAGER_PROVIDER: signoz
+       SIGNOZ_TELEMETRYSTORE_CLICKHOUSE_DSN: tcp://signoz-clickhouse:9000
+       SIGNOZ_SQLSTORE_SQLITE_PATH: /var/lib/signoz/signoz.db
+       DASHBOARDS_PATH: /root/config/dashboards
+       STORAGE: clickhouse
+       GODEBUG: netdns=go
+       TELEMETRY_ENABLED: true
+       DEPLOYMENT_TYPE: docker-standalone-amd
+    healthcheck:
+      test:
+        - CMD
+        - wget
+        - --spider
+        - -q
+        - localhost:8080/api/v1/health
+      interval: 30s
+      timeout: 5s
+      retries: 3
+    image: signoz/signoz:${VERSION:-v0.86.2}
+    labels:
+      homepage.group: Infrastructure/App Performance Monitoring
+      homepage.name: Signoz
+      homepage.href: https://apm.${MY_TLD}
+      homepage.icon: signoz.svg
+      homepage.description: Logs, metrics, and traces in a single pane
+      swag: enable
+      swag_proto: http
+      swag_port: 8080
+      swag_url: apm.${MY_TLD}
+      swag.uptime-kuma.enabled: true
+      swag.uptime-kuma.monitor.url: https://apm.${MY_TLD}
+      swag.uptime-kuma.monitor.interval: 300
+    ports:
+      - 36113:8080 # signoz port
+      # - "6060:6060"     # pprof port
+    volumes:
+      - ${DOCKER_VOLUME_CONFIG}/signoz/common/prometheus.yml:/root/config/prometheus.yml
+      - ${DOCKER_VOLUME_CONFIG}/signoz/common/dashboards:/root/config/dashboards
+      - signoz-sqlite:/var/lib/signoz/
+  signoz-otel-collector:
+    <<: *signoz-db-depend
+    container_name: signoz-otel-collector
+    command:
+      - --config=/etc/otel-collector-config.yaml
+      - --manager-config=/etc/manager-config.yaml
+      - --copy-path=/var/tmp/collector-config.yaml
+      - --feature-gates=-pkg.translator.prometheus.NormalizeName
+    depends_on:
+      signoz-app:
+        condition: service_healthy
+    environment:
+      OTEL_RESOURCE_ATTRIBUTES: host.name=signoz-host,os.type=linux
+      LOW_CARDINAL_EXCEPTION_GROUPING: false
+    image: signoz/signoz-otel-collector:${OTELCOL_TAG:-v0.111.42}
+    ports:
+      # - "1777:1777"     # pprof extension
+      - "4317:4317" # OTLP gRPC receiver
+      - "4318:4318" # OTLP HTTP receiver
+    volumes:
+      - ${DOCKER_VOLUME_CONFIG}/signoz/common/otel/otel-collector-config.yaml:/etc/otel-collector-config.yaml
+      - ${DOCKER_VOLUME_CONFIG}/signoz/common/otel/otel-collector-opamp-config.yaml:/etc/manager-config.yaml
+  signoz-schema-migrator-sync:
+    <<: *signoz-common
+    image: signoz/signoz-schema-migrator:${OTELCOL_TAG:-v0.111.42}
+    container_name: signoz-schema-migrator-sync
+    command:
+      - sync
+      - --dsn=tcp://signoz-clickhouse:9000
+      - --up=
+    depends_on:
+      signoz-clickhouse:
+        condition: service_healthy
+    restart: on-failure
+  signoz-schema-migrator-async:
+    <<: *signoz-db-depend
+    image: signoz/signoz-schema-migrator:${OTELCOL_TAG:-v0.111.42}
+    container_name: signoz-schema-migrator-async
+    command:
+      - async
+      - --dsn=tcp://signoz-clickhouse:9000
+      - --up=
+    restart: on-failure
   sonarqube:
     container_name: sonarqube
     depends_on:
@@ -4683,7 +4942,7 @@ services:
       VALIDATION: dns
       CROWDSEC_API_KEY: ${CROWDSEC_API_KEY}
       CROWDSEC_LAPI_URL: http://crowdsec:8080
-      DOCKER_MODS: linuxserver/mods:universal-docker|linuxserver/mods:swag-auto-reload|linuxserver/mods:swag-auto-proxy|linuxserver/mods:swag-dashboard|linuxserver/mods:swag-maxmind|linuxserver/mods:universal-stdout-logs|linuxserver/mods:universal-package-install #|ghcr.io/linuxserver/mods:swag-crowdsec#|linuxserver/mods:swag-auto-uptime-kuma
+      DOCKER_MODS: linuxserver/mods:universal-docker|linuxserver/mods:swag-auto-reload|linuxserver/mods:swag-auto-proxy|linuxserver/mods:swag-dashboard|linuxserver/mods:swag-maxmind|linuxserver/mods:universal-stdout-logs|linuxserver/mods:universal-package-install|ghcr.io/linuxserver/mods:swag-crowdsec #|linuxserver/mods:swag-auto-uptime-kuma
       INSTALL_PACKAGES: nginx-mod-http-js
       PROPAGATION: 30
       UPTIME_KUMA_PASSWORD: ${UPTIME_KUMA_PASSWORD}
@@ -5230,6 +5489,8 @@ volumes:
     name: dawarich_public
   dawarich_watched:
     name: dawarich_watched
+  dockflare_data:
+    name: dockflare_data
   fastenhealth-cache:
     name: fastenhealth-cache
   fastenhealth-db:
@@ -5308,6 +5569,12 @@ volumes:
     name: semaphore_data
   semaphore_tmp:
     name: semaphore_tmp
+  signoz-clickhouse:
+    name: signoz-clickhouse
+  signoz-sqlite:
+    name: signoz-sqlite
+  signoz-zookeeper-1:
+    name: signoz-zookeeper-1
   sonarqube-data:
     name: sonarqube-data
   sonarqube-db: