Add new geoip2influx files from upstream (#955)

* Add new geoip2influx files from upstream

Dockerfile

@@ -2,7 +2,14 @@
 
 FROM scratch as buildstage
 
-ADD https://raw.githubusercontent.com/gilbN/geoip2influx/master/geoip2influx.py /root-layer/geoip2influx/geoip2influx.py
+ADD https://raw.githubusercontent.com/gilbN/geoip2influx/master/run.py /root-layer/geoip2influx/run.py
+ADD https://raw.githubusercontent.com/gilbN/geoip2influx/master/geoip2influx/__init__.py /root-layer/geoip2influx/geoip2influx/__init__.py
+ADD https://raw.githubusercontent.com/GilbN/geoip2influx/master/geoip2influx/constants.py /root-layer/geoip2influx/geoip2influx/constants.py
+ADD https://raw.githubusercontent.com/gilbN/geoip2influx/master/geoip2influx/influx_base.py /root-layer/geoip2influx/geoip2influx/influx_base.py
+ADD https://raw.githubusercontent.com/gilbN/geoip2influx/master/geoip2influx/influx.py /root-layer/geoip2influx/geoip2influx/influx.py
+ADD https://raw.githubusercontent.com/gilbN/geoip2influx/master/geoip2influx/influxv2.py /root-layer/geoip2influx/geoip2influx/influxv2.py
+ADD https://raw.githubusercontent.com/gilbN/geoip2influx/master/geoip2influx/logger.py /root-layer/geoip2influx/geoip2influx/logger.py
+ADD https://raw.githubusercontent.com/gilbN/geoip2influx/master/geoip2influx/logparser.py /root-layer/geoip2influx/geoip2influx/logparser.py
 COPY root/ /root-layer/
 
 # runtime stage

README.md

@@ -17,31 +17,74 @@ Add the ones that differ on your system.
 
 | Environment Variable | Example Value | Description |
 | -------------------- | ------------- | ----------- |
-| NGINX_LOG_PATH | /config/log/nginx/access.log | Optional, defaults to the example. |
+| NGINX_LOG_PATH | /config/log/nginx/access.log | Container path for Nginx logfile , defaults to the example. |
+| GEO_MEASUREMENT | geoip2influx | InfluxDB measurement name for geohashes. Optional, defaults to the example. |
+| LOG_MEASUREMENT | nginx_access_logs | InfluxDB measurement name for nginx logs. Optional, defaults to the example. |
+| SEND_NGINX_LOGS | true | Set to `false` to disable nginx logs. Optional, defaults to `true`. |
+| GEOIP2INFLUX_LOG_LEVEL | info | Sets the log level in geoip2influx.log. Use `debug` for verbose logging Optional, defaults to info. |
+| GEOIP2INFLUX_LOG_PATH | /config/log/geoip2influx/geoip2influx.log | Optional. Defaults to example. |
+| GEOIP_DB_PATH | /config/geoip2db/GeoLite2-City.mmdb | Optional. Defaults to example. |
+| MAXMINDDB_LICENSE_KEY | xxxxxxx | Add your Maxmind licence key |
+| MAXMINDDB_USER_ID | xxxxxxx| Add your Maxmind account id |
+
+**InfluxDB v1.8.x values**
+
+| Environment Variable | Example Value | Description |
+| -------------------- | ------------- | ----------- |
 | INFLUX_HOST | localhost | Host running InfluxDB. |
 | INFLUX_HOST_PORT | 8086 | Optional, defaults to 8086. |
 | INFLUX_DATABASE | geoip2influx | Optional, defaults to geoip2influx. |
 | INFLUX_USER | root | Optional, defaults to root. |
 | INFLUX_PASS | root | Optional, defaults to root. |
-| GEO_MEASUREMENT | geoip2influx | Optional, InfluxDB measurement name for geohashes. Defaults to the example. |
+| INFLUX_RETENTION | 7d | Sets the retention for the database. Optional, defaults to example.|
-| LOG_MEASUREMENT | nginx_access_logs | Optional, InfluxDB measurement name for nginx logs. Defaults to the example. |
+| INFLUX_SHARD | 1d | Set the shard for the database. Optional, defaults to example. |
-| SEND_NGINX_LOGS | true | Optional, set to `false` to disable nginx logs. Defaults to `true`. |
-| GEOIP2INFLUX_LOG_LEVEL | info | Optional. Sets the log level in geoip2influx.log. Use `debug` for verbose logging Optional, defaults to info. |
-| INFLUX_RETENTION | 7d | Optional. Sets the retention for the database. Defaults to example.|
-| INFLUX_SHARD | 1d | Optional. Set the shard for the database. Defaults to example. |
 
-***
+**InfluxDB v2.x values**
 
-### InfluxDB 
+| Environment Variable | Example Value | Description |
+| -------------------- | ------------- | ----------- |
+| USE_INFLUXDB_V2 | true | Required if using InfluxDB2. Defaults to false |
+| INFLUXDB_V2_TOKEN | secret-token | Required |
+| INFLUXDB_V2_URL | http://localhost:8086 | Optional, defaults to http://localhost:8086 |
+| INFLUXDB_V2_ORG | geoip2influx | Optional, defaults to geoip2influx. Will be created if not exists. |
+| INFLUXDB_V2_BUCKET | geoip2influx | Optional, defaults to geoip2influx. Will be created if not exists. |
+| INFLUXDB_V2_RETENTION | 604800 | Optional, defaults to 604800. 7 days in seconds |
+| INFLUXDB_V2_DEBUG | false | Optional, defaults to false. Enables the debug mode for the influxdb-client package. |
+| INFLUXDB_V2_BATCHING | true | Optional, defaults to false. Enables batch writing of data. |
+| INFLUXDB_V2_BATCH_SIZE | 100 | Optional, defaults to 10. |
+| INFLUXDB_V2_FLUSH_INTERVAL | 30000 | Optional, defaults to 15000. How often in milliseconds to write a batch |
 
-#### InfluxDB v2.x is not supported. Use v1.8.x. 
+### INFLUXDB_V2_TOKEN
 
-The InfluxDB database will be created automatically with the name you choose.
+If the organization or bucket does not exist, it will try and create them with the token.
+
+> [!NOTE]
+> The minimim level of rights needed is write access to the bucket.
+
+### MaxMind Geolite2
+
+Get your licence key here: https://www.maxmind.com/en/geolite2/signup
+
+## InfluxDB 
+
+### InfluxDB v2.x and v1.8x is supported.
+
+#### Note: The Grafana dashboard currently only supports InfluxDB v1.8.x
+
+The InfluxDB database/bucket and retention rules will be created automatically with the name you choose.
+
+```
+-e INFLUX_DATABASE=geoip2influx or -e INFLUXDB_V2_BUCKET=geoip2influx
+```
 
 ***
 
 ## Grafana dashboard: 
-### [Grafana Dashboard Link](https://grafana.com/grafana/dashboards/12268/)
+
+Use [https://github.com/GilbN/geoip2influx/blob/master/nginx_logs_geo_map.json](https://github.com/GilbN/geoip2influx/blob/master/nginx_logs_geo_map.json)
+
+> [!NOTE]
+> Dashboard currently only supports InfluxDB 1.8.x.
 
 ***
 

root/etc/s6-overlay/s6-rc.d/init-mod-swag-geoip2influx-add-package/run

@@ -1,14 +1,19 @@
 #!/usr/bin/with-contenv bash
 
+echo -e "[init-geoip2influx-setup] Starting"
+
+
 if ! pip list 2>&1 | grep -q "influxdb"; then
   echo '-------------------------------------------------------------'
   echo '| Mod by Gilbn                                              |'
   echo '| Adding required modules for Geoip2Influx to install list  |'
   echo '-------------------------------------------------------------'
   echo "\
-    geoip2==3.0.0 \
+    geoip2==4.8.0 \
     geohash2==1.1 \
-    influxdb==5.3.0 \
+    influxdb==5.3.2 \
+    python-dotenv \
+    influxdb-client==1.44.0 \
     IPy==1.01" >> /mod-pip-packages-to-install.list
 fi
 
@@ -25,19 +30,35 @@ lsiown -R abc:abc \
   /geoip2influx \
   /config/log/geoip2influx
 
-chmod +x /geoip2influx/geoip2influx.py
+chmod +x /geoip2influx/run.py
 
 # Display variables for troubleshooting
-echo -e "Variables set:\\n\
+echo -e "[init-geoip2influx-setup] Variables set:\\n\\n
-NGINX_LOG_PATH=${NGINX_LOG_PATH}\\n\
 INFLUX_HOST=${INFLUX_HOST}\\n\
 INFLUX_HOST_PORT=${INFLUX_HOST_PORT}\\n\
 INFLUX_DATABASE=${INFLUX_DATABASE}\\n\
 INFLUX_USER=${INFLUX_USER}\\n\
 INFLUX_PASS=${INFLUX_PASS}\\n\
 INFLUX_RETENTION=${INFLUX_RETENTION}\\n\
-INFLUX_SHARD=${INFLUX_SHARD}\\n\
+INFLUX_SHARD=${INFLUX_SHARD}\\n\\n
+INFLUXDB_V2_TOKEN=${INFLUXDB_V2_TOKEN}\\n\
+INFLUXDB_V2_URL=${INFLUXDB_V2_URL}\\n\
+INFLUXDB_V2_ORG=${INFLUXDB_V2_ORG}\\n\
+INFLUXDB_V2_BUCKET=${INFLUXDB_V2_BUCKET}\\n\
+INFLUXDB_V2_RETENTION=${INFLUXDB_V2_RETENTION}\\n\
+INFLUXDB_V2_DEBUG=${INFLUXDB_V2_DEBUG}\\n\
+INFLUXDB_V2_BATCHING=${INFLUXDB_V2_BATCHING}\\n\
+INFLUXDB_V2_BATCH_SIZE=${INFLUXDB_V2_BATCH_SIZE}\\n\
+INFLUXDB_V2_FLUSH_INTERVAL=${INFLUXDB_V2_FLUSH_INTERVAL}\\n\\n
 GEO_MEASUREMENT=${GEO_MEASUREMENT}\\n\
 LOG_MEASUREMENT=${LOG_MEASUREMENT}\\n\
+NGINX_LOG_PATH=${NGINX_LOG_PATH}\\n\
 SEND_NGINX_LOGS=${SEND_NGINX_LOGS}\\n\
-GEOIP2INFLUX_LOG_LEVEL=${GEOIP2INFLUX_LOG_LEVEL}\\n"
+GEOIP2INFLUX_LOG_LEVEL=${GEOIP2INFLUX_LOG_LEVEL}\\n\
+GEOIP2INFLUX_LOG_PATH=${GEOIP2INFLUX_LOG_PATH}\\n\
+GEOIP_DB_PATH=${GEOIP_DB_PATH}\\n\
+USE_INFLUXDB_V2=${USE_INFLUXDB_V2}\\n\
+MAXMINDDB_USER_ID=${MAXMINDDB_USER_ID}\\n\
+MAXMINDDB_LICENSE_KEY=${MAXMINDDB_LICENSE_KEY}\\n"
+
+echo -e "[init-geoip2influx-setup] Finished"

root/etc/s6-overlay/s6-rc.d/svc-mod-swag-geoip2influx/run

@@ -1,4 +1,4 @@
 #!/usr/bin/with-contenv bash
 
 exec \
-    python3 /geoip2influx/geoip2influx.py
+    python3 /geoip2influx/run.py