Add new geoip2influx files from upstream (#955)

* Add new geoip2influx files from upstream
This commit is contained in:
GilbN
2024-09-11 20:47:53 +02:00
committed by GitHub
parent 6a87a2cb63
commit c8b2d94afe
4 changed files with 92 additions and 21 deletions
+8 -1
View File
@@ -2,7 +2,14 @@
FROM scratch as buildstage
ADD https://raw.githubusercontent.com/gilbN/geoip2influx/master/geoip2influx.py /root-layer/geoip2influx/geoip2influx.py
ADD https://raw.githubusercontent.com/gilbN/geoip2influx/master/run.py /root-layer/geoip2influx/run.py
ADD https://raw.githubusercontent.com/gilbN/geoip2influx/master/geoip2influx/__init__.py /root-layer/geoip2influx/geoip2influx/__init__.py
ADD https://raw.githubusercontent.com/GilbN/geoip2influx/master/geoip2influx/constants.py /root-layer/geoip2influx/geoip2influx/constants.py
ADD https://raw.githubusercontent.com/gilbN/geoip2influx/master/geoip2influx/influx_base.py /root-layer/geoip2influx/geoip2influx/influx_base.py
ADD https://raw.githubusercontent.com/gilbN/geoip2influx/master/geoip2influx/influx.py /root-layer/geoip2influx/geoip2influx/influx.py
ADD https://raw.githubusercontent.com/gilbN/geoip2influx/master/geoip2influx/influxv2.py /root-layer/geoip2influx/geoip2influx/influxv2.py
ADD https://raw.githubusercontent.com/gilbN/geoip2influx/master/geoip2influx/logger.py /root-layer/geoip2influx/geoip2influx/logger.py
ADD https://raw.githubusercontent.com/gilbN/geoip2influx/master/geoip2influx/logparser.py /root-layer/geoip2influx/geoip2influx/logparser.py
COPY root/ /root-layer/
# runtime stage
+55 -12
View File
@@ -17,31 +17,74 @@ Add the ones that differ on your system.
| Environment Variable | Example Value | Description |
| -------------------- | ------------- | ----------- |
| NGINX_LOG_PATH | /config/log/nginx/access.log | Optional, defaults to the example. |
| NGINX_LOG_PATH | /config/log/nginx/access.log | Container path for Nginx logfile , defaults to the example. |
| GEO_MEASUREMENT | geoip2influx | InfluxDB measurement name for geohashes. Optional, defaults to the example. |
| LOG_MEASUREMENT | nginx_access_logs | InfluxDB measurement name for nginx logs. Optional, defaults to the example. |
| SEND_NGINX_LOGS | true | Set to `false` to disable nginx logs. Optional, defaults to `true`. |
| GEOIP2INFLUX_LOG_LEVEL | info | Sets the log level in geoip2influx.log. Use `debug` for verbose logging Optional, defaults to info. |
| GEOIP2INFLUX_LOG_PATH | /config/log/geoip2influx/geoip2influx.log | Optional. Defaults to example. |
| GEOIP_DB_PATH | /config/geoip2db/GeoLite2-City.mmdb | Optional. Defaults to example. |
| MAXMINDDB_LICENSE_KEY | xxxxxxx | Add your Maxmind licence key |
| MAXMINDDB_USER_ID | xxxxxxx| Add your Maxmind account id |
**InfluxDB v1.8.x values**
| Environment Variable | Example Value | Description |
| -------------------- | ------------- | ----------- |
| INFLUX_HOST | localhost | Host running InfluxDB. |
| INFLUX_HOST_PORT | 8086 | Optional, defaults to 8086. |
| INFLUX_DATABASE | geoip2influx | Optional, defaults to geoip2influx. |
| INFLUX_USER | root | Optional, defaults to root. |
| INFLUX_PASS | root | Optional, defaults to root. |
| GEO_MEASUREMENT | geoip2influx | Optional, InfluxDB measurement name for geohashes. Defaults to the example. |
| LOG_MEASUREMENT | nginx_access_logs | Optional, InfluxDB measurement name for nginx logs. Defaults to the example. |
| SEND_NGINX_LOGS | true | Optional, set to `false` to disable nginx logs. Defaults to `true`. |
| GEOIP2INFLUX_LOG_LEVEL | info | Optional. Sets the log level in geoip2influx.log. Use `debug` for verbose logging Optional, defaults to info. |
| INFLUX_RETENTION | 7d | Optional. Sets the retention for the database. Defaults to example.|
| INFLUX_SHARD | 1d | Optional. Set the shard for the database. Defaults to example. |
| INFLUX_RETENTION | 7d | Sets the retention for the database. Optional, defaults to example.|
| INFLUX_SHARD | 1d | Set the shard for the database. Optional, defaults to example. |
***
**InfluxDB v2.x values**
### InfluxDB
| Environment Variable | Example Value | Description |
| -------------------- | ------------- | ----------- |
| USE_INFLUXDB_V2 | true | Required if using InfluxDB2. Defaults to false |
| INFLUXDB_V2_TOKEN | secret-token | Required |
| INFLUXDB_V2_URL | http://localhost:8086 | Optional, defaults to http://localhost:8086 |
| INFLUXDB_V2_ORG | geoip2influx | Optional, defaults to geoip2influx. Will be created if not exists. |
| INFLUXDB_V2_BUCKET | geoip2influx | Optional, defaults to geoip2influx. Will be created if not exists. |
| INFLUXDB_V2_RETENTION | 604800 | Optional, defaults to 604800. 7 days in seconds |
| INFLUXDB_V2_DEBUG | false | Optional, defaults to false. Enables the debug mode for the influxdb-client package. |
| INFLUXDB_V2_BATCHING | true | Optional, defaults to false. Enables batch writing of data. |
| INFLUXDB_V2_BATCH_SIZE | 100 | Optional, defaults to 10. |
| INFLUXDB_V2_FLUSH_INTERVAL | 30000 | Optional, defaults to 15000. How often in milliseconds to write a batch |
#### InfluxDB v2.x is not supported. Use v1.8.x.
### INFLUXDB_V2_TOKEN
The InfluxDB database will be created automatically with the name you choose.
If the organization or bucket does not exist, it will try and create them with the token.
> [!NOTE]
> The minimim level of rights needed is write access to the bucket.
### MaxMind Geolite2
Get your licence key here: https://www.maxmind.com/en/geolite2/signup
## InfluxDB
### InfluxDB v2.x and v1.8x is supported.
#### Note: The Grafana dashboard currently only supports InfluxDB v1.8.x
The InfluxDB database/bucket and retention rules will be created automatically with the name you choose.
```
-e INFLUX_DATABASE=geoip2influx or -e INFLUXDB_V2_BUCKET=geoip2influx
```
***
## Grafana dashboard:
### [Grafana Dashboard Link](https://grafana.com/grafana/dashboards/12268/)
Use [https://github.com/GilbN/geoip2influx/blob/master/nginx_logs_geo_map.json](https://github.com/GilbN/geoip2influx/blob/master/nginx_logs_geo_map.json)
> [!NOTE]
> Dashboard currently only supports InfluxDB 1.8.x.
***
@@ -1,14 +1,19 @@
#!/usr/bin/with-contenv bash
echo -e "[init-geoip2influx-setup] Starting"
if ! pip list 2>&1 | grep -q "influxdb"; then
echo '-------------------------------------------------------------'
echo '| Mod by Gilbn |'
echo '| Adding required modules for Geoip2Influx to install list |'
echo '-------------------------------------------------------------'
echo "\
geoip2==3.0.0 \
geoip2==4.8.0 \
geohash2==1.1 \
influxdb==5.3.0 \
influxdb==5.3.2 \
python-dotenv \
influxdb-client==1.44.0 \
IPy==1.01" >> /mod-pip-packages-to-install.list
fi
@@ -25,19 +30,35 @@ lsiown -R abc:abc \
/geoip2influx \
/config/log/geoip2influx
chmod +x /geoip2influx/geoip2influx.py
chmod +x /geoip2influx/run.py
# Display variables for troubleshooting
echo -e "Variables set:\\n\
NGINX_LOG_PATH=${NGINX_LOG_PATH}\\n\
echo -e "[init-geoip2influx-setup] Variables set:\\n\\n
INFLUX_HOST=${INFLUX_HOST}\\n\
INFLUX_HOST_PORT=${INFLUX_HOST_PORT}\\n\
INFLUX_DATABASE=${INFLUX_DATABASE}\\n\
INFLUX_USER=${INFLUX_USER}\\n\
INFLUX_PASS=${INFLUX_PASS}\\n\
INFLUX_RETENTION=${INFLUX_RETENTION}\\n\
INFLUX_SHARD=${INFLUX_SHARD}\\n\
INFLUX_SHARD=${INFLUX_SHARD}\\n\\n
INFLUXDB_V2_TOKEN=${INFLUXDB_V2_TOKEN}\\n\
INFLUXDB_V2_URL=${INFLUXDB_V2_URL}\\n\
INFLUXDB_V2_ORG=${INFLUXDB_V2_ORG}\\n\
INFLUXDB_V2_BUCKET=${INFLUXDB_V2_BUCKET}\\n\
INFLUXDB_V2_RETENTION=${INFLUXDB_V2_RETENTION}\\n\
INFLUXDB_V2_DEBUG=${INFLUXDB_V2_DEBUG}\\n\
INFLUXDB_V2_BATCHING=${INFLUXDB_V2_BATCHING}\\n\
INFLUXDB_V2_BATCH_SIZE=${INFLUXDB_V2_BATCH_SIZE}\\n\
INFLUXDB_V2_FLUSH_INTERVAL=${INFLUXDB_V2_FLUSH_INTERVAL}\\n\\n
GEO_MEASUREMENT=${GEO_MEASUREMENT}\\n\
LOG_MEASUREMENT=${LOG_MEASUREMENT}\\n\
NGINX_LOG_PATH=${NGINX_LOG_PATH}\\n\
SEND_NGINX_LOGS=${SEND_NGINX_LOGS}\\n\
GEOIP2INFLUX_LOG_LEVEL=${GEOIP2INFLUX_LOG_LEVEL}\\n"
GEOIP2INFLUX_LOG_LEVEL=${GEOIP2INFLUX_LOG_LEVEL}\\n\
GEOIP2INFLUX_LOG_PATH=${GEOIP2INFLUX_LOG_PATH}\\n\
GEOIP_DB_PATH=${GEOIP_DB_PATH}\\n\
USE_INFLUXDB_V2=${USE_INFLUXDB_V2}\\n\
MAXMINDDB_USER_ID=${MAXMINDDB_USER_ID}\\n\
MAXMINDDB_LICENSE_KEY=${MAXMINDDB_LICENSE_KEY}\\n"
echo -e "[init-geoip2influx-setup] Finished"
@@ -1,4 +1,4 @@
#!/usr/bin/with-contenv bash
exec \
python3 /geoip2influx/geoip2influx.py
python3 /geoip2influx/run.py