Merge pull request #66 from gilbN/swag-geoip2influx

swag:geoip2influx adds geoip2 python script

.travis.yml

@@ -4,16 +4,16 @@ language: shell
 
 branches:
   only:
-    - <baseimagename>-<modname> #replace variables, omit brackets
+    - swag-geoip2influx
 
 services:
   - docker
 
 env:
   global:
-    - DOCKERHUB="linuxserver/mods" #don't modify
+    - DOCKERHUB="linuxserver/mods"
-    - BASEIMAGE="baseimagename" #replace
+    - BASEIMAGE="swag"
-    - MODNAME="modname" #replace
+    - MODNAME="geoip2influx"
 
 jobs:
   include:

Dockerfile

@@ -1,4 +1,12 @@
+FROM lsiobase/alpine:3.12 as buildstage
+
+ADD https://raw.githubusercontent.com/gilbN/geoip2influx/master/geoip2influx.py /root-layer/geoip2influx/geoip2influx.py
+COPY root/ /root-layer/
+
+# runtime stage
 FROM scratch
 
-# copy local files
+LABEL maintainer="GilbN"
-COPY root/ /
+
+# Add files from buildstage
+COPY --from=buildstage /root-layer/ /

Dockerfile.complex

@@ -1,21 +0,0 @@
-## Buildstage ##
-FROM lsiobase/alpine:3.9 as buildstage
-
-RUN \
- echo "**** install packages ****" && \
- apk add --no-cache \
-	curl && \
- echo "**** grab rclone ****" && \
- mkdir -p /root-layer && \
- curl -o \
-	/root-layer/rclone.deb -L \
-	"https://downloads.rclone.org/v1.47.0/rclone-v1.47.0-linux-amd64.deb"
-
-# copy local files
-COPY root/ /root-layer/
-
-## Single layer deployed image ##
-FROM scratch
-
-# Add files from buildstage
-COPY --from=buildstage /root-layer/ /

README.md

@@ -1,17 +1,89 @@
-# Rsync - Docker mod for openssh-server
+# Geoip2Influx - Docker mod for nginx/swag
 
-This mod adds rsync to openssh-server, to be installed/updated during container start.
 
-In openssh-server docker arguments, set an environment variable `DOCKER_MODS=linuxserver/mods:openssh-server-rsync`
+This mod adds a python script that sends geo location metrics from the nginx access log to InfluxDB
 
-If adding multiple mods, enter them in an array separated by `|`, such as `DOCKER_MODS=linuxserver/mods:openssh-server-rsync|linuxserver/mods:openssh-server-mod2`
+![](.assets/geoip2influx.png)
 
-# Mod creation instructions
 
-* Ask the team to create a new branch named `<baseimagename>-<modname>`. Baseimage should be the name of the image the mod will be applied to. The new branch will be based on the `template` branch.
+The mod will parse the access log for IPs and and convert them into geo metrics for InfluxDB. It will also send log metrics if enabled.
-* Fork the repo, checkout the newly created branch.
+
-* Edit the `Dockerfile` for the mod. `Dockerfile.complex` is only an example and included for reference; it should be deleted when done.
+Add `-e DOCKER_MODS=linuxserver/mods:swag-geoip2influx`
-* Inspect the `root` folder contents. Edit, add and remove as necessary.
+
-* Edit this readme with pertinent info, delete these instructions.
+## Enviroment variables:
-* Finally edit the `travis.yml`. Customize the build branch, and the vars for `BASEIMAGE` and `MODNAME`.
+
-* Submit PR against the branch created by the team.
+These are the **default** values for all envs. 
+Add the ones that differ on your system. 
+
+| Environment Variable | Example Value | Description |
+| -------------------- | ------------- | ----------- |
+| NGINX_LOG_PATH | /config/log/nginx/access.log | Optional, defaults to the example. |
+| INFLUX_HOST | localhost | Host running InfluxDB. |
+| INFLUX_HOST_PORT | 8086 | Optional, defaults to 8086. |
+| INFLUX_DATABASE | geoip2influx | Optional, defaults to geoip2influx. |
+| INFLUX_USER | root | Optional, defaults to root. |
+| INFLUX_PASS | root | Optional, defaults to root. |
+| GEO_MEASUREMENT | geoip2influx | Optional, InfluxDB measurement name for geohashes. Defaults to the example. |
+| LOG_MEASUREMENT | nginx_access_logs | Optional, InfluxDB measurement name for nginx logs. Defaults to the example. |
+| SEND_NGINX_LOGS | true | Optional, set to `false` to disable nginx logs. Defaults to `true`. |
+| GEOIP2INFLUX_LOG_LEVEL | info | Optional. Sets the log level in geoip2influx.log. Use `debug` for verbose logging Optional, defaults to info. |
+| INFLUX_RETENTION | 7d | Optional. Sets the retention for the database. Defaults to example.|
+| INFLUX_SHARD | 1d | Optional. Set the shard for the database. Defaults to example. |
+| MAXMINDDB_LICENSE_KEY | xxxxxxx | Add your Maxmind licence key |
+
+***
+### MaxMind Geolite2
+
+Default download location is `/config/geoip2db/GeoLite2-City.mmdb`
+
+Get your licence key here: https://www.maxmind.com/en/geolite2/signup
+
+### InfluxDB 
+
+The InfluxDB database will be created automatically with the name you choose.
+
+***
+
+## Grafana dashboard: 
+### [Grafana Dashboard Link](https://grafana.com/grafana/dashboards/12268/)
+
+***
+
+## Sending Nginx log metrics
+
+1. Add the following to the http block in your `nginx.conf`file:
+
+```nginx
+geoip2 /config/geoip2db/GeoLite2-City.mmdb {
+auto_reload 5m;
+$geoip2_data_country_code country iso_code;
+$geoip2_data_city_name city names en;
+}
+
+log_format geoip2influx '$remote_addr - $remote_user [$time_local]'
+           '"$request" $status $body_bytes_sent'
+           '"$http_referer" $host "$http_user_agent"'
+           '"$request_time" "$upstream_connect_time"'
+           '"$geoip2_data_city_name" "$geoip2_data_country_code"';
+ ```
+ 
+ 2. Set the access log use the `geoip2influx` log format. 
+ ```nginx
+ access_log /config/log/nginx/access.log geoip2influx;
+ ```
+
+### Multiple log files
+
+If you separate your nginx log files but want this mod to parse all of them you can do the following:
+
+As nginx can have multiple `access log` directives in a block, just add another one in the server block. 
+
+**Example**
+
+```nginx
+	access_log /config/log/nginx/technicalramblings/access.log;
+	access_log /config/log/nginx/access.log geoip2influx;
+```
+This will log the same lines to both files.
+
+Then use the `/config/log/nginx/access.log` file in the `NGINX_LOG_PATH` variable. 

root/etc/cont-init.d/98-geoip2influx

@@ -0,0 +1,43 @@
+#!/usr/bin/with-contenv bash
+
+if [ ! -d /usr/lib/python3.8/site-packages/influxdb ]; then
+  echo '-------------------------------------------------------------'
+  echo '| Mod by Gilbn                                              |'
+  echo '| Running installation of required modules for Geoip2Influx |'
+  echo '-------------------------------------------------------------'
+  pip3 install --no-cache-dir -U \
+    geoip2==3.0.0 \
+    geohash2==1.1 \
+    influxdb==5.3.0 \
+    IPy==1.0
+fi
+
+# Create log folder
+mkdir -p \
+  /config/log/geoip2influx
+# move old log if needed
+if [ -f /config/geoip2db/geoip2influx.log ]; then
+  mv /config/geoip2db/geoip2influx.log /config/log/geoip2influx
+fi
+
+# permissions
+chown -R abc:abc \
+  /geoip2influx \
+  /config/log/geoip2influx
+
+chmod +x /geoip2influx/geoip2influx.py
+
+# Display variables for troubleshooting
+echo -e "Variables set:\\n\
+NGINX_LOG_PATH=${NGINX_LOG_PATH}\\n\
+INFLUX_HOST=${INFLUX_HOST}\\n\
+INFLUX_HOST_PORT=${INFLUX_HOST_PORT}\\n\
+INFLUX_DATABASE=${INFLUX_DATABASE}\\n\
+INFLUX_USER=${INFLUX_USER}\\n\
+INFLUX_PASS=${INFLUX_PASS}\\n\
+INFLUX_RETENTION=${INFLUX_RETENTION}\\n\
+INFLUX_SHARD=${INFLUX_SHARD}\\n\
+GEO_MEASUREMENT=${GEO_MEASUREMENT}\\n\
+LOG_MEASUREMENT=${LOG_MEASUREMENT}\\n\
+SEND_NGINX_LOGS=${SEND_NGINX_LOGS}\\n\
+GEOIP2INFLUX_LOG_LEVEL=${GEOIP2INFLUX_LOG_LEVEL}\\n"

root/etc/cont-init.d/98-vpn-config

@@ -1,27 +0,0 @@
-#!/usr/bin/with-contenv bash
-
-# Determine if setup is needed
-if [ ! -f /usr/local/lib/python***/dist-packages/sshuttle ] && \
-[ -f /usr/bin/apt ]; then
-  ## Ubuntu
-  apt-get update
-  apt-get install --no-install-recommends -y \
-    iptables \
-    openssh-client \
-    python3 \
-    python3-pip 
-  pip3 install sshuttle
-fi
-if [ ! -f /usr/lib/python***/site-packages/sshuttle ] && \
-[ -f /sbin/apk ]; then
-  # Alpine
-  apk add --no-cache \
-    iptables \
-    openssh \
-    py3-pip \
-    python3 
-  pip3 install sshuttle
-fi
-
-chown -R root:root /root
-chmod -R 600 /root/.ssh

root/etc/logrotate.d/geoip2influx

@@ -0,0 +1,14 @@
+/config/log/geoip2influx/geoip2influx.log {
+ daily
+ rotate 7
+ size 25M
+ compress
+ delaycompress
+ nodateext
+ missingok
+ notifempty
+ postrotate
+    s6-svc -r /var/run/s6/services/geoip2influx
+ endscript
+ su abc abc
+}

root/etc/services.d/geoip2influx/run

@@ -0,0 +1,4 @@
+#!/usr/bin/with-contenv bash
+
+  exec \
+        python3 /geoip2influx/geoip2influx.py

root/etc/services.d/sshvpn/run

@@ -1,3 +0,0 @@
-#!/usr/bin/with-contenv bash
-
-sshuttle --dns --remote root@${HOST}:${PORT} 0/0 -x 172.17.0.0/16