Introducing logs monitoring using Graylog2

Readme.md

@@ -261,6 +261,14 @@ You can also mount your own bash aliases by mounting a .bash_aliases file in /ho
 
 Solr admin interface can be accessed either through port 8983, i.e http://localhost:8983, or with the /solr/ url, i.e http://www.mysite.ezdev/solr/ .
 
+### Check containers logs
+
+You can various logs for debugging through Graylog stacks.
+
+Just go to http://127.0.0.1:9000/ and fill the form with admin/admin.
+
+For further explanations see [this documentation](./docs/graylog.md)
+
 ## Stopping the Environment
 
     ./stack.sh down

config/graylog/filebeat/conf.d/filebeat_apache.conf.yml

@@ -0,0 +1,21 @@
+###################### Filebeat Configuration for Apache2 #########################
+
+# Retrieve logs from services
+# @author Andréas 'Screamz' HANSS <ahanss@kaliop.com>
+
+#=========================== Filebeat prospectors =============================
+
+filebeat.prospectors:
+- input_type: log
+  paths:
+    - /app-logs/apache/access.log
+  document_type: apache_access
+
+- input_type: log
+  paths:
+    - /app-logs/apache/error.log
+  document_type: apache_error
+
+#----------------------------- Graylog2 output --------------------------------
+output.logstash:
+  hosts: ["graylog-server:5044"]

config/graylog/filebeat/conf.d/filebeat_solr.conf.yml

@@ -0,0 +1,16 @@
+###################### Filebeat Configuration for Apache SOLR #########################
+
+# Retrieve logs from services
+# @author Andréas 'Screamz' HANSS <ahanss@kaliop.com>
+
+#=========================== Filebeat prospectors =============================
+
+filebeat.prospectors:
+- input_type: log
+  paths:
+    - /app-logs/solr/solr.log
+  document_type: solr
+
+#----------------------------- Graylog2 output --------------------------------
+output.logstash:
+  hosts: ["graylog-server:5047"]

config/graylog/filebeat/conf.d/filebeat_varnish.conf.yml

@@ -0,0 +1,17 @@
+###################### Filebeat Configuration for Varnish NCSA #########################
+
+# Retrieve logs from services
+# @author Andréas 'Screamz' HANSS <ahanss@kaliop.com>
+
+#=========================== Filebeat prospectors =============================
+
+filebeat.prospectors:
+- input_type: log
+  paths:
+    - /app-logs/varnish/varnishncsa.log
+  document_type: varnish
+  include_lines: ['BAN', 'PURGE']
+
+#----------------------------- Graylog2 output --------------------------------
+output.logstash:
+  hosts: ["graylog-server:5046"]

config/graylog/inject-graylog-config.sh

@@ -0,0 +1 @@
+mongorestore --gzip --archive=/docker-entrypoint-initdb.d/dump.graylog.gz --db graylog

docker-compose-dev-graylog-template.yml

@@ -0,0 +1,173 @@
+
+version: '2'
+services:
+    web:
+      image: klabs/apache_${DOCKER_PHP_VERSION}
+      hostname: ${DOCKER_PROJECT_NAME}_web
+      container_name: web
+      ports:
+          - "82:82"
+          - "88:88"
+      volumes:
+          - ./config/apache/sites-available/001-dynamic-vhost-ez5.conf:/etc/apache2/sites-available/001-dynamic-vhost-ez5.conf
+          - ./config/apache/sites-available/002-dynamic-vhost-ezplatform.conf:/etc/apache2/sites-available/002-dynamic-vhost-ezplatform.conf
+          - ./config/apache/sites-available/003-dynamic-vhost-ez4.conf:/etc/apache2/sites-available/003-dynamic-vhost-ez4.conf
+          - ./config/apache/sites-available/ez5-common.conf:/etc/apache2/sites-available/ez5-common.conf
+          - ./config/apache/sites-available/ezplatform-common.conf:/etc/apache2/sites-available/ezplatform-common.conf
+          - ./logs/apache/:/var/log/apache2
+          - "./config/apache/php5/custom_vars.ini:$DOCKER_PHP_CONF_PATH/apache2/conf.d/custom_vars.ini"
+          - "./config/apache/php5/timezone.ini:$DOCKER_PHP_CONF_PATH/apache2/conf.d/timezone.ini"
+          - "$DOCKER_WWW_ROOT:$DOCKER_WWW_DEST"
+          - "$DOCKER_STORAGE_LOCAL_PATH:$DOCKER_STORAGE_MOUNT_POINT"
+      env_file:
+          - docker-compose.env
+          - docker-compose.env.local
+      cap_add:
+          - SYS_PTRACE
+    cli:
+      image: klabs/cli_${DOCKER_PHP_VERSION}
+      hostname: ${DOCKER_PROJECT_NAME}_cli
+      container_name: cli
+      privileged: true
+      volumes:
+          - ./logs/cli:/var/log/php
+          # files in /tmp/cron.d get managed by bootstrap.sh
+          - ./config/cli/cron.d:/tmp/cron.d
+          - ~/.gitconfig:/home/site/.gitconfig
+          - ~/.ssh/config:/home/site/.ssh/config
+          - "$SSH_AUTH_SOCK:/ssh-agent"
+          - "./config/cli/php5/custom_vars.ini:$DOCKER_PHP_CONF_PATH/cli/conf.d/custom_vars.ini"
+          - "./config/cli/php5/timezone.ini:$DOCKER_PHP_CONF_PATH/cli/conf.d/timezone.ini"
+          - "$DOCKER_WWW_ROOT:$DOCKER_WWW_DEST"
+          - "$DOCKER_STORAGE_LOCAL_PATH:$DOCKER_STORAGE_MOUNT_POINT"
+      environment:
+          - SSH_AUTH_SOCK=/ssh-agent
+      env_file:
+          - docker-compose.env
+          - docker-compose.env.local
+      cap_add:
+          - SYS_PTRACE
+    memcached:
+      image: klabs/memcached
+      hostname:  ${DOCKER_PROJECT_NAME}_memcached
+      container_name: memcached
+      ports:
+          - "11211:11211"
+      volumes:
+          - ./logs/memcache/:/var/log/memcache
+      env_file:
+          - docker-compose.env
+          - docker-compose.env.local
+      cap_add:
+          - SYS_PTRACE
+    solr:
+      image: klabs/solr4
+      hostname: ${DOCKER_PROJECT_NAME}_solr
+      container_name: solr
+      ports:
+          - "8983:8983"
+      volumes:
+          - "${DOCKER_SOLR_CONF_PATH}:/opt/solr/solr/"
+          - ./data/solr/:/opt/solr/data/
+          - ./logs/solr/:/opt/solr/logs
+      env_file:
+          - docker-compose.env
+          - docker-compose.env.local
+      cap_add:
+          - SYS_PTRACE
+    varnish:
+      image: klabs/varnish
+      hostname: ${DOCKER_PROJECT_NAME}_varnish
+      container_name: varnish
+      ports:
+          - "81:81"
+      volumes:
+          - "${DOCKER_VARNISH_VCL_FILE}:/etc/varnish/default.vcl"
+          - ./logs/varnish/:/var/log/varnish
+      env_file:
+          - docker-compose.env
+          - docker-compose.env.local
+      cap_add:
+          - SYS_PTRACE
+          # varnish init daemon wants to set ulimits
+          - SYS_RESOURCE
+      security_opt:
+        - apparmor:unconfined
+      depends_on:
+          - web
+          - cli
+    haproxy:
+      image: klabs/haproxy
+      hostname: ${DOCKER_PROJECT_NAME}_haproxy
+      container_name: haproxy
+      ports:
+          - "80:80"
+      volumes:
+          - ./config/haproxy/haproxy.cfg:/etc/haproxy/haproxy.cfg
+      env_file:
+          - docker-compose.env
+          - docker-compose.env.local
+      depends_on:
+          - web
+          - varnish
+          - solr
+    mysql:
+      image: klabs/mysql
+      hostname: ${DOCKER_PROJECT_NAME}_mysql
+      container_name: mysql
+      # nb: this prevents the image to start on some ubuntu installs because of apparmor config...
+      #privileged: true
+      ports:
+          - "3307:3306"
+      volumes:
+          - ./config/mysql/:/etc/mysql/conf.d/
+          - ./data/mysql/:/var/lib/mysql
+          - ./logs/mysql/:/var/log/mysql
+      env_file:
+          - docker-compose.env
+          - docker-compose.env.local
+      cap_add:
+          - SYS_PTRACE
+      command: ["/root/bootstrap.sh", "mysqld --character-set-server=utf8 --collation-server=utf8_unicode_ci"]
+
+    # official phpmyadmin image
+    phpmyadmin:
+      image: phpmyadmin/phpmyadmin
+      hostname: ${DOCKER_PROJECT_NAME}_phpmyadmin
+      container_name: phpmyadmin
+      env_file:
+          - docker-compose.env
+          - docker-compose.env.local
+      depends_on:
+          - mysql
+   # Graylog stack
+    mongo:
+      container_name: graylog-mongo
+      image: "mongo:3"
+      volumes: 
+        - ./config/graylog:/docker-entrypoint-initdb.d
+    elasticsearch:
+      container_name: graylog-elastic
+      image: "elasticsearch:2"
+      command: "elasticsearch -Des.cluster.name='graylog'"
+    graylog:
+      container_name: graylog-server
+      image: 'graylog2/server:latest'
+      environment:
+        GRAYLOG_PASSWORD_SECRET: somepasswordpepper
+        GRAYLOG_ROOT_PASSWORD_SHA2: 8c6976e5b5410415bde908bd4dee15dfb167a9c873fc4bb8a81f6f2ab448a918
+        GRAYLOG_WEB_ENDPOINT_URI: http://127.0.0.1:9000/api
+      depends_on:
+        - mongo
+        - elasticsearch
+      ports:
+        - "9000:9000"
+    filebeats:
+        container_name: graylog-filebeat
+        #image: TODO_ON_DOCKERHUB_UPLOAD
+        build: ./images/filebeat
+        depends_on:
+            - graylog
+        volumes: 
+            - ./config/graylog/filebeat:/mnt
+            - ./logs:/app-logs

docs/graylog.md

@@ -0,0 +1,88 @@
+<h1 align="center">
+<img
+    width="150"
+    src="https://www.graylog.fr/wp-content/uploads/2014/05/graylog2_logo.png"
+    alt="graylog logo"
+    >
+  <br>
+
+  Graylog - Logs Management Stack
+  <br>
+</h1>
+
+<h4 align="center">Various log aggregator in order to enhance debugging and monitoring</h4>
+
+<p align="center">
+  <img 
+  width="150" src="http://www.kaliop.fr/sites/default/files/inline-images/visuel-k-new.jpg" 
+  alt="Kaliop logo">
+</p>
+<br>
+
+This stack is a poke in order to demonstrate the power of such tool. Please follow the guide to accomplish the walkthrough.
+
+This library is mainly maintained by [Andréas HANSS](https://github.com/ScreamZ), feel free to contact if you are having some questions.
+
+# Requirements
+
+- **Kaliop eZ Docker stack.** https://github.com/kaliop/ezdocker-stack
+- **Docker & Docker-compose.** Recent version
+
+# Introduction
+
+**Why GRAYLOG ?**
+
+- Open source log management.
+- Scale well in large architecture.
+- Embedded user authentication system.
+- Embedded alert system on various metrics.
+- Support for various format, especially well with [GELF](http://docs.graylog.org/en/2.2/pages/gelf.html).
+- Awesome admin & UI.
+- Support for LDAP.
+
+**Why not ELK (Elastic Logstash Kibana)**
+- Require [Shield/security](https://www.elastic.co/products/x-pack/security) for user authentication handling (Not free).
+- Require [Watcher](https://www.elastic.co/products/x-pack/alerting) for user authentication handling (Not free).
+- Heavy costly resources consumption and poor performances.
+- Logstash has no UI out-of-the-box, require manual configuration.
+
+# Setup
+
+## What is logged
+
+At the moment you're able to receive following logs :
+
+- **Varnish BAN / PURGE -** While you're not banning using the Varnish agent tool, take note that it works with back-office BANs.
+
+- **Apache2 Access/Errors**
+
+- **Nginx Access/Errors**
+
+- **SOLR**
+
+
+# Troubleshooting
+
+**Check docker network**
+Your containers might be on different network, feel free to update the docker-compose.yml file accordingly in order to match.
+
+# Future improvements
+
+**Pre-bootstrap Mongo with config**
+Instead of doing the restore, bootstrap the database at compilation with given parameters.
+
+**Use log aggregator system**
+
+- ElasticBEAT with graylog sidecar mode or without. http://docs.graylog.org/en/2.2/pages/collector.html (Deprecated) see http://docs.graylog.org/en/2.2/pages/collector_sidecar.html
+- FluentD
+- Rsylog / Syslog-NG
+
+**Think about using LDAP system**
+
+**Optimize index and stream internally**
+
+**Find a way to use GELF everywhere**
+
+- In symfony use monolog with specific wrapper that ignore exception in case of not responsing server
+
+**Format Apache/NGINX logs upstream in config**

graylog.yml

@@ -0,0 +1,36 @@
+# Graylog stack
+version: '2'
+services:
+    mongo:
+      container_name: graylog-mongo
+      image: "mongo:3"
+      volumes:
+        - ./config/graylog:/docker-entrypoint-initdb.d
+
+    elasticsearch:
+      container_name: graylog-elastic
+      image: "elasticsearch:2"
+      command: "elasticsearch -Des.cluster.name='graylog'"
+
+    graylog:
+      container_name: graylog-server
+      image: 'graylog2/server:latest'
+      environment:
+        GRAYLOG_PASSWORD_SECRET: somepasswordpepper
+        GRAYLOG_ROOT_PASSWORD_SHA2: 8c6976e5b5410415bde908bd4dee15dfb167a9c873fc4bb8a81f6f2ab448a918
+        GRAYLOG_WEB_ENDPOINT_URI: http://127.0.0.1:9000/api
+      depends_on:
+        - mongo
+        - elasticsearch
+      ports:
+        - "9000:9000"
+
+    filebeats:
+        container_name: graylog-filebeat
+        #image: TODO_ON_DOCKERHUB_UPLOAD
+        build: ./images/filebeat
+        depends_on:
+            - graylog
+        volumes:
+            - ./config/graylog/filebeat:/mnt
+            - ./logs:/app-logs

images/apache_php54/bootstrap.sh

@@ -55,9 +55,8 @@ trap clean_up SIGTERM
 
 service apache2 restart
 
-
 echo [`date`] Bootstrap finished
 
 tail -f /dev/null &
 child=$!
-wait "$child"
+wait "$child"