ELK + Grafana

.env

@@ -0,0 +1,41 @@
+# Project namespace (defaults to the current folder name if not set)
+#COMPOSE_PROJECT_NAME=myproject
+
+
+# Password for the 'elastic' user (at least 6 characters)
+ELASTIC_PASSWORD=changeme
+
+
+# Password for the 'kibana_system' user (at least 6 characters)
+KIBANA_PASSWORD=changeme
+
+
+# Version of Elastic products
+STACK_VERSION=8.7.1
+
+
+# Set the cluster name
+CLUSTER_NAME=snort-cluster
+
+
+# Set to 'basic' or 'trial' to automatically start the 30-day trial
+LICENSE=basic
+#LICENSE=trial
+
+
+# Port to expose Elasticsearch HTTP API to the host
+ES_PORT=9200
+
+
+# Port to expose Kibana to the host
+KIBANA_PORT=5601
+
+
+# Increase or decrease based on the available host memory (in bytes)
+ES_MEM_LIMIT=1073741824
+KB_MEM_LIMIT=1073741824
+LS_MEM_LIMIT=1073741824
+
+
+# SAMPLE Predefined Key only to be used in POC environments
+ENCRYPTION_KEY=c34d38b3a14956121ff2170e5030b471551370178f43e5626eec58b04a30fae2

.gitignore

@@ -0,0 +1,2 @@
+./logs/*.txt
+./logs/*.log

Dockerfile

@@ -39,10 +39,10 @@ RUN mkdir ${PREFIX_DIR}/etc/rules && \
   touch ${PREFIX_DIR}/etc/rules/local.rules && \
   touch ${PREFIX_DIR}/etc/lists/default.blocklist && \
   mkdir /var/log/snort
-COPY snort3-community-rules.tar.gz ${HOME}/snort3-community-rules.tar.gz
+COPY ./tars/snort3-community-rules.tar.gz ${HOME}/snort3-community-rules.tar.gz
-COPY feodotracker.tar.gz ${HOME}/feodotracker.tar.gz
+COPY ./tars/feodotracker.tar.gz ${HOME}/feodotracker.tar.gz
-COPY appid-rules.tar.gz ${HOME}/appid-rules.tar.gz
+COPY ./tars/appid-rules.tar.gz ${HOME}/appid-rules.tar.gz
-COPY emerging-rules.tar.gz ${HOME}/emerging-rules.tar.gz
+COPY ./tars/emerging-rules.tar.gz ${HOME}/emerging-rules.tar.gz
 RUN tar -xvzf snort3-community-rules.tar.gz && cd snort3-community-rules && mkdir ${PREFIX_DIR}/etc/rules/snort3-community-rules/ && cp * ${PREFIX_DIR}/etc/rules/snort3-community-rules/
 
 WORKDIR $HOME
@@ -58,15 +58,15 @@ RUN snort --version
 
 # Install OpenAppID
 WORKDIR $HOME
-COPY snort-openappid.tar.gz ${HOME}/OpenAppId-23020.tar.gz
+COPY ./tars/snort-openappid.tar.gz ${HOME}/OpenAppId-23020.tar.gz
 RUN tar -xzvf OpenAppId-23020.tar.gz && mkdir -p /usr/local/lib/openappid && cp -r odp /usr/local/lib/openappid
 
 WORKDIR $HOME
-COPY healthcheck.sh ${HOME}/healthcheck.sh
+COPY ./scripts/healthcheck.sh ${HOME}/healthcheck.sh
 RUN chmod +x ${HOME}/healthcheck.sh
 HEALTHCHECK --interval=30s CMD ${HOME}/healthcheck.sh
 
-COPY supervisord.conf /etc/supervisor/conf.d/supervisord.conf
+COPY ./configs/supervisord.conf /etc/supervisor/conf.d/supervisord.conf
-COPY entrypoint.sh ${HOME}/entrypoint.sh
+COPY ./scripts/entrypoint.sh ${HOME}/entrypoint.sh
 
 ENTRYPOINT ["/bin/bash", "/root/entrypoint.sh"]

configs/filebeat.yml

@@ -0,0 +1,26 @@
+filebeat.inputs:
+  - type: filestream
+    id: default-filestream
+    paths:
+      - ingest_data/*.txt
+      - ingest_data/*.log
+
+filebeat.autodiscover:
+  providers:
+    - type: docker
+      hints.enabled: true
+
+processors:
+  - add_docker_metadata: ~
+
+setup.kibana:
+  host: ${KIBANA_HOSTS}
+  username: ${ELASTIC_USER}
+  password: ${ELASTIC_PASSWORD}
+
+output.elasticsearch:
+  hosts: ${ELASTIC_HOSTS}
+  username: ${ELASTIC_USER}
+  password: ${ELASTIC_PASSWORD}
+  ssl.enabled: true
+  ssl.certificate_authorities: "certs/ca/ca.crt"

configs/logstash.conf

@@ -0,0 +1,39 @@
+input {
+ file {
+   mode => "tail"
+   type => "snort-json"
+   path => "/usr/share/logstash/ingest_data/alert_json.txt"
+ }
+ file {
+  mode => "tail"
+  type => "snort-appid"
+  path => "/usr/share/logstash/ingest_data/appid_stats.log"
+ }
+}
+
+
+filter {
+}
+
+
+output {
+  if [type] == "snort-json" {
+    elasticsearch {
+      index => "snort-json-%{+YYYY.MM.dd}"
+      hosts=> "${ELASTIC_HOSTS}"
+      user=> "${ELASTIC_USER}"
+      password=> "${ELASTIC_PASSWORD}"
+      cacert=> "certs/ca/ca.crt"
+    }
+  }
+  if [type] == "snort-appid" {
+    elasticsearch {
+      index => "snort-appid-%{+YYYY.MM.dd}"
+      hosts=> "${ELASTIC_HOSTS}"
+      user=> "${ELASTIC_USER}"
+      password=> "${ELASTIC_PASSWORD}"
+      cacert=> "certs/ca/ca.crt"
+    }
+    }
+ 
+}

configs/snort.rules

@@ -0,0 +1 @@
+#alert icmp any any -> $HOME_NET any (msg:"ICMP connection test"; sid:1000001; rev:1;)

docker-compose.yml

@@ -1,60 +1,245 @@
 name: snort
 services:
-    snort:
+  snort:
-        restart: always
+    restart: always
-        cap_add:
+    cap_add:
-            - NET_ADMIN
+      - NET_ADMIN
-        build:
+    build:
-            dockerfile: Dockerfile
+      dockerfile: Dockerfile
-            context: ./
+      context: ./
-        volumes:
+    volumes:
-            - ./snort-conf:/usr/local/etc/snort
+      - ./configs/snort-conf:/usr/local/etc/snort
-            - ./snort.rules:/usr/local/etc/rules/local.rules
+      - ./configs/snort.rules:/usr/local/etc/rules/local.rules
-            - ./logs:/var/log/snort
+      - ./logs:/var/log/snort
-        network_mode: host
+    network_mode: host
-    promtail:
+  promtail:
-        image: grafana/promtail:1.4.1
+    image: grafana/promtail:1.4.1
-        restart: always
+    restart: always
-        volumes:
+    volumes:
-            - promtail-data:/var/lib/promtail/positions
+      - ./grafana/promtail-data:/var/lib/promtail/positions
-            - ./promtail/docker.yml:/etc/promtail/promtail.yml
+      - ./configs/promtail.yml:/etc/promtail/promtail.yml
-            - ./logs:/var/log/snort
+      - ./logs:/var/log/snort
-        command:
+    command:
-            - '-config.file=/etc/promtail/promtail.yml'
+      - "-config.file=/etc/promtail/promtail.yml"
-        networks:
+    networks:
-            snort_lan:
+      snort_lan:
-    loki:
+  loki:
-        hostname: loki
+    hostname: loki
-        image: grafana/loki:latest
+    image: grafana/loki:latest
-        restart: always
+    restart: always
-        environment:
+    environment:
-            TZ: Europe/Moscow
+      TZ: Europe/Moscow
-        ports:
+    ports:
-            - "3100:3100"
+      - "3100:3100"
-        command: -config.file=/etc/loki/local-config.yaml
+    command: -config.file=/etc/loki/local-config.yaml
-        networks:
+    networks:
-            snort_lan:
+      snort_lan:
-    grafana:
+  grafana:
-        image: grafana/grafana-enterprise
+    image: grafana/grafana-enterprise
-        container_name: grafana
+    hostname: grafana
-        hostname: grafana
+    environment:
-        environment:
+      - GF_PATHS_PROVISIONING=/etc/grafana/provisioning
-            - GF_PATHS_PROVISIONING=/etc/grafana/provisioning
+      - GF_AUTH_ANONYMOUS_ENABLED=true
-            - GF_AUTH_ANONYMOUS_ENABLED=true
+      - GF_AUTH_ANONYMOUS_ORG_ROLE=Admin
-            - GF_AUTH_ANONYMOUS_ORG_ROLE=Admin
+      - TZ=Europe/Moscow
-            - TZ=Europe/Moscow
+    restart: always
-        restart: always
+    ports:
-        ports:
+      - "3000:3000"
-        - '3000:3000'
+    networks:
-        networks:
+      snort_lan:
-            snort_lan:
+    volumes:
-        volumes:
+      - grafana-storage:/var/lib/grafana
-            - grafana-storage:/var/lib/grafana
+      - ./grafana/provisioning/datasources:/etc/grafana/provisioning/datasources
-            - ./grafana/provisioning/datasources:/etc/grafana/provisioning/datasources
+
+  setup:
+    image: elasticsearch:${STACK_VERSION}
+    volumes:
+      - certs:/usr/share/elasticsearch/config/certs
+    user: "0"
+    command: >
+      bash -c '
+      if [ x${ELASTIC_PASSWORD} == x ]; then
+      echo "Set the ELASTIC_PASSWORD environment variable in the .env file";
+      exit 1;
+      elif [ x${KIBANA_PASSWORD} == x ]; then
+      echo "Set the KIBANA_PASSWORD environment variable in the .env file";
+      exit 1;
+      fi;
+      if [ ! -f config/certs/ca.zip ]; then
+      echo "Creating CA";
+      bin/elasticsearch-certutil ca --silent --pem -out config/certs/ca.zip;
+      unzip config/certs/ca.zip -d config/certs;
+      fi;
+      if [ ! -f config/certs/certs.zip ]; then
+      echo "Creating certs";
+      echo -ne \
+      "instances:\n"\
+      "  - name: es01\n"\
+      "    dns:\n"\
+      "      - es01\n"\
+      "      - localhost\n"\
+      "    ip:\n"\
+      "      - 127.0.0.1\n"\
+      "  - name: kibana\n"\
+      "    dns:\n"\
+      "      - kibana\n"\
+      "      - localhost\n"\
+      "    ip:\n"\
+      "      - 127.0.0.1\n"\
+      > config/certs/instances.yml;
+      bin/elasticsearch-certutil cert --silent --pem -out config/certs/certs.zip --in config/certs/instances.yml --ca-cert config/certs/ca/ca.crt --ca-key config/certs/ca/ca.key;
+      unzip config/certs/certs.zip -d config/certs;
+      fi;
+      echo "Setting file permissions"
+      chown -R root:root config/certs;
+      find . -type d -exec chmod 750 \{\} \;;
+      find . -type f -exec chmod 640 \{\} \;;
+      echo "Waiting for Elasticsearch availability";
+      until curl -s --cacert config/certs/ca/ca.crt https://es01:9200 | grep -q "missing authentication credentials"; do sleep 30; done;
+      echo "Setting kibana_system password";
+      until curl -s -X POST --cacert config/certs/ca/ca.crt -u "elastic:${ELASTIC_PASSWORD}" -H "Content-Type: application/json" https://es01:9200/_security/user/kibana_system/_password -d "{\"password\":\"${KIBANA_PASSWORD}\"}" | grep -q "^{}"; do sleep 10; done;
+      echo "All done!";
+      '
+    healthcheck:
+      test: ["CMD-SHELL", "[ -f config/certs/es01/es01.crt ]"]
+      interval: 1s
+      timeout: 5s
+      retries: 120
+    networks:
+      snort_lan:
+
+  es01:
+    depends_on:
+      setup:
+        condition: service_healthy
+    image: elasticsearch:${STACK_VERSION}
+    labels:
+      co.elastic.logs/module: elasticsearch
+    volumes:
+      - certs:/usr/share/elasticsearch/config/certs
+      - esdata01:/usr/share/elasticsearch/data
+    ports:
+      - ${ES_PORT}:9200
+    environment:
+      - node.name=es01
+      - cluster.name=${CLUSTER_NAME}
+      - discovery.type=single-node
+      - ELASTIC_PASSWORD=${ELASTIC_PASSWORD}
+      - bootstrap.memory_lock=true
+      - xpack.security.enabled=true
+      - xpack.security.http.ssl.enabled=true
+      - xpack.security.http.ssl.key=certs/es01/es01.key
+      - xpack.security.http.ssl.certificate=certs/es01/es01.crt
+      - xpack.security.http.ssl.certificate_authorities=certs/ca/ca.crt
+      - xpack.security.transport.ssl.enabled=true
+      - xpack.security.transport.ssl.key=certs/es01/es01.key
+      - xpack.security.transport.ssl.certificate=certs/es01/es01.crt
+      - xpack.security.transport.ssl.certificate_authorities=certs/ca/ca.crt
+      - xpack.security.transport.ssl.verification_mode=certificate
+      - xpack.license.self_generated.type=${LICENSE}
+    mem_limit: ${ES_MEM_LIMIT}
+    ulimits:
+      memlock:
+        soft: -1
+        hard: -1
+    healthcheck:
+      test:
+        [
+          "CMD-SHELL",
+          "curl -s --cacert config/certs/ca/ca.crt https://localhost:9200 | grep -q 'missing authentication credentials'",
+        ]
+      interval: 10s
+      timeout: 10s
+      retries: 120
+    networks:
+      snort_lan:
+
+  kibana:
+    depends_on:
+      es01:
+        condition: service_healthy
+    image: kibana:${STACK_VERSION}
+    labels:
+      co.elastic.logs/module: kibana
+    volumes:
+      - certs:/usr/share/kibana/config/certs
+      - kibanadata:/usr/share/kibana/data
+    ports:
+      - 22:22
+      - ${KIBANA_PORT}:5601
+    environment:
+      - SERVERNAME=kibana
+      - ELASTICSEARCH_HOSTS=https://es01:9200
+      - ELASTICSEARCH_USERNAME=kibana_system
+      - ELASTICSEARCH_PASSWORD=${KIBANA_PASSWORD}
+      - ELASTICSEARCH_SSL_CERTIFICATEAUTHORITIES=config/certs/ca/ca.crt
+      - XPACK_SECURITY_ENCRYPTIONKEY=${ENCRYPTION_KEY}
+      - XPACK_ENCRYPTEDSAVEDOBJECTS_ENCRYPTIONKEY=${ENCRYPTION_KEY}
+      - XPACK_REPORTING_ENCRYPTIONKEY=${ENCRYPTION_KEY}
+    mem_limit: ${KB_MEM_LIMIT}
+    healthcheck:
+      test: ["CMD-SHELL", "curl -s -I http://localhost:5601 | grep -q 'HTTP/1.1 302 Found'"]
+      interval: 10s
+      timeout: 10s
+      retries: 120
+    networks:
+      snort_lan:
+
+  filebeat01:
+    depends_on:
+      es01:
+        condition: service_healthy
+    image: elastic/filebeat:${STACK_VERSION}
+    user: root
+    command: filebeat -e -strict.perms=false
+    volumes:
+      - certs:/usr/share/filebeat/certs
+      - filebeatdata01:/usr/share/filebeat/data
+      - "/var/lib/docker/containers:/var/lib/docker/containers:ro"
+      - "/var/run/docker.sock:/var/run/docker.sock:ro"
+      - "./configs/filebeat.yml:/usr/share/filebeat/filebeat.yml:ro"
+      - "./logs:/usr/share/filebeat/ingest_data/"
+    environment:
+      - ELASTIC_USER=elastic
+      - ELASTIC_PASSWORD=${ELASTIC_PASSWORD}
+      - ELASTIC_HOSTS=https://es01:9200
+      - KIBANA_HOSTS=http://kibana:5601
+      - LOGSTASH_HOSTS=http://logstash01:9600
+    networks:
+      snort_lan:
+
+  logstash01:
+    depends_on:
+      es01:
+        condition: service_healthy
+      kibana:
+        condition: service_healthy
+    image: logstash:${STACK_VERSION}
+    labels:
+      co.elastic.logs/module: logstash
+    user: root
+    volumes:
+      - certs:/usr/share/logstash/certs
+      - logstashdata01:/usr/share/logstash/data
+      - "./configs/logstash.conf:/usr/share/logstash/pipeline/logstash.conf:ro"
+      - "./logs:/usr/share/logstash/ingest_data/"
+    environment:
+      - xpack.monitoring.enabled=false
+      - ELASTIC_USER=elastic
+      - ELASTIC_PASSWORD=${ELASTIC_PASSWORD}
+      - ELASTIC_HOSTS=https://es01:9200
+    networks:
+      snort_lan:
 
 networks:
-    snort_lan:
+  snort_lan:
-        driver: bridge
+    driver: bridge
 volumes:
-    grafana-storage: {}
+  grafana-storage: {}
-    promtail-data: {}
+  promtail-data: {}
+  certs: {}
+  esdata01: {}
+  kibanadata: {}
+  logstashdata01: {}
+  filebeatdata01: {}
+  metricbeatdata01: {}

scripts/entrypoint.sh

@@ -4,5 +4,6 @@ ip route del default
 ip add sh $interface
 ip route add default dev $interface
 ip link set dev $interface promisc on
+ethtool -K $interface gro off lro off
 
 /usr/bin/supervisord -c /etc/supervisor/conf.d/supervisord.conf

snort.rules

@@ -1 +0,0 @@
-alert icmp any any -> $HOME_NET any (msg:"ICMP connection test"; sid:1000001; rev:1;)