From 970b65c5c9e372669d7939721a55502574143bf7 Mon Sep 17 00:00:00 2001 From: Maxim Malakhov Date: Tue, 20 Aug 2024 15:19:28 +0300 Subject: [PATCH] Removed elasticsearch and added healthcheck --- .env | 41 ------ Dockerfile | 5 + docker-compose.yml | 202 +--------------------------- filebeat.yml | 30 ----- healthcheck.sh | 6 + logstash.conf | 24 ---- logstash_ingest_data/alert_csv.txt | 0 logstash_ingest_data/alert_fast.txt | 6 - logstash_ingest_data/supervisor.log | 154 --------------------- snort-rules.txt => snort.rules | 0 10 files changed, 14 insertions(+), 454 deletions(-) delete mode 100644 .env delete mode 100644 filebeat.yml create mode 100644 healthcheck.sh delete mode 100644 logstash.conf delete mode 100644 logstash_ingest_data/alert_csv.txt delete mode 100644 logstash_ingest_data/alert_fast.txt delete mode 100644 logstash_ingest_data/supervisor.log rename snort-rules.txt => snort.rules (100%) diff --git a/.env b/.env deleted file mode 100644 index 4e372fd..0000000 --- a/.env +++ /dev/null @@ -1,41 +0,0 @@ -# Project namespace (defaults to the current folder name if not set) -#COMPOSE_PROJECT_NAME=myproject - - -# Password for the 'elastic' user (at least 6 characters) -ELASTIC_PASSWORD=changeme - - -# Password for the 'kibana_system' user (at least 6 characters) -KIBANA_PASSWORD=changeme - - -# Version of Elastic products -STACK_VERSION=8.7.1 - - -# Set the cluster name -CLUSTER_NAME=snort-cluster - - -# Set to 'basic' or 'trial' to automatically start the 30-day trial -LICENSE=basic -#LICENSE=trial - - -# Port to expose Elasticsearch HTTP API to the host -ES_PORT=9200 - - -# Port to expose Kibana to the host -KIBANA_PORT=5601 - - -# Increase or decrease based on the available host memory (in bytes) -ES_MEM_LIMIT=1073741824 -KB_MEM_LIMIT=1073741824 -LS_MEM_LIMIT=1073741824 - - -# SAMPLE Predefined Key only to be used in POC environments -ENCRYPTION_KEY=c34d38b3a14956121ff2170e5030b471551370178f43e5626eec58b04a30fae2 diff --git a/Dockerfile b/Dockerfile index 71cf515..7eef1c5 100644 --- a/Dockerfile +++ b/Dockerfile @@ -61,6 +61,11 @@ WORKDIR $HOME COPY snort-openappid.tar.gz ${HOME}/OpenAppId-23020.tar.gz RUN tar -xzvf OpenAppId-23020.tar.gz && mkdir -p /usr/local/lib/openappid && cp -r odp /usr/local/lib/openappid +WORKDIR $HOME +COPY healthcheck.sh ${HOME}/healthcheck.sh +RUN chmod +x ${HOME}/healthcheck.sh +HEALTHCHECK --interval=30s CMD ${HOME}/healthcheck.sh + COPY supervisord.conf /etc/supervisor/conf.d/supervisord.conf COPY entrypoint.sh ${HOME}/entrypoint.sh diff --git a/docker-compose.yml b/docker-compose.yml index 444e479..2e29309 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -9,207 +9,11 @@ services: context: ./ volumes: - ./snort-conf:/usr/local/etc/snort - - ./snort-rules.txt:/usr/local/etc/rules/local.rules - #- ./logs:/var/log/snort - - ./logstash_ingest_data:/var/log/snort - networks: - snort_lan: - - setup: - image: elasticsearch:${STACK_VERSION} - volumes: - - certs:/usr/share/elasticsearch/config/certs - user: "0" - command: > - bash -c ' - if [ x${ELASTIC_PASSWORD} == x ]; then - echo "Set the ELASTIC_PASSWORD environment variable in the .env file"; - exit 1; - elif [ x${KIBANA_PASSWORD} == x ]; then - echo "Set the KIBANA_PASSWORD environment variable in the .env file"; - exit 1; - fi; - if [ ! -f config/certs/ca.zip ]; then - echo "Creating CA"; - bin/elasticsearch-certutil ca --silent --pem -out config/certs/ca.zip; - unzip config/certs/ca.zip -d config/certs; - fi; - if [ ! -f config/certs/certs.zip ]; then - echo "Creating certs"; - echo -ne \ - "instances:\n"\ - " - name: es01\n"\ - " dns:\n"\ - " - es01\n"\ - " - localhost\n"\ - " ip:\n"\ - " - 127.0.0.1\n"\ - " - name: kibana\n"\ - " dns:\n"\ - " - kibana\n"\ - " - localhost\n"\ - " ip:\n"\ - " - 127.0.0.1\n"\ - > config/certs/instances.yml; - bin/elasticsearch-certutil cert --silent --pem -out config/certs/certs.zip --in config/certs/instances.yml --ca-cert config/certs/ca/ca.crt --ca-key config/certs/ca/ca.key; - unzip config/certs/certs.zip -d config/certs; - fi; - echo "Setting file permissions" - chown -R root:root config/certs; - find . -type d -exec chmod 750 \{\} \;; - find . -type f -exec chmod 640 \{\} \;; - echo "Waiting for Elasticsearch availability"; - until curl -s --cacert config/certs/ca/ca.crt https://es01:9200 | grep -q "missing authentication credentials"; do sleep 30; done; - echo "Setting kibana_system password"; - until curl -s -X POST --cacert config/certs/ca/ca.crt -u "elastic:${ELASTIC_PASSWORD}" -H "Content-Type: application/json" https://es01:9200/_security/user/kibana_system/_password -d "{\"password\":\"${KIBANA_PASSWORD}\"}" | grep -q "^{}"; do sleep 10; done; - echo "All done!"; - ' - healthcheck: - test: ["CMD-SHELL", "[ -f config/certs/es01/es01.crt ]"] - interval: 1s - timeout: 5s - retries: 120 + - ./snort.rules:/usr/local/etc/rules/local.rules + - ./logs:/var/log/snort networks: snort_lan: - es01: - depends_on: - setup: - condition: service_healthy - image: elasticsearch:${STACK_VERSION} - labels: - co.elastic.logs/module: elasticsearch - volumes: - - certs:/usr/share/elasticsearch/config/certs - - esdata01:/usr/share/elasticsearch/data - ports: - - ${ES_PORT}:9200 - environment: - - node.name=es01 - - cluster.name=${CLUSTER_NAME} - - discovery.type=single-node - - ELASTIC_PASSWORD=${ELASTIC_PASSWORD} - - bootstrap.memory_lock=true - - xpack.security.enabled=true - - xpack.security.http.ssl.enabled=true - - xpack.security.http.ssl.key=certs/es01/es01.key - - xpack.security.http.ssl.certificate=certs/es01/es01.crt - - xpack.security.http.ssl.certificate_authorities=certs/ca/ca.crt - - xpack.security.transport.ssl.enabled=true - - xpack.security.transport.ssl.key=certs/es01/es01.key - - xpack.security.transport.ssl.certificate=certs/es01/es01.crt - - xpack.security.transport.ssl.certificate_authorities=certs/ca/ca.crt - - xpack.security.transport.ssl.verification_mode=certificate - - xpack.license.self_generated.type=${LICENSE} - mem_limit: ${ES_MEM_LIMIT} - ulimits: - memlock: - soft: -1 - hard: -1 - healthcheck: - test: - [ - "CMD-SHELL", - "curl -s --cacert config/certs/ca/ca.crt https://localhost:9200 | grep -q 'missing authentication credentials'", - ] - interval: 10s - timeout: 10s - retries: 120 - networks: - snort_lan: - - kibana: - depends_on: - es01: - condition: service_healthy - image: kibana:${STACK_VERSION} - labels: - co.elastic.logs/module: kibana - volumes: - - certs:/usr/share/kibana/config/certs - - kibanadata:/usr/share/kibana/data - ports: - - ${KIBANA_PORT}:5601 - environment: - - SERVERNAME=kibana - - ELASTICSEARCH_HOSTS=https://es01:9200 - - ELASTICSEARCH_USERNAME=kibana_system - - ELASTICSEARCH_PASSWORD=${KIBANA_PASSWORD} - - ELASTICSEARCH_SSL_CERTIFICATEAUTHORITIES=config/certs/ca/ca.crt - - XPACK_SECURITY_ENCRYPTIONKEY=${ENCRYPTION_KEY} - - XPACK_ENCRYPTEDSAVEDOBJECTS_ENCRYPTIONKEY=${ENCRYPTION_KEY} - - XPACK_REPORTING_ENCRYPTIONKEY=${ENCRYPTION_KEY} - mem_limit: ${KB_MEM_LIMIT} - healthcheck: - test: - [ - "CMD-SHELL", - "curl -s -I http://localhost:5601 | grep -q 'HTTP/1.1 302 Found'", - ] - interval: 10s - timeout: 10s - retries: 120 - networks: - snort_lan: - - filebeat01: - depends_on: - es01: - condition: service_healthy - image: elastic/filebeat:${STACK_VERSION} - user: root - volumes: - - certs:/usr/share/filebeat/certs - - filebeatdata01:/usr/share/filebeat/data - - "./filebeat_ingest_data/:/usr/share/filebeat/ingest_data/" - - "./filebeat.yml:/usr/share/filebeat/filebeat.yml:ro" - - "/var/lib/docker/containers:/var/lib/docker/containers:ro" - - "/var/run/docker.sock:/var/run/docker.sock:ro" - environment: - - ELASTIC_USER=elastic - - ELASTIC_PASSWORD=${ELASTIC_PASSWORD} - - ELASTIC_HOSTS=https://es01:9200 - - KIBANA_HOSTS=http://kibana:5601 - - LOGSTASH_HOSTS=http://logstash01:9600 - networks: - snort_lan: - - logstash01: - depends_on: - es01: - condition: service_healthy - kibana: - condition: service_healthy - image: logstash:${STACK_VERSION} - labels: - co.elastic.logs/module: logstash - user: root - volumes: - - certs:/usr/share/logstash/certs - - logstashdata01:/usr/share/logstash/data - - "./logstash_ingest_data/:/usr/share/logstash/ingest_data/" - - "./logstash.conf:/usr/share/logstash/pipeline/logstash.conf:ro" - environment: - - xpack.monitoring.enabled=false - - ELASTIC_USER=elastic - - ELASTIC_PASSWORD=${ELASTIC_PASSWORD} - - ELASTIC_HOSTS=https://es01:9200 - networks: - snort_lan: - - networks: snort_lan: - driver: bridge - -volumes: - certs: - driver: local - esdata01: - driver: local - kibanadata: - driver: local - logstashdata01: - driver: local - filebeatdata01: - driver: local \ No newline at end of file + driver: bridge \ No newline at end of file diff --git a/filebeat.yml b/filebeat.yml deleted file mode 100644 index 6cebf9b..0000000 --- a/filebeat.yml +++ /dev/null @@ -1,30 +0,0 @@ -filebeat.inputs: -- type: filestream - id: default-filestream - paths: - - ingest_data/*.txt - - ingest_data/*.log - - -filebeat.autodiscover: - providers: - - type: docker - hints.enabled: true - - -processors: -- add_docker_metadata: ~ - - -setup.kibana: - host: ${KIBANA_HOSTS} - username: ${ELASTIC_USER} - password: ${ELASTIC_PASSWORD} - - -output.elasticsearch: - hosts: ${ELASTIC_HOSTS} - username: ${ELASTIC_USER} - password: ${ELASTIC_PASSWORD} - ssl.enabled: true - ssl.certificate_authorities: "certs/ca/ca.crt" \ No newline at end of file diff --git a/healthcheck.sh b/healthcheck.sh new file mode 100644 index 0000000..78a4a39 --- /dev/null +++ b/healthcheck.sh @@ -0,0 +1,6 @@ +#!/bin/bash + +set -e -o pipefail + +log "Checking if $(basename "${0}") is healthy ..." +[[ $(pgrep --count --full /usr/bin/supervisord) -gt 0 ]] \ No newline at end of file diff --git a/logstash.conf b/logstash.conf deleted file mode 100644 index 6f65bad..0000000 --- a/logstash.conf +++ /dev/null @@ -1,24 +0,0 @@ -input { - file { - #https://www.elastic.co/guide/en/logstash/current/plugins-inputs-file.html - #default is TAIL which assumes more data will come into the file. - #change to mode => "read" if the file is a compelte file. by default, the file will be removed once reading is complete -- backup your files if you need them. - mode => "tail" - path => "/usr/share/logstash/ingest_data/*" - } -} - - -filter { -} - - -output { - elasticsearch { - index => "logstash-%{+YYYY.MM.dd}" - hosts=> "${ELASTIC_HOSTS}" - user=> "${ELASTIC_USER}" - password=> "${ELASTIC_PASSWORD}" - cacert=> "certs/ca/ca.crt" - } -} diff --git a/logstash_ingest_data/alert_csv.txt b/logstash_ingest_data/alert_csv.txt deleted file mode 100644 index e69de29..0000000 diff --git a/logstash_ingest_data/alert_fast.txt b/logstash_ingest_data/alert_fast.txt deleted file mode 100644 index 6ed4c6d..0000000 --- a/logstash_ingest_data/alert_fast.txt +++ /dev/null @@ -1,6 +0,0 @@ -08/22-19:30:41.554941 [**] [1:1000001:1] "ICMP connection test" [**] [Priority: 0] [AppID: ICMP] {ICMP} 192.168.57.1 -> 192.168.57.3 -08/22-19:30:42.578554 [**] [1:1000001:1] "ICMP connection test" [**] [Priority: 0] [AppID: ICMP] {ICMP} 192.168.57.1 -> 192.168.57.3 -08/22-19:30:43.602594 [**] [1:1000001:1] "ICMP connection test" [**] [Priority: 0] [AppID: ICMP] {ICMP} 192.168.57.1 -> 192.168.57.3 -08/22-19:30:44.626660 [**] [1:1000001:1] "ICMP connection test" [**] [Priority: 0] [AppID: ICMP] {ICMP} 192.168.57.1 -> 192.168.57.3 -08/22-19:30:45.650654 [**] [1:1000001:1] "ICMP connection test" [**] [Priority: 0] [AppID: ICMP] {ICMP} 192.168.57.1 -> 192.168.57.3 -08/22-19:30:46.674630 [**] [1:1000001:1] "ICMP connection test" [**] [Priority: 0] [AppID: ICMP] {ICMP} 192.168.57.1 -> 192.168.57.3 \ No newline at end of file diff --git a/logstash_ingest_data/supervisor.log b/logstash_ingest_data/supervisor.log deleted file mode 100644 index 8cb779f..0000000 --- a/logstash_ingest_data/supervisor.log +++ /dev/null @@ -1,154 +0,0 @@ --------------------------------------------------- -o")~ Snort++ 3.3.2.0 --------------------------------------------------- -Loading /usr/local/etc/snort/snort.lua: -Loading snort_defaults.lua: -Finished snort_defaults.lua: - ssh - host_cache - pop - so_proxy - stream_tcp - mms - smtp - gtp_inspect - packets - dce_http_proxy - alert_fast - alert_csv - ips - stream_icmp - hosts - normalizer - binder - wizard - appid - js_norm - file_id - http2_inspect - http_inspect - stream_udp - ftp_data - ftp_server - search_engine - port_scan - dce_http_server - dce_tcp - dce_smb - iec104 - cip - telnet - ssl - sip - rpc_decode - netflow - modbus - host_tracker - stream_user - stream_ip - trace - back_orifice - classifications - dnp3 - active - process - ftp_client - daq - decode - alerts - stream - references - arp_spoof - output - network - dns - dce_udp - imap - file_policy - s7commplus - stream_file -Finished /usr/local/etc/snort/snort.lua: -Loading file_id.rules_file: -Loading file_magic.rules: -Finished file_magic.rules: -Finished file_id.rules_file: -Loading ips.rules: -Loading ../rules/snort3-community-rules/snort3-community.rules: -Finished ../rules/snort3-community-rules/snort3-community.rules: -Loading ../rules/local.rules: -Finished ../rules/local.rules: -Finished ips.rules: --------------------------------------------------- -ips policies rule stats - id loaded shared enabled file - 0 4239 0 4239 /usr/local/etc/snort/snort.lua --------------------------------------------------- -rule counts - total rules loaded: 4239 - text rules: 4239 - option chains: 4239 - chain headers: 325 - flowbits: 48 - flowbits not checked: 23 --------------------------------------------------- -port rule counts - tcp udp icmp ip - any 472 58 148 22 - src 170 15 0 0 - dst 775 150 0 0 - both 6 11 0 0 - total 1423 234 148 22 --------------------------------------------------- -service rule counts to-srv to-cli - dcerpc: 72 20 - dhcp: 2 2 - dns: 28 7 - file_id: 219 219 - ftp: 90 4 - ftp-data: 1 96 - http: 2084 255 - http2: 2084 255 - http3: 2084 255 - imap: 35 117 - irc: 5 2 - kerberos: 3 0 - ldap: 0 1 - mysql: 3 0 - netbios-dgm: 1 1 - netbios-ns: 4 3 - netbios-ssn: 69 17 - nntp: 2 0 - pop3: 23 117 - rdp: 5 0 - sip: 5 5 - smtp: 129 2 - snmp: 18 7 - ssdp: 3 0 - ssl: 20 42 - sunrpc: 68 4 - telnet: 12 6 - tftp: 1 0 - wins: 1 0 - total: 7071 1437 --------------------------------------------------- -fast pattern groups - src: 114 - dst: 312 - any: 8 - to_server: 69 - to_client: 49 --------------------------------------------------- -search engine (ac_bnfa) - instances: 335 - patterns: 10790 - pattern chars: 175259 - num states: 123288 - num match states: 10510 - memory scale: MB - total memory: 3.68362 - pattern memory: 0.578426 - match list memory: 1.33591 - transition memory: 1.72839 - fast pattern only: 7096 -appid: MaxRss diff: 230232 -appid: p \ No newline at end of file diff --git a/snort-rules.txt b/snort.rules similarity index 100% rename from snort-rules.txt rename to snort.rules