diff --git a/.gitignore b/.gitignore index 5ec79f0..68191ca 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,16 @@ -.idea -.env -*~ -Local -*.txt -!docs/requirements.txt +* +!bin +!configuration +!data/.gitignore +!docker/compose +!docker/conf +!docker/dockerfiles +!docs +!logs +!.gitignore +!.gitattributes +!.readthedocs.yaml +!lds +!lds.bat +!LICENSE +!README.md diff --git a/docker/compose/companion.yaml b/docker/compose/companion.yaml index 8ab7ef7..b824041 100644 --- a/docker/compose/companion.yaml +++ b/docker/compose/companion.yaml @@ -23,6 +23,8 @@ services: networks: backend: ipv4_address: 172.29.0.10 + datastore: + ipv4_address: 172.30.0.10 runner: container_name: RUNNER @@ -36,7 +38,17 @@ services: - ../../configuration/scheduler/cron-jobs:/etc/cron.d:ro - ../../logs/runner:/var/log/supervisor - ../../logs/apache:/global/log/apache + - ../../logs/cloudbeaver:/global/log/cloudbeaver + - ../../logs/elasticsearch:/global/log/elasticsearch + - ../../logs/kibana:/global/log/kibana + - ../../logs/mariadb:/global/log/mariadb + - ../../logs/mongo-express:/global/log/mongo-express + - ../../logs/mongodb:/global/log/mongodb + - ../../logs/mysql:/global/log/mysql - ../../logs/nginx:/global/log/nginx + - ../../logs/postgresql:/global/log/postgresql + - ../../logs/redis:/global/log/redis + - ../../logs/redis-insight:/global/log/redis-insight - /var/run/docker.sock:/var/run/docker.sock depends_on: - server-tools diff --git a/docker/compose/db-client.yaml b/docker/compose/db-client.yaml index 462c621..395a10a 100644 --- a/docker/compose/db-client.yaml +++ b/docker/compose/db-client.yaml @@ -17,6 +17,7 @@ services: - RI_REDIS_HOST=redis volumes: - ../../data/redis-insight:/data + - ../../logs/redis-insight:/var/log/redis-insight networks: datastore: ipv4_address: 172.30.0.150 @@ -33,6 +34,7 @@ services: - TZ=${TZ:-} volumes: - ../../data/cloudbeaver:/opt/cloudbeaver/workspace + - ../../logs/cloudbeaver:/opt/cloudbeaver/logs networks: datastore: ipv4_address: 172.30.0.151 @@ -53,6 +55,8 @@ services: - ME_CONFIG_MONGODB_ADMINUSERNAME=${MONGODB_ROOT_USERNAME:-root} - ME_CONFIG_MONGODB_ADMINPASSWORD=${MONGODB_ROOT_PASSWORD:-12345} - ME_CONFIG_MONGODB_URL=mongodb://${MONGODB_ROOT_USERNAME:-root}:${MONGODB_ROOT_PASSWORD:-12345}@mongodb:${MONGODB_PORT:-27017}/admin + volumes: + - ../../logs/mongo-express:/var/log/mongo-express networks: datastore: ipv4_address: 172.30.0.152 @@ -63,7 +67,7 @@ services: <<: *db-client-service container_name: KIBANA hostname: kibana - image: kibana:${ELASTICSEARCH_VERSION:-9.2.4} + image: kibana:${ELASTICSEARCH_VERSION:-9.3.0} profiles: [elasticsearch] depends_on: - elasticsearch @@ -72,8 +76,26 @@ services: - "ELASTICSEARCH_HOSTS=http://elasticsearch:9200" volumes: - ../../data/kibana:/usr/share/kibana/data + - ../../logs/kibana:/usr/share/kibana/logs networks: datastore: ipv4_address: 172.30.0.153 frontend: ipv4_address: 172.28.0.153 + + filebeat: + image: docker.elastic.co/beats/filebeat:${ELASTICSEARCH_VERSION:-9.3.0} + container_name: FILEBEAT + restart: unless-stopped + user: root + profiles: [elasticsearch] + depends_on: + - elasticsearch + volumes: + - ../conf/filebeat.yml:/usr/share/filebeat/filebeat.yml:ro + - ../../logs:/global/log:ro + - ../../data/filebeat:/usr/share/filebeat/data + command: ["--strict.perms=false"] + networks: + datastore: + ipv4_address: 172.30.0.154 diff --git a/docker/compose/db.yaml b/docker/compose/db.yaml index aa961b8..51b004d 100644 --- a/docker/compose/db.yaml +++ b/docker/compose/db.yaml @@ -12,6 +12,7 @@ services: profiles: [redis] volumes: - ../../data/redis:/data + - ../../logs/redis:/var/log/redis environment: - TZ=${TZ:-} - REDIS_ARGS=--save 10 1 --appendonly yes @@ -37,6 +38,7 @@ services: - POSTGRES_DB=${POSTGRES_DATABASE:-postgres} volumes: - ../../data/postgresql:/var/lib/postgresql + - ../../logs/postgresql:/var/log/postgresql - ../conf/pg_hba.conf:/etc/postgresql/pg_hba.conf # - ../conf/postgresql.conf:/etc/postgresql/postgresql.conf command: @@ -94,6 +96,7 @@ services: - MONGO_INITDB_ROOT_PASSWORD=${MONGODB_ROOT_PASSWORD:-12345} volumes: - ../../data/mongo:/data/db + - ../../logs/mongodb:/var/log/mongodb healthcheck: test: [ @@ -135,7 +138,7 @@ services: <<: *db-service container_name: ELASTICSEARCH hostname: elasticsearch - image: elasticsearch:${ELASTICSEARCH_VERSION:-9.2.4} + image: elasticsearch:${ELASTICSEARCH_VERSION:-9.3.0} profiles: [elasticsearch] environment: - TZ=${TZ:-} @@ -145,6 +148,7 @@ services: - "node.name=elasticsearch-node-single" volumes: - ../../data/elasticsearch:/usr/share/elasticsearch/data + - ../../logs/elasticsearch:/usr/share/elasticsearch/logs healthcheck: test: ["CMD-SHELL", "curl -f http://localhost:9200/_cluster/health || exit 1"] interval: 30s diff --git a/docker/conf/filebeat.yml b/docker/conf/filebeat.yml new file mode 100644 index 0000000..9fc7b3e --- /dev/null +++ b/docker/conf/filebeat.yml @@ -0,0 +1,31 @@ +filebeat.inputs: + - type: filestream + id: lds-global-logs + enabled: true + paths: + - /global/log/*/*.log + - /global/log/*/*/*.log + exclude_files: ['-\d{8}(\.gz)?$'] + processors: + - dissect: + tokenizer: "/global/log/%{service}/%{file}" + field: "log.file.path" + target_prefix: "" + - convert: + fields: + - {from: "service", to: "service", type: "string"} + ignore_missing: true + fail_on_error: false +output.elasticsearch: + hosts: ["http://elasticsearch:9200"] + index: "lds-%{[service]}-%{+yyyyMMdd}" +setup.template.enabled: true +setup.template.name: "lds" +setup.template.pattern: "lds-*-*" +setup.template.overwrite: false +setup.dashboards.enabled: true +setup.dashboards.retry.enabled: true +setup.dashboards.retry.interval: 10s +setup.dashboards.retry.maximum: 30 +setup.kibana: + host: "http://kibana:5601" \ No newline at end of file diff --git a/lds b/lds index 32aeaba..b4734e4 100755 --- a/lds +++ b/lds @@ -371,7 +371,7 @@ fix_perms() { chmod 2777 "$DIR/data" mkdir -p "$DIR/data/cloudbeaver" "$DIR/data/mysql" "$DIR/data/postgresql" "$DIR/data/mongo" \ "$DIR/data/mariadb" "$DIR/data/elasticsearch" "$DIR/data/redis" "$DIR/data/mailpit" \ - "$DIR/data/redis-insight" "$DIR/data/kibana" + "$DIR/data/redis-insight" "$DIR/data/kibana" "$DIR/data/filebeat" find "$DIR/data" -mindepth 1 -maxdepth 1 -type d -exec chmod 2777 {} + find "$DIR/data" -type f -exec chmod 666 {} + @@ -469,7 +469,7 @@ declare -A SERVICES=( declare -a SERVICE_ORDER=(POSTGRESQL MYSQL MARIADB ELASTICSEARCH MONGODB REDIS) declare -A PROFILE_ENV=( - [elasticsearch]="ELASTICSEARCH_VERSION=9.2.4" + [elasticsearch]="ELASTICSEARCH_VERSION=9.3.0" [mysql]="MYSQL_VERSION=latest MYSQL_ROOT_PASSWORD=12345 MYSQL_USER=infocyph MYSQL_PASSWORD=12345 MYSQL_DATABASE=localdb" [mariadb]="MARIADB_VERSION=latest MARIADB_ROOT_PASSWORD=12345 MARIADB_USER=infocyph MARIADB_PASSWORD=12345 MARIADB_DATABASE=localdb" [mongodb]="MONGODB_VERSION=latest MONGODB_ROOT_USERNAME=root MONGODB_ROOT_PASSWORD=12345" diff --git a/logs/.gitignore b/logs/.gitignore index e5704d6..8ba10b5 100755 --- a/logs/.gitignore +++ b/logs/.gitignore @@ -4,5 +4,13 @@ !nginx !mariadb !mysql +!cloudbeaver +!kibana +!mongo-express +!redis-insight +!elasticsearch +!mongodb +!postgresql +!redis !olds !.gitignore \ No newline at end of file diff --git a/logs/cloudbeaver/.gitignore b/logs/cloudbeaver/.gitignore new file mode 100644 index 0000000..c96a04f --- /dev/null +++ b/logs/cloudbeaver/.gitignore @@ -0,0 +1,2 @@ +* +!.gitignore \ No newline at end of file diff --git a/logs/elasticsearch/.gitignore b/logs/elasticsearch/.gitignore new file mode 100644 index 0000000..c96a04f --- /dev/null +++ b/logs/elasticsearch/.gitignore @@ -0,0 +1,2 @@ +* +!.gitignore \ No newline at end of file diff --git a/logs/kibana/.gitignore b/logs/kibana/.gitignore new file mode 100644 index 0000000..c96a04f --- /dev/null +++ b/logs/kibana/.gitignore @@ -0,0 +1,2 @@ +* +!.gitignore \ No newline at end of file diff --git a/logs/mongo-express/.gitignore b/logs/mongo-express/.gitignore new file mode 100644 index 0000000..c96a04f --- /dev/null +++ b/logs/mongo-express/.gitignore @@ -0,0 +1,2 @@ +* +!.gitignore \ No newline at end of file diff --git a/logs/mongodb/.gitignore b/logs/mongodb/.gitignore new file mode 100644 index 0000000..c96a04f --- /dev/null +++ b/logs/mongodb/.gitignore @@ -0,0 +1,2 @@ +* +!.gitignore \ No newline at end of file diff --git a/logs/postgresql/.gitignore b/logs/postgresql/.gitignore new file mode 100644 index 0000000..c96a04f --- /dev/null +++ b/logs/postgresql/.gitignore @@ -0,0 +1,2 @@ +* +!.gitignore \ No newline at end of file diff --git a/logs/redis-insight/.gitignore b/logs/redis-insight/.gitignore new file mode 100644 index 0000000..c96a04f --- /dev/null +++ b/logs/redis-insight/.gitignore @@ -0,0 +1,2 @@ +* +!.gitignore \ No newline at end of file diff --git a/logs/redis/.gitignore b/logs/redis/.gitignore new file mode 100644 index 0000000..c96a04f --- /dev/null +++ b/logs/redis/.gitignore @@ -0,0 +1,2 @@ +* +!.gitignore \ No newline at end of file