From 7e049676e43f3681f546b412bd3cae0e3a482c5b Mon Sep 17 00:00:00 2001 From: "A. B. M. Mahmudul Hasan" Date: Sat, 21 Feb 2026 12:01:36 +0600 Subject: [PATCH 1/6] lds split --- .gitignore | 22 ++++++++++++++++------ 1 file changed, 16 insertions(+), 6 deletions(-) diff --git a/.gitignore b/.gitignore index 5ec79f0..68191ca 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,16 @@ -.idea -.env -*~ -Local -*.txt -!docs/requirements.txt +* +!bin +!configuration +!data/.gitignore +!docker/compose +!docker/conf +!docker/dockerfiles +!docs +!logs +!.gitignore +!.gitattributes +!.readthedocs.yaml +!lds +!lds.bat +!LICENSE +!README.md From b6760ba3a024659c46a9412455cafa0103a93a32 Mon Sep 17 00:00:00 2001 From: "A. B. M. Mahmudul Hasan" Date: Sat, 21 Feb 2026 12:48:34 +0600 Subject: [PATCH 2/6] updated directories --- docker/compose/companion.yaml | 10 ++++++++++ docker/compose/db-client.yaml | 5 +++++ docker/compose/db.yaml | 4 ++++ logs/.gitignore | 8 ++++++++ logs/cloudbeaver/.gitignore | 2 ++ logs/elasticsearch/.gitignore | 2 ++ logs/kibana/.gitignore | 2 ++ logs/mongo-express/.gitignore | 2 ++ logs/mongodb/.gitignore | 2 ++ logs/postgresql/.gitignore | 2 ++ logs/redis-insight/.gitignore | 2 ++ logs/redis/.gitignore | 2 ++ 12 files changed, 43 insertions(+) create mode 100644 logs/cloudbeaver/.gitignore create mode 100644 logs/elasticsearch/.gitignore create mode 100644 logs/kibana/.gitignore create mode 100644 logs/mongo-express/.gitignore create mode 100644 logs/mongodb/.gitignore create mode 100644 logs/postgresql/.gitignore create mode 100644 logs/redis-insight/.gitignore create mode 100644 logs/redis/.gitignore diff --git a/docker/compose/companion.yaml b/docker/compose/companion.yaml index 8ab7ef7..fc7a722 100644 --- a/docker/compose/companion.yaml +++ b/docker/compose/companion.yaml @@ -36,7 +36,17 @@ services: - ../../configuration/scheduler/cron-jobs:/etc/cron.d:ro - ../../logs/runner:/var/log/supervisor - ../../logs/apache:/global/log/apache + - ../../logs/cloudbeaver:/global/log/cloudbeaver + - ../../logs/elasticsearch:/global/log/elasticsearch + - ../../logs/kibana:/global/log/kibana + - ../../logs/mariadb:/global/log/mariadb + - ../../logs/mongo-express:/global/log/mongo-express + - ../../logs/mongodb:/global/log/mongodb + - ../../logs/mysql:/global/log/mysql - ../../logs/nginx:/global/log/nginx + - ../../logs/postgresql:/global/log/postgresql + - ../../logs/redis:/global/log/redis + - ../../logs/redis-insight:/global/log/redis-insight - /var/run/docker.sock:/var/run/docker.sock depends_on: - server-tools diff --git a/docker/compose/db-client.yaml b/docker/compose/db-client.yaml index 462c621..0aa05df 100644 --- a/docker/compose/db-client.yaml +++ b/docker/compose/db-client.yaml @@ -17,6 +17,7 @@ services: - RI_REDIS_HOST=redis volumes: - ../../data/redis-insight:/data + - ../../logs/redis-insight:/var/log/redis-insight networks: datastore: ipv4_address: 172.30.0.150 @@ -33,6 +34,7 @@ services: - TZ=${TZ:-} volumes: - ../../data/cloudbeaver:/opt/cloudbeaver/workspace + - ../../logs/cloudbeaver:/opt/cloudbeaver/logs networks: datastore: ipv4_address: 172.30.0.151 @@ -53,6 +55,8 @@ services: - ME_CONFIG_MONGODB_ADMINUSERNAME=${MONGODB_ROOT_USERNAME:-root} - ME_CONFIG_MONGODB_ADMINPASSWORD=${MONGODB_ROOT_PASSWORD:-12345} - ME_CONFIG_MONGODB_URL=mongodb://${MONGODB_ROOT_USERNAME:-root}:${MONGODB_ROOT_PASSWORD:-12345}@mongodb:${MONGODB_PORT:-27017}/admin + volumes: + - ../../logs/mongo-express:/var/log/mongo-express networks: datastore: ipv4_address: 172.30.0.152 @@ -72,6 +76,7 @@ services: - "ELASTICSEARCH_HOSTS=http://elasticsearch:9200" volumes: - ../../data/kibana:/usr/share/kibana/data + - ../../logs/kibana:/usr/share/kibana/logs networks: datastore: ipv4_address: 172.30.0.153 diff --git a/docker/compose/db.yaml b/docker/compose/db.yaml index aa961b8..8010878 100644 --- a/docker/compose/db.yaml +++ b/docker/compose/db.yaml @@ -12,6 +12,7 @@ services: profiles: [redis] volumes: - ../../data/redis:/data + - ../../logs/redis:/var/log/redis environment: - TZ=${TZ:-} - REDIS_ARGS=--save 10 1 --appendonly yes @@ -37,6 +38,7 @@ services: - POSTGRES_DB=${POSTGRES_DATABASE:-postgres} volumes: - ../../data/postgresql:/var/lib/postgresql + - ../../logs/postgresql:/var/log/postgresql - ../conf/pg_hba.conf:/etc/postgresql/pg_hba.conf # - ../conf/postgresql.conf:/etc/postgresql/postgresql.conf command: @@ -94,6 +96,7 @@ services: - MONGO_INITDB_ROOT_PASSWORD=${MONGODB_ROOT_PASSWORD:-12345} volumes: - ../../data/mongo:/data/db + - ../../logs/mongodb:/var/log/mongodb healthcheck: test: [ @@ -145,6 +148,7 @@ services: - "node.name=elasticsearch-node-single" volumes: - ../../data/elasticsearch:/usr/share/elasticsearch/data + - ../../logs/elasticsearch:/usr/share/elasticsearch/logs healthcheck: test: ["CMD-SHELL", "curl -f http://localhost:9200/_cluster/health || exit 1"] interval: 30s diff --git a/logs/.gitignore b/logs/.gitignore index e5704d6..8ba10b5 100755 --- a/logs/.gitignore +++ b/logs/.gitignore @@ -4,5 +4,13 @@ !nginx !mariadb !mysql +!cloudbeaver +!kibana +!mongo-express +!redis-insight +!elasticsearch +!mongodb +!postgresql +!redis !olds !.gitignore \ No newline at end of file diff --git a/logs/cloudbeaver/.gitignore b/logs/cloudbeaver/.gitignore new file mode 100644 index 0000000..c96a04f --- /dev/null +++ b/logs/cloudbeaver/.gitignore @@ -0,0 +1,2 @@ +* +!.gitignore \ No newline at end of file diff --git a/logs/elasticsearch/.gitignore b/logs/elasticsearch/.gitignore new file mode 100644 index 0000000..c96a04f --- /dev/null +++ b/logs/elasticsearch/.gitignore @@ -0,0 +1,2 @@ +* +!.gitignore \ No newline at end of file diff --git a/logs/kibana/.gitignore b/logs/kibana/.gitignore new file mode 100644 index 0000000..c96a04f --- /dev/null +++ b/logs/kibana/.gitignore @@ -0,0 +1,2 @@ +* +!.gitignore \ No newline at end of file diff --git a/logs/mongo-express/.gitignore b/logs/mongo-express/.gitignore new file mode 100644 index 0000000..c96a04f --- /dev/null +++ b/logs/mongo-express/.gitignore @@ -0,0 +1,2 @@ +* +!.gitignore \ No newline at end of file diff --git a/logs/mongodb/.gitignore b/logs/mongodb/.gitignore new file mode 100644 index 0000000..c96a04f --- /dev/null +++ b/logs/mongodb/.gitignore @@ -0,0 +1,2 @@ +* +!.gitignore \ No newline at end of file diff --git a/logs/postgresql/.gitignore b/logs/postgresql/.gitignore new file mode 100644 index 0000000..c96a04f --- /dev/null +++ b/logs/postgresql/.gitignore @@ -0,0 +1,2 @@ +* +!.gitignore \ No newline at end of file diff --git a/logs/redis-insight/.gitignore b/logs/redis-insight/.gitignore new file mode 100644 index 0000000..c96a04f --- /dev/null +++ b/logs/redis-insight/.gitignore @@ -0,0 +1,2 @@ +* +!.gitignore \ No newline at end of file diff --git a/logs/redis/.gitignore b/logs/redis/.gitignore new file mode 100644 index 0000000..c96a04f --- /dev/null +++ b/logs/redis/.gitignore @@ -0,0 +1,2 @@ +* +!.gitignore \ No newline at end of file From a2209b65ca26998ceeb4b1c1ca4509424386bba6 Mon Sep 17 00:00:00 2001 From: "A. B. M. Mahmudul Hasan" Date: Sat, 21 Feb 2026 20:56:51 +0600 Subject: [PATCH 3/6] update log --- docker/compose/db-client.yaml | 20 ++++++++++++++++++-- docker/compose/db.yaml | 3 +-- docker/conf/filebeat.yml | 23 +++++++++++++++++++++++ lds | 4 +--- 4 files changed, 43 insertions(+), 7 deletions(-) create mode 100644 docker/conf/filebeat.yml diff --git a/docker/compose/db-client.yaml b/docker/compose/db-client.yaml index 0aa05df..d9ea2ce 100644 --- a/docker/compose/db-client.yaml +++ b/docker/compose/db-client.yaml @@ -67,8 +67,7 @@ services: <<: *db-client-service container_name: KIBANA hostname: kibana - image: kibana:${ELASTICSEARCH_VERSION:-9.2.4} - profiles: [elasticsearch] + image: kibana:${ELASTICSEARCH_VERSION:-9.3.0} depends_on: - elasticsearch environment: @@ -82,3 +81,20 @@ services: ipv4_address: 172.30.0.153 frontend: ipv4_address: 172.28.0.153 + + filebeat: + image: docker.elastic.co/beats/filebeat:${ELASTICSEARCH_VERSION:-9.3.0} + container_name: FILEBEAT + restart: unless-stopped + depends_on: + - elasticsearch + volumes: + - ../conf/filebeat.yml:/usr/share/filebeat/filebeat.yml:ro + - ../../logs:/global/log:ro + - ../../data/filebeat:/usr/share/filebeat/data + command: ["--strict.perms=false"] + networks: + datastore: + ipv4_address: 172.30.0.154 + frontend: + ipv4_address: 172.28.0.154 diff --git a/docker/compose/db.yaml b/docker/compose/db.yaml index 8010878..a614161 100644 --- a/docker/compose/db.yaml +++ b/docker/compose/db.yaml @@ -138,8 +138,7 @@ services: <<: *db-service container_name: ELASTICSEARCH hostname: elasticsearch - image: elasticsearch:${ELASTICSEARCH_VERSION:-9.2.4} - profiles: [elasticsearch] + image: elasticsearch:${ELASTICSEARCH_VERSION:-9.3.0} environment: - TZ=${TZ:-} - "discovery.type=single-node" diff --git a/docker/conf/filebeat.yml b/docker/conf/filebeat.yml new file mode 100644 index 0000000..0d1dd5f --- /dev/null +++ b/docker/conf/filebeat.yml @@ -0,0 +1,23 @@ +filebeat.inputs: + - type: filestream + id: lds-global-logs + enabled: true + paths: + - /global/log/*/*.log + - /global/log/*/*/*.log + exclude_files: ['-\d{8}(\.gz)?$'] + processors: + - dissect: + tokenizer: "/global/log/%{service}/%{file}" + field: "log.file.path" + target_prefix: "" + - convert: + fields: + - {from: "service", to: "service", type: "string"} + ignore_missing: true + fail_on_error: false +output.elasticsearch: + hosts: ["http://elasticsearch:9200"] + index: "lds-%{[service]}-%{+yyyyMMdd}" +setup.kibana: + host: "http://kibana:5601" \ No newline at end of file diff --git a/lds b/lds index 32aeaba..c5466a7 100755 --- a/lds +++ b/lds @@ -371,7 +371,7 @@ fix_perms() { chmod 2777 "$DIR/data" mkdir -p "$DIR/data/cloudbeaver" "$DIR/data/mysql" "$DIR/data/postgresql" "$DIR/data/mongo" \ "$DIR/data/mariadb" "$DIR/data/elasticsearch" "$DIR/data/redis" "$DIR/data/mailpit" \ - "$DIR/data/redis-insight" "$DIR/data/kibana" + "$DIR/data/redis-insight" "$DIR/data/kibana" "$DIR/data/filebeat" find "$DIR/data" -mindepth 1 -maxdepth 1 -type d -exec chmod 2777 {} + find "$DIR/data" -type f -exec chmod 666 {} + @@ -461,7 +461,6 @@ declare -A SERVICES=( [POSTGRESQL]="postgresql" [MYSQL]="mysql" [MARIADB]="mariadb" - [ELASTICSEARCH]="elasticsearch" [MONGODB]="mongodb" [REDIS]="redis" ) @@ -469,7 +468,6 @@ declare -A SERVICES=( declare -a SERVICE_ORDER=(POSTGRESQL MYSQL MARIADB ELASTICSEARCH MONGODB REDIS) declare -A PROFILE_ENV=( - [elasticsearch]="ELASTICSEARCH_VERSION=9.2.4" [mysql]="MYSQL_VERSION=latest MYSQL_ROOT_PASSWORD=12345 MYSQL_USER=infocyph MYSQL_PASSWORD=12345 MYSQL_DATABASE=localdb" [mariadb]="MARIADB_VERSION=latest MARIADB_ROOT_PASSWORD=12345 MARIADB_USER=infocyph MARIADB_PASSWORD=12345 MARIADB_DATABASE=localdb" [mongodb]="MONGODB_VERSION=latest MONGODB_ROOT_USERNAME=root MONGODB_ROOT_PASSWORD=12345" From ea16158457d35d195b6fb38ef3936f5a1bbfce4f Mon Sep 17 00:00:00 2001 From: "A. B. M. Mahmudul Hasan" Date: Sat, 21 Feb 2026 21:10:59 +0600 Subject: [PATCH 4/6] filebeat: run as root and add ES template settings --- docker/compose/db-client.yaml | 1 + docker/conf/filebeat.yml | 4 ++++ 2 files changed, 5 insertions(+) diff --git a/docker/compose/db-client.yaml b/docker/compose/db-client.yaml index d9ea2ce..6cf0879 100644 --- a/docker/compose/db-client.yaml +++ b/docker/compose/db-client.yaml @@ -86,6 +86,7 @@ services: image: docker.elastic.co/beats/filebeat:${ELASTICSEARCH_VERSION:-9.3.0} container_name: FILEBEAT restart: unless-stopped + user: root depends_on: - elasticsearch volumes: diff --git a/docker/conf/filebeat.yml b/docker/conf/filebeat.yml index 0d1dd5f..6e5f2f2 100644 --- a/docker/conf/filebeat.yml +++ b/docker/conf/filebeat.yml @@ -19,5 +19,9 @@ filebeat.inputs: output.elasticsearch: hosts: ["http://elasticsearch:9200"] index: "lds-%{[service]}-%{+yyyyMMdd}" +setup.template.enabled: true +setup.template.name: "lds" +setup.template.pattern: "lds-*-*" +setup.template.overwrite: false setup.kibana: host: "http://kibana:5601" \ No newline at end of file From 6e1dbdf743fbee0f6e506db7513de2b3a61e34ff Mon Sep 17 00:00:00 2001 From: "A. B. M. Mahmudul Hasan" Date: Sun, 22 Feb 2026 09:05:10 +0600 Subject: [PATCH 5/6] dash update --- docker/conf/filebeat.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docker/conf/filebeat.yml b/docker/conf/filebeat.yml index 6e5f2f2..9fc7b3e 100644 --- a/docker/conf/filebeat.yml +++ b/docker/conf/filebeat.yml @@ -23,5 +23,9 @@ setup.template.enabled: true setup.template.name: "lds" setup.template.pattern: "lds-*-*" setup.template.overwrite: false +setup.dashboards.enabled: true +setup.dashboards.retry.enabled: true +setup.dashboards.retry.interval: 10s +setup.dashboards.retry.maximum: 30 setup.kibana: host: "http://kibana:5601" \ No newline at end of file From 5caadace822866c520e60b08ba68a11f79c4a523 Mon Sep 17 00:00:00 2001 From: "A. B. M. Mahmudul Hasan" Date: Sun, 22 Feb 2026 14:09:31 +0600 Subject: [PATCH 6/6] dash update --- docker/compose/companion.yaml | 2 ++ docker/compose/db-client.yaml | 4 ++-- docker/compose/db.yaml | 1 + lds | 2 ++ 4 files changed, 7 insertions(+), 2 deletions(-) diff --git a/docker/compose/companion.yaml b/docker/compose/companion.yaml index fc7a722..b824041 100644 --- a/docker/compose/companion.yaml +++ b/docker/compose/companion.yaml @@ -23,6 +23,8 @@ services: networks: backend: ipv4_address: 172.29.0.10 + datastore: + ipv4_address: 172.30.0.10 runner: container_name: RUNNER diff --git a/docker/compose/db-client.yaml b/docker/compose/db-client.yaml index 6cf0879..395a10a 100644 --- a/docker/compose/db-client.yaml +++ b/docker/compose/db-client.yaml @@ -68,6 +68,7 @@ services: container_name: KIBANA hostname: kibana image: kibana:${ELASTICSEARCH_VERSION:-9.3.0} + profiles: [elasticsearch] depends_on: - elasticsearch environment: @@ -87,6 +88,7 @@ services: container_name: FILEBEAT restart: unless-stopped user: root + profiles: [elasticsearch] depends_on: - elasticsearch volumes: @@ -97,5 +99,3 @@ services: networks: datastore: ipv4_address: 172.30.0.154 - frontend: - ipv4_address: 172.28.0.154 diff --git a/docker/compose/db.yaml b/docker/compose/db.yaml index a614161..51b004d 100644 --- a/docker/compose/db.yaml +++ b/docker/compose/db.yaml @@ -139,6 +139,7 @@ services: container_name: ELASTICSEARCH hostname: elasticsearch image: elasticsearch:${ELASTICSEARCH_VERSION:-9.3.0} + profiles: [elasticsearch] environment: - TZ=${TZ:-} - "discovery.type=single-node" diff --git a/lds b/lds index c5466a7..b4734e4 100755 --- a/lds +++ b/lds @@ -461,6 +461,7 @@ declare -A SERVICES=( [POSTGRESQL]="postgresql" [MYSQL]="mysql" [MARIADB]="mariadb" + [ELASTICSEARCH]="elasticsearch" [MONGODB]="mongodb" [REDIS]="redis" ) @@ -468,6 +469,7 @@ declare -A SERVICES=( declare -a SERVICE_ORDER=(POSTGRESQL MYSQL MARIADB ELASTICSEARCH MONGODB REDIS) declare -A PROFILE_ENV=( + [elasticsearch]="ELASTICSEARCH_VERSION=9.3.0" [mysql]="MYSQL_VERSION=latest MYSQL_ROOT_PASSWORD=12345 MYSQL_USER=infocyph MYSQL_PASSWORD=12345 MYSQL_DATABASE=localdb" [mariadb]="MARIADB_VERSION=latest MARIADB_ROOT_PASSWORD=12345 MARIADB_USER=infocyph MARIADB_PASSWORD=12345 MARIADB_DATABASE=localdb" [mongodb]="MONGODB_VERSION=latest MONGODB_ROOT_USERNAME=root MONGODB_ROOT_PASSWORD=12345"