fs-based logs

docker-compose-mktxp-stack-fs.yml

@@ -0,0 +1,99 @@
+networks:
+  mktxp: {}
+services:
+  mktxp:
+    container_name: mktxp
+    image: ghcr.io/akpw/mktxp:latest
+    volumes:
+      - './mktxp/:/home/mktxp/mktxp/'
+    networks:
+      mktxp: null
+    restart: unless-stopped  
+  grafana:
+    container_name: grafana
+    environment:
+      GF_AUTH_ANONYMOUS_ENABLED: "true"
+      GF_AUTH_ANONYMOUS_ORG_ROLE: Admin
+      GF_AUTH_BASIC_ENABLED: "false"
+      GF_AUTH_DISABLE_LOGIN_FORM: "true"
+      GF_USERS_DEFAULT_THEME: "light"
+      GF_INSTALL_PLUGINS: "flant-statusmap-panel"
+      GF_DASHBOARDS_DEFAULT_HOME_DASHBOARD_PATH: /var/lib/grafana/dashboards/mikrotik/mikrotik_mktxp_exporter.json
+    image: grafana/grafana:latest
+    networks:
+      mktxp: null
+    ports:
+    - protocol: tcp
+      published: 3000
+      target: 3000
+    restart: always
+    volumes:
+    - ./grafana/provisioning/datasources:/etc/grafana/provisioning/datasources:ro
+    - ./grafana/provisioning/dashboards:/etc/grafana/provisioning/dashboards:ro
+    - ./grafana/dashboards:/var/lib/grafana/dashboards
+  loki:
+    command: -config.file=/etc/loki/loki-config.yml
+    container_name: loki
+    image: grafana/loki:2.3.0
+    networks:
+      mktxp: null
+    ports:
+    - protocol: tcp
+      published: 3100
+      target: 3100
+    restart: always
+    volumes:
+    - ./loki/loki-config.yml:/etc/loki/loki-config.yml
+  prometheus:
+    command:
+    - --config.file=/etc/prometheus/prometheus.yml
+    - --storage.tsdb.path=/prometheus
+    - --web.enable-admin-api
+    - --web.enable-lifecycle
+    container_name: prometheus
+    image: prom/prometheus:latest
+    networks:
+      mktxp: null
+    ports:
+    - published: 9090
+      target: 9090
+    restart: always
+    volumes:
+    - ./prometheus/prometheus.yml:/etc/prometheus/prometheus.yml:ro
+  promtail:
+    command: -config.file=/etc/promtail/promtail-config.yml
+    container_name: promtail
+    image: grafana/promtail:2.3.0
+    networks:
+      mktxp: null
+    ports:
+    - protocol: tcp
+      published: 1514
+      target: 1514
+    - protocol: tcp
+      published: 9080
+      target: 9080
+    restart: always
+    volumes:
+    - ./promtail/promtail-config-fs.yml:/etc/promtail/promtail-config.yml:ro
+    - ./syslog-ng/logs:/var/log:ro
+  syslog-ng:
+    command: -edv
+    container_name: syslog-ng
+    depends_on:
+    - promtail
+    image: balabit/syslog-ng:latest
+    networks:
+      mktxp: null
+    ports:
+    - protocol: udp
+      published: 514
+      target: 514
+    - protocol: tcp
+      published: 601
+      target: 601
+    restart: always
+    volumes:
+    - ./syslog-ng/syslog-ng-fs.conf:/etc/syslog-ng/syslog-ng.conf:ro
+    - ./syslog-ng/logs:/var/log/syslog-ng
+version: '3.3'

promtail/promtail-config-fs.yml

@@ -0,0 +1,28 @@
+server:
+  http_listen_port: 9080
+  grpc_listen_port: 0
+
+positions:
+  filename: /tmp/positions.yaml
+
+clients:
+  - url: http://loki:3100/loki/api/v1/push
+
+scrape_configs:
+
+  - job_name: mktxp-syslog
+    static_configs:
+    - targets:
+        - localhost
+      labels:
+        job: "syslog"
+        __path__: /var/log/*.log
+    pipeline_stages:
+    - match:
+        selector: '{job="syslog"}'
+        stages:
+        - regex:
+            source: filename
+            expression: "/var/log/(?P<routerboard>.+).log"
+        - labels:
+            routerboard:

syslog-ng/syslog-ng-fs.conf

@@ -0,0 +1,42 @@
+#############################################################################
+# Default syslog-ng.conf file which collects all local logs into a
+# single file called /var/log/messages tailored to container usage.
+#
+# The changes from the stock, default syslog-ng.conf file is that we've
+# dropped the system() source that is not needed and that we enabled network
+# connections using default-network-drivers(). Customize as needed and
+# override using the -v option to docker, such as:
+#
+#  docker run ...  -v "$PWD/syslog-ng.conf":/etc/syslog-ng/syslog-ng.conf
+#
+
+@version: 3.29
+@include "scl.conf"
+
+source s_local {
+	internal();
+};
+
+source s_network {
+	default-network-drivers(
+		# NOTE: TLS support
+		#
+		# the default-network-drivers() source driver opens the TLS
+		# enabled ports as well, however without an actual key/cert
+		# pair they will not operate and syslog-ng would display a
+		# warning at startup.
+		#
+		#tls(key-file("/path/to/ssl-private-key") cert-file("/path/to/ssl-cert"))
+	);
+};
+
+destination d_netlog { file("/var/log/syslog-ng/${HOST}.log"); };
+
+
+log {
+        source(s_local);
+        source(s_network);
+        destination(d_netlog);
+};
+
+