cjones/mktxp-stack
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Initial Commit

Browse Source
This commit is contained in:
Arseniy Kuznetsov
2022-10-17 15:35:36 +01:00
parent 98b0eeb134
commit d970f8a697
16 changed files with 11169 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

11
LICENSE Normal file
View File

@@ -0,0 +1,11 @@
## Copyright (c) 2022 Arseniy Kuznetsov
##
## This program is free software; you can redistribute it and/or
## modify it under the terms of the GNU General Public License
## as published by the Free Software Foundation; either version 2
## of the License, or (at your option) any later version.
##
## This program is distributed in the hope that it will be useful,
## but WITHOUT ANY WARRANTY; without even the implied warranty of
## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
## GNU General Public License for more details.

97
docker-compose-mktxp.yml Normal file
View File

@@ -0,0 +1,97 @@
networks:
mktxp: {}
services:
mktxp:
container_name: mktxp
image: ghcr.io/akpw/mktxp:latest
volumes:
- './mktxp/:/home/mktxp/mktxp/'
networks:
mktxp: null
restart: unless-stopped
grafana:
container_name: grafana
environment:
GF_AUTH_ANONYMOUS_ENABLED: "true"
GF_AUTH_ANONYMOUS_ORG_ROLE: Admin
GF_AUTH_BASIC_ENABLED: "false"
GF_AUTH_DISABLE_LOGIN_FORM: "true"
GF_USERS_DEFAULT_THEME: "light"
GF_INSTALL_PLUGINS: "flant-statusmap-panel"
GF_DASHBOARDS_DEFAULT_HOME_DASHBOARD_PATH: /var/lib/grafana/dashboards/mikrotik/mikrotik_mktxp_exporter.json
image: grafana/grafana:latest
networks:
mktxp: null
ports:
- protocol: tcp
published: 3000
target: 3000
restart: always
volumes:
- ./grafana/provisioning/datasources:/etc/grafana/provisioning/datasources:ro
- ./grafana/provisioning/dashboards:/etc/grafana/provisioning/dashboards:ro
- ./grafana/dashboards:/var/lib/grafana/dashboards
loki:
command: -config.file=/etc/loki/loki-config.yml
container_name: loki
image: grafana/loki:2.3.0
networks:
mktxp: null
ports:
- protocol: tcp
published: 3100
target: 3100
restart: always
volumes:
- ./loki/loki-config.yml:/etc/loki/loki-config.yml
prometheus:
command:
- --config.file=/etc/prometheus/prometheus.yml
- --storage.tsdb.path=/prometheus
- --web.enable-admin-api
- --web.enable-lifecycle
container_name: prometheus
image: prom/prometheus:latest
networks:
mktxp: null
ports:
- published: 9090
target: 9090
restart: always
volumes:
- ./prometheus/prometheus.yml:/etc/prometheus/prometheus.yml:ro
promtail:
command: -config.file=/etc/promtail/promtail-config.yml
container_name: promtail
image: grafana/promtail:2.3.0
networks:
mktxp: null
ports:
- protocol: tcp
published: 1514
target: 1514
- protocol: tcp
published: 9080
target: 9080
restart: always
volumes:
- ./promtail/promtail-config.yml:/etc/promtail/promtail-config.yml:ro
syslog-ng:
command: -edv
container_name: syslog-ng
depends_on:
- promtail
image: balabit/syslog-ng:latest
networks:
mktxp: null
ports:
- protocol: udp
published: 514
target: 514
- protocol: tcp
published: 601
target: 601
restart: always
volumes:
- ./syslog-ng/syslog-ng.conf:/etc/syslog-ng/syslog-ng.conf:ro
version: '3.3'

437
grafana/dashboards/mikrotik/mikrotik_loki_logs.json Normal file
View File

@@ -0,0 +1,437 @@
{
"annotations": {
"list": [
{
"builtIn": 1,
"datasource": {
"type": "datasource",
"uid": "grafana"
},
"enable": true,
"hide": true,
"iconColor": "rgba(0, 211, 255, 1)",
"name": "Annotations & Alerts",
"target": {
"limit": 100,
"matchAny": false,
"tags": [],
"type": "dashboard"
},
"type": "dashboard"
}
]
},
"description": "Mikrotik Loki Logs",
"editable": true,
"fiscalYearStartMonth": 0,
"gnetId": 17139,
"graphTooltip": 0,
"links": [
{
"asDropdown": false,
"icon": "external link",
"includeVars": false,
"keepTime": false,
"tags": [
"mktxp"
],
"targetBlank": false,
"title": "MKTXP",
"tooltip": "",
"type": "dashboards",
"url": ""
},
{
"asDropdown": true,
"icon": "external link",
"includeVars": false,
"keepTime": false,
"tags": [
"system"
],
"targetBlank": false,
"title": "System",
"tooltip": "",
"type": "dashboards",
"url": ""
}
],
"liveNow": false,
"panels": [
{
"datasource": {
"type": "loki",
"uid": "P8E80F9AEF21F6940"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"drawStyle": "line",
"fillOpacity": 10,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"lineInterpolation": "linear",
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": {
"log": 10,
"type": "log"
},
"showPoints": "never",
"spanNulls": false,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "red",
"value": 80
}
]
},
"unit": "short"
},
"overrides": []
},
"gridPos": {
"h": 12,
"w": 24,
"x": 0,
"y": 0
},
"id": 4,
"links": [],
"maxDataPoints": 100,
"options": {
"legend": {
"calcs": [
"mean",
"lastNotNull",
"max"
],
"displayMode": "table",
"placement": "right",
"showLegend": true
},
"tooltip": {
"mode": "multi",
"sort": "none"
}
},
"pluginVersion": "9.2.0",
"targets": [
{
"datasource": {
"type": "loki",
"uid": "P8E80F9AEF21F6940"
},
"expr": "count_over_time({job=\"syslog\"} |=\"GT Critical\" [5m])",
"legendFormat": "Critical",
"refId": "A"
},
{
"datasource": {
"type": "loki",
"uid": "P8E80F9AEF21F6940"
},
"expr": "count_over_time({job=\"syslog\"} |=\"GT Error\" !=\"critical\" [5m])",
"legendFormat": "Error",
"refId": "B"
},
{
"datasource": {
"type": "loki",
"uid": "P8E80F9AEF21F6940"
},
"expr": "count_over_time({job=\"syslog\"} |=\"GT Warning\" !=\"critical\" [5m])",
"legendFormat": "Warning",
"refId": "C"
},
{
"datasource": {
"type": "loki",
"uid": "P8E80F9AEF21F6940"
},
"expr": "count_over_time({job=\"syslog\"} |=\"GT Firewall\" [5m])",
"legendFormat": "Firewall",
"refId": "D"
},
{
"datasource": {
"type": "loki",
"uid": "P8E80F9AEF21F6940"
},
"expr": "count_over_time({job=\"syslog\"} |=\"GT Info\" !=\"firewall\" [5m])",
"legendFormat": "Info",
"refId": "E"
}
],
"title": "Logs counts",
"type": "timeseries"
},
{
"datasource": {
"type": "loki",
"uid": "P8E80F9AEF21F6940"
},
"gridPos": {
"h": 15,
"w": 15,
"x": 0,
"y": 12
},
"id": 12,
"links": [],
"maxDataPoints": 100,
"options": {
"dedupStrategy": "none",
"enableLogDetails": true,
"prettifyLogMessage": false,
"showCommonLabels": false,
"showLabels": false,
"showTime": true,
"sortOrder": "Descending",
"wrapLogMessage": false
},
"pluginVersion": "7.3.5",
"targets": [
{
"datasource": {
"type": "loki",
"uid": "P8E80F9AEF21F6940"
},
"expr": "{job=\"syslog\"} |=\"GT Firewall\" |~ \"(?i)$searchable_pattern\"",
"legendFormat": "",
"refId": "A"
}
],
"title": "Firewall ",
"type": "logs"
},
{
"datasource": {
"type": "loki",
"uid": "P8E80F9AEF21F6940"
},
"gridPos": {
"h": 9,
"w": 9,
"x": 15,
"y": 12
},
"id": 8,
"links": [],
"maxDataPoints": 100,
"options": {
"dedupStrategy": "none",
"enableLogDetails": true,
"prettifyLogMessage": false,
"showCommonLabels": false,
"showLabels": false,
"showTime": true,
"sortOrder": "Descending",
"wrapLogMessage": false
},
"pluginVersion": "7.3.5",
"targets": [
{
"datasource": {
"type": "loki",
"uid": "P8E80F9AEF21F6940"
},
"expr": "{job=\"syslog\"} |=\"GT Critical\" |~ \"(?i)$searchable_pattern\"",
"legendFormat": "",
"refId": "A"
}
],
"title": "Critical ",
"type": "logs"
},
{
"datasource": {
"type": "loki",
"uid": "P8E80F9AEF21F6940"
},
"gridPos": {
"h": 8,
"w": 9,
"x": 15,
"y": 21
},
"id": 10,
"links": [],
"maxDataPoints": 100,
"options": {
"dedupStrategy": "none",
"enableLogDetails": true,
"prettifyLogMessage": false,
"showCommonLabels": false,
"showLabels": false,
"showTime": true,
"sortOrder": "Descending",
"wrapLogMessage": false
},
"pluginVersion": "7.3.5",
"targets": [
{
"datasource": {
"type": "loki",
"uid": "P8E80F9AEF21F6940"
},
"expr": "{job=\"syslog\"} |=\"GT Error\" |~ \"(?i)$searchable_pattern\"",
"legendFormat": "",
"refId": "A"
}
],
"title": "Error ",
"type": "logs"
},
{
"datasource": {
"type": "loki",
"uid": "P8E80F9AEF21F6940"
},
"gridPos": {
"h": 13,
"w": 15,
"x": 0,
"y": 27
},
"id": 6,
"links": [],
"maxDataPoints": 100,
"options": {
"dedupStrategy": "none",
"enableLogDetails": true,
"prettifyLogMessage": false,
"showCommonLabels": false,
"showLabels": false,
"showTime": true,
"sortOrder": "Descending",
"wrapLogMessage": false
},
"pluginVersion": "7.3.5",
"targets": [
{
"datasource": {
"type": "loki",
"uid": "P8E80F9AEF21F6940"
},
"expr": "{job=\"syslog\"} |=\"GT Info\" !=\"firewall\"|~ \"(?i)$searchable_pattern\"",
"legendFormat": "",
"refId": "A"
}
],
"title": "Info ",
"transformations": [],
"type": "logs"
},
{
"datasource": {
"type": "loki",
"uid": "P8E80F9AEF21F6940"
},
"gridPos": {
"h": 11,
"w": 9,
"x": 15,
"y": 29
},
"id": 14,
"links": [],
"maxDataPoints": 100,
"options": {
"dedupStrategy": "none",
"enableLogDetails": true,
"prettifyLogMessage": false,
"showCommonLabels": false,
"showLabels": false,
"showTime": true,
"sortOrder": "Descending",
"wrapLogMessage": false
},
"pluginVersion": "7.3.5",
"targets": [
{
"datasource": {
"type": "loki",
"uid": "P8E80F9AEF21F6940"
},
"expr": "{job=\"syslog\"} |=\"GT Warning\" !=\"critical\"|~ \"(?i)$searchable_pattern\"",
"legendFormat": "",
"refId": "A"
}
],
"title": "Warning ",
"type": "logs"
}
],
"refresh": "10s",
"schemaVersion": 37,
"style": "dark",
"tags": [
"mikrotik",
"loki",
"dashboard"
],
"templating": {
"list": [
{
"current": {
"selected": false,
"text": "",
"value": ""
},
"hide": 0,
"label": "Search (case insensitive)",
"name": "searchable_pattern",
"options": [
{
"selected": true,
"text": "",
"value": ""
}
],
"query": "",
"skipUrlSync": false,
"type": "textbox"
}
]
},
"time": {
"from": "now-12h",
"to": "now"
},
"timepicker": {
"hidden": false
},
"timezone": "",
"title": "Mikrotik Loki Logs",
"uid": "L4lAcEaGz",
"version": 1,
"weekStart": ""
}

5536
grafana/dashboards/mikrotik/mikrotik_mktxp_exporter.json Normal file
View File

File diff suppressed because it is too large Load Diff

3169
grafana/dashboards/system/system_overview_grafana.json Normal file
View File

File diff suppressed because it is too large Load Diff

1647
grafana/dashboards/system/system_overview_prometheus.json Normal file
View File

File diff suppressed because it is too large Load Diff

12
grafana/provisioning/dashboards/mikrotik.yml Executable file
View File

@@ -0,0 +1,12 @@
apiVersion: 1
providers:
- name: 'Mikrotik Dashboards'
orgId: 1
folder: 'Mikrotik'
type: file
disableDeletion: false
editable: true
updateIntervalSeconds: 10
options:
path: /var/lib/grafana/dashboards/mikrotik

12
grafana/provisioning/dashboards/system.yml Executable file
View File

@@ -0,0 +1,12 @@
apiVersion: 1
providers:
- name: 'System Dashboards'
orgId: 1
folder: 'System'
type: file
disableDeletion: false
editable: true
updateIntervalSeconds: 10
options:
path: /var/lib/grafana/dashboards/system

10
grafana/provisioning/datasources/loki.yaml Executable file
View File

@@ -0,0 +1,10 @@
apiVersion: 1
datasources:
-
access: proxy
basicAuth: false
jsonData:
maxLines: 1000
name: Loki
type: loki
url: "http://loki:3100/"

8
grafana/provisioning/datasources/prometheus.yaml Executable file
View File

@@ -0,0 +1,8 @@
apiVersion: 1
datasources:
-
access: proxy
basicAuth: false
name: Prometheus
type: prometheus
url: "http://prometheus:9090/"

64
loki/loki-config.yml Executable file
View File

@@ -0,0 +1,64 @@
auth_enabled: false
server:
http_listen_port: 3100
ingester:
lifecycler:
address: 127.0.0.1
ring:
kvstore:
store: inmemory
replication_factor: 1
final_sleep: 0s
chunk_idle_period: 1h # Any chunk not receiving new logs in this time will be flushed
max_chunk_age: 1h # All chunks will be flushed when they hit this age, default is 1h
chunk_target_size: 1048576 # Loki will attempt to build chunks up to 1.5MB, flushing first if chunk_idle_period or max_chunk_age is reached first
chunk_retain_period: 30s # Must be greater than index read cache TTL if using an index cache (Default index read cache TTL is 5m)
max_transfer_retries: 0 # Chunk transfers disabled
schema_config:
configs:
- from: 2020-10-24
store: boltdb-shipper
object_store: filesystem
schema: v11
index:
prefix: index_
period: 24h
storage_config:
boltdb_shipper:
active_index_directory: /tmp/loki/boltdb-shipper-active
cache_location: /tmp/loki/boltdb-shipper-cache
cache_ttl: 24h # Can be increased for faster performance over longer query periods, uses more disk space
shared_store: filesystem
filesystem:
directory: /tmp/loki/chunks
compactor:
working_directory: /tmp/loki/boltdb-shipper-compactor
shared_store: filesystem
limits_config:
reject_old_samples: true
reject_old_samples_max_age: 168h
chunk_store_config:
max_look_back_period: 0s
table_manager:
retention_deletes_enabled: false
retention_period: 0s
ruler:
storage:
type: local
local:
directory: /tmp/loki/rules
rule_path: /tmp/loki/rules-temp
alertmanager_url: http://localhost:9093
ring:
kvstore:
store: inmemory
enable_api: true

22
mktxp/_mktxp.conf Normal file
View File

@@ -0,0 +1,22 @@
## Copyright (c) 2020 Arseniy Kuznetsov
##
## This program is free software; you can redistribute it and/or
## modify it under the terms of the GNU General Public License
## as published by the Free Software Foundation; either version 2
## of the License, or (at your option) any later version.
##
## This program is distributed in the hope that it will be useful,
## but WITHOUT ANY WARRANTY; without even the implied warranty of
## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
## GNU General Public License for more details.
[MKTXP]
port = 49090
socket_timeout = 2
initial_delay_on_failure = 120
max_delay_on_failure = 900
delay_inc_div = 5
bandwidth_test_interval = 420
verbose_mode = False
bandwidth = True

42
mktxp/mktxp.conf Normal file
View File

@@ -0,0 +1,42 @@
## Copyright (c) 2020 Arseniy Kuznetsov
##
## This program is free software; you can redistribute it and/or
## modify it under the terms of the GNU General Public License
## as published by the Free Software Foundation; either version 2
## of the License, or (at your option) any later version.
##
## This program is distributed in the hope that it will be useful,
## but WITHOUT ANY WARRANTY; without even the implied warranty of
## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
## GNU General Public License for more details.
[Sample-Router]
enabled = False # turns metrics collection for this RouterOS device on / off
hostname = localhost # RouterOS IP address
port = 8728 # RouterOS IP Port
username = username # RouterOS user, needs to have 'read' and 'api' permissions
password = password
use_ssl = False # enables connection via API-SSL servis
no_ssl_certificate = False # enables API_SSL connect without router SSL certificate
ssl_certificate_verify = False # turns SSL certificate verification on / off
dhcp = True # DHCP general metrics
dhcp_lease = True # DHCP lease metrics
connections = True # IP connections metrics
pool = True # Pool metrics
interface = True # Interfaces traffic metrics
firewall = True # Firewall rules traffic metrics
monitor = True # Interface monitor metrics
poe = True # POE metrics
public_ip = True # Public IP metrics
route = True # Routes metrics
wireless = True # WLAN general metrics
wireless_clients = True # WLAN clients metrics
capsman = True # CAPsMAN general metrics
capsman_clients = True # CAPsMAN clients metrics
use_comments_over_names = True # when available, forces using comments over the interfaces names

38
prometheus/prometheus.yml Executable file
View File

@@ -0,0 +1,38 @@
# my global config
global:
scrape_interval: 15s # Set the scrape interval to every 15 seconds. Default is every 1 minute.
evaluation_interval: 15s # Evaluate rules every 15 seconds. The default is every 1 minute.
external_labels:
origin_prometheus: mktxp
# Alertmanager configuration
alerting:
alertmanagers:
- static_configs:
- targets:
# - alertmanager:9093
# Load rules once and periodically evaluate them according to the global 'evaluation_interval'.
rule_files:
scrape_configs:
- job_name: 'prometheus'
static_configs:
- targets: ['prometheus:9090']
- job_name: 'grafana'
static_configs:
- targets: ['grafana:3000']
- job_name: 'loki'
static_configs:
- targets: ['loki:3100']
- job_name: 'promtail'
static_configs:
- targets: ['promtail:9080']
- job_name: 'mktxp'
static_configs:
- targets: ['mktxp:49090']

22
promtail/promtail-config.yml Executable file
View File

@@ -0,0 +1,22 @@
server:
http_listen_port: 9080
grpc_listen_port: 0
positions:
filename: /tmp/positions.yaml
clients:
- url: http://loki:3100/loki/api/v1/push
scrape_configs:
- job_name: syslog
syslog:
listen_address: 0.0.0.0:1514
idle_timeout: 60s
label_structured_data: yes
labels:
job: "syslog"
relabel_configs:
- source_labels: ['__syslog_message_hostname']
target_label: 'host'

42
syslog-ng/syslog-ng.conf Executable file
View File

@@ -0,0 +1,42 @@
#############################################################################
# Default syslog-ng.conf file which collects all local logs into a
# single file called /var/log/messages tailored to container usage.
#
# The changes from the stock, default syslog-ng.conf file is that we've
# dropped the system() source that is not needed and that we enabled network
# connections using default-network-drivers(). Customize as needed and
# override using the -v option to docker, such as:
#
# docker run ... -v "$PWD/syslog-ng.conf":/etc/syslog-ng/syslog-ng.conf
#
@version: 3.29
@include "scl.conf"
source s_local {
internal();
};
source s_network {
default-network-drivers(
# NOTE: TLS support
#
# the default-network-drivers() source driver opens the TLS
# enabled ports as well, however without an actual key/cert
# pair they will not operate and syslog-ng would display a
# warning at startup.
#
#tls(key-file("/path/to/ssl-private-key") cert-file("/path/to/ssl-cert"))
);
};
destination d_loki {
syslog("promtail" transport("tcp") port("1514"));
};
log {
source(s_local);
source(s_network);
destination(d_loki);
};