mirror of
https://github.com/FAUSheppy/icinga-webhook-gateway
synced 2026-04-26 22:22:30 +02:00
Compare commits
31 Commits
smart-dev
...
b0752c2984
| Author | SHA1 | Date | |
|---|---|---|---|
| b0752c2984 | |||
| 1af07b90cc | |||
| ed039833c5 | |||
| bb5aaa47ad | |||
| 87b8de01d7 | |||
| ef8e1e6a81 | |||
| 6680f4769c | |||
| 95c3551a5c | |||
| 31db0c22d2 | |||
| ce5328da53 | |||
| 1d36a9aaed | |||
| 7fea3bf315 | |||
| 2e37ddcb8e | |||
| 7d612c0ccd | |||
| 52569c7687 | |||
| d70a37f42c | |||
| 8d6590364f | |||
| dd7a81fd0f | |||
| 7b5f28651b | |||
| a7f4788291 | |||
| 74b48a2477 | |||
| 18f8436078 | |||
| 3df3ddb08e | |||
| 72e0210d26 | |||
| edc454f154 | |||
| 824c108678 | |||
| 08fc17efe0 | |||
| 683ebefbb0 | |||
| 0842818cbc | |||
| d6ea667733 | |||
| 935bfa3eef |
5
.github/workflows/main.yaml
vendored
5
.github/workflows/main.yaml
vendored
@@ -4,6 +4,8 @@ on:
|
|||||||
push:
|
push:
|
||||||
branches:
|
branches:
|
||||||
- "master"
|
- "master"
|
||||||
|
schedule:
|
||||||
|
- cron: "0 2 * * 0"
|
||||||
|
|
||||||
jobs:
|
jobs:
|
||||||
docker:
|
docker:
|
||||||
@@ -14,9 +16,6 @@ jobs:
|
|||||||
-
|
-
|
||||||
name: Checkout
|
name: Checkout
|
||||||
uses: actions/checkout@v3
|
uses: actions/checkout@v3
|
||||||
-
|
|
||||||
name: Set up QEMU
|
|
||||||
uses: docker/setup-qemu-action@v2
|
|
||||||
-
|
-
|
||||||
name: Set up Docker Buildx
|
name: Set up Docker Buildx
|
||||||
uses: docker/setup-buildx-action@v2
|
uses: docker/setup-buildx-action@v2
|
||||||
|
|||||||
@@ -1,5 +1,15 @@
|
|||||||
import icinga2api
|
import icinga2api
|
||||||
import icinga2api.client
|
import icinga2api.client
|
||||||
|
from urllib.parse import urlparse
|
||||||
|
|
||||||
|
def split_url(url):
|
||||||
|
parsed = urlparse(url)
|
||||||
|
|
||||||
|
http_vhost = parsed.hostname
|
||||||
|
http_uri = parsed.path or "/"
|
||||||
|
http_ssl = parsed.scheme == "https"
|
||||||
|
|
||||||
|
return http_vhost, http_uri, http_ssl
|
||||||
|
|
||||||
def _create_client(app):
|
def _create_client(app):
|
||||||
|
|
||||||
@@ -39,26 +49,52 @@ def _build_service_name(user, async_service_name):
|
|||||||
|
|
||||||
return "{}_async_{}".format(user, async_service_name)
|
return "{}_async_{}".format(user, async_service_name)
|
||||||
|
|
||||||
def create_service(user, async_service_name, app):
|
def create_service(user, async_service_name, app, webcheck=False):
|
||||||
|
|
||||||
|
if not app.config.get("ICINGA_API_URL"):
|
||||||
|
return
|
||||||
|
|
||||||
client = _create_client(app)
|
client = _create_client(app)
|
||||||
name = _build_service_name(user, async_service_name)
|
name = _build_service_name(user, async_service_name)
|
||||||
host_name = app.config["ASYNC_ICINGA_DUMMY_HOST"]
|
host_name = app.config["ASYNC_ICINGA_DUMMY_HOST"]
|
||||||
|
|
||||||
service_config = {
|
# TODO: query service from DB
|
||||||
"templates": ["generic-service"],
|
accepted_return_codes = [200, 204]
|
||||||
"attrs": {
|
|
||||||
"display_name": name,
|
|
||||||
"check_command": "gateway",
|
if webcheck:
|
||||||
"host_name" : host_name,
|
http_vhost, http_uri, http_ssl = split_url(url)
|
||||||
"vars" : {
|
service_config = {
|
||||||
"host" : "async-icinga.atlantishq.de",
|
"templates": ["generic-service"],
|
||||||
"service_name" : async_service_name,
|
"attrs": {
|
||||||
"protocol" : "https",
|
"display_name": name,
|
||||||
"owner" : user
|
"check_command": "http",
|
||||||
|
"host_name": host_name,
|
||||||
|
"vars": {
|
||||||
|
"http_vhost": http_vhost,
|
||||||
|
"http_uri": http_uri,
|
||||||
|
"http_expect": http_expect,
|
||||||
|
"http_accept_status": accepted_return_codes, # array
|
||||||
|
"http_ssl": True,
|
||||||
|
"http_sni": True
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
else:
|
||||||
|
service_config = {
|
||||||
|
"templates": ["generic-service"],
|
||||||
|
"attrs": {
|
||||||
|
"display_name": name,
|
||||||
|
"check_command": "gateway",
|
||||||
|
"host_name" : host_name,
|
||||||
|
"vars" : {
|
||||||
|
"host" : "async-icinga.atlantishq.de",
|
||||||
|
"service_name" : async_service_name,
|
||||||
|
"protocol" : "https",
|
||||||
|
"owner" : user
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
|
||||||
|
|
||||||
# Create the service (name is required in this format)
|
# Create the service (name is required in this format)
|
||||||
service_api_helper_name = "{}!{}".format(host_name, name)
|
service_api_helper_name = "{}!{}".format(host_name, name)
|
||||||
@@ -67,6 +103,9 @@ def create_service(user, async_service_name, app):
|
|||||||
|
|
||||||
def delete_service(user, async_service_name, app):
|
def delete_service(user, async_service_name, app):
|
||||||
|
|
||||||
|
if not app.config.get("ICINGA_API_URL"):
|
||||||
|
return
|
||||||
|
|
||||||
client = _create_client(app)
|
client = _create_client(app)
|
||||||
name = _build_service_name(user, async_service_name)
|
name = _build_service_name(user, async_service_name)
|
||||||
host_name = app.config["ASYNC_ICINGA_DUMMY_HOST"]
|
host_name = app.config["ASYNC_ICINGA_DUMMY_HOST"]
|
||||||
@@ -83,7 +122,15 @@ def build_icinga_link_for_service(user, service_name, static_configured, app):
|
|||||||
url_fmt = "{base}/icingaweb2/monitoring/list/services?service={service}&modifyFilter=1"
|
url_fmt = "{base}/icingaweb2/monitoring/list/services?service={service}&modifyFilter=1"
|
||||||
name = service_name
|
name = service_name
|
||||||
|
|
||||||
return url_fmt.format(base=app.config["ICINGA_WEB_URL"],
|
icinga_web_url = app.config.get("ICINGA_WEB_URL")
|
||||||
host=app.config["ASYNC_ICINGA_DUMMY_HOST"],
|
if not icinga_web_url:
|
||||||
|
icinga_web_url = "ICINGA_WEB_URL_NOT_SET:"
|
||||||
|
|
||||||
|
dummy_host=app.config.get("ASYNC_ICINGA_DUMMY_HOST")
|
||||||
|
if not dummy_host:
|
||||||
|
dummy_host = "ASYNC_ICINGA_DUMMY_HOST_NOT_SET:"
|
||||||
|
|
||||||
|
return url_fmt.format(base=icinga_web_url,
|
||||||
|
host=dummy_host,
|
||||||
service=name)
|
service=name)
|
||||||
|
|
||||||
|
|||||||
2
req.txt
2
req.txt
@@ -5,3 +5,5 @@ flask-wtf
|
|||||||
waitress
|
waitress
|
||||||
requests
|
requests
|
||||||
icinga2api
|
icinga2api
|
||||||
|
psycopg2-binary
|
||||||
|
tzdata
|
||||||
|
|||||||
268
server.py
268
server.py
@@ -9,10 +9,11 @@ import datetime
|
|||||||
import pytimeparse.timeparse as timeparse
|
import pytimeparse.timeparse as timeparse
|
||||||
import sys
|
import sys
|
||||||
import secrets
|
import secrets
|
||||||
|
import zoneinfo
|
||||||
|
|
||||||
import flask_wtf
|
import flask_wtf
|
||||||
from flask_wtf import FlaskForm
|
from flask_wtf import FlaskForm
|
||||||
from wtforms import StringField, SubmitField, BooleanField, DecimalField, HiddenField
|
from wtforms import StringField, SubmitField, BooleanField, DecimalField, HiddenField, SelectField
|
||||||
from wtforms.validators import DataRequired, Length
|
from wtforms.validators import DataRequired, Length
|
||||||
|
|
||||||
from sqlalchemy import Column, Integer, String, Boolean, or_, and_
|
from sqlalchemy import Column, Integer, String, Boolean, or_, and_
|
||||||
@@ -22,17 +23,37 @@ from sqlalchemy.sql import func
|
|||||||
import sqlalchemy
|
import sqlalchemy
|
||||||
from flask_sqlalchemy import SQLAlchemy
|
from flask_sqlalchemy import SQLAlchemy
|
||||||
from sqlalchemy.sql.expression import func
|
from sqlalchemy.sql.expression import func
|
||||||
|
from sqlalchemy.exc import IntegrityError
|
||||||
|
from psycopg2.errors import UniqueViolation
|
||||||
|
|
||||||
import icingatools
|
import icingatools
|
||||||
|
import smarttools
|
||||||
|
|
||||||
app = flask.Flask("Icinga Report In Gateway")
|
app = flask.Flask("Icinga Report In Gateway")
|
||||||
|
|
||||||
|
|
||||||
app.config['SQLALCHEMY_DATABASE_URI'] = 'sqlite:///database.sqlite'
|
app.config['SQLALCHEMY_DATABASE_URI'] = os.environ.get('SQLALCHEMY_DATABASE_URI') or 'sqlite:///database.sqlite'
|
||||||
app.config['JSON_CONFIG_FILE'] = 'services.json'
|
app.config['JSON_CONFIG_FILE'] = 'services.json'
|
||||||
app.config['JSON_CONFIG_DIR'] = 'config'
|
app.config['JSON_CONFIG_DIR'] = 'config'
|
||||||
|
app.config['TIME_ZONE'] = zoneinfo.ZoneInfo(os.getenv("TIME_ZONE", "UTC"))
|
||||||
|
app.config['AUTH_HEADER'] = os.environ.get("AUTH_HEADER") or "X-Forwarded-Preferred-Username"
|
||||||
db = SQLAlchemy(app)
|
db = SQLAlchemy(app)
|
||||||
|
|
||||||
|
from urllib.parse import urlparse
|
||||||
|
|
||||||
|
def split_url(url: str):
|
||||||
|
parsed = urlparse(url)
|
||||||
|
|
||||||
|
http_vhost = parsed.hostname
|
||||||
|
http_uri = parsed.path or "/"
|
||||||
|
http_ssl = parsed.scheme == "https"
|
||||||
|
|
||||||
|
return {
|
||||||
|
"http_vhost": http_vhost,
|
||||||
|
"http_uri": http_uri,
|
||||||
|
"http_ssl": http_ssl
|
||||||
|
}
|
||||||
|
|
||||||
class Service(db.Model):
|
class Service(db.Model):
|
||||||
|
|
||||||
__tablename__ = "services"
|
__tablename__ = "services"
|
||||||
@@ -41,6 +62,12 @@ class Service(db.Model):
|
|||||||
token = Column(String)
|
token = Column(String)
|
||||||
timeout = Column(Integer)
|
timeout = Column(Integer)
|
||||||
owner = Column(String)
|
owner = Column(String)
|
||||||
|
special_type = Column(String)
|
||||||
|
|
||||||
|
# web checks #
|
||||||
|
url = Column(String)
|
||||||
|
accepted_codes = Column(String)
|
||||||
|
http_expect = Column(String)
|
||||||
|
|
||||||
staticly_configured = Column(Boolean)
|
staticly_configured = Column(Boolean)
|
||||||
|
|
||||||
@@ -54,9 +81,25 @@ class Status(db.Model):
|
|||||||
info_text = Column(String)
|
info_text = Column(String)
|
||||||
|
|
||||||
def human_date(self):
|
def human_date(self):
|
||||||
dt = datetime.datetime.fromtimestamp(self.timestamp)
|
dt = datetime.datetime.fromtimestamp(self.timestamp, app.config["TIME_ZONE"])
|
||||||
return dt.strftime("%d. %B %Y at %H:%M")
|
return dt.strftime("%d. %B %Y at %H:%M")
|
||||||
|
|
||||||
|
class SMARTStatus(db.Model):
|
||||||
|
|
||||||
|
__tablename__ = "smart"
|
||||||
|
|
||||||
|
service = Column(String, primary_key=True)
|
||||||
|
timestamp = Column(Integer, primary_key=True)
|
||||||
|
model_number = Column(String, primary_key=True)
|
||||||
|
power_cycles = Column(Integer)
|
||||||
|
temperature = Column(Integer)
|
||||||
|
available_spare = Column(Integer)
|
||||||
|
unsafe_shutdowns = Column(Integer)
|
||||||
|
critical_warning = Column(Integer)
|
||||||
|
power_cycles = Column(Integer)
|
||||||
|
power_on_hours = Column(Integer)
|
||||||
|
wearleveling_count = Column(Integer)
|
||||||
|
|
||||||
def buildReponseDict(status, service=None):
|
def buildReponseDict(status, service=None):
|
||||||
|
|
||||||
if not status:
|
if not status:
|
||||||
@@ -73,7 +116,7 @@ def buildReponseDict(status, service=None):
|
|||||||
@app.route('/overview')
|
@app.route('/overview')
|
||||||
def overview():
|
def overview():
|
||||||
|
|
||||||
user = str(flask.request.headers.get("X-Forwarded-Preferred-Username"))
|
user = str(flask.request.headers.get(app.config['AUTH_HEADER']))
|
||||||
|
|
||||||
# query all services #
|
# query all services #
|
||||||
services = db.session.query(Service).filter(Service.owner == user).all()
|
services = db.session.query(Service).filter(Service.owner == user).all()
|
||||||
@@ -108,6 +151,7 @@ class EntryForm(FlaskForm):
|
|||||||
|
|
||||||
service = StringField("Service Name")
|
service = StringField("Service Name")
|
||||||
service_hidden = HiddenField("service_hidden")
|
service_hidden = HiddenField("service_hidden")
|
||||||
|
special_type = SelectField("Type", choices=["Default", "SMART"])
|
||||||
timeout = DecimalField("Timeout in days", default=30)
|
timeout = DecimalField("Timeout in days", default=30)
|
||||||
|
|
||||||
def create_entry(form, user):
|
def create_entry(form, user):
|
||||||
@@ -116,9 +160,21 @@ def create_entry(form, user):
|
|||||||
|
|
||||||
service_name = form.service.data or form.service_hidden.data
|
service_name = form.service.data or form.service_hidden.data
|
||||||
|
|
||||||
|
# keep token if modification #
|
||||||
|
s_tmp = db.session.query(Service).filter(Service.service == service_name).first()
|
||||||
|
if s_tmp:
|
||||||
|
token = s_tmp.token
|
||||||
|
if not token:
|
||||||
|
raise AssertionError("WTF Service without Token {}".format(service_name))
|
||||||
|
|
||||||
day_delta = datetime.timedelta(days=int(form.timeout.data))
|
day_delta = datetime.timedelta(days=int(form.timeout.data))
|
||||||
|
|
||||||
|
special_type = form.special_type.data
|
||||||
|
if form.special_type == "Default":
|
||||||
|
special_type = None
|
||||||
|
|
||||||
service = Service(service=service_name, timeout=day_delta.total_seconds(),
|
service = Service(service=service_name, timeout=day_delta.total_seconds(),
|
||||||
owner=user, token=token)
|
owner=user, token=token, special_type=special_type)
|
||||||
|
|
||||||
# service.data set = create, service_hidden.data = modify #
|
# service.data set = create, service_hidden.data = modify #
|
||||||
if form.service.data:
|
if form.service.data:
|
||||||
@@ -130,7 +186,7 @@ def create_entry(form, user):
|
|||||||
@app.route("/service-details")
|
@app.route("/service-details")
|
||||||
def service_details():
|
def service_details():
|
||||||
|
|
||||||
user = str(flask.request.headers.get("X-Forwarded-Preferred-Username"))
|
user = flask.request.headers.get(app.config['AUTH_HEADER'])
|
||||||
service = flask.request.args.get("service")
|
service = flask.request.args.get("service")
|
||||||
|
|
||||||
# query service #
|
# query service #
|
||||||
@@ -143,22 +199,45 @@ def service_details():
|
|||||||
return ("Services is not owned by {}".format(user))
|
return ("Services is not owned by {}".format(user))
|
||||||
|
|
||||||
status_list_query = db.session.query(Status).filter(Status.service==service.service)
|
status_list_query = db.session.query(Status).filter(Status.service==service.service)
|
||||||
status_list = status_list_query.order_by(sqlalchemy.desc(Status.timestamp)).limit(20).all()
|
status_list = status_list_query.order_by(sqlalchemy.desc(Status.timestamp)).limit(200).all()
|
||||||
|
|
||||||
|
# build status tupel (repeats, status) #
|
||||||
|
current_tupel = None
|
||||||
|
prev_status = None
|
||||||
|
tupel_list = []
|
||||||
|
for s in status_list:
|
||||||
|
|
||||||
|
# set initial #
|
||||||
|
if not current_tupel:
|
||||||
|
current_tupel = [1, s]
|
||||||
|
tupel_list.append(current_tupel)
|
||||||
|
continue
|
||||||
|
|
||||||
|
if current_tupel[1].info_text == s.info_text:
|
||||||
|
current_tupel[0] += 1
|
||||||
|
else:
|
||||||
|
current_tupel = [1, s]
|
||||||
|
tupel_list.append(current_tupel)
|
||||||
|
|
||||||
|
|
||||||
|
print(tupel_list)
|
||||||
icinga_link = icingatools.build_icinga_link_for_service(user, service.service,
|
icinga_link = icingatools.build_icinga_link_for_service(user, service.service,
|
||||||
service.staticly_configured, app)
|
service.staticly_configured, app)
|
||||||
|
|
||||||
|
smart_entry_list = db.session.query(SMARTStatus).filter(SMARTStatus.service==service.service)
|
||||||
|
smart_entry = smart_entry_list.order_by(SMARTStatus.timestamp.desc()).first()
|
||||||
|
|
||||||
return flask.render_template("service_info.html", service=service, flask=flask,
|
return flask.render_template("service_info.html", service=service, flask=flask,
|
||||||
user=user, status_list=status_list, icinga_link=icinga_link)
|
user=user, status_list=tupel_list, icinga_link=icinga_link, smart=smart_entry)
|
||||||
|
|
||||||
|
|
||||||
@app.route("/entry-form", methods=["GET", "POST", "DELETE"])
|
@app.route("/entry-form", methods=["GET", "POST", "DELETE"])
|
||||||
def create_interface():
|
def create_interface():
|
||||||
|
|
||||||
user = str(flask.request.headers.get("X-Forwarded-Preferred-Username"))
|
user = flask.request.headers.get(app.config['AUTH_HEADER'])
|
||||||
|
|
||||||
# check if is delete #
|
# check if is delete #
|
||||||
operation = flask.request.args.get("operation")
|
operation = flask.request.args.get("operation")
|
||||||
if operation and operation == "delete" :
|
if operation and operation == "delete" :
|
||||||
|
|
||||||
service_delete_name = flask.request.args.get("service")
|
service_delete_name = flask.request.args.get("service")
|
||||||
@@ -175,13 +254,14 @@ def create_interface():
|
|||||||
return flask.redirect("/overview")
|
return flask.redirect("/overview")
|
||||||
|
|
||||||
form = EntryForm()
|
form = EntryForm()
|
||||||
|
|
||||||
# handle modification #
|
# handle modification #
|
||||||
modify_service_name = flask.request.args.get("service")
|
modify_service_name = flask.request.args.get("service")
|
||||||
if modify_service_name:
|
if modify_service_name:
|
||||||
service = db.session.query(Service).filter(Service.service == modify_service_name).first()
|
service = db.session.query(Service).filter(Service.service == modify_service_name).first()
|
||||||
if service and service.owner == user:
|
if service and service.owner == user:
|
||||||
form.service.default = service.service
|
form.service.default = service.service
|
||||||
|
form.special_type.default = service.special_type
|
||||||
form.timeout.default = datetime.timedelta(seconds=service.timeout).days
|
form.timeout.default = datetime.timedelta(seconds=service.timeout).days
|
||||||
form.service_hidden.default = service.service
|
form.service_hidden.default = service.service
|
||||||
form.process()
|
form.process()
|
||||||
@@ -189,9 +269,21 @@ def create_interface():
|
|||||||
return ("Not a valid service to modify", 404)
|
return ("Not a valid service to modify", 404)
|
||||||
|
|
||||||
if flask.request.method == "POST":
|
if flask.request.method == "POST":
|
||||||
create_entry(form, user)
|
|
||||||
|
try:
|
||||||
|
create_entry(form, user)
|
||||||
|
except IntegrityError as e:
|
||||||
|
db.session.rollback()
|
||||||
|
# TODO: this only works for PG
|
||||||
|
if isinstance(e.orig, UniqueViolation):
|
||||||
|
return ("A service with this name already exists (possibly by another user)", 409)
|
||||||
|
else:
|
||||||
|
return (f"Error: {e}", 500)
|
||||||
|
|
||||||
|
# service created successfully #
|
||||||
service_name = form.service.data or form.service_hidden.data
|
service_name = form.service.data or form.service_hidden.data
|
||||||
return flask.redirect('/service-details?service={}'.format(service_name))
|
return flask.redirect('/service-details?service={}'.format(service_name))
|
||||||
|
|
||||||
else:
|
else:
|
||||||
return flask.render_template('add_modify_service.html', form=form,
|
return flask.render_template('add_modify_service.html', form=form,
|
||||||
is_modification=bool(modify_service_name))
|
is_modification=bool(modify_service_name))
|
||||||
@@ -252,7 +344,7 @@ def default():
|
|||||||
if not lastSuccess.timestamp == 0 and delta > timeout and latestInfoIsSuccess:
|
if not lastSuccess.timestamp == 0 and delta > timeout and latestInfoIsSuccess:
|
||||||
|
|
||||||
# lastes info is success but timed out #
|
# lastes info is success but timed out #
|
||||||
lastSuccess.info_text = "Service {} overdue since {}".format(service, str(delta))
|
lastSuccess.info_text = "Service {} overdue since {}".format(service, str(delta))
|
||||||
if timeout/delta > 0.9 or (delta - timeout) < datetime.timedelta(hours=12):
|
if timeout/delta > 0.9 or (delta - timeout) < datetime.timedelta(hours=12):
|
||||||
lastSuccess.status = "WARNING"
|
lastSuccess.status = "WARNING"
|
||||||
else:
|
else:
|
||||||
@@ -270,16 +362,27 @@ def default():
|
|||||||
elif flask.request.method == "POST":
|
elif flask.request.method == "POST":
|
||||||
|
|
||||||
# get variables #
|
# get variables #
|
||||||
service = flask.request.json["service"]
|
service = flask.request.json.get("service")
|
||||||
token = flask.request.json["token"]
|
token = flask.request.json.get("token")
|
||||||
status = flask.request.json["status"]
|
status = flask.request.json.get("status")
|
||||||
text = flask.request.json["info"]
|
text = flask.request.json.get("info") or "no_info"
|
||||||
timestamp = datetime.datetime.now().timestamp()
|
timestamp = datetime.datetime.now().timestamp()
|
||||||
|
|
||||||
|
smart = flask.request.json.get("smart")
|
||||||
|
|
||||||
|
# check smart json quoting problems #
|
||||||
|
if smart and type(smart) == str:
|
||||||
|
try:
|
||||||
|
smart = json.loads(smart)
|
||||||
|
except json.decoder.JSONDecodeError as e:
|
||||||
|
return ("Error in SMART-json {}".format(e), 415)
|
||||||
|
|
||||||
if not service:
|
if not service:
|
||||||
return ("'service' ist empty field in json", 400)
|
return ("'service' ist empty field in json", 400)
|
||||||
elif not token:
|
elif not token:
|
||||||
return ("'token' ist empty field in json", 400)
|
return ("'token' ist empty field in json", 400)
|
||||||
|
elif not status and not smart:
|
||||||
|
return ("'status' is empty field in json", 400)
|
||||||
|
|
||||||
# verify token & service in config #
|
# verify token & service in config #
|
||||||
verifiedServiceObj = db.session.query(Service).filter(
|
verifiedServiceObj = db.session.query(Service).filter(
|
||||||
@@ -288,15 +391,104 @@ def default():
|
|||||||
if not verifiedServiceObj:
|
if not verifiedServiceObj:
|
||||||
return ("Service ({}) with this token ({}) not found in DB".format(service, token), 401)
|
return ("Service ({}) with this token ({}) not found in DB".format(service, token), 401)
|
||||||
else:
|
else:
|
||||||
status = Status(service=service, timestamp=timestamp, status=status, info_text=text)
|
|
||||||
|
# handle a SMART-record submission (with errorhandling) #
|
||||||
|
if smart and not verifiedServiceObj.special_type == "SMART":
|
||||||
|
return ("SMART Field for non-SMART type service", 415)
|
||||||
|
elif smart:
|
||||||
|
text, status = record_and_check_smart(verifiedServiceObj,
|
||||||
|
timestamp, smart)
|
||||||
|
|
||||||
|
status = Status(service=service, timestamp=timestamp, status=status,
|
||||||
|
info_text=text)
|
||||||
db.session.merge(status)
|
db.session.merge(status)
|
||||||
db.session.commit()
|
|
||||||
|
try:
|
||||||
|
db.session.commit()
|
||||||
|
except IntegrityError as e:
|
||||||
|
db.session.rollback()
|
||||||
|
# TODO: this only works for PG
|
||||||
|
if isinstance(e.orig, UniqueViolation):
|
||||||
|
return ("Status at this time already submitted", 409)
|
||||||
|
else:
|
||||||
|
return (f"Error: {e}", 500)
|
||||||
|
|
||||||
return ("", 204)
|
return ("", 204)
|
||||||
else:
|
else:
|
||||||
return ("Method not implemented: {}".format(flask.request.method), 405)
|
return ("Method not implemented: {}".format(flask.request.method), 405)
|
||||||
|
|
||||||
|
def record_and_check_smart(service, timestamp, smart):
|
||||||
|
|
||||||
|
if "nvme_smart_health_information_log" in smart:
|
||||||
|
health_info = smart["nvme_smart_health_information_log"]
|
||||||
|
else:
|
||||||
|
health_info = smarttools.normalize(smart)
|
||||||
|
|
||||||
|
if not service.special_type == "SMART":
|
||||||
|
raise AssertionError("Trying to record SMART-record for non-SMART service")
|
||||||
|
|
||||||
|
# record the status #
|
||||||
|
smart_status = SMARTStatus(service=service.service, timestamp=timestamp,
|
||||||
|
temperature=health_info["temperature"],
|
||||||
|
critical_warning=health_info["critical_warning"],
|
||||||
|
unsafe_shutdowns=health_info["unsafe_shutdowns"],
|
||||||
|
power_cycles=health_info["power_cycles"],
|
||||||
|
power_on_hours=health_info["power_on_hours"],
|
||||||
|
available_spare=health_info.get("available_spare"),
|
||||||
|
model_number=smart.get("model_name"),
|
||||||
|
wearleveling_count=health_info.get("wearleveling_count"))
|
||||||
|
|
||||||
|
db.session.add(smart_status)
|
||||||
|
db.session.commit()
|
||||||
|
|
||||||
|
# check the status #
|
||||||
|
smart_last_query = db.session.query(SMARTStatus)
|
||||||
|
smart_last_query = smart_last_query.filter(SMARTStatus.service==service.service)
|
||||||
|
smart_last = smart_last_query.order_by(sqlalchemy.desc(SMARTStatus.timestamp)).first()
|
||||||
|
smart_second_last = smart_last_query.order_by(sqlalchemy.desc(
|
||||||
|
SMARTStatus.timestamp)).offset(1).first()
|
||||||
|
|
||||||
|
# last record (max 6 months ago) #
|
||||||
|
timestampt_minus_6m = datetime.datetime.now() - datetime.timedelta(days=180)
|
||||||
|
smart_old_query = smart_last_query.filter(
|
||||||
|
SMARTStatus.timestamp > timestampt_minus_6m.timestamp())
|
||||||
|
smart_old = smart_old_query.order_by(sqlalchemy.asc(SMARTStatus.timestamp)).first()
|
||||||
|
|
||||||
|
# critial != 0 #
|
||||||
|
if smart_last.critical_warning != 0:
|
||||||
|
return ("SMART reports disk critical => oO better do something about this", "CRITICAL")
|
||||||
|
|
||||||
|
# wearleveling < 20% (SAMSUNG only) #
|
||||||
|
if smart_last.wearleveling_count and smart_last.wearleveling_count <= 20:
|
||||||
|
return ("SMART report prefail disk (wear_level < 20%)", "CRITICAL")
|
||||||
|
|
||||||
|
# temp max > X #
|
||||||
|
if smart_last.temperature > 60:
|
||||||
|
return ("Disk Temperatur {}".format(smart_last.temperature), "CRITICAL")
|
||||||
|
|
||||||
|
# available_SSD spare #
|
||||||
|
spare_change = smart_old.available_spare - smart_last.available_spare
|
||||||
|
|
||||||
|
if smart_last.available_spare <= 25:
|
||||||
|
return ("SSD spare <25 ({}) YOUR DISK WILL DIE SOON".format(spare_change),
|
||||||
|
"CRITICAL")
|
||||||
|
elif smart_last.available_spare <= 50:
|
||||||
|
return ("SSD spare <50 ({})".format(spare_change), "WARNING")
|
||||||
|
elif spare_change >= 10:
|
||||||
|
return ("Strong degration in SSD spare space ({} in under 6 months)".format(
|
||||||
|
spare_change), "WARNING")
|
||||||
|
|
||||||
|
# unsafe_shutdowns +1 #
|
||||||
|
if(smart_second_last and
|
||||||
|
smart_second_last.unsafe_shutdowns - smart_last.unsafe_shutdowns >= 1):
|
||||||
|
return ("Disk had {} unsafe shutdowns".format(smart_last.unsafe_shutdowns),
|
||||||
|
"WARNING")
|
||||||
|
|
||||||
|
return ("{} - no problems detected".format(smart_last.model_number), "OK")
|
||||||
|
|
||||||
|
|
||||||
def create_app():
|
def create_app():
|
||||||
|
|
||||||
db.create_all()
|
db.create_all()
|
||||||
config = {}
|
config = {}
|
||||||
|
|
||||||
@@ -323,20 +515,44 @@ def create_app():
|
|||||||
config |= json.load(f)
|
config |= json.load(f)
|
||||||
|
|
||||||
if not config:
|
if not config:
|
||||||
print("No valid configuration found - need at least one service")
|
print("No static services configuration found - loading finished.")
|
||||||
return
|
|
||||||
|
|
||||||
for key in config:
|
for key in config:
|
||||||
timeout = timeparse.timeparse(config[key]["timeout"])
|
timeout = timeparse.timeparse(config[key]["timeout"])
|
||||||
staticly_configured = True
|
staticly_configured = True
|
||||||
db.session.merge(Service(service=key, token=config[key]["token"],
|
db.session.merge(Service(service=key, token=config[key]["token"],
|
||||||
staticly_configured=staticly_configured, timeout=timeout,
|
staticly_configured=staticly_configured, timeout=timeout,
|
||||||
owner=config[key]["owner"]))
|
owner=config[key]["owner"]))
|
||||||
db.session.commit()
|
db.session.commit()
|
||||||
|
|
||||||
# create dummy host #
|
LOAD_FROM_ENV = [
|
||||||
icingatools.create_master_host(app)
|
"ICINGA_API_USER",
|
||||||
|
"ICINGA_API_PASS",
|
||||||
|
"ICINGA_API_URL",
|
||||||
|
"ICINGA_WEB_URL",
|
||||||
|
"ASYNC_ICINGA_DUMMY_HOST"
|
||||||
|
]
|
||||||
|
|
||||||
|
enforce_load_from_env = os.environ.get("ENFORCE_LOAD_FROM_ENV") or ""
|
||||||
|
missing = [k for k in LOAD_FROM_ENV if k not in os.environ]
|
||||||
|
if missing and enforce_load_from_env.lower() == "true":
|
||||||
|
print(f"ENFORCE_LOAD_FROM_ENV is 'true' but we are missing: {missing} - Abort.")
|
||||||
|
sys.exit(1)
|
||||||
|
|
||||||
|
for key in LOAD_FROM_ENV:
|
||||||
|
if key in os.environ:
|
||||||
|
print(f"Loading/Overwriting {key} from environment", file=sys.stderr)
|
||||||
|
app.config[key] = os.environ[key]
|
||||||
|
|
||||||
|
|
||||||
|
# create icinga host #
|
||||||
|
if not app.config.get("ICINGA_API_URL"):
|
||||||
|
print("ICINGA_API_URL not defined. Not connecting Icinga", file=sys.stderr)
|
||||||
|
else:
|
||||||
|
icingatools.create_master_host(app)
|
||||||
|
|
||||||
|
print(f"Expected AUTH_HEADER is: {app.config['AUTH_HEADER']}")
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
|
|
||||||
|
|||||||
54
smarttools.py
Normal file
54
smarttools.py
Normal file
@@ -0,0 +1,54 @@
|
|||||||
|
def normalize(smart):
|
||||||
|
'''Load different types of SMART outputs'''
|
||||||
|
|
||||||
|
ret = dict()
|
||||||
|
ret.update({ "temperature" : 0 })
|
||||||
|
ret.update({ "critical_warning" : 0 })
|
||||||
|
ret.update({ "unsafe_shutdowns" : 0 })
|
||||||
|
ret.update({ "power_cycles" : 0 })
|
||||||
|
ret.update({ "power_on_hours" : 0 })
|
||||||
|
ret.update({ "available_spare" : 100 })
|
||||||
|
ret.update({ "wearleveling_count" : 100 })
|
||||||
|
|
||||||
|
if "ata_smart_attributes" in smart:
|
||||||
|
|
||||||
|
# get main table #
|
||||||
|
table = smart["ata_smart_attributes"]["table"]
|
||||||
|
|
||||||
|
# temperatur #
|
||||||
|
ret["temperature"] = smart["temperature"]["current"]
|
||||||
|
|
||||||
|
for el in table:
|
||||||
|
|
||||||
|
# look for relevant metrics #
|
||||||
|
name = el["name"].lower()
|
||||||
|
target_name = el["name"].lower() # name in return map
|
||||||
|
|
||||||
|
# handle value mapping #
|
||||||
|
use_raw = False
|
||||||
|
if name == "used_rsvd_blk_cnt_tot":
|
||||||
|
target_name = "available_spare"
|
||||||
|
elif name == "power_cylce_count":
|
||||||
|
target_name = "power_cycles"
|
||||||
|
use_raw = True
|
||||||
|
elif name == "power_on_hours":
|
||||||
|
target_name = "power_on_hours"
|
||||||
|
use_raw = True
|
||||||
|
elif name == "perc_avail_resrvd_space":
|
||||||
|
target_name = "available_spare"
|
||||||
|
|
||||||
|
# check if metric should be recorded #
|
||||||
|
if target_name in ret:
|
||||||
|
|
||||||
|
# set return dict #
|
||||||
|
if use_raw:
|
||||||
|
value = el["raw"]["value"]
|
||||||
|
else:
|
||||||
|
value = el["value"]
|
||||||
|
|
||||||
|
ret[target_name] = value
|
||||||
|
|
||||||
|
if ret["critical_warning"] == 0 and "smart_status" in smart:
|
||||||
|
ret["critical_warning"] = int(not smart["smart_status"]["passed"])
|
||||||
|
|
||||||
|
return ret
|
||||||
@@ -1,5 +1,6 @@
|
|||||||
body{
|
body{
|
||||||
background: radial-gradient(ellipse at center, #47918a 0%, #0b3161 100%);
|
background: radial-gradient(ellipse at center, #47918a 0%, #0b3161 100%);
|
||||||
|
background-attachment: fixed;
|
||||||
color: aliceblue !important;
|
color: aliceblue !important;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -155,6 +156,22 @@ body{
|
|||||||
cursor: pointer;
|
cursor: pointer;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
.smart-info{
|
||||||
|
font-family: monospace;
|
||||||
|
|
||||||
|
padding-top: 2px;
|
||||||
|
padding-top: 2px;
|
||||||
|
padding-left: 5px;
|
||||||
|
|
||||||
|
padding-left: 5px;
|
||||||
|
|
||||||
|
color: black;
|
||||||
|
|
||||||
|
border: none;
|
||||||
|
outline: none;
|
||||||
|
cursor: auto;
|
||||||
|
}
|
||||||
|
|
||||||
.box{
|
.box{
|
||||||
border-style: solid;
|
border-style: solid;
|
||||||
border-width: 1px;
|
border-width: 1px;
|
||||||
|
|||||||
@@ -37,6 +37,9 @@
|
|||||||
</br>
|
</br>
|
||||||
{{ form.timeout.label }} {{ form.timeout() }} </br>
|
{{ form.timeout.label }} {{ form.timeout() }} </br>
|
||||||
|
|
||||||
|
</br>
|
||||||
|
{{ form.special_type.label }} {{ form.special_type() }} </br>
|
||||||
|
|
||||||
{% if is_modification %}
|
{% if is_modification %}
|
||||||
<input class="form-button mt-4" type="submit" value="Send Modification">
|
<input class="form-button mt-4" type="submit" value="Send Modification">
|
||||||
{% else %}
|
{% else %}
|
||||||
|
|||||||
@@ -8,7 +8,11 @@
|
|||||||
<a href="/service-details?service={{ status.service}}"
|
<a href="/service-details?service={{ status.service}}"
|
||||||
class="col-md-5 m-3 p-2 border rounded overview-tile"
|
class="col-md-5 m-3 p-2 border rounded overview-tile"
|
||||||
{% if status.status == "OK" %}
|
{% if status.status == "OK" %}
|
||||||
style="background-color: lightgreen;"
|
{% if status.info_text == "Submitted from Web-Interface" %}
|
||||||
|
style="background-color: #5cffe0;"
|
||||||
|
{% else %}
|
||||||
|
style="background-color: lightgreen;"
|
||||||
|
{% endif %}
|
||||||
{% elif status.status == "WARNING" %}
|
{% elif status.status == "WARNING" %}
|
||||||
style="background-color: orange;"
|
style="background-color: orange;"
|
||||||
{% elif status.status == "CRITICAL" %}
|
{% elif status.status == "CRITICAL" %}
|
||||||
|
|||||||
@@ -56,8 +56,8 @@
|
|||||||
|
|
||||||
<div class="last-status">
|
<div class="last-status">
|
||||||
{% if status_list | length > 0 %}
|
{% if status_list | length > 0 %}
|
||||||
<p class="{{ status_list[0].status }}">
|
<p class="{{ status_list[0][1].status }}">
|
||||||
{{ status_list[0].status }} submitted on {{ status_list[0].human_date() }}
|
{{ status_list[0][1].status }} submitted on {{ status_list[0][1].human_date() }}
|
||||||
</p>
|
</p>
|
||||||
{% else %}
|
{% else %}
|
||||||
<p style="color: darkred;">No status for this service submitted</p>
|
<p style="color: darkred;">No status for this service submitted</p>
|
||||||
@@ -71,8 +71,39 @@
|
|||||||
class="service-token">Secret Token: {{ service.token }}</div>
|
class="service-token">Secret Token: {{ service.token }}</div>
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
|
{% if service.special_type == "SMART" %}
|
||||||
|
<div class="clear smart-info mt-3" style="background-color: orange;">
|
||||||
|
Smart Monitor {% if smart %} for: {{ smart.model_number }} {% endif %}
|
||||||
|
</div>
|
||||||
|
<div class="clear smart-info mt-3" style="background-color: orange;">
|
||||||
|
Example below requires smartmontools ("smartctl") in PATH.
|
||||||
|
On Linux this is usually available via the package manager,
|
||||||
|
on Windows install it from the
|
||||||
|
<a style="text-decoration: underline; color: #5000e1; font-weight: bold;" href="https://www.smartmontools.org/wiki/Download#InstalltheWindowspackage">offical page</a>.
|
||||||
|
</div>
|
||||||
|
{% endif %}
|
||||||
|
|
||||||
|
{% if service.special_type == "SMART" %}
|
||||||
|
<h5 class="clear my-4">Linux</h5>
|
||||||
|
{% else %}
|
||||||
<h5 class="clear my-4">Curl</h5>
|
<h5 class="clear my-4">Curl</h5>
|
||||||
|
{% endif %}
|
||||||
<div class="ml-3 example">
|
<div class="ml-3 example">
|
||||||
|
{% if service.special_type == "SMART" %}
|
||||||
|
SMART='{ <br>
|
||||||
|
<div class="example-indent">
|
||||||
|
"service" : "{{ service.service }}", <br>
|
||||||
|
"token" : "{{ service.token }}", <br>
|
||||||
|
"status" : "N/A", <br>
|
||||||
|
"smart" : '$(/sbin/smartctl -a /dev/nvme0n1 --json)' <br>
|
||||||
|
</div>
|
||||||
|
}' <br><br>
|
||||||
|
curl -X POST -H "Content-Type: application/json" \ <br>
|
||||||
|
<div class="example-indent">
|
||||||
|
--data "${SMART}" \ <br>
|
||||||
|
{{ flask.request.url_root.replace("http://", "https://" )}}report
|
||||||
|
</div>
|
||||||
|
{% else %}
|
||||||
curl -X POST \ <br>
|
curl -X POST \ <br>
|
||||||
<div class="example-indent">
|
<div class="example-indent">
|
||||||
-H "Content-Type: application/json" \ <br>
|
-H "Content-Type: application/json" \ <br>
|
||||||
@@ -81,8 +112,39 @@
|
|||||||
"status" : "OK", "info" : "Free Text Information here" }' \<br>
|
"status" : "OK", "info" : "Free Text Information here" }' \<br>
|
||||||
{{ flask.request.url_root.replace("http://", "https://" )}}report
|
{{ flask.request.url_root.replace("http://", "https://" )}}report
|
||||||
</div>
|
</div>
|
||||||
|
{% endif %}
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
|
{% if service.special_type == "SMART" %}
|
||||||
|
<h5 class="my-4">Windows</h5>
|
||||||
|
<div class="ml-3 example">
|
||||||
|
$SMART = @{ <br>
|
||||||
|
<div class="example-indent">
|
||||||
|
service = "{{ service.service }}"<br>
|
||||||
|
token = "{{ service.token }}"<br>
|
||||||
|
status = "N/A"<br>
|
||||||
|
smart = "$(smartctl -a C: --json | Out-String)"<br>
|
||||||
|
</div>
|
||||||
|
} | ConvertTo-Json<br><br>
|
||||||
|
Invoke-RestMethod -TimeoutSec 2 -Uri "{{ flask.request.url_root.replace("http://", "https://" )}}report" -Method Post -Headers @{"Content-Type"="application/json"} -Body $SMART
|
||||||
|
</div>
|
||||||
|
<!-- register task example -->
|
||||||
|
<h5 class="my-4">Windows Task (requires Admin-Powershell)</h5>
|
||||||
|
<div class="ml-3 example">
|
||||||
|
$ScriptPath = Join-Path $HOME -ChildPath "smart_monitor.ps1" <br>
|
||||||
|
echo '$SMART = @{ <br>
|
||||||
|
<div class="example-indent">
|
||||||
|
service = "{{ service.service }}"<br>
|
||||||
|
token = "{{ service.token }}"<br>
|
||||||
|
status = "N/A"<br>
|
||||||
|
smart = "$(smartctl -a C: --json | Out-String)"<br>
|
||||||
|
</div>
|
||||||
|
} | ConvertTo-Json<br><br>
|
||||||
|
Invoke-RestMethod -TimeoutSec 2 -Uri "{{ flask.request.url_root.replace("http://", "https://" )}}report" -Method Post -Headers @{"Content-Type"="application/json"} -Body $SMART' > $ScriptPath <br>
|
||||||
|
schtasks /create /tn SMART_Monitor /tr "powershell.exe -executionpolicy bypass -File '$ScriptPath'" /sc hourly /mo 1 /ru "Administratoren"<br>
|
||||||
|
echo "Done" <br>
|
||||||
|
</div>
|
||||||
|
{% else %}
|
||||||
<h5 class="my-4">Python</h5>
|
<h5 class="my-4">Python</h5>
|
||||||
<div class="ml-3 example">
|
<div class="ml-3 example">
|
||||||
import requests<br>
|
import requests<br>
|
||||||
@@ -97,6 +159,7 @@
|
|||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
|
{% endif %}
|
||||||
|
|
||||||
<table class="mb-4 mt-5 status-table">
|
<table class="mb-4 mt-5 status-table">
|
||||||
<thead>
|
<thead>
|
||||||
@@ -106,12 +169,19 @@
|
|||||||
</thead>
|
</thead>
|
||||||
|
|
||||||
<tbody class="mt-2">
|
<tbody class="mt-2">
|
||||||
{% for status in status_list %}
|
{% for status_tupel in status_list %}
|
||||||
<tr>
|
<tr>
|
||||||
<td>{{ status.human_date() }}</td>
|
<td>{{ status_tupel[1].human_date() }}</td>
|
||||||
<td class="{{ status.status }}">{{ status.status }}</td>
|
<td class="{{ status_tupel[1].status }}">{{ status_tupel[1].status }}</td>
|
||||||
<td>{{ status.info_text }}</td>
|
<td>{{ status_tupel[1].info_text }}</td>
|
||||||
</tr>
|
</tr>
|
||||||
|
{% if status_tupel[0] > 1 %}
|
||||||
|
<tr>
|
||||||
|
<td>---</td>
|
||||||
|
<td><i> + {{ status_tupel[0] }} identical reports</i></td>
|
||||||
|
<td>|</td>
|
||||||
|
</tr>
|
||||||
|
{% endif %}
|
||||||
{% endfor %}
|
{% endfor %}
|
||||||
</tbody>
|
</tbody>
|
||||||
</table>
|
</table>
|
||||||
|
|||||||
Reference in New Issue
Block a user