update: add build schedule

.github/workflows/main.yaml

@@ -4,6 +4,8 @@ on:
   push:
     branches:
       - "master"
+  schedule:
+      - cron: "0 2 * * 0"
 
 jobs:
   docker:

icingatools.py

@@ -41,6 +41,9 @@ def _build_service_name(user, async_service_name):
 
 def create_service(user, async_service_name, app):
 
+    if not app.config.get("ICINGA_API_URL"):
+        return
+
     client = _create_client(app)
     name = _build_service_name(user, async_service_name)
     host_name = app.config["ASYNC_ICINGA_DUMMY_HOST"]
@@ -67,6 +70,9 @@ def create_service(user, async_service_name, app):
 
 def delete_service(user, async_service_name, app):
 
+    if not app.config.get("ICINGA_API_URL"):
+        return
+
     client = _create_client(app)
     name = _build_service_name(user, async_service_name)
     host_name = app.config["ASYNC_ICINGA_DUMMY_HOST"]
@@ -83,7 +89,15 @@ def build_icinga_link_for_service(user, service_name, static_configured, app):
         url_fmt = "{base}/icingaweb2/monitoring/list/services?service={service}&modifyFilter=1"
         name = service_name
 
-    return url_fmt.format(base=app.config["ICINGA_WEB_URL"],
-                            host=app.config["ASYNC_ICINGA_DUMMY_HOST"],
+    icinga_web_url = app.config.get("ICINGA_WEB_URL")
+    if not icinga_web_url:
+        icinga_web_url = "ICINGA_WEB_URL_NOT_SET:"
+
+    dummy_host=app.config.get("ASYNC_ICINGA_DUMMY_HOST")
+    if not dummy_host:
+        dummy_host = "ASYNC_ICINGA_DUMMY_HOST_NOT_SET:"
+
+    return url_fmt.format(base=icinga_web_url,
+                            host=dummy_host,
                             service=name)
 

server.py

@@ -12,7 +12,7 @@ import secrets
 
 import flask_wtf
 from flask_wtf import FlaskForm
-from wtforms import StringField, SubmitField, BooleanField, DecimalField, HiddenField
+from wtforms import StringField, SubmitField, BooleanField, DecimalField, HiddenField, SelectField
 from wtforms.validators import DataRequired, Length
 
 from sqlalchemy import Column, Integer, String, Boolean, or_, and_
@@ -24,6 +24,7 @@ from flask_sqlalchemy import SQLAlchemy
 from sqlalchemy.sql.expression import func
 
 import icingatools
+import smarttools
 
 app = flask.Flask("Icinga Report In Gateway")
 
@@ -41,6 +42,7 @@ class Service(db.Model):
     token   = Column(String)
     timeout = Column(Integer)
     owner   = Column(String)
+    special_type = Column(String)
 
     staticly_configured = Column(Boolean)
 
@@ -57,6 +59,22 @@ class Status(db.Model):
         dt = datetime.datetime.fromtimestamp(self.timestamp)
         return dt.strftime("%d. %B %Y at %H:%M")
 
+class SMARTStatus(db.Model):
+
+    __tablename__ = "smart"
+
+    service     = Column(String, primary_key=True)
+    timestamp   = Column(Integer, primary_key=True)
+    power_cycles = Column(Integer)
+    temperature = Column(Integer)
+    available_spare  = Column(Integer)
+    unsafe_shutdowns  = Column(Integer)
+    critical_warning = Column(Integer)
+    model_number = Column(String)
+    power_cycles = Column(Integer)
+    power_on_hours = Column(Integer)
+    wearleveling_count = Column(Integer)
+
 def buildReponseDict(status, service=None):
 
     if not status:
@@ -108,6 +126,7 @@ class EntryForm(FlaskForm):
 
     service = StringField("Service Name")
     service_hidden = HiddenField("service_hidden")
+    special_type = SelectField("Type", choices=["Default", "SMART"])
     timeout = DecimalField("Timeout in days", default=30)
 
 def create_entry(form, user):
@@ -116,9 +135,21 @@ def create_entry(form, user):
 
     service_name = form.service.data or form.service_hidden.data
 
+    # keep token if modification #
+    s_tmp = db.session.query(Service).filter(Service.service == service_name).first()
+    if s_tmp:
+        token = s_tmp.token
+        if not token:
+            raise AssertionError("WTF Service without Token {}".format(service_name))
+
     day_delta = datetime.timedelta(days=int(form.timeout.data))
+
+    special_type = form.special_type.data
+    if form.special_type == "Default":
+        special_type = None
+
     service = Service(service=service_name, timeout=day_delta.total_seconds(),
-                        owner=user, token=token)
+                        owner=user, token=token, special_type=special_type)
 
     # service.data set = create, service_hidden.data = modify #
     if form.service.data:
@@ -148,8 +179,11 @@ def service_details():
     icinga_link = icingatools.build_icinga_link_for_service(user, service.service,
                         service.staticly_configured, app)
 
+    smart_entry_list = db.session.query(SMARTStatus).filter(SMARTStatus.service==service.service)
+    smart_entry = smart_entry_list.order_by(SMARTStatus.timestamp.desc()).first()
+
     return flask.render_template("service_info.html", service=service, flask=flask,
-                                    user=user, status_list=status_list, icinga_link=icinga_link)
+                    user=user, status_list=status_list, icinga_link=icinga_link, smart=smart_entry)
 
 
 @app.route("/entry-form", methods=["GET", "POST", "DELETE"])
@@ -158,7 +192,7 @@ def create_interface():
     user = str(flask.request.headers.get("X-Forwarded-Preferred-Username"))
 
     # check if is delete #
-    operation = flask.request.args.get("operation") 
+    operation = flask.request.args.get("operation")
     if operation and operation == "delete" :
 
         service_delete_name = flask.request.args.get("service")
@@ -175,13 +209,14 @@ def create_interface():
         return flask.redirect("/overview")
 
     form = EntryForm()
-   
+
     # handle modification #
     modify_service_name = flask.request.args.get("service")
     if modify_service_name:
         service = db.session.query(Service).filter(Service.service == modify_service_name).first()
         if service and service.owner == user:
             form.service.default = service.service
+            form.special_type.default = service.special_type
             form.timeout.default = datetime.timedelta(seconds=service.timeout).days
             form.service_hidden.default = service.service
             form.process()
@@ -252,7 +287,7 @@ def default():
             if not lastSuccess.timestamp == 0 and delta > timeout and latestInfoIsSuccess:
 
                 # lastes info is success but timed out #
-                lastSuccess.info_text = "Service {} overdue since {}".format(service, str(delta)) 
+                lastSuccess.info_text = "Service {} overdue since {}".format(service, str(delta))
                 if timeout/delta > 0.9 or (delta - timeout) < datetime.timedelta(hours=12):
                     lastSuccess.status = "WARNING"
                 else:
@@ -270,16 +305,27 @@ def default():
     elif flask.request.method == "POST":
 
         # get variables #
-        service   = flask.request.json["service"]
-        token     = flask.request.json["token"]
-        status    = flask.request.json["status"]
-        text      = flask.request.json["info"]
+        service   = flask.request.json.get("service")
+        token     = flask.request.json.get("token")
+        status    = flask.request.json.get("status")
+        text      = flask.request.json.get("info") or "no_info"
         timestamp = datetime.datetime.now().timestamp()
 
+        smart = flask.request.json.get("smart")
+
+        # check smart json quoting problems #
+        if smart and type(smart) == str:
+            try:
+                smart = json.loads(smart)
+            except json.decoder.JSONDecodeError as e:
+                return ("Error in SMART-json {}".format(e), 415)
+
         if not service:
             return ("'service' ist empty field in json", 400)
         elif not token:
             return ("'token' ist empty field in json", 400)
+        elif not status and not smart:
+            return ("'status' is empty field in json", 400)
 
         # verify token & service in config #
         verifiedServiceObj = db.session.query(Service).filter(
@@ -288,15 +334,94 @@ def default():
         if not verifiedServiceObj:
             return ("Service ({}) with this token ({}) not found in DB".format(service, token), 401)
         else:
-            status = Status(service=service, timestamp=timestamp, status=status, info_text=text)
+
+            # handle a SMART-record submission (with errorhandling) #
+            if smart and not verifiedServiceObj.special_type == "SMART":
+                return ("SMART Field for non-SMART type service", 415)
+            elif smart:
+                text, status = record_and_check_smart(verifiedServiceObj,
+                                                        timestamp, smart)
+
+            status = Status(service=service, timestamp=timestamp, status=status, 
+                                info_text=text)
             db.session.merge(status)
             db.session.commit()
             return ("", 204)
     else:
         return ("Method not implemented: {}".format(flask.request.method), 405)
 
+def record_and_check_smart(service, timestamp, smart):
+
+    if "nvme_smart_health_information_log" in smart:
+        health_info = smart["nvme_smart_health_information_log"]
+    else:
+        health_info = smarttools.normalize(smart)
+
+    if not service.special_type == "SMART":
+        raise AssertionError("Trying to record SMART-record for non-SMART service")
+
+    # record the status #
+    smart_status = SMARTStatus(service=service.service, timestamp=timestamp,
+                        temperature=health_info["temperature"],
+                        critical_warning=health_info["critical_warning"],
+                        unsafe_shutdowns=health_info["unsafe_shutdowns"],
+                        power_cycles=health_info["power_cycles"],
+                        power_on_hours=health_info["power_on_hours"],
+                        available_spare=health_info.get("available_spare"),
+                        model_number=smart.get("model_name"),
+                        wearleveling_count=health_info.get("wearleveling_count"))
+
+    db.session.add(smart_status)
+    db.session.commit()
+
+    # check the status #
+    smart_last_query = db.session.query(SMARTStatus)
+    smart_last_query = smart_last_query.filter(SMARTStatus.service==service.service)
+    smart_last = smart_last_query.order_by(sqlalchemy.desc(SMARTStatus.timestamp)).first()
+    smart_second_last = smart_last_query.order_by(sqlalchemy.desc(
+                            SMARTStatus.timestamp)).offset(1).first()
+
+    # last record (max 6 months ago) #
+    timestampt_minus_6m = datetime.datetime.now() - datetime.timedelta(days=180)
+    smart_old_query = smart_last_query.filter(
+                        SMARTStatus.timestamp > timestampt_minus_6m.timestamp())
+    smart_old = smart_old_query.order_by(sqlalchemy.asc(SMARTStatus.timestamp)).first()
+
+    # critial != 0 #
+    if smart_last.critical_warning != 0:
+        return ("SMART reports disk critical => oO better do something about this", "CRITICAL")
+
+    # wearleveling < 20% (SAMSUNG only) #
+    if smart_last.wearleveling_count and smart_last.wearleveling_count <= 20:
+        return ("SMART report prefail disk (wear_level < 20%)", "CRITICAL")
+
+    # temp max > X #
+    if smart_last.temperature > 60:
+        return ("Disk Temperatur {}".format(smart_last.temperature), "CRITICAL")
+
+    # available_SSD spare #
+    spare_change = smart_old.available_spare - smart_last.available_spare
+
+    if smart_last.available_spare <= 25:
+        return ("SSD spare <25 ({}) YOUR DISK WILL DIE SOON".format(spare_change),
+                    "CRITICAL")
+    elif smart_last.available_spare <= 50:
+        return ("SSD spare <50 ({})".format(spare_change), "WARNING")
+    elif spare_change >= 10:
+        return ("Strong degration in SSD spare space ({} in under 6 months)".format(
+                    spare_change), "WARNING")
+
+    # unsafe_shutdowns +1 #
+    if(smart_second_last and 
+          smart_second_last.unsafe_shutdowns - smart_last.unsafe_shutdowns >= 1):
+        return ("Disk had {} unsafe shutdowns".format(smart_last.unsafe_shutdowns),
+                    "WARNING")
+
+    return ("{} - no problems detected".format(smart_last.model_number), "OK")
+
+
 def create_app():
-    
+
     db.create_all()
     config = {}
 
@@ -329,14 +454,17 @@ def create_app():
     for key in config:
         timeout = timeparse.timeparse(config[key]["timeout"])
         staticly_configured = True
-        db.session.merge(Service(service=key, token=config[key]["token"], 
+        db.session.merge(Service(service=key, token=config[key]["token"],
                                     staticly_configured=staticly_configured, timeout=timeout,
                                     owner=config[key]["owner"]))
         db.session.commit()
 
-    # create dummy host #
-    icingatools.create_master_host(app)
-        
+    # create icinga host #
+    if not app.config.get("ICINGA_API_URL"):
+        print("ICINGA_API_URL not defined. Not connecting Icinga", file=sys.stderr)
+    else:
+        icingatools.create_master_host(app)
+
 
 if __name__ == "__main__":
 

smarttools.py

@@ -0,0 +1,54 @@
+def normalize(smart):
+    '''Load different types of SMART outputs'''
+
+    ret = dict()
+    ret.update({ "temperature"      : 0 })
+    ret.update({ "critical_warning" : 0 })
+    ret.update({ "unsafe_shutdowns" : 0 })
+    ret.update({ "power_cycles"     : 0 })
+    ret.update({ "power_on_hours"   : 0 })
+    ret.update({ "available_spare"  : 100 })
+    ret.update({ "wearleveling_count" : 100 })
+
+    if "ata_smart_attributes" in smart:
+
+        # get main table #
+        table = smart["ata_smart_attributes"]["table"]
+        
+        # temperatur #
+        ret["temperature"] = smart["temperature"]["current"]
+
+        for el in table:
+        
+            # look for relevant metrics #
+            name = el["name"].lower()
+            target_name = el["name"].lower() # name in return map
+
+            # handle value mapping #
+            use_raw = False
+            if name == "used_rsvd_blk_cnt_tot":
+                target_name = "available_spare"
+            elif name == "power_cylce_count":
+                target_name = "power_cycles"
+                use_raw = True
+            elif name == "power_on_hours":
+                target_name = "power_on_hours"
+                use_raw = True
+            elif name == "perc_avail_resrvd_space":
+                target_name = "available_spare"
+
+            # check if metric should be recorded #
+            if target_name in ret:
+
+                # set return dict #
+                if use_raw:
+                    value = el["raw"]["value"]
+                else:
+                    value = el["value"]
+
+                ret[target_name] = value
+
+        if ret["critical_warning"] == 0 and "smart_status" in smart:
+            ret["critical_warning"] = int(not smart["smart_status"]["passed"])
+
+    return ret

static/site.css

@@ -1,5 +1,6 @@
 body{
     background: radial-gradient(ellipse at center, #47918a 0%, #0b3161 100%);
+    background-attachment: fixed;
     color: aliceblue !important;
 }
 
@@ -155,6 +156,22 @@ body{
     cursor: pointer;
 }
 
+.smart-info{
+    font-family: monospace;
+
+    padding-top: 2px;
+    padding-top: 2px;
+    padding-left: 5px;
+    
+    padding-left: 5px;
+
+    color: black;
+
+    border: none;
+    outline: none;
+    cursor: auto;
+}
+
 .box{
     border-style: solid;
     border-width: 1px;

templates/add_modify_service.html

@@ -37,6 +37,9 @@
                 </br>
                 {{ form.timeout.label }}    {{ form.timeout() }}        </br>
 
+                </br>
+                {{ form.special_type.label }}    {{ form.special_type() }}        </br>
+
                 {% if is_modification %}
                 <input class="form-button mt-4" type="submit" value="Send Modification">
                 {% else %}

templates/service_info.html

@@ -71,8 +71,39 @@
                      class="service-token">Secret Token: {{ service.token }}</div>
             </div>
 
+            {% if service.special_type == "SMART" %}
+            <div class="clear smart-info mt-3" style="background-color: orange;">
+                Smart Monitor {% if smart %} for: {{ smart.model_number }} {% endif %}
+            </div>
+            <div class="clear smart-info mt-3" style="background-color: orange;">
+                Example below requires smartmontools ("smartctl") in PATH.
+                On Linux this is usually available via the package manager,
+                on Windows install it from the 
+                <a style="text-decoration: underline; color: #5000e1; font-weight: bold;" href="https://www.smartmontools.org/wiki/Download#InstalltheWindowspackage">offical page</a>.
+            </div>
+            {% endif %}
+
+            {% if service.special_type == "SMART" %}
+            <h5 class="clear my-4">Linux</h5>
+            {% else %}
             <h5 class="clear my-4">Curl</h5>
+            {% endif %}
             <div class="ml-3 example">
+                {% if service.special_type == "SMART" %}
+                SMART='{ <br>
+                <div class="example-indent">
+                    "service" : "{{ service.service }}", <br>
+                    "token"   : "{{ service.token }}", <br>
+                    "status"  : "N/A", <br>
+                    "smart"   : '$(/sbin/smartctl -a /dev/nvme0n1 --json)' <br>
+                </div>
+                }' <br><br>
+                curl -X POST -H "Content-Type: application/json" \ <br>
+                    <div class="example-indent">
+                    --data "${SMART}" \ <br>
+                    {{ flask.request.url_root.replace("http://", "https://" )}}report
+                    </div>
+                {% else %}
                 curl -X POST \ <br>
                 <div class="example-indent">
                     -H "Content-Type: application/json" \ <br>
@@ -81,8 +112,39 @@
                     "status" : "OK", "info" : "Free Text Information here" }' \<br>
                     {{ flask.request.url_root.replace("http://", "https://" )}}report
                 </div>
+                {% endif %}
             </div>
 
+            {% if service.special_type == "SMART" %}
+            <h5 class="my-4">Windows</h5>
+            <div class="ml-3 example">
+                $SMART = @{ <br>
+                <div class="example-indent">
+                  service = "{{ service.service }}"<br>
+                  token = "{{ service.token }}"<br>
+                  status = "N/A"<br>
+                  smart = "$(smartctl -a C: --json | Out-String)"<br>
+                </div>
+                } | ConvertTo-Json<br><br>
+                Invoke-RestMethod -TimeoutSec 2 -Uri "{{ flask.request.url_root.replace("http://", "https://" )}}report" -Method Post -Headers @{"Content-Type"="application/json"} -Body $SMART
+            </div>
+            <!-- register task example -->
+            <h5 class="my-4">Windows Task (requires Admin-Powershell)</h5>
+            <div class="ml-3 example">
+                $ScriptPath = Join-Path $HOME -ChildPath "smart_monitor.ps1" <br>
+                echo '$SMART = @{ <br>
+                <div class="example-indent">
+                  service = "{{ service.service }}"<br>
+                  token = "{{ service.token }}"<br>
+                  status = "N/A"<br>
+                  smart = "$(smartctl -a C: --json | Out-String)"<br>
+                </div>
+                } | ConvertTo-Json<br><br>
+                Invoke-RestMethod -TimeoutSec 2 -Uri "{{ flask.request.url_root.replace("http://", "https://" )}}report" -Method Post -Headers @{"Content-Type"="application/json"} -Body $SMART' &gt; $ScriptPath <br>
+                schtasks /create /tn SMART_Monitor /tr "powershell.exe -executionpolicy bypass -File '$ScriptPath'" /sc hourly /mo 1 /ru "Administratoren"<br>
+            echo "Done" <br>
+            </div>
+            {% else %}
             <h5 class="my-4">Python</h5>
             <div class="ml-3 example">
                     import requests<br>
@@ -97,6 +159,7 @@
                     </div>
                 </div>
             </div>
+            {% endif %}
 
             <table class="mb-4 mt-5 status-table">
             <thead>