fix: rename status_list/recen_entries

.github/workflows/main.yaml

@@ -4,6 +4,8 @@ on:
   push:
     branches:
       - "master"
+  schedule:
+      - cron: "0 2 * * 0"
 
 jobs:
   docker:
@@ -14,9 +16,6 @@ jobs:
       -
         name: Checkout
         uses: actions/checkout@v3
-      -
-        name: Set up QEMU
-        uses: docker/setup-qemu-action@v2
       -
         name: Set up Docker Buildx
         uses: docker/setup-buildx-action@v2

icingatools.py

@@ -1,5 +1,15 @@
 import icinga2api
 import icinga2api.client
+from urllib.parse import urlparse
+
+def split_url(url):
+    parsed = urlparse(url)
+
+    http_vhost = parsed.hostname
+    http_uri = parsed.path or "/"
+    http_ssl = parsed.scheme == "https"
+
+    return http_vhost, http_uri, http_ssl
 
 def _create_client(app):
 
@@ -39,7 +49,7 @@ def _build_service_name(user, async_service_name):
     
     return "{}_async_{}".format(user, async_service_name)
 
-def create_service(user, async_service_name, app):
+def create_service(user, async_service_name, app, webcheck=False):
 
     if not app.config.get("ICINGA_API_URL"):
         return
@@ -48,20 +58,43 @@ def create_service(user, async_service_name, app):
     name = _build_service_name(user, async_service_name)
     host_name = app.config["ASYNC_ICINGA_DUMMY_HOST"]
 
-    service_config = {
+    # TODO: query service from DB
-        "templates": ["generic-service"],
+    accepted_return_codes = [200, 204]
-        "attrs": {
+
-            "display_name": name,
+
-            "check_command": "gateway",
+    if webcheck:
-            "host_name" : host_name,
+        http_vhost, http_uri, http_ssl = split_url(url)
-            "vars" : {
+        service_config = {
-                "host" : "async-icinga.atlantishq.de",
+            "templates": ["generic-service"],
-                "service_name" : async_service_name,
+            "attrs": {
-                "protocol" : "https",
+                "display_name": name,
-                "owner" : user
+                "check_command": "http",
+                "host_name": host_name,
+                "vars": {
+                    "http_vhost": http_vhost,
+                    "http_uri": http_uri,
+                    "http_expect": http_expect,
+                    "http_accept_status": accepted_return_codes, # array
+                    "http_ssl": True,
+                    "http_sni": True
+                }
+            }
+        }
+    else:
+        service_config = {
+            "templates": ["generic-service"],
+            "attrs": {
+                "display_name": name,
+                "check_command": "gateway",
+                "host_name" : host_name,
+                "vars" : {
+                    "host" : "async-icinga.atlantishq.de",
+                    "service_name" : async_service_name,
+                    "protocol" : "https",
+                    "owner" : user
+                }
             }
         }
-    }
     
     # Create the service (name is required in this format)
     service_api_helper_name = "{}!{}".format(host_name, name)

req.txt

@@ -5,3 +5,5 @@ flask-wtf
 waitress
 requests
 icinga2api
+psycopg2-binary
+tzdata

server.py

@@ -9,19 +9,22 @@ import datetime
 import pytimeparse.timeparse as timeparse
 import sys
 import secrets
+import zoneinfo
 
 import flask_wtf
 from flask_wtf import FlaskForm
 from wtforms import StringField, SubmitField, BooleanField, DecimalField, HiddenField, SelectField
 from wtforms.validators import DataRequired, Length
 
-from sqlalchemy import Column, Integer, String, Boolean, or_, and_
+from sqlalchemy import Column, Integer, String, Boolean, or_, and_, desc
 from sqlalchemy.orm import sessionmaker
 from sqlalchemy.exc import IntegrityError
 from sqlalchemy.sql import func
 import sqlalchemy
 from flask_sqlalchemy import SQLAlchemy
 from sqlalchemy.sql.expression import func
+from sqlalchemy.exc import IntegrityError
+from psycopg2.errors import UniqueViolation
 
 import icingatools
 import smarttools
@@ -29,11 +32,28 @@ import smarttools
 app = flask.Flask("Icinga Report In Gateway")
 
 
-app.config['SQLALCHEMY_DATABASE_URI'] = 'sqlite:///database.sqlite'
+app.config['SQLALCHEMY_DATABASE_URI'] = os.environ.get('SQLALCHEMY_DATABASE_URI') or 'sqlite:///database.sqlite'
 app.config['JSON_CONFIG_FILE'] = 'services.json'
 app.config['JSON_CONFIG_DIR'] = 'config'
+app.config['TIME_ZONE'] = zoneinfo.ZoneInfo(os.getenv("TIME_ZONE", "UTC"))
+app.config['AUTH_HEADER'] = os.environ.get("AUTH_HEADER") or "X-Forwarded-Preferred-Username"
 db = SQLAlchemy(app)
 
+from urllib.parse import urlparse
+
+def split_url(url: str):
+    parsed = urlparse(url)
+
+    http_vhost = parsed.hostname
+    http_uri = parsed.path or "/"
+    http_ssl = parsed.scheme == "https"
+
+    return {
+        "http_vhost": http_vhost,
+        "http_uri": http_uri,
+        "http_ssl": http_ssl
+    }
+
 class Service(db.Model):
 
     __tablename__ = "services"
@@ -44,6 +64,11 @@ class Service(db.Model):
     owner   = Column(String)
     special_type = Column(String)
 
+    # web checks #
+    url = Column(String)
+    accepted_codes = Column(String)
+    http_expect = Column(String)
+
     staticly_configured = Column(Boolean)
 
 class Status(db.Model):
@@ -56,7 +81,7 @@ class Status(db.Model):
     info_text   = Column(String)
 
     def human_date(self):
-        dt = datetime.datetime.fromtimestamp(self.timestamp)
+        dt = datetime.datetime.fromtimestamp(self.timestamp, app.config["TIME_ZONE"])
         return dt.strftime("%d. %B %Y at %H:%M")
 
 class SMARTStatus(db.Model):
@@ -65,12 +90,12 @@ class SMARTStatus(db.Model):
 
     service     = Column(String, primary_key=True)
     timestamp   = Column(Integer, primary_key=True)
+    model_number = Column(String, primary_key=True)
     power_cycles = Column(Integer)
     temperature = Column(Integer)
     available_spare  = Column(Integer)
     unsafe_shutdowns  = Column(Integer)
     critical_warning = Column(Integer)
-    model_number = Column(String)
     power_cycles = Column(Integer)
     power_on_hours = Column(Integer)
     wearleveling_count = Column(Integer)
@@ -91,7 +116,7 @@ def buildReponseDict(status, service=None):
 @app.route('/overview')
 def overview():
 
-    user = str(flask.request.headers.get("X-Forwarded-Preferred-Username"))
+    user = str(flask.request.headers.get(app.config['AUTH_HEADER']))
 
     # query all services #
     services = db.session.query(Service).filter(Service.owner == user).all()
@@ -161,7 +186,7 @@ def create_entry(form, user):
 @app.route("/service-details")
 def service_details():
 
-    user = str(flask.request.headers.get("X-Forwarded-Preferred-Username"))
+    user = flask.request.headers.get(app.config['AUTH_HEADER'])
     service = flask.request.args.get("service")
 
     # query service #
@@ -173,9 +198,38 @@ def service_details():
     if service.owner and str(service.owner) != user:
         return ("Services is not owned by {}".format(user))
 
+    two_weeks_ago_ts = int((datetime.datetime.now() - datetime.timedelta(days=14)).timestamp())
     status_list_query = db.session.query(Status).filter(Status.service==service.service)
-    status_list = status_list_query.order_by(sqlalchemy.desc(Status.timestamp)).limit(20).all()
+    recent_query = (
+        status_list_query
+        .filter(Status.timestamp >= two_weeks_ago_ts, Status.service==service.service)
+        .order_by(desc(Status.timestamp))
+    ).limit(1000)
+    status_list = recent_query.all()
 
+    if not status_list:
+        status_list = status_list_query.order_by(sqlalchemy.desc(Status.timestamp)).limit(1000).all()
+
+    # build status tupel (repeats, status) #
+    current_tupel = None
+    prev_status = None
+    tupel_list = []
+    for s in status_list:
+
+        # set initial #
+        if not current_tupel:
+            current_tupel = [1, s]
+            tupel_list.append(current_tupel)
+            continue
+
+        if current_tupel[1].info_text == s.info_text:
+            current_tupel[0] += 1
+        else:
+            current_tupel = [1, s]
+            tupel_list.append(current_tupel)
+
+
+    print(tupel_list)
     icinga_link = icingatools.build_icinga_link_for_service(user, service.service,
                         service.staticly_configured, app)
 
@@ -183,13 +237,13 @@ def service_details():
     smart_entry = smart_entry_list.order_by(SMARTStatus.timestamp.desc()).first()
 
     return flask.render_template("service_info.html", service=service, flask=flask,
-                    user=user, status_list=status_list, icinga_link=icinga_link, smart=smart_entry)
+                    user=user, status_list=tupel_list, icinga_link=icinga_link, smart=smart_entry)
 
 
 @app.route("/entry-form", methods=["GET", "POST", "DELETE"])
 def create_interface():
 
-    user = str(flask.request.headers.get("X-Forwarded-Preferred-Username"))
+    user = flask.request.headers.get(app.config['AUTH_HEADER'])
 
     # check if is delete #
     operation = flask.request.args.get("operation")
@@ -224,9 +278,21 @@ def create_interface():
             return ("Not a valid service to modify", 404)
 
     if flask.request.method == "POST":
-        create_entry(form, user)
+
+        try:
+            create_entry(form, user)
+        except IntegrityError as e:
+            db.session.rollback()
+            # TODO: this only works for PG
+            if isinstance(e.orig, UniqueViolation):
+                return ("A service with this name already exists (possibly by another user)", 409)
+            else:
+                return (f"Error: {e}", 500)
+
+        # service created successfully #
         service_name = form.service.data or form.service_hidden.data
         return flask.redirect('/service-details?service={}'.format(service_name))
+
     else:
         return flask.render_template('add_modify_service.html', form=form,
                     is_modification=bool(modify_service_name))
@@ -345,7 +411,17 @@ def default():
             status = Status(service=service, timestamp=timestamp, status=status,
                                 info_text=text)
             db.session.merge(status)
-            db.session.commit()
+
+            try:
+                db.session.commit()
+            except IntegrityError as e:
+                db.session.rollback()
+                # TODO: this only works for PG
+                if isinstance(e.orig, UniqueViolation):
+                    return ("Status at this time already submitted", 409)
+                else:
+                    return (f"Error: {e}", 500)
+
             return ("", 204)
     else:
         return ("Method not implemented: {}".format(flask.request.method), 405)
@@ -396,7 +472,7 @@ def record_and_check_smart(service, timestamp, smart):
         return ("SMART report prefail disk (wear_level < 20%)", "CRITICAL")
 
     # temp max > X #
-    if smart_last.temperature > 50:
+    if smart_last.temperature > 60:
         return ("Disk Temperatur {}".format(smart_last.temperature), "CRITICAL")
 
     # available_SSD spare #
@@ -412,11 +488,12 @@ def record_and_check_smart(service, timestamp, smart):
                     spare_change), "WARNING")
 
     # unsafe_shutdowns +1 #
-    if smart_second_last.unsafe_shutdowns - smart_last.unsafe_shutdowns >= 1:
+    if(smart_second_last and
+          smart_second_last.unsafe_shutdowns - smart_last.unsafe_shutdowns >= 1):
         return ("Disk had {} unsafe shutdowns".format(smart_last.unsafe_shutdowns),
                     "WARNING")
 
-    return ("", "OK")
+    return ("{} - no problems detected".format(smart_last.model_number), "OK")
 
 
 def create_app():
@@ -447,8 +524,7 @@ def create_app():
                     config |= json.load(f)
 
     if not config:
-        print("No valid configuration found - need at least one service")
+        print("No static services configuration found - loading finished.")
-        return
 
     for key in config:
         timeout = timeparse.timeparse(config[key]["timeout"])
@@ -458,12 +534,34 @@ def create_app():
                                     owner=config[key]["owner"]))
         db.session.commit()
 
+    LOAD_FROM_ENV = [
+        "ICINGA_API_USER",
+        "ICINGA_API_PASS",
+        "ICINGA_API_URL",
+        "ICINGA_WEB_URL",
+        "ASYNC_ICINGA_DUMMY_HOST"
+    ]
+
+    enforce_load_from_env = os.environ.get("ENFORCE_LOAD_FROM_ENV") or ""
+    missing = [k for k in LOAD_FROM_ENV if k not in os.environ]
+    if missing and enforce_load_from_env.lower() == "true":
+        print(f"ENFORCE_LOAD_FROM_ENV is 'true' but we are missing: {missing} - Abort.")
+        sys.exit(1)
+
+    for key in LOAD_FROM_ENV:
+        if key in os.environ:
+            print(f"Loading/Overwriting {key} from environment", file=sys.stderr)
+            app.config[key] = os.environ[key]
+
+
     # create icinga host #
     if not app.config.get("ICINGA_API_URL"):
         print("ICINGA_API_URL not defined. Not connecting Icinga", file=sys.stderr)
     else:
         icingatools.create_master_host(app)
 
+    print(f"Expected AUTH_HEADER is: {app.config['AUTH_HEADER']}")
+
 
 if __name__ == "__main__":
 

smarttools.py

@@ -34,6 +34,8 @@ def normalize(smart):
             elif name == "power_on_hours":
                 target_name = "power_on_hours"
                 use_raw = True
+            elif name == "perc_avail_resrvd_space":
+                target_name = "available_spare"
 
             # check if metric should be recorded #
             if target_name in ret:
@@ -46,4 +48,7 @@ def normalize(smart):
 
                 ret[target_name] = value
 
+        if ret["critical_warning"] == 0 and "smart_status" in smart:
+            ret["critical_warning"] = int(not smart["smart_status"]["passed"])
+
     return ret

static/site.css

@@ -1,5 +1,6 @@
 body{
     background: radial-gradient(ellipse at center, #47918a 0%, #0b3161 100%);
+    background-attachment: fixed;
     color: aliceblue !important;
 }
 

templates/overview.html

@@ -8,7 +8,11 @@
                 <a href="/service-details?service={{ status.service}}"
                    class="col-md-5 m-3 p-2 border rounded overview-tile"
                         {% if status.status == "OK"  %}
-                            style="background-color: lightgreen;"
+                            {% if status.info_text == "Submitted from Web-Interface" %}
+                                style="background-color: #5cffe0;"
+                            {% else %}
+                                style="background-color: lightgreen;"
+                            {% endif %}
                         {% elif status.status == "WARNING"  %}
                             style="background-color: orange;"
                         {% elif status.status == "CRITICAL"  %}

templates/service_info.html

@@ -56,8 +56,8 @@
 
             <div class="last-status">
             {% if status_list | length > 0 %}
-            <p class="{{ status_list[0].status }}">
+            <p class="{{ status_list[0][1].status }}">
-                {{ status_list[0].status }} submitted on {{ status_list[0].human_date() }}
+                {{ status_list[0][1].status }} submitted on {{ status_list[0][1].human_date() }}
             </p>
             {% else %}
             <p style="color: darkred;">No status for this service submitted</p>
@@ -83,13 +83,13 @@
             </div>
             {% endif %}
 
-            {% if smart %}
+            {% if service.special_type == "SMART" %}
             <h5 class="clear my-4">Linux</h5>
             {% else %}
             <h5 class="clear my-4">Curl</h5>
             {% endif %}
             <div class="ml-3 example">
-                {% if smart %}
+                {% if service.special_type == "SMART" %}
                 SMART='{ <br>
                 <div class="example-indent">
                     "service" : "{{ service.service }}", <br>
@@ -115,7 +115,7 @@
                 {% endif %}
             </div>
 
-            {% if smart %}
+            {% if service.special_type == "SMART" %}
             <h5 class="my-4">Windows</h5>
             <div class="ml-3 example">
                 $SMART = @{ <br>
@@ -128,6 +128,22 @@
                 } | ConvertTo-Json<br><br>
                 Invoke-RestMethod -TimeoutSec 2 -Uri "{{ flask.request.url_root.replace("http://", "https://" )}}report" -Method Post -Headers @{"Content-Type"="application/json"} -Body $SMART
             </div>
+            <!-- register task example -->
+            <h5 class="my-4">Windows Task (requires Admin-Powershell)</h5>
+            <div class="ml-3 example">
+                $ScriptPath = Join-Path $HOME -ChildPath "smart_monitor.ps1" <br>
+                echo '$SMART = @{ <br>
+                <div class="example-indent">
+                  service = "{{ service.service }}"<br>
+                  token = "{{ service.token }}"<br>
+                  status = "N/A"<br>
+                  smart = "$(smartctl -a C: --json | Out-String)"<br>
+                </div>
+                } | ConvertTo-Json<br><br>
+                Invoke-RestMethod -TimeoutSec 2 -Uri "{{ flask.request.url_root.replace("http://", "https://" )}}report" -Method Post -Headers @{"Content-Type"="application/json"} -Body $SMART' &gt; $ScriptPath <br>
+                schtasks /create /tn SMART_Monitor /tr "powershell.exe -executionpolicy bypass -File '$ScriptPath'" /sc hourly /mo 1 /ru "Administratoren"<br>
+            echo "Done" <br>
+            </div>
             {% else %}
             <h5 class="my-4">Python</h5>
             <div class="ml-3 example">
@@ -153,12 +169,19 @@
             </thead>
 
             <tbody class="mt-2">
-                {% for status in status_list %}
+                {% for status_tupel in status_list %}
                 <tr>
-                    <td>{{ status.human_date()  }}</td>
+                    <td>{{ status_tupel[1].human_date()  }}</td>
-                    <td class="{{ status.status }}">{{ status.status }}</td>
+                    <td class="{{ status_tupel[1].status }}">{{ status_tupel[1].status }}</td>
-                    <td>{{ status.info_text     }}</td>
+                    <td>{{ status_tupel[1].info_text     }}</td>
                 </tr>
+                {% if status_tupel[0] > 1 %}
+                <tr>
+                    <td>---</td>
+                    <td><i> + {{ status_tupel[0] }} identical reports</i></td>
+                    <td>|</td>
+                </tr>
+                {% endif %}
                 {% endfor %}
             </tbody>
             </table>