fix: improve db request rtt

.github/workflows/main.yaml

@@ -4,6 +4,8 @@ on:
   push:
     branches:
       - "master"
+  schedule:
+    - cron: "0 2 * * 0"
 
 jobs:
   docker:

Dockerfile

@@ -3,7 +3,6 @@ FROM python:3-alpine
 RUN apk add --no-cache curl lzo-dev gcc libc-dev
 
 WORKDIR /app
-COPY ./ .
 
 RUN pip install --no-cache-dir -U pip
 RUN pip install --no-cache-dir --break-system-packages waitress
@@ -11,6 +10,7 @@ RUN pip install --no-cache-dir --break-system-packages waitress
 COPY req.txt .
 RUN pip install --no-cache-dir -r req.txt
 
+COPY ./ .
 RUN ln -s /app/uploads/ /app/static/uploads
 
 EXPOSE 5000/tcp

README.md

@@ -1 +1 @@
-# v1.0
+# v1.01

notifications.py

@@ -10,15 +10,22 @@ def send_notification(app, target_user, mapname, old_replay, new_replay):
         return
 
     # send to event dispatcher #
-    message = "Trackmania: Record broken on {}".format(mapname)
-    message += "Old time: {}".format(old_replay.get_human_readable_time())
-    message += "New time: {}".format(new_replay.get_human_readable_time())
-    message += "by {}".format(new_replay.clean_login())
+    message = "TM: Record broken on {}\n\n".format(mapname)
+    message += "Old time:   {}\n".format(old_replay.get_human_readable_time())
+    message += "New time: {}\n".format(new_replay.get_human_readable_time())
+    message += "\nby {}".format(new_replay.clean_login())
 
-    payload = { "users": [user], "msg" : message }
+    payload = {
+        "users": [target_user],
+        "msg" : message,
+        "method" : "any"
+    }
 
-    r = requests.post(app.config["DISPATCH_SERVER"] + "/smart-send",
-                    json=payload, auth=app.config["DISPATCH_AUTH"])
+    url_and_token = "/smart-send?dispatch-access-token={}".format(app.config["DISPATCH_TOKEN"])
+    r = requests.post(app.config["DISPATCH_SERVER"] + url_and_token, json=payload)
 
     if not r.ok:
-        print("Error handing off notification to dispatch ({})".format(r.status_code), file=sys.stderr)
+        msg = "Error handing off notification to dispatch ({} {})".format(r.status_code, r.content)
+        print(msg, file=sys.stderr)
+    else:
+        print("Handed off notification for {} to dispatch".format(target_user), file=sys.stderr)

req.txt

@@ -3,3 +3,6 @@ flask-sqlalchemy
 pygbx
 werkzeug
 xmltodict
+requests
+psycopg2-binary
+boto3

server.py

@@ -9,76 +9,99 @@ import json
 import datetime
 
 from pygbx import Gbx, GbxType
+import pygbx
 import tm2020parser
+import notifications
 
 import sqlalchemy
 from sqlalchemy import Column, Integer, String, Boolean, or_, and_, asc, desc
 from flask_sqlalchemy import SQLAlchemy
 
+import os
+from flask import send_from_directory, abort
+
 app = flask.Flask("TM Friends Replay Server")
 
 app.config["SQLALCHEMY_TRACK_MODIFICATIONS"] = False
-app.config["SQLALCHEMY_DATABASE_URI"] = os.environ.get("SQLITE_LOCATION") or "sqlite:///sqlite.db"
+app.config["SQLALCHEMY_DATABASE_URI"] = os.environ.get("DB_URL") or "sqlite:///sqlite.db"
+app.config["AUTH_HEADER"] = os.environ.get("AUTH_HEADER") or "X-Forwarded-Preferred-Username"
+
+S3_ENDPOINT_URL = os.getenv("S3_ENDPOINT_URL")
 db = SQLAlchemy(app)
 
-class Map(db.Model):
+SEASON_ORDERING = ["Winter", "Spring", "Summer", "Fall"]
+def filter_for_current_season(maps):
 
+    maps = list(maps)
+
+    year = str(datetime.datetime.now().year)
+
+    season = 0
+    for m in maps:
+
+        if year not in m.mapname:
+            continue
+        else:
+            for i, season_name in enumerate(SEASON_ORDERING):
+                if season_name in m.mapname and i > season:
+                    season = i
+
+    filter_func = lambda m: SEASON_ORDERING[season] in m.mapname and year in m.mapname
+    maps_season = list(filter(filter_func, maps))
+    return maps_season
+
+from sqlalchemy import Column, String, Integer, Boolean, ForeignKey, Index, or_, func
+from sqlalchemy.orm import relationship, selectinload
+
+
+class Map(db.Model):
     __tablename__ = "maps"
 
     map_uid = Column(String, primary_key=True)
     mapname = Column(String)
     game = Column(String)
 
-    def get_best_replay(self):
+    replays = relationship(
+        "ParsedReplay",
+        primaryjoin="Map.map_uid == foreign(ParsedReplay.map_uid)",
+        order_by="ParsedReplay.race_time",
+        lazy="selectin",
+        viewonly=True,
+    )
 
-        q = db.session.query(ParsedReplay).filter(ParsedReplay.map_uid == self.map_uid)
-        r = q.order_by(asc(ParsedReplay.race_time)).first()
-        return r
+    def get_best_replay(self):
+        return self.replays[0] if self.replays else None
 
     def get_best_replay_for_player(self, player):
-
-        q = db.session.query(ParsedReplay).filter(ParsedReplay.map_uid == self.map_uid)
-        q = q.filter(or_(ParsedReplay.uploader == player, ParsedReplay.login == player))
-        r = q.order_by(asc(ParsedReplay.race_time)).first()
-        return r
+        return next((r for r in self.replays if r.uploader == player or r.login == player), None)
 
     def get_second_best_replay(self):
-
-        q = db.session.query(ParsedReplay).filter(ParsedReplay.map_uid == self.map_uid)
-        q = q.filter(ParsedReplay.login != self.get_best_replay().login)
-        results = q.order_by(asc(ParsedReplay.race_time)).all()
-        if not results or len(results) < 1:
-            return None
-        return results[0] # because first is already filtered out by login filter
+        best = self.get_best_replay()
+        return next((r for r in self.replays if best and r.login != best.login), None)
 
     def get_record_replay_percent_diff(self):
-
         best = self.get_best_replay()
         second = self.get_second_best_replay()
 
-        if not second:
+        if not best or not second:
             return ""
-        elif best.race_time == second.race_time:
-            return "Tied by {}".format(second.clean_login())
-        else:
-            dif = second.race_time - best.race_time
-            percent = dif/best.race_time*100
-            return "+ {:.2f}% by {}".format(percent, second.clean_login())
+        if best.race_time == second.race_time:
+            return f"Tied by {second.clean_login()}"
+
+        return f"+ {(second.race_time - best.race_time) / best.race_time * 100:.2f}% by {second.clean_login()}"
 
     def get_best_replay_repr(self):
-        r = self.get_best_replay()
-        if not r:
-            return "-"
-        return str(r)
+        return str(self.get_best_replay()) if self.get_best_replay() else "-"
 
     def get_best_replay_age(self):
+        best = self.get_best_replay()
+        if not best:
+            return None
+
+        return (datetime.datetime.now() - datetime.datetime.fromisoformat(best.upload_dt)).days
 
-        parsed = datetime.datetime.fromisoformat(self.get_best_replay().upload_dt)
-        delta = datetime.datetime.now() - parsed
-        return delta.days
 
 class UserSettings(db.Model):
-
     __tablename__ = "user_settings"
 
     user = Column(String, primary_key=True)
@@ -87,28 +110,28 @@ class UserSettings(db.Model):
     show_tmnf = Column(Boolean)
     show_tm_2020_current = Column(Boolean)
 
-    notifcations_all     = Column(Boolean)
-    notifcations_self    = Column(Boolean)
+    notifications_all = Column(Boolean)
+    notifications_self = Column(Boolean)
+
 
 class ParsedReplay(db.Model):
-
     __tablename__ = "replays"
 
     filehash = Column(String, primary_key=True)
-
     ghost_id = Column(Integer)
     race_time = Column(Integer)
-
     uploader = Column(String)
     filepath = Column(String)
     upload_dt = Column(String)
-
-    map_uid     = Column(String) # ghost_uid
+    map_uid = Column(String, ForeignKey("maps.map_uid"), index=True)
     login = Column(String)
     cp_times = Column(String)
 
-    login_uid_tm2020 = Column(String)
-    game = Column(String)
+    __table_args__ = (
+        Index("ix_replays_map_uid_race_time", "map_uid", "race_time"),
+        Index("ix_replays_map_uid_login_race_time", "map_uid", "login", "race_time"),
+    )
+
 
     def clean_login(self):
         if "/" in self.login:
@@ -116,13 +139,16 @@ class ParsedReplay(db.Model):
         else:
             return self.login
 
-    def get_human_readable_time(self, thousands=False):
+    def get_human_readable_time(self):
         t = datetime.timedelta(microseconds=self.race_time*1000)
         t_string = str(t)
         if t.seconds < 60*60:
             t_string =  t_string[2:]
         if t.microseconds != 0:
+            if self.game == "tmnf":
                 return t_string[:-4]
+            else:
+                return t_string[:-3]
         return t_string + ".00"
 
     def __repr__(self):
@@ -133,11 +159,73 @@ class ParsedReplay(db.Model):
     def to_dict(self):
         d = dict()
         d.update({ "login" : self.login })
+        d.update({ "filehash" : self.filehash })
         d.update({ "race_time" : self.get_human_readable_time() })
         d.update({ "filepath" : self.filepath })
         d.update({ "upload_dt" : self.upload_dt })
         return d
 
+
+@app.route("/update-user-settings", methods=["GET", "POST"])
+def update_user_settings():
+
+    user = flask.request.headers.get(app.config["AUTH_HEADER"])
+    user_helper = user or "anonymous"
+
+    settings = db.session.query(UserSettings).filter(UserSettings.user==user_helper).first()
+
+    # handle new settings #/
+    if not settings:
+        settings = UserSettings(user=user_helper, show_tm_2020=False, show_tmnf=False,
+                                    show_tm_2020_current=True, notifications_self=True,
+                                    notifications_all=False)
+        db.session.add(settings)
+        db.session.commit()
+        settings = db.session.query(UserSettings).filter(UserSettings.user==user_helper).first()
+
+
+    if flask.request.method == "GET":
+
+        key = flask.request.args.get("key")
+
+        # some sanity checks #
+        if not key:
+            return ("key missing in args", 422)
+        if key == "user" or key.startswith("_") or "sql" in key:
+            return ("key {} is not allowed".format(key), 422)
+
+        # return attribute #
+        return (str(getattr(settings, key)), 200)
+
+    elif flask.request.method == "POST":
+        json_dict = flask.request.json
+        key_value_list = json_dict.get("payload")
+
+        if not key_value_list:
+            return ("'payload' field empty", 422)
+
+        for el in key_value_list:
+
+            key = el.get("key")
+            value = el.get("value")
+
+            if key == "user" or key.startswith("_") or "sql" in key:
+                return ("key {} is not allowed".format(key), 422)
+
+            if key is None or value is None:
+                return ("element in payload list does not contain key and value", 422)
+
+            try:
+                getattr(settings, key)
+                setattr(settings, key, bool(value))
+                db.session.merge(settings)
+                db.session.commit()
+                return ("", 204)
+            except AttributeError:
+                return ("key {} not part of user settings".format(key), 422)
+    else:
+        raise AssertionError("Unsupported Method: {}".format(flask.request.method))
+
 class DataTable():
 
     def __init__(self, d, cols):
@@ -254,11 +342,13 @@ def _extracted_login_from_file(fullpath):
         login_from_filename = "its_a_sheppy"
     else:
         login_from_filename = os.path.basename(fullpath).split("_")[0]
+
     with open(fullpath, "rb") as f:
         content = f.read()
         decoded_string = content.decode("ascii", errors="ignore")
         if login_from_filename not in decoded_string:
             raise ValueError("Login indicated by filename does not match login in file")
+
     return login_from_filename
 
 
@@ -324,8 +414,8 @@ def ranks():
 
 
 @app.route("/map-info")
-def list():
-    player = flask.request.headers.get("X-Forwarded-Preferred-Username")
+def map_info():
+    player = flask.request.headers.get(app.config["AUTH_HEADER"])
     header_col = ["Player", "Time", "Date", "Replay"]
     map_uid = flask.request.args.get("map_uid")
     return flask.render_template("map-info.html", header_col=header_col, map_uid=map_uid,
@@ -336,15 +426,22 @@ def mapnames():
     '''Index Location'''
 
     # TODO list by user
-    player = flask.request.headers.get("X-Forwarded-Preferred-Username")
+    player = flask.request.headers.get(app.config["AUTH_HEADER"]) or "anonymous"
     maps_query = db.session.query(Map).order_by(asc(Map.mapname))
 
     # limit leaderboard to game #
-    game = flask.request.args.get("game")
-    if game == "tm2020":
+    settings = db.session.query(UserSettings).filter(UserSettings.user==player).first()
+    if settings:
+        if not settings.show_tm_2020 and not settings.show_tmnf:
             maps_query = maps_query.filter(Map.game=="tm2020")
-    elif game=="tmnf":
-        maps_query = maps_query.filter(Map.game!="tm2020")
+            latest_season = tm2020parser.get_latest_season_from_maps(maps_query.all())
+            # handle no replays #
+            if latest_season:
+                maps_query = maps_query.filter(Map.map_uid.like("{}%".format(latest_season)))
+        elif settings.show_tm_2020 and not settings.show_tmnf:
+            maps_query = maps_query.filter(Map.game=="tm2020")
+        elif not settings.show_tm_2020 and settings.show_tmnf:
+            maps_query = maps_query.filter(Map.game=="tmnf")
         else:
             pass
 
@@ -354,6 +451,9 @@ def mapnames():
     allowed = ("A", "B", "C", "D", "E", "Fall", "Winter", "Spring", "Summer")
     maps_filtered = filter(lambda m: m.mapname.startswith(allowed), maps)
 
+    if settings and settings.show_tm_2020_current:
+        maps_filtered = filter_for_current_season(maps_filtered)
+
     return flask.render_template("index.html", maps=maps_filtered, player=player)
 
 @app.route("/open-info")
@@ -374,7 +474,7 @@ def openinfo():
 def source(map_uid):
 
     # path = map_uid
-    dt = DataTable(flask.request.form.to_dict(), ["login", "race_time", "upload_dt", "filepath" ])
+    dt = DataTable(flask.request.form.to_dict(), ["login", "race_time", "upload_dt", "filehash" ])
     jsonDict = dt.get(map_uid=map_uid)
     return flask.Response(json.dumps(jsonDict), 200, mimetype='application/json')
 
@@ -386,33 +486,90 @@ def index_source(map_uid):
     jsonDict = dt.get(map_uid=map_uid)
     return flask.Response(json.dumps(jsonDict), 200, mimetype='application/json')
 
-@app.route("/upload", methods = ['GET', 'POST'])
+import os
+import boto3
+import flask
+import werkzeug
+import sqlalchemy
+
+S3_BUCKET = os.getenv("S3_BUCKET")
+
+def s3_enabled():
+    return all([
+        os.getenv("AWS_ACCESS_KEY_ID"),
+        os.getenv("AWS_SECRET_ACCESS_KEY"),
+        S3_BUCKET
+    ])
+
+def get_s3_client():
+    kwargs = {}
+
+    if S3_ENDPOINT_URL:
+        kwargs["endpoint_url"] = S3_ENDPOINT_URL
+
+    return boto3.client("s3", **kwargs)
+
+
+def upload_to_s3(local_path, replay):
+    s3 = get_s3_client()
+    key = f"{replay.filehash}"
+    s3.upload_file(local_path, S3_BUCKET, key)
+    return key
+
+
+@app.route("/upload", methods=['GET', 'POST'])
 def upload():
 
     results = []
 
-    uploader = flask.request.headers.get("X-Forwarded-Preferred-Username")
+    uploader = flask.request.headers.get(app.config["AUTH_HEADER"])
+
     if flask.request.method == 'POST':
-        #f = flask.request.files['file']
+
         f_list = flask.request.files.getlist("file[]")
+
         for f_storage in f_list:
+
             fname = werkzeug.utils.secure_filename(f_storage.filename)
-            fullpath = os.path.join("uploads/", fname)
-            f_storage.save(fullpath)
+
+            os.makedirs("uploads", exist_ok=True)
+
+            # temporary save
+            tmp_path = os.path.join("uploads", fname)
+            f_storage.save(tmp_path)
+
             try:
-                replay = replay_from_path(fullpath, uploader=uploader)
+                replay = replay_from_path(tmp_path, uploader=uploader)
+
+                new_basename = f"{replay.filehash}"
+                fullpath = os.path.join("uploads", new_basename)
+
+                os.rename(tmp_path, fullpath)
+
+                replay.filepath = fullpath
+
+                if s3_enabled():
+                    s3_key = upload_to_s3(fullpath, replay)
+                    os.remove(fullpath)
+
                 db.session.add(replay)
                 db.session.commit()
+
                 check_replay_trigger(replay)
+
             except ValueError as e:
-                results += [(fname, str(e))]
+                results.append((fname, str(e)))
                 continue
+            except pygbx.GbxLoadError as e:
+                print(f"Failed to load Replay: {e}")
+                continue
+
             except sqlalchemy.exc.IntegrityError as e:
-                results += [(fname, str(e.args))]
+                results.append((fname, str(e.args)))
                 db.session.rollback()
                 continue
 
-            results += [(fname, None)]
+            results.append((fname, None))
 
         return flask.render_template("upload-post.html", results=results)
 
@@ -421,12 +578,13 @@ def upload():
 
 def check_replay_trigger(replay):
 
-    map_obj = db.session.get(Map).filter(Map.map_uid == replay.map_uid).first()
-    assert(map_uid)
-
+    map_obj = db.session.query(Map).filter(Map.map_uid == replay.map_uid).first()
     best = map_obj.get_best_replay()
     second = map_obj.get_second_best_replay()
 
+    if not second:
+        return
+
     if replay.filehash != best.filehash:
         return
 
@@ -434,16 +592,48 @@ def check_replay_trigger(replay):
         return
 
     settings = db.session.query(UserSettings).filter(UserSettings.user == second.uploader).first()
-    if settings and settings.notifcations_self:
+    if settings and settings.notifications_self:
         notifications.send_notification(app, settings.user, map_obj.map_uid, second, replay)
 
+@app.route("/downloads/<path:filename>")
+def downloads(filename):
+
+    # Ensure directory exists
+    os.makedirs("uploads", exist_ok=True)
+    local_path = os.path.join("uploads/", filename)
+
+    if not os.path.isfile(local_path):
+        print(f"{local_path} missing, attempting to retrieve from S3")
+        s3 = get_s3_client()
+
+        try:
+            s3.download_file(
+                S3_BUCKET,
+                f"{filename}",
+                os.path.join("uploads/", filename)
+            )
+
+        except Exception:
+            print(f"{filename} not found on S3")
+            abort(404)
+
+    print(f"Sending {filename}")
+
+    replay = db.session.query(ParsedReplay).filter(ParsedReplay.filehash==filename).first()
+
+    return send_from_directory("uploads/",
+                filename,
+                as_attachment=True,
+                download_name=os.path.basename(replay.filepath))
+
 def create_app():
 
     db.create_all()
 
+    print(f"S3 enabled: {s3_enabled()} (if true will only write tmp/cache to disk")
     app.config["DISPATCH_SERVER"] = os.environ.get("DISPATCH_SERVER")
     if app.config["DISPATCH_SERVER"]:
-        app.config["DISPATCH_AUTH"] = (os.environ["DISPATCH_AUTH_USER"], os.environ["DISPATCH_AUTH_PASSWORD"])
+        app.config["DISPATCH_TOKEN"] = os.environ["DISPATCH_TOKEN"]
 
 if __name__ == "__main__":
 

static/site.css

@@ -27,6 +27,15 @@ table{
     border-color: transparent;
 }
 
+.user-settings{
+    display: inline-flex;
+    color: white;
+}
+
+.form-check{
+    margin-right: 15px;
+}
+
 .m-auto{
     margin: auto;
 }

static/user_settings.js

@@ -0,0 +1,47 @@
+/* event listener for all sliders */
+const sliders = Array.from(document.getElementsByClassName("form-check-input"))
+
+/* safety switch -> never run set before get */
+var sliders_set = false
+
+/* defer */
+sliders_load_all()
+
+/* get initial values & set listeners */
+function sliders_load_all(){
+    Promise.all(sliders.map(s => {
+        fetch("/update-user-settings?key=" + s.id, { credentials: "include" }).then(response => {
+            response.text().then(data => {
+                if(data == "True"){
+                    s.checked = true
+                }
+            })
+        })
+        s.addEventListener("change", submit)
+    })).then(
+        sliders_set = true
+    )
+}
+
+/* submit settings */
+function submit(e){
+
+    console.log("submit")
+    if(!sliders_set){
+        return
+    }
+
+    const s = e.target
+    const json_data = JSON.stringify({ payload : [ { key : s.id, value : s.checked } ] })
+    fetch("/update-user-settings", { 
+                method: "POST",
+                credentials: "include",
+                headers: {'Content-Type': 'application/json'},
+                body: json_data
+            }
+    ).then(response => {
+        if(s.id.startsWith("show")){
+            window.location.reload()
+        }
+    })
+}

templates/datatable.html

@@ -37,7 +37,7 @@
   	              {
                     "targets": 3,
                     "render": function ( data, type, full, meta ) {
-                        	     return '<a href=\"/static/'+data+'\" download>Download</a>';
+                        	     return '<a href=\"/downloads/'+data+'\" download>Download</a>';
                               }
                   },
                   {

templates/index.html

@@ -3,15 +3,28 @@
 </head>
 <body>
     {% include "upload-button.html" %}
-    <button class="ml-4 mt-4 mb-4 btn btn-info" onclick="window.location.href='/?game=tmnf'">
-        TMNF
-    </button>
-    <button class="mt-4 mb-4 btn btn-info" onclick="window.location.href='/?game=tm2020'">
-        TM2020
-    </button>
-    <button class="mt-4 mb-4 btn btn-info" onclick="window.location.href='/'">
-        All
-    </button>
+
+    <div class="user-settings">
+        <div class="form-check form-switch">
+            <input class="form-check-input" type="checkbox" role="switch" id="show_tm_2020_current">
+            <label class="form-check-label" for="show_tm_2020_current">Only current TM2020</label>
+        </div>
+        <div class="form-check form-switch">
+            <input class="form-check-input" type="checkbox" role="switch" id="show_tm_2020">
+            <label class="form-check-label" for="show_tm_2020">Show All TM2020</label>
+        </div>
+        <div class="form-check form-switch">
+            <input class="form-check-input" type="checkbox" role="switch" id="show_tmnf">
+            <label class="form-check-label" for="show_tmnf">Show All TMNF</label>
+        </div>
+        <div class="form-check form-switch">
+            <input class="form-check-input" type="checkbox" role="switch" id="notifications_self">
+            <label class="form-check-label" for="notifications_self">Send notifcations</label>
+        </div>
+    </div>
+
+    <script src="/static/user_settings.js" defer></script>
+    
     <table class="m-auto">
 	<thead>
 		<tr>

tm2020parser.py

@@ -5,6 +5,41 @@ import hashlib
 import pygbx
 import xmltodict
 
+def get_latest_season_from_maps(maps):
+    '''Determine the latest season in DB'''
+
+    order = ["Winter", "Spring", "Summer", "Fall"]
+
+    max_year = 0
+    max_season = "Winter"
+
+    if not maps:
+        return
+
+    for m in maps:
+        splitted = m.map_uid.split(" ")
+
+        # not a campain map #
+        if len(splitted) < 3:
+            return
+
+        season = splitted[0]
+        year = splitted[1]
+
+        # also not a campaing map #
+        if not season in order:
+            return
+        try:
+            year = int(year)
+        except ValueError:
+            return
+
+        if year > max_year or year == max_year and order.index(season) >= order.index(max_season):
+            max_year = year
+            max_season = season
+
+    return "{} {}".format(max_season, max_year)
+
 class GhostWrapper():
 
     def __init__(self, fullpath, uploader):