feat: implement downloading from s3

.github/workflows/main.yaml

@@ -4,6 +4,8 @@ on:
   push:
     branches:
       - "master"
+  schedule:
+    - cron: "0 2 * * 0"
 
 jobs:
   docker:

Dockerfile

@@ -3,7 +3,6 @@ FROM python:3-alpine
 RUN apk add --no-cache curl lzo-dev gcc libc-dev
 
 WORKDIR /app
-COPY ./ .
 
 RUN pip install --no-cache-dir -U pip
 RUN pip install --no-cache-dir --break-system-packages waitress
@@ -11,6 +10,7 @@ RUN pip install --no-cache-dir --break-system-packages waitress
 COPY req.txt .
 RUN pip install --no-cache-dir -r req.txt
 
+COPY ./ .
 RUN ln -s /app/uploads/ /app/static/uploads
 
 EXPOSE 5000/tcp

README.md

@@ -1 +1 @@
-# v1.0
+# v1.01

notifications.py

@@ -10,15 +10,22 @@ def send_notification(app, target_user, mapname, old_replay, new_replay):
         return
 
     # send to event dispatcher #
-    message = "Trackmania: Record broken on {}".format(mapname)
+    message = "TM: Record broken on {}\n\n".format(mapname)
-    message += "Old time: {}".format(old_replay.get_human_readable_time())
+    message += "Old time:   {}\n".format(old_replay.get_human_readable_time())
-    message += "New time: {}".format(new_replay.get_human_readable_time())
+    message += "New time: {}\n".format(new_replay.get_human_readable_time())
-    message += "by {}".format(new_replay.clean_login())
+    message += "\nby {}".format(new_replay.clean_login())
 
-    payload = { "users": [user], "msg" : message }
+    payload = {
+        "users": [target_user],
+        "msg" : message,
+        "method" : "any"
+    }
 
-    r = requests.post(app.config["DISPATCH_SERVER"] + "/smart-send",
+    url_and_token = "/smart-send?dispatch-access-token={}".format(app.config["DISPATCH_TOKEN"])
-                    json=payload, auth=app.config["DISPATCH_AUTH"])
+    r = requests.post(app.config["DISPATCH_SERVER"] + url_and_token, json=payload)
 
     if not r.ok:
-        print("Error handing off notification to dispatch ({})".format(r.status_code), file=sys.stderr)
+        msg = "Error handing off notification to dispatch ({} {})".format(r.status_code, r.content)
+        print(msg, file=sys.stderr)
+    else:
+        print("Handed off notification for {} to dispatch".format(target_user), file=sys.stderr)

req.txt

@@ -3,3 +3,6 @@ flask-sqlalchemy
 pygbx
 werkzeug
 xmltodict
+requests
+psycopg2-binary
+boto3

server.py

@@ -9,18 +9,47 @@ import json
 import datetime
 
 from pygbx import Gbx, GbxType
+import pygbx
 import tm2020parser
+import notifications
 
 import sqlalchemy
 from sqlalchemy import Column, Integer, String, Boolean, or_, and_, asc, desc
 from flask_sqlalchemy import SQLAlchemy
 
+import os
+from flask import send_from_directory, abort
+
 app = flask.Flask("TM Friends Replay Server")
 
 app.config["SQLALCHEMY_TRACK_MODIFICATIONS"] = False
-app.config["SQLALCHEMY_DATABASE_URI"] = os.environ.get("SQLITE_LOCATION") or "sqlite:///sqlite.db"
+app.config["SQLALCHEMY_DATABASE_URI"] = os.environ.get("DB_URL") or "sqlite:///sqlite.db"
+app.config["AUTH_HEADER"] = os.environ.get("AUTH_HEADER") or "X-Forwarded-Preferred-Username"
+
+S3_ENDPOINT_URL = os.getenv("S3_ENDPOINT_URL")
 db = SQLAlchemy(app)
 
+SEASON_ORDERING = ["Winter", "Spring", "Summer", "Fall"]
+def filter_for_current_season(maps):
+
+    maps = list(maps)
+
+    year = str(datetime.datetime.now().year)
+
+    season = 0
+    for m in maps:
+
+        if year not in m.mapname:
+            continue
+        else:
+            for i, season_name in enumerate(SEASON_ORDERING):
+                if season_name in m.mapname and i > season:
+                    season = i
+
+    filter_func = lambda m: SEASON_ORDERING[season] in m.mapname and year in m.mapname
+    maps_season = list(filter(filter_func, maps))
+    return maps_season
+
 class Map(db.Model):
 
     __tablename__ = "maps"
@@ -87,8 +116,68 @@ class UserSettings(db.Model):
     show_tmnf            = Column(Boolean)
     show_tm_2020_current = Column(Boolean)
 
-    notifcations_all     = Column(Boolean)
+    notifications_all     = Column(Boolean)
-    notifcations_self    = Column(Boolean)
+    notifications_self    = Column(Boolean)
+
+@app.route("/update-user-settings", methods=["GET", "POST"])
+def update_user_settings():
+
+    user = flask.request.headers.get(app.config["AUTH_HEADER"])
+    user_helper = user or "anonymous"
+
+    settings = db.session.query(UserSettings).filter(UserSettings.user==user_helper).first()
+
+    # handle new settings #/
+    if not settings:
+        settings = UserSettings(user=user_helper, show_tm_2020=False, show_tmnf=False,
+                                    show_tm_2020_current=True, notifications_self=True,
+                                    notifications_all=False)
+        db.session.add(settings)
+        db.session.commit()
+        settings = db.session.query(UserSettings).filter(UserSettings.user==user_helper).first()
+
+
+    if flask.request.method == "GET":
+
+        key = flask.request.args.get("key")
+
+        # some sanity checks #
+        if not key:
+            return ("key missing in args", 422)
+        if key == "user" or key.startswith("_") or "sql" in key:
+            return ("key {} is not allowed".format(key), 422)
+
+        # return attribute #
+        return (str(getattr(settings, key)), 200)
+
+    elif flask.request.method == "POST":
+        json_dict = flask.request.json
+        key_value_list = json_dict.get("payload")
+
+        if not key_value_list:
+            return ("'payload' field empty", 422)
+
+        for el in key_value_list:
+
+            key = el.get("key")
+            value = el.get("value")
+
+            if key == "user" or key.startswith("_") or "sql" in key:
+                return ("key {} is not allowed".format(key), 422)
+
+            if key is None or value is None:
+                return ("element in payload list does not contain key and value", 422)
+
+            try:
+                getattr(settings, key)
+                setattr(settings, key, bool(value))
+                db.session.merge(settings)
+                db.session.commit()
+                return ("", 204)
+            except AttributeError:
+                return ("key {} not part of user settings".format(key), 422)
+    else:
+        raise AssertionError("Unsupported Method: {}".format(flask.request.method))
 
 class ParsedReplay(db.Model):
 
@@ -116,13 +205,16 @@ class ParsedReplay(db.Model):
         else:
             return self.login
 
-    def get_human_readable_time(self, thousands=False):
+    def get_human_readable_time(self):
         t = datetime.timedelta(microseconds=self.race_time*1000)
         t_string = str(t)
         if t.seconds < 60*60:
             t_string =  t_string[2:]
         if t.microseconds != 0:
-            return t_string[:-4]
+            if self.game == "tmnf":
+                return t_string[:-4]
+            else:
+                return t_string[:-3]
         return t_string + ".00"
 
     def __repr__(self):
@@ -133,6 +225,7 @@ class ParsedReplay(db.Model):
     def to_dict(self):
         d = dict()
         d.update({ "login" : self.login })
+        d.update({ "filehash" : self.filehash })
         d.update({ "race_time" : self.get_human_readable_time() })
         d.update({ "filepath" : self.filepath })
         d.update({ "upload_dt" : self.upload_dt })
@@ -196,7 +289,7 @@ class DataTable():
         if map_uid:
             print("Filter for map: {}".format(map_uid))
             query = query.filter(ParsedReplay.map_uid == map_uid)
-            
+
         total = query.count()
         if self.searchValue:
 
@@ -248,17 +341,19 @@ class DataTable():
 
 def _extracted_login_from_file(fullpath):
     '''Extract a login from a tmnf 2020 replay manually'''
-    
+
     # TODO fix underscores in filenames #
     if "its_a_sheppy" in fullpath:
         login_from_filename = "its_a_sheppy"
     else:
         login_from_filename = os.path.basename(fullpath).split("_")[0]
+
     with open(fullpath, "rb") as f:
         content = f.read()
         decoded_string = content.decode("ascii", errors="ignore")
         if login_from_filename not in decoded_string:
             raise ValueError("Login indicated by filename does not match login in file")
+
     return login_from_filename
 
 
@@ -280,7 +375,7 @@ def replay_from_path(fullpath, uploader=None):
                         upload_dt=ghost.upload_dt,
                         cp_times=ghost.cp_times,
                         game=ghost.game)
-    
+
     # build database map object from replay #
     m = Map(map_uid=replay.map_uid, mapname=replay.map_uid, game=replay.game)
 
@@ -324,8 +419,8 @@ def ranks():
 
 
 @app.route("/map-info")
-def list():
+def map_info():
-    player = flask.request.headers.get("X-Forwarded-Preferred-Username")
+    player = flask.request.headers.get(app.config["AUTH_HEADER"])
     header_col = ["Player", "Time", "Date", "Replay"]
     map_uid = flask.request.args.get("map_uid")
     return flask.render_template("map-info.html", header_col=header_col, map_uid=map_uid,
@@ -336,17 +431,24 @@ def mapnames():
     '''Index Location'''
 
     # TODO list by user
-    player = flask.request.headers.get("X-Forwarded-Preferred-Username")
+    player = flask.request.headers.get(app.config["AUTH_HEADER"]) or "anonymous"
     maps_query = db.session.query(Map).order_by(asc(Map.mapname))
 
     # limit leaderboard to game #
-    game = flask.request.args.get("game")
+    settings = db.session.query(UserSettings).filter(UserSettings.user==player).first()
-    if game == "tm2020":
+    if settings:
-        maps_query = maps_query.filter(Map.game=="tm2020")
+        if not settings.show_tm_2020 and not settings.show_tmnf:
-    elif game=="tmnf":
+            maps_query = maps_query.filter(Map.game=="tm2020")
-        maps_query = maps_query.filter(Map.game!="tm2020")
+            latest_season = tm2020parser.get_latest_season_from_maps(maps_query.all())
-    else:
+            # handle no replays #
-        pass
+            if latest_season:
+                maps_query = maps_query.filter(Map.map_uid.like("{}%".format(latest_season)))
+        elif settings.show_tm_2020 and not settings.show_tmnf:
+            maps_query = maps_query.filter(Map.game=="tm2020")
+        elif not settings.show_tm_2020 and settings.show_tmnf:
+            maps_query = maps_query.filter(Map.game=="tmnf")
+        else:
+            pass
 
     maps = maps_query.all()
 
@@ -354,6 +456,9 @@ def mapnames():
     allowed = ("A", "B", "C", "D", "E", "Fall", "Winter", "Spring", "Summer")
     maps_filtered = filter(lambda m: m.mapname.startswith(allowed), maps)
 
+    if settings and settings.show_tm_2020_current:
+        maps_filtered = filter_for_current_season(maps_filtered)
+
     return flask.render_template("index.html", maps=maps_filtered, player=player)
 
 @app.route("/open-info")
@@ -374,7 +479,7 @@ def openinfo():
 def source(map_uid):
 
     # path = map_uid
-    dt = DataTable(flask.request.form.to_dict(), ["login", "race_time", "upload_dt", "filepath" ])
+    dt = DataTable(flask.request.form.to_dict(), ["login", "race_time", "upload_dt", "filehash" ])
     jsonDict = dt.get(map_uid=map_uid)
     return flask.Response(json.dumps(jsonDict), 200, mimetype='application/json')
 
@@ -386,33 +491,90 @@ def index_source(map_uid):
     jsonDict = dt.get(map_uid=map_uid)
     return flask.Response(json.dumps(jsonDict), 200, mimetype='application/json')
 
-@app.route("/upload", methods = ['GET', 'POST'])
+import os
+import boto3
+import flask
+import werkzeug
+import sqlalchemy
+
+S3_BUCKET = os.getenv("S3_BUCKET")
+
+def s3_enabled():
+    return all([
+        os.getenv("AWS_ACCESS_KEY_ID"),
+        os.getenv("AWS_SECRET_ACCESS_KEY"),
+        S3_BUCKET
+    ])
+
+def get_s3_client():
+    kwargs = {}
+
+    if S3_ENDPOINT_URL:
+        kwargs["endpoint_url"] = S3_ENDPOINT_URL
+
+    return boto3.client("s3", **kwargs)
+
+
+def upload_to_s3(local_path, replay):
+    s3 = get_s3_client()
+    key = f"{replay.filehash}"
+    s3.upload_file(local_path, S3_BUCKET, key)
+    return key
+
+
+@app.route("/upload", methods=['GET', 'POST'])
 def upload():
 
     results = []
 
-    uploader = flask.request.headers.get("X-Forwarded-Preferred-Username")
+    uploader = flask.request.headers.get(app.config["AUTH_HEADER"])
+
     if flask.request.method == 'POST':
-        #f = flask.request.files['file']
+
         f_list = flask.request.files.getlist("file[]")
+
         for f_storage in f_list:
+
             fname = werkzeug.utils.secure_filename(f_storage.filename)
-            fullpath = os.path.join("uploads/", fname)
+
-            f_storage.save(fullpath)
+            os.makedirs("uploads", exist_ok=True)
+
+            # temporary save
+            tmp_path = os.path.join("uploads", fname)
+            f_storage.save(tmp_path)
+
             try:
-                replay = replay_from_path(fullpath, uploader=uploader)
+                replay = replay_from_path(tmp_path, uploader=uploader)
+
+                new_basename = f"{replay.filehash}"
+                fullpath = os.path.join("uploads", new_basename)
+
+                os.rename(tmp_path, fullpath)
+
+                replay.filepath = fullpath
+
+                if s3_enabled():
+                    s3_key = upload_to_s3(fullpath, replay)
+                    os.remove(fullpath)
+
                 db.session.add(replay)
                 db.session.commit()
+
                 check_replay_trigger(replay)
+
             except ValueError as e:
-                results += [(fname, str(e))]
+                results.append((fname, str(e)))
                 continue
+            except pygbx.GbxLoadError as e:
+                print(f"Failed to load Replay: {e}")
+                continue
+
             except sqlalchemy.exc.IntegrityError as e:
-                results += [(fname, str(e.args))]
+                results.append((fname, str(e.args)))
                 db.session.rollback()
                 continue
 
-            results += [(fname, None)]
+            results.append((fname, None))
 
         return flask.render_template("upload-post.html", results=results)
 
@@ -421,12 +583,13 @@ def upload():
 
 def check_replay_trigger(replay):
 
-    map_obj = db.session.get(Map).filter(Map.map_uid == replay.map_uid).first()
+    map_obj = db.session.query(Map).filter(Map.map_uid == replay.map_uid).first()
-    assert(map_uid)
-
     best = map_obj.get_best_replay()
     second = map_obj.get_second_best_replay()
 
+    if not second:
+        return
+
     if replay.filehash != best.filehash:
         return
 
@@ -434,16 +597,42 @@ def check_replay_trigger(replay):
         return
 
     settings = db.session.query(UserSettings).filter(UserSettings.user == second.uploader).first()
-    if settings and settings.notifcations_self:
+    if settings and settings.notifications_self:
         notifications.send_notification(app, settings.user, map_obj.map_uid, second, replay)
 
+@app.route("/downloads/<path:filename>")
+def downloads(filename):
+
+    # Ensure directory exists
+    os.makedirs("uploads", exist_ok=True)
+    local_path = os.path.join("uploads/", filename)
+
+    if not os.path.isfile(local_path):
+        print(f"{local_path} missing, attempting to retrieve from S3")
+        s3 = get_s3_client()
+
+        try:
+            s3.download_file(
+                S3_BUCKET,
+                f"{filename}",
+                os.path.join("uploads/", filename)
+            )
+
+        except Exception:
+            print(f"{filename} not found on S3")
+            abort(404)
+
+    print(f"Sending {filename}")
+    return send_from_directory("uploads/", filename)
+
 def create_app():
 
     db.create_all()
 
+    print(f"S3 enabled: {s3_enabled()} (if true will only write tmp/cache to disk")
     app.config["DISPATCH_SERVER"] = os.environ.get("DISPATCH_SERVER")
     if app.config["DISPATCH_SERVER"]:
-        app.config["DISPATCH_AUTH"] = (os.environ["DISPATCH_AUTH_USER"], os.environ["DISPATCH_AUTH_PASSWORD"])
+        app.config["DISPATCH_TOKEN"] = os.environ["DISPATCH_TOKEN"]
 
 if __name__ == "__main__":
 

static/site.css

@@ -27,6 +27,15 @@ table{
     border-color: transparent;
 }
 
+.user-settings{
+    display: inline-flex;
+    color: white;
+}
+
+.form-check{
+    margin-right: 15px;
+}
+
 .m-auto{
     margin: auto;
 }

static/user_settings.js

@@ -0,0 +1,47 @@
+/* event listener for all sliders */
+const sliders = Array.from(document.getElementsByClassName("form-check-input"))
+
+/* safety switch -> never run set before get */
+var sliders_set = false
+
+/* defer */
+sliders_load_all()
+
+/* get initial values & set listeners */
+function sliders_load_all(){
+    Promise.all(sliders.map(s => {
+        fetch("/update-user-settings?key=" + s.id, { credentials: "include" }).then(response => {
+            response.text().then(data => {
+                if(data == "True"){
+                    s.checked = true
+                }
+            })
+        })
+        s.addEventListener("change", submit)
+    })).then(
+        sliders_set = true
+    )
+}
+
+/* submit settings */
+function submit(e){
+
+    console.log("submit")
+    if(!sliders_set){
+        return
+    }
+
+    const s = e.target
+    const json_data = JSON.stringify({ payload : [ { key : s.id, value : s.checked } ] })
+    fetch("/update-user-settings", { 
+                method: "POST",
+                credentials: "include",
+                headers: {'Content-Type': 'application/json'},
+                body: json_data
+            }
+    ).then(response => {
+        if(s.id.startsWith("show")){
+            window.location.reload()
+        }
+    })
+}

templates/datatable.html

@@ -37,7 +37,7 @@
   	              {
                     "targets": 3,
                     "render": function ( data, type, full, meta ) {
-                        	     return '<a href=\"/static/'+data+'\" download>Download</a>';
+                        	     return '<a href=\"/downloads/'+data+'\" download>Download</a>';
                               }
                   },
                   {

templates/index.html

@@ -3,15 +3,28 @@
 </head>
 <body>
     {% include "upload-button.html" %}
-    <button class="ml-4 mt-4 mb-4 btn btn-info" onclick="window.location.href='/?game=tmnf'">
+
-        TMNF
+    <div class="user-settings">
-    </button>
+        <div class="form-check form-switch">
-    <button class="mt-4 mb-4 btn btn-info" onclick="window.location.href='/?game=tm2020'">
+            <input class="form-check-input" type="checkbox" role="switch" id="show_tm_2020_current">
-        TM2020
+            <label class="form-check-label" for="show_tm_2020_current">Only current TM2020</label>
-    </button>
+        </div>
-    <button class="mt-4 mb-4 btn btn-info" onclick="window.location.href='/'">
+        <div class="form-check form-switch">
-        All
+            <input class="form-check-input" type="checkbox" role="switch" id="show_tm_2020">
-    </button>
+            <label class="form-check-label" for="show_tm_2020">Show All TM2020</label>
+        </div>
+        <div class="form-check form-switch">
+            <input class="form-check-input" type="checkbox" role="switch" id="show_tmnf">
+            <label class="form-check-label" for="show_tmnf">Show All TMNF</label>
+        </div>
+        <div class="form-check form-switch">
+            <input class="form-check-input" type="checkbox" role="switch" id="notifications_self">
+            <label class="form-check-label" for="notifications_self">Send notifcations</label>
+        </div>
+    </div>
+
+    <script src="/static/user_settings.js" defer></script>
+    
     <table class="m-auto">
 	<thead>
 		<tr>

tm2020parser.py

@@ -5,6 +5,41 @@ import hashlib
 import pygbx
 import xmltodict
 
+def get_latest_season_from_maps(maps):
+    '''Determine the latest season in DB'''
+
+    order = ["Winter", "Spring", "Summer", "Fall"]
+
+    max_year = 0
+    max_season = "Winter"
+
+    if not maps:
+        return
+
+    for m in maps:
+        splitted = m.map_uid.split(" ")
+
+        # not a campain map #
+        if len(splitted) < 3:
+            return
+
+        season = splitted[0]
+        year = splitted[1]
+
+        # also not a campaing map #
+        if not season in order:
+            return
+        try:
+            year = int(year)
+        except ValueError:
+            return
+
+        if year > max_year or year == max_year and order.index(season) >= order.index(max_season):
+            max_year = year
+            max_season = season
+
+    return "{} {}".format(max_season, max_year)
+
 class GhostWrapper():
 
     def __init__(self, fullpath, uploader):