feat: implement downloading from s3

Dockerfile

@@ -3,7 +3,6 @@ FROM python:3-alpine
 RUN apk add --no-cache curl lzo-dev gcc libc-dev
 
 WORKDIR /app
-COPY ./ .
 
 RUN pip install --no-cache-dir -U pip
 RUN pip install --no-cache-dir --break-system-packages waitress
@@ -11,6 +10,7 @@ RUN pip install --no-cache-dir --break-system-packages waitress
 COPY req.txt .
 RUN pip install --no-cache-dir -r req.txt
 
+COPY ./ .
 RUN ln -s /app/uploads/ /app/static/uploads
 
 EXPOSE 5000/tcp

req.txt

@@ -4,3 +4,5 @@ pygbx
 werkzeug
 xmltodict
 requests
+psycopg2-binary
+boto3

server.py

@@ -9,6 +9,7 @@ import json
 import datetime
 
 from pygbx import Gbx, GbxType
+import pygbx
 import tm2020parser
 import notifications
 
@@ -16,10 +17,16 @@ import sqlalchemy
 from sqlalchemy import Column, Integer, String, Boolean, or_, and_, asc, desc
 from flask_sqlalchemy import SQLAlchemy
 
+import os
+from flask import send_from_directory, abort
+
 app = flask.Flask("TM Friends Replay Server")
 
 app.config["SQLALCHEMY_TRACK_MODIFICATIONS"] = False
-app.config["SQLALCHEMY_DATABASE_URI"] = os.environ.get("SQLITE_LOCATION") or "sqlite:///sqlite.db"
+app.config["SQLALCHEMY_DATABASE_URI"] = os.environ.get("DB_URL") or "sqlite:///sqlite.db"
+app.config["AUTH_HEADER"] = os.environ.get("AUTH_HEADER") or "X-Forwarded-Preferred-Username"
+
+S3_ENDPOINT_URL = os.getenv("S3_ENDPOINT_URL")
 db = SQLAlchemy(app)
 
 SEASON_ORDERING = ["Winter", "Spring", "Summer", "Fall"]
@@ -115,7 +122,7 @@ class UserSettings(db.Model):
 @app.route("/update-user-settings", methods=["GET", "POST"])
 def update_user_settings():
 
-    user = flask.request.headers.get("X-Forwarded-Preferred-Username")
+    user = flask.request.headers.get(app.config["AUTH_HEADER"])
     user_helper = user or "anonymous"
 
     settings = db.session.query(UserSettings).filter(UserSettings.user==user_helper).first()
@@ -218,6 +225,7 @@ class ParsedReplay(db.Model):
     def to_dict(self):
         d = dict()
         d.update({ "login" : self.login })
+        d.update({ "filehash" : self.filehash })
         d.update({ "race_time" : self.get_human_readable_time() })
         d.update({ "filepath" : self.filepath })
         d.update({ "upload_dt" : self.upload_dt })
@@ -339,11 +347,13 @@ def _extracted_login_from_file(fullpath):
         login_from_filename = "its_a_sheppy"
     else:
         login_from_filename = os.path.basename(fullpath).split("_")[0]
+
     with open(fullpath, "rb") as f:
         content = f.read()
         decoded_string = content.decode("ascii", errors="ignore")
         if login_from_filename not in decoded_string:
             raise ValueError("Login indicated by filename does not match login in file")
+
     return login_from_filename
 
 
@@ -410,7 +420,7 @@ def ranks():
 
 @app.route("/map-info")
 def map_info():
-    player = flask.request.headers.get("X-Forwarded-Preferred-Username")
+    player = flask.request.headers.get(app.config["AUTH_HEADER"])
     header_col = ["Player", "Time", "Date", "Replay"]
     map_uid = flask.request.args.get("map_uid")
     return flask.render_template("map-info.html", header_col=header_col, map_uid=map_uid,
@@ -421,7 +431,7 @@ def mapnames():
     '''Index Location'''
 
     # TODO list by user
-    player = flask.request.headers.get("X-Forwarded-Preferred-Username") or "anonymous"
+    player = flask.request.headers.get(app.config["AUTH_HEADER"]) or "anonymous"
     maps_query = db.session.query(Map).order_by(asc(Map.mapname))
 
     # limit leaderboard to game #
@@ -446,7 +456,7 @@ def mapnames():
     allowed = ("A", "B", "C", "D", "E", "Fall", "Winter", "Spring", "Summer")
     maps_filtered = filter(lambda m: m.mapname.startswith(allowed), maps)
 
-    if settings.show_tm_2020_current:
+    if settings and settings.show_tm_2020_current:
         maps_filtered = filter_for_current_season(maps_filtered)
 
     return flask.render_template("index.html", maps=maps_filtered, player=player)
@@ -469,7 +479,7 @@ def openinfo():
 def source(map_uid):
 
     # path = map_uid
-    dt = DataTable(flask.request.form.to_dict(), ["login", "race_time", "upload_dt", "filepath" ])
+    dt = DataTable(flask.request.form.to_dict(), ["login", "race_time", "upload_dt", "filehash" ])
     jsonDict = dt.get(map_uid=map_uid)
     return flask.Response(json.dumps(jsonDict), 200, mimetype='application/json')
 
@@ -481,33 +491,90 @@ def index_source(map_uid):
     jsonDict = dt.get(map_uid=map_uid)
     return flask.Response(json.dumps(jsonDict), 200, mimetype='application/json')
 
-@app.route("/upload", methods = ['GET', 'POST'])
+import os
+import boto3
+import flask
+import werkzeug
+import sqlalchemy
+
+S3_BUCKET = os.getenv("S3_BUCKET")
+
+def s3_enabled():
+    return all([
+        os.getenv("AWS_ACCESS_KEY_ID"),
+        os.getenv("AWS_SECRET_ACCESS_KEY"),
+        S3_BUCKET
+    ])
+
+def get_s3_client():
+    kwargs = {}
+
+    if S3_ENDPOINT_URL:
+        kwargs["endpoint_url"] = S3_ENDPOINT_URL
+
+    return boto3.client("s3", **kwargs)
+
+
+def upload_to_s3(local_path, replay):
+    s3 = get_s3_client()
+    key = f"{replay.filehash}"
+    s3.upload_file(local_path, S3_BUCKET, key)
+    return key
+
+
+@app.route("/upload", methods=['GET', 'POST'])
 def upload():
 
     results = []
 
-    uploader = flask.request.headers.get("X-Forwarded-Preferred-Username")
+    uploader = flask.request.headers.get(app.config["AUTH_HEADER"])
+
     if flask.request.method == 'POST':
-        #f = flask.request.files['file']
+
         f_list = flask.request.files.getlist("file[]")
+
         for f_storage in f_list:
+
             fname = werkzeug.utils.secure_filename(f_storage.filename)
-            fullpath = os.path.join("uploads/", fname)
+
-            f_storage.save(fullpath)
+            os.makedirs("uploads", exist_ok=True)
+
+            # temporary save
+            tmp_path = os.path.join("uploads", fname)
+            f_storage.save(tmp_path)
+
             try:
-                replay = replay_from_path(fullpath, uploader=uploader)
+                replay = replay_from_path(tmp_path, uploader=uploader)
+
+                new_basename = f"{replay.filehash}"
+                fullpath = os.path.join("uploads", new_basename)
+
+                os.rename(tmp_path, fullpath)
+
+                replay.filepath = fullpath
+
+                if s3_enabled():
+                    s3_key = upload_to_s3(fullpath, replay)
+                    os.remove(fullpath)
+
                 db.session.add(replay)
                 db.session.commit()
+
                 check_replay_trigger(replay)
+
             except ValueError as e:
-                results += [(fname, str(e))]
+                results.append((fname, str(e)))
                 continue
+            except pygbx.GbxLoadError as e:
+                print(f"Failed to load Replay: {e}")
+                continue
+
             except sqlalchemy.exc.IntegrityError as e:
-                results += [(fname, str(e.args))]
+                results.append((fname, str(e.args)))
                 db.session.rollback()
                 continue
 
-            results += [(fname, None)]
+            results.append((fname, None))
 
         return flask.render_template("upload-post.html", results=results)
 
@@ -533,10 +600,36 @@ def check_replay_trigger(replay):
     if settings and settings.notifications_self:
         notifications.send_notification(app, settings.user, map_obj.map_uid, second, replay)
 
+@app.route("/downloads/<path:filename>")
+def downloads(filename):
+
+    # Ensure directory exists
+    os.makedirs("uploads", exist_ok=True)
+    local_path = os.path.join("uploads/", filename)
+
+    if not os.path.isfile(local_path):
+        print(f"{local_path} missing, attempting to retrieve from S3")
+        s3 = get_s3_client()
+
+        try:
+            s3.download_file(
+                S3_BUCKET,
+                f"{filename}",
+                os.path.join("uploads/", filename)
+            )
+
+        except Exception:
+            print(f"{filename} not found on S3")
+            abort(404)
+
+    print(f"Sending {filename}")
+    return send_from_directory("uploads/", filename)
+
 def create_app():
 
     db.create_all()
 
+    print(f"S3 enabled: {s3_enabled()} (if true will only write tmp/cache to disk")
     app.config["DISPATCH_SERVER"] = os.environ.get("DISPATCH_SERVER")
     if app.config["DISPATCH_SERVER"]:
         app.config["DISPATCH_TOKEN"] = os.environ["DISPATCH_TOKEN"]

templates/datatable.html

@@ -37,7 +37,7 @@
   	              {
                     "targets": 3,
                     "render": function ( data, type, full, meta ) {
-                        	     return '<a href=\"/static/'+data+'\" download>Download</a>';
+                        	     return '<a href=\"/downloads/'+data+'\" download>Download</a>';
                               }
                   },
                   {