wip: asnyc http (1)

.gitignore

@@ -14,3 +14,4 @@ example_software_root/
 
 # windows dir on linux ä
 **\\install-dir/
+server/data/

client.py

@@ -9,6 +9,7 @@ import os
 import cache_utils
 import imagetools
 import webbrowser
+import statekeeper
 
 customtkinter.set_appearance_mode("dark")
 customtkinter.set_default_color_theme("blue")
@@ -183,6 +184,7 @@ def load_main():
     # create tiles from meta files #
     cache_dir_size = 0
     for software in db.find_all_metadata():
+
         create_main_window_tile(software, scrollable_frame)
 
         # retrieve cache dir from any software #
@@ -222,24 +224,35 @@ def load_details(app, software):
 def create_main_window_tile(software, parent):
     '''Create the main window tile'''
 
-    if software.get_thumbnail():
+    img = PIL.Image.new('RGB', (200, 300))
-        try:
+    imgTk = PIL.ImageTk.PhotoImage(img)
-            print("Loading thumbnail:",  software.get_thumbnail())
+    button = customtkinter.CTkButton(parent, image=imgTk,
-            img = PIL.Image.open(software.get_thumbnail())
-            img = imagetools.smart_resize(img, 200, 300)
-        except PIL.UnidentifiedImageError:
-            print("Failed to load thumbnail:", software.get_thumbnail())
-            img = PIL.Image.new('RGB', (200, 300))
-    else:
-        img = PIL.Image.new('RGB', (200, 300))
-
-    img = PIL.ImageTk.PhotoImage(img)
-    button = customtkinter.CTkButton(parent, image=img,
                                         width=200, height=300,
                                         command=lambda: switch_to_game_details(software),
                                         border_width=0, corner_radius=0, border_spacing=0,
                                         text=software.title,
                                         fg_color="transparent", compound="top", anchor="s")
+
+
+    def callback_update_thumbnail():
+
+        # TODO: bind button & software into this callback
+
+        try:
+            target_file = software.get_thumbnail()
+            print("Loading thumbnail (async):",  )
+            img = PIL.Image.open(software.get_thumbnail())
+            img = imagetools.smart_resize(img, 200, 300)
+        except PIL.UnidentifiedImageError:
+            print("Failed to load thumbnail:", software.get_thumbnail())
+            img = PIL.Image.new('RGB', (200, 300))
+        
+        # TODO: button reconfigure
+
+    # register the update task for the image #
+    statekeeper.add_task(callback_update_thumbnail)
+
+    # cache button and return #
     buttons.append(button)
     return button
 
@@ -320,7 +333,9 @@ if __name__ == "__main__":
         print(user, password, install_dir, remote_root_dir, server, config_loaded["Server/Path:"])
 
         # add db backend #
-        if backend_type == "FTP/FTPS":
+        if True:
+            db = data_backend.HTTP(None, None, install_dir, remote_root_dir="./", server="http://localhost:5000", tkinter_root=app)
+        elif backend_type == "FTP/FTPS":
             db = data_backend.FTP(user, password, install_dir, server=server,
                 remote_root_dir=remote_root_dir, progress_bar_wrapper=pgw, tkinter_root=app)
         elif backend_type == "Local Filesystem":

client_details.py

@@ -3,6 +3,7 @@ import tkinter
 import customtkinter
 import imagetools
 import os
+import statekeeper
 
 def show_large_picture(app, path):
     '''Show a full-window version of the clicked picture'''
@@ -21,6 +22,7 @@ def show_large_picture(app, path):
     large_image = customtkinter.CTkButton(app, text="", image=img, width=x-2*30, height=y-2*30,
                     fg_color="transparent", hover_color="black", corner_radius=0, border_width=0, border_spacing=0,
                     command=lambda: large_image.destroy())
+
     large_image.place(x=30, y=30)
 
 def create_details_page(app, software, backswitch_function):
@@ -170,21 +172,24 @@ def create_details_page(app, software, backswitch_function):
     # add other pictures #
     if software.pictures:
 
-        picture_frame = customtkinter.CTkScrollableFrame(info_frame, height=200, width=300, orientation="horizontal", fg_color="transparent")
+        def callback_add_pictures():
-        picture_frame.grid(column=0, row=7, sticky="we")
+            picture_frame = customtkinter.CTkScrollableFrame(info_frame, height=200, width=300, orientation="horizontal", fg_color="transparent")
+            picture_frame.grid(column=0, row=7, sticky="we")
 
-        i = 0
+            i = 0
-        for path in software.pictures[1:]:
+            for path in software.pictures[1:]:
-            img = PIL.Image.open(path)
+                img = PIL.Image.open(path)
-            img = imagetools.smart_resize(img, 180, 180)
+                img = imagetools.smart_resize(img, 180, 180)
-            img = PIL.ImageTk.PhotoImage(img)
+                img = PIL.ImageTk.PhotoImage(img)
-            extra_pic_button = customtkinter.CTkButton(picture_frame, text="", image=img, command=lambda path=path: show_large_picture(app, path),
+                extra_pic_button = customtkinter.CTkButton(picture_frame, text="", image=img, command=lambda path=path: show_large_picture(app, path),
-                                     hover_color="black", corner_radius=0,)
+                                        hover_color="black", corner_radius=0,)
-            extra_pic_button.configure(fg_color="transparent")
+                extra_pic_button.configure(fg_color="transparent")
-            extra_pic_button.grid(pady=10, row=0, column=i)
+                extra_pic_button.grid(pady=10, row=0, column=i)
-            elements.append(extra_pic_button)
+                elements.append(extra_pic_button)
-            i += 1
+                i += 1
-        
+            
-        elements.append(picture_frame)
+            elements.append(picture_frame)
+
+        statekeeper.add_to_task_queue(callback_add_pictures)
 
     return elements

data_backend.py

@@ -6,19 +6,9 @@ import ftplib
 import tqdm
 import ssl
 import concurrent.futures
+import statekeeper
+import requests
 
-class SESSION_REUSE_FTP_TLS(ftplib.FTP_TLS):
-    """Explicit FTPS, with shared TLS session"""
-
-    def ntransfercmd(self, cmd, rest=None):
-
-        conn, size = ftplib.FTP.ntransfercmd(self, cmd, rest)
-        if self._prot_p:
-            conn = self.context.wrap_socket(
-                conn,
-                server_hostname=self.host,
-                session=self.sock.session)  # this is the fix
-        return conn, size
 
 class DataBackend:
 
@@ -36,7 +26,6 @@ class DataBackend:
         self.progress_bar_wrapper = progress_bar_wrapper
         self.root = tkinter_root
         self.cache_dir = "./cache/"
-        self.ftp = None # ftp connection object
 
     def get(self, path, return_content=False):
         '''Return the contents of this path'''
@@ -102,70 +91,16 @@ class LocalFS(DataBackend):
                 meta_info_list.append(software.Software(meta_file, self, self.progress_bar_wrapper))
 
         return list(filter(lambda x: not x.invalid, meta_info_list))
-
+class HTTP(DataBackend):
-class FTP(DataBackend):
 
     paths_listed = {}
+    REMOTE_PATH = "/get-path"
 
-    def _connect(self, individual_connection=False):
+    def _get_url(self):
+        print(self.server + HTTP.REMOTE_PATH)
+        return self.server + HTTP.REMOTE_PATH
 
-        if self.ftp and not individual_connection:
+    def get(self, path, cache_dir=None, return_content=False):
-            try:
-                self.ftp.voidcmd("NOOP")
-                return self.ftp
-            except ssl.SSLError:
-                pass # reconnect
-
-        if self.server.startswith("ftp://"):
-            tls = False
-        elif self.server.startswith("ftps://"):
-            tls = True
-        else:
-            raise ValueError("FTP Server must start with ftp:// or ftps://")
-
-        # build connection parameters #
-        server = self.server.split("://")[1]
-        port = None
-        try:
-            server = server.split(":")[0]
-        except (IndexError, ValueError):
-            port = 0
-
-        # try extract server #
-        try:
-            server = server.split(":")[0]
-        except (IndexError, ValueError):
-            server = self.server
-
-        print("Connecting to:", server, "on port:", port, "ssl =", tls)
-
-        # connect #
-        if not tls:
-            ftp = ftplib.FTP()
-        else:
-            ftp = SESSION_REUSE_FTP_TLS()
-            ftp.ssl_version = ssl.PROTOCOL_TLSv1_2
-
-        ftp.connect(server, port=port or 0)
-
-        if self.user:
-            ftp.login(self.user, self.password)
-        else:
-            ftp.login()
-
-        # open a secure session for tls #
-        if tls:
-            ftp.prot_p()
-
-        # cache dir is automatically set #
-        self.cache_dir = None
-
-        if not individual_connection:
-            self.ftp = ftp
-        return ftp
-
-
-    def get(self, path, cache_dir=None, return_content=False, new_connection=False):
 
         # check the load cache dir #
         if cache_dir:
@@ -177,102 +112,56 @@ class FTP(DataBackend):
         fullpath = path
         if self.remote_root_dir and not path.startswith(self.remote_root_dir):
             fullpath = os.path.join(self.remote_root_dir, path)
-            #print(self.remote_root_dir, path, fullpath)
+
         fullpath = fullpath.replace("\\", "/")
         local_file = os.path.join(cache_dir, os.path.basename(path))
 
-        # print("Cachedir:", cache_dir, os.path.basename(path), local_file)
+        print("Requiring:", local_file)
 
         if not os.path.isfile(local_file):
 
-            ftp = self._connect(individual_connection=True)
+            if return_content:
-            ftp.sendcmd('TYPE I')
 
-            # load the file on remote #
+                # the content is needed for the UI now and not cached, it's needs to be downloaded synchroniously #
-            if not new_connection:
+                # as there cannot be a meaningful UI-draw without it. #
+                r = requests.get(self._get_url(), params={ "path" : path, "as_string": True })
 
-                total_size = ftp.size(fullpath)
+                # cache the download imediatelly #
-                print("Total Size:", total_size)
+                with open(local_file, encoding="utf-8", mode="w") as f:
-                self.progress_bar_wrapper.get_pb()["maximum"] = total_size
+                    f.write(r.json()["content"])
 
-                print(local_file, "not in cache, retriving..")
+                # return the content #
-                with open(local_file, "w") as f:
+                return r.json()["content"]
-                    f.write(local_file)
-                with open(local_file, 'wb') as local_file_open, tqdm.tqdm(
-                    desc="Downloading",
-                    total=total_size,
-                    unit='B',
-                    unit_scale=True
-                ) as cmd_progress_bar:
 
-                    # Define a callback function to update the progress bar #
-                    def callback(data):
-                        local_file_open.write(data)
-                        if new_connection: # return if parralell
-                            return
-                        self.root.update_idletasks() # Update the GUI
-                        current_total = self.progress_bar_wrapper.get_pb().get() + len(data)/total_size
-                        self.progress_bar_wrapper.get_pb().set(current_total)
-                        self.progress_bar_wrapper.set_text(
-                                        text="Downloading: {:.2f}%".format(current_total*100))
-                        cmd_progress_bar.update(len(data))
-
-                    # run with callback #
-                    ftp.retrbinary('RETR ' + fullpath, callback)
             else:
-                with open(local_file, 'wb') as fp:
+                statekeeper.add_to_download_queue(self._get_url(), path, first=return_content)
-                    ftp.retrbinary('RETR ' + fullpath, fp.write)
 
-            if new_connection:
+        elif return_content:
-                ftp.close()
-
-        if return_content:
             with open(local_file, encoding="utf-8") as fr:
                 return fr.read()
+        else:
+            return local_file
 
-        return local_file
+    def list(self, path, fullpaths=False):
 
-    def list(self, path, fullpaths=False, new_connection=False):
-
-        # prepend root dir if not given #
         fullpath = path
         if self.remote_root_dir and not path.startswith(self.remote_root_dir):
             fullpath = os.path.join(self.remote_root_dir, path)
         fullpath = fullpath.replace("\\", "/")
-        #print(fullpath)
 
-        # if not os.path.isdir(fullpath):
+        # retrieve session cached paths #
-        #     return []
+        if fullpath in self.paths_listed:
+            paths = self.paths_listed[fullpath]
+        else:
 
-        try:
+            r = requests.get(self._get_url(), params={ "path" : path })
-            # retrieve session cached paths #
+            print(r, r.status_code, r.content)
-            if fullpath in self.paths_listed:
+            paths = r.json()["contents"]
-                paths = self.paths_listed[fullpath]
-                #print("Retrieved paths from cache:", fullpath, paths)
-            else:
-                ftp = self._connect(individual_connection=new_connection)
-                print("Listing previously unlisted path: {}".format(fullpath))
-                self.paths_listed.update({fullpath: []}) # in case dir does not exit
-                paths = ftp.nlst(fullpath)
-                self.paths_listed.update({fullpath: paths})
-
-                if new_connection: # close individual connections
-                    ftp.close()
 
             if not fullpaths:
                 return paths
-
-            return [ os.path.join(path, filename).replace("\\", "/") for filename in paths ]
-
-        except ftplib.error_perm as e:
-            if "550 No files found" in str(e):
-                print("No files in this directory: {}".format(fullpath))
-                return []
-            elif "550 No such file or directory" in str(e):
-                print("File or dir does not exist: {}".format(fullpath))
-                return []
             else:
-                raise e
+                return [ os.path.join(path, filename).replace("\\", "/") for filename in paths ]
 
     def find_all_metadata(self):
 
@@ -282,24 +171,21 @@ class FTP(DataBackend):
         with concurrent.futures.ThreadPoolExecutor(max_workers=os.cpu_count()*5) as executor:
 
             software_dir_contents = list(executor.map(
-                    lambda s: self.list(s, fullpaths=True, new_connection=True), root_elements))
+                    lambda s: self.list(s, fullpaths=True), root_elements))
 
             # this caches the paths, done remove it #
             cache_list = [os.path.join(s, "registry_files") for s in root_elements ]
             cache_list += [os.path.join(s, "pictures") for s in root_elements ]
+
+            # THIS PRELOAD IMAGES-paths, DO NOT REMOVE IT #
             picture_contents_async_cache = list(executor.map(
-                    lambda s: self.list(s, fullpaths=True, new_connection=True), cache_list))
+                    lambda s: self.list(s, fullpaths=True), cache_list))
 
         for files in software_dir_contents:
-            #print(s)
-            #files = self.list(s, fullpaths=True)
-            print(files)
             for f in files:
                 if f.endswith("meta.yaml"):
-                    meta_file_content = self.get(f, cache_dir="cache", return_content=True)
-                    #print(meta_file_content)
                     local_meta_file_list.append(f)
 
         with concurrent.futures.ThreadPoolExecutor(max_workers=os.cpu_count()*5) as executor:
             software_list = executor.map(lambda meta_file: software.Software(meta_file, self, self.progress_bar_wrapper), local_meta_file_list)
-            return list(filter(lambda x: not x.invalid, software_list))
+            return list(filter(lambda x: not x.invalid, software_list))

server/main.py

@@ -0,0 +1,65 @@
+from flask import Flask, request, jsonify, send_file, abort
+import os
+import sys
+
+from werkzeug.utils import secure_filename
+
+app = Flask(__name__)
+
+# Base directory constraint
+BASE_DIR = os.path.abspath("./data")
+
+@app.route('/get-path', methods=['GET'])
+def get_path():
+
+    # Get the "path" and "info" arguments from the URL
+    path = request.args.get('path')
+
+    # replace windows paths
+    path = path.replace("\\", "/")
+    if path.startswith("/"):
+        path = path[1:]
+    print("path", path, file=sys.stderr)
+
+    info = request.args.get('info')
+
+    if not path:
+        return jsonify({"error": "Missing 'path' parameter."}), 400
+
+    # Ensure the path is secure and resolve it within the BASE_DIR
+    #secure_path = secure_filename(path)
+    full_path = os.path.abspath(os.path.join(BASE_DIR, path))
+
+    if not full_path.startswith(BASE_DIR):
+        return jsonify({"error": "Access to the specified path is not allowed."}), 403
+
+    print(full_path, file=sys.stderr)
+
+    # Check if the path exists
+    if not os.path.exists(full_path):
+        print("missing file", file=sys.stderr)
+        return jsonify({"contents": list()})
+
+    # If the path is a directory, return a JSON list of its contents
+    if os.path.isdir(full_path):
+        contents = filter(lambda x: not x.startswith("."), os.listdir(full_path))
+        return jsonify({"contents": list(contents)})
+
+    # If the path is a file
+    if os.path.isfile(full_path):
+        if info == '1':
+            # Return the file size if 'info=1' is specified
+            file_size = os.path.getsize(full_path)
+            return jsonify({"size": file_size})
+        else:
+            # Return the file as a download
+            try:
+                return send_file(full_path, as_attachment=True)
+            except Exception as e:
+                return jsonify({"error": str(e)}), 500
+
+    # If the path is neither a file nor a directory, return an error
+    return jsonify({"error": "Invalid path type."}), 400
+
+if __name__ == '__main__':
+    app.run(debug=True)

software.py

@@ -32,7 +32,7 @@ class Software:
 
     def _load_from_yaml(self):
 
-        content = self.backend.get(self.meta_file, self.cache_dir, return_content=True, new_connection=True)
+        content = self.backend.get(self.meta_file, self.cache_dir, return_content=True)
 
         meta = yaml.safe_load(content)
         if not meta:
@@ -48,10 +48,10 @@ class Software:
         self.run_exe = meta.get("run_exe")
         self.installer = meta.get("installer")
 
-        self.pictures = [ self.backend.get(pp, self.cache_dir, new_connection=True) for pp in
+        self.pictures = [ self.backend.get(pp, self.cache_dir) for pp in
-                self.backend.list(os.path.join(self.directory, "pictures"), fullpaths=True, new_connection=True) ]
+                self.backend.list(os.path.join(self.directory, "pictures"), fullpaths=True) ]
 
-        self.reg_files = self.backend.list(os.path.join(self.directory, "registry_files"), fullpaths=True, new_connection=True)
+        self.reg_files = self.backend.list(os.path.join(self.directory, "registry_files"), fullpaths=True)
 
 
     def get_thumbnail(self):

statekeeper.py

@@ -0,0 +1,25 @@
+import requests
+import os
+
+def add_to_download_queue(url, path):
+    '''The download is added to the global queue and downloaded eventually'''
+    _download(url, path)
+
+def add_to_task_queue(task):
+    '''Add a callback to background execution queue'''
+    task()
+
+def _download(url, path):
+
+    response = requests.get(url + path, stream=True)
+
+    # Check if the request was successful
+    if response.status_code == 200:
+        # Save the file locally
+        local_filename = os.path.join("./cache", path)
+
+        with open(local_filename, 'wb') as f:
+            for chunk in response.iter_content(chunk_size=8192):  # Download in chunks
+                f.write(chunk)
+
+        print(f"File downloaded successfully as {local_filename}")