From f46fce18248b1c15c5271aee0762e27773fc1093 Mon Sep 17 00:00:00 2001 From: Yannik Schmidt Date: Wed, 8 Jan 2025 19:39:23 +0100 Subject: [PATCH] wip: asnyc http (1) --- .gitignore | 1 + client.py | 43 +++++++---- client_details.py | 35 +++++---- data_backend.py | 188 +++++++++------------------------------------- server/main.py | 65 ++++++++++++++++ software.py | 8 +- statekeeper.py | 25 ++++++ 7 files changed, 181 insertions(+), 184 deletions(-) create mode 100644 server/main.py create mode 100644 statekeeper.py diff --git a/.gitignore b/.gitignore index 2012580..57f4421 100644 --- a/.gitignore +++ b/.gitignore @@ -14,3 +14,4 @@ example_software_root/ # windows dir on linux รค **\\install-dir/ +server/data/ diff --git a/client.py b/client.py index e5af13a..d442d90 100644 --- a/client.py +++ b/client.py @@ -9,6 +9,7 @@ import os import cache_utils import imagetools import webbrowser +import statekeeper customtkinter.set_appearance_mode("dark") customtkinter.set_default_color_theme("blue") @@ -183,6 +184,7 @@ def load_main(): # create tiles from meta files # cache_dir_size = 0 for software in db.find_all_metadata(): + create_main_window_tile(software, scrollable_frame) # retrieve cache dir from any software # @@ -222,24 +224,35 @@ def load_details(app, software): def create_main_window_tile(software, parent): '''Create the main window tile''' - if software.get_thumbnail(): - try: - print("Loading thumbnail:", software.get_thumbnail()) - img = PIL.Image.open(software.get_thumbnail()) - img = imagetools.smart_resize(img, 200, 300) - except PIL.UnidentifiedImageError: - print("Failed to load thumbnail:", software.get_thumbnail()) - img = PIL.Image.new('RGB', (200, 300)) - else: - img = PIL.Image.new('RGB', (200, 300)) - - img = PIL.ImageTk.PhotoImage(img) - button = customtkinter.CTkButton(parent, image=img, + img = PIL.Image.new('RGB', (200, 300)) + imgTk = PIL.ImageTk.PhotoImage(img) + button = customtkinter.CTkButton(parent, image=imgTk, width=200, height=300, command=lambda: switch_to_game_details(software), border_width=0, corner_radius=0, border_spacing=0, text=software.title, fg_color="transparent", compound="top", anchor="s") + + + def callback_update_thumbnail(): + + # TODO: bind button & software into this callback + + try: + target_file = software.get_thumbnail() + print("Loading thumbnail (async):", ) + img = PIL.Image.open(software.get_thumbnail()) + img = imagetools.smart_resize(img, 200, 300) + except PIL.UnidentifiedImageError: + print("Failed to load thumbnail:", software.get_thumbnail()) + img = PIL.Image.new('RGB', (200, 300)) + + # TODO: button reconfigure + + # register the update task for the image # + statekeeper.add_task(callback_update_thumbnail) + + # cache button and return # buttons.append(button) return button @@ -320,7 +333,9 @@ if __name__ == "__main__": print(user, password, install_dir, remote_root_dir, server, config_loaded["Server/Path:"]) # add db backend # - if backend_type == "FTP/FTPS": + if True: + db = data_backend.HTTP(None, None, install_dir, remote_root_dir="./", server="http://localhost:5000", tkinter_root=app) + elif backend_type == "FTP/FTPS": db = data_backend.FTP(user, password, install_dir, server=server, remote_root_dir=remote_root_dir, progress_bar_wrapper=pgw, tkinter_root=app) elif backend_type == "Local Filesystem": diff --git a/client_details.py b/client_details.py index 0dc6a88..ea22a1e 100644 --- a/client_details.py +++ b/client_details.py @@ -3,6 +3,7 @@ import tkinter import customtkinter import imagetools import os +import statekeeper def show_large_picture(app, path): '''Show a full-window version of the clicked picture''' @@ -21,6 +22,7 @@ def show_large_picture(app, path): large_image = customtkinter.CTkButton(app, text="", image=img, width=x-2*30, height=y-2*30, fg_color="transparent", hover_color="black", corner_radius=0, border_width=0, border_spacing=0, command=lambda: large_image.destroy()) + large_image.place(x=30, y=30) def create_details_page(app, software, backswitch_function): @@ -170,21 +172,24 @@ def create_details_page(app, software, backswitch_function): # add other pictures # if software.pictures: - picture_frame = customtkinter.CTkScrollableFrame(info_frame, height=200, width=300, orientation="horizontal", fg_color="transparent") - picture_frame.grid(column=0, row=7, sticky="we") + def callback_add_pictures(): + picture_frame = customtkinter.CTkScrollableFrame(info_frame, height=200, width=300, orientation="horizontal", fg_color="transparent") + picture_frame.grid(column=0, row=7, sticky="we") - i = 0 - for path in software.pictures[1:]: - img = PIL.Image.open(path) - img = imagetools.smart_resize(img, 180, 180) - img = PIL.ImageTk.PhotoImage(img) - extra_pic_button = customtkinter.CTkButton(picture_frame, text="", image=img, command=lambda path=path: show_large_picture(app, path), - hover_color="black", corner_radius=0,) - extra_pic_button.configure(fg_color="transparent") - extra_pic_button.grid(pady=10, row=0, column=i) - elements.append(extra_pic_button) - i += 1 - - elements.append(picture_frame) + i = 0 + for path in software.pictures[1:]: + img = PIL.Image.open(path) + img = imagetools.smart_resize(img, 180, 180) + img = PIL.ImageTk.PhotoImage(img) + extra_pic_button = customtkinter.CTkButton(picture_frame, text="", image=img, command=lambda path=path: show_large_picture(app, path), + hover_color="black", corner_radius=0,) + extra_pic_button.configure(fg_color="transparent") + extra_pic_button.grid(pady=10, row=0, column=i) + elements.append(extra_pic_button) + i += 1 + + elements.append(picture_frame) + + statekeeper.add_to_task_queue(callback_add_pictures) return elements diff --git a/data_backend.py b/data_backend.py index bb91ede..9e6c111 100644 --- a/data_backend.py +++ b/data_backend.py @@ -6,19 +6,9 @@ import ftplib import tqdm import ssl import concurrent.futures +import statekeeper +import requests -class SESSION_REUSE_FTP_TLS(ftplib.FTP_TLS): - """Explicit FTPS, with shared TLS session""" - - def ntransfercmd(self, cmd, rest=None): - - conn, size = ftplib.FTP.ntransfercmd(self, cmd, rest) - if self._prot_p: - conn = self.context.wrap_socket( - conn, - server_hostname=self.host, - session=self.sock.session) # this is the fix - return conn, size class DataBackend: @@ -36,7 +26,6 @@ class DataBackend: self.progress_bar_wrapper = progress_bar_wrapper self.root = tkinter_root self.cache_dir = "./cache/" - self.ftp = None # ftp connection object def get(self, path, return_content=False): '''Return the contents of this path''' @@ -102,70 +91,16 @@ class LocalFS(DataBackend): meta_info_list.append(software.Software(meta_file, self, self.progress_bar_wrapper)) return list(filter(lambda x: not x.invalid, meta_info_list)) - -class FTP(DataBackend): +class HTTP(DataBackend): paths_listed = {} + REMOTE_PATH = "/get-path" - def _connect(self, individual_connection=False): + def _get_url(self): + print(self.server + HTTP.REMOTE_PATH) + return self.server + HTTP.REMOTE_PATH - if self.ftp and not individual_connection: - try: - self.ftp.voidcmd("NOOP") - return self.ftp - except ssl.SSLError: - pass # reconnect - - if self.server.startswith("ftp://"): - tls = False - elif self.server.startswith("ftps://"): - tls = True - else: - raise ValueError("FTP Server must start with ftp:// or ftps://") - - # build connection parameters # - server = self.server.split("://")[1] - port = None - try: - server = server.split(":")[0] - except (IndexError, ValueError): - port = 0 - - # try extract server # - try: - server = server.split(":")[0] - except (IndexError, ValueError): - server = self.server - - print("Connecting to:", server, "on port:", port, "ssl =", tls) - - # connect # - if not tls: - ftp = ftplib.FTP() - else: - ftp = SESSION_REUSE_FTP_TLS() - ftp.ssl_version = ssl.PROTOCOL_TLSv1_2 - - ftp.connect(server, port=port or 0) - - if self.user: - ftp.login(self.user, self.password) - else: - ftp.login() - - # open a secure session for tls # - if tls: - ftp.prot_p() - - # cache dir is automatically set # - self.cache_dir = None - - if not individual_connection: - self.ftp = ftp - return ftp - - - def get(self, path, cache_dir=None, return_content=False, new_connection=False): + def get(self, path, cache_dir=None, return_content=False): # check the load cache dir # if cache_dir: @@ -177,102 +112,56 @@ class FTP(DataBackend): fullpath = path if self.remote_root_dir and not path.startswith(self.remote_root_dir): fullpath = os.path.join(self.remote_root_dir, path) - #print(self.remote_root_dir, path, fullpath) + fullpath = fullpath.replace("\\", "/") local_file = os.path.join(cache_dir, os.path.basename(path)) - # print("Cachedir:", cache_dir, os.path.basename(path), local_file) + print("Requiring:", local_file) if not os.path.isfile(local_file): - ftp = self._connect(individual_connection=True) - ftp.sendcmd('TYPE I') + if return_content: - # load the file on remote # - if not new_connection: + # the content is needed for the UI now and not cached, it's needs to be downloaded synchroniously # + # as there cannot be a meaningful UI-draw without it. # + r = requests.get(self._get_url(), params={ "path" : path, "as_string": True }) - total_size = ftp.size(fullpath) - print("Total Size:", total_size) - self.progress_bar_wrapper.get_pb()["maximum"] = total_size + # cache the download imediatelly # + with open(local_file, encoding="utf-8", mode="w") as f: + f.write(r.json()["content"]) - print(local_file, "not in cache, retriving..") - with open(local_file, "w") as f: - f.write(local_file) - with open(local_file, 'wb') as local_file_open, tqdm.tqdm( - desc="Downloading", - total=total_size, - unit='B', - unit_scale=True - ) as cmd_progress_bar: + # return the content # + return r.json()["content"] - # Define a callback function to update the progress bar # - def callback(data): - local_file_open.write(data) - if new_connection: # return if parralell - return - self.root.update_idletasks() # Update the GUI - current_total = self.progress_bar_wrapper.get_pb().get() + len(data)/total_size - self.progress_bar_wrapper.get_pb().set(current_total) - self.progress_bar_wrapper.set_text( - text="Downloading: {:.2f}%".format(current_total*100)) - cmd_progress_bar.update(len(data)) - - # run with callback # - ftp.retrbinary('RETR ' + fullpath, callback) else: - with open(local_file, 'wb') as fp: - ftp.retrbinary('RETR ' + fullpath, fp.write) + statekeeper.add_to_download_queue(self._get_url(), path, first=return_content) - if new_connection: - ftp.close() - - if return_content: + elif return_content: with open(local_file, encoding="utf-8") as fr: return fr.read() + else: + return local_file - return local_file + def list(self, path, fullpaths=False): - def list(self, path, fullpaths=False, new_connection=False): - - # prepend root dir if not given # fullpath = path if self.remote_root_dir and not path.startswith(self.remote_root_dir): fullpath = os.path.join(self.remote_root_dir, path) fullpath = fullpath.replace("\\", "/") - #print(fullpath) - # if not os.path.isdir(fullpath): - # return [] + # retrieve session cached paths # + if fullpath in self.paths_listed: + paths = self.paths_listed[fullpath] + else: - try: - # retrieve session cached paths # - if fullpath in self.paths_listed: - paths = self.paths_listed[fullpath] - #print("Retrieved paths from cache:", fullpath, paths) - else: - ftp = self._connect(individual_connection=new_connection) - print("Listing previously unlisted path: {}".format(fullpath)) - self.paths_listed.update({fullpath: []}) # in case dir does not exit - paths = ftp.nlst(fullpath) - self.paths_listed.update({fullpath: paths}) - - if new_connection: # close individual connections - ftp.close() + r = requests.get(self._get_url(), params={ "path" : path }) + print(r, r.status_code, r.content) + paths = r.json()["contents"] if not fullpaths: return paths - - return [ os.path.join(path, filename).replace("\\", "/") for filename in paths ] - - except ftplib.error_perm as e: - if "550 No files found" in str(e): - print("No files in this directory: {}".format(fullpath)) - return [] - elif "550 No such file or directory" in str(e): - print("File or dir does not exist: {}".format(fullpath)) - return [] else: - raise e + return [ os.path.join(path, filename).replace("\\", "/") for filename in paths ] def find_all_metadata(self): @@ -282,24 +171,21 @@ class FTP(DataBackend): with concurrent.futures.ThreadPoolExecutor(max_workers=os.cpu_count()*5) as executor: software_dir_contents = list(executor.map( - lambda s: self.list(s, fullpaths=True, new_connection=True), root_elements)) + lambda s: self.list(s, fullpaths=True), root_elements)) # this caches the paths, done remove it # cache_list = [os.path.join(s, "registry_files") for s in root_elements ] cache_list += [os.path.join(s, "pictures") for s in root_elements ] + + # THIS PRELOAD IMAGES-paths, DO NOT REMOVE IT # picture_contents_async_cache = list(executor.map( - lambda s: self.list(s, fullpaths=True, new_connection=True), cache_list)) + lambda s: self.list(s, fullpaths=True), cache_list)) for files in software_dir_contents: - #print(s) - #files = self.list(s, fullpaths=True) - print(files) for f in files: if f.endswith("meta.yaml"): - meta_file_content = self.get(f, cache_dir="cache", return_content=True) - #print(meta_file_content) local_meta_file_list.append(f) with concurrent.futures.ThreadPoolExecutor(max_workers=os.cpu_count()*5) as executor: software_list = executor.map(lambda meta_file: software.Software(meta_file, self, self.progress_bar_wrapper), local_meta_file_list) - return list(filter(lambda x: not x.invalid, software_list)) + return list(filter(lambda x: not x.invalid, software_list)) \ No newline at end of file diff --git a/server/main.py b/server/main.py new file mode 100644 index 0000000..c169c09 --- /dev/null +++ b/server/main.py @@ -0,0 +1,65 @@ +from flask import Flask, request, jsonify, send_file, abort +import os +import sys + +from werkzeug.utils import secure_filename + +app = Flask(__name__) + +# Base directory constraint +BASE_DIR = os.path.abspath("./data") + +@app.route('/get-path', methods=['GET']) +def get_path(): + + # Get the "path" and "info" arguments from the URL + path = request.args.get('path') + + # replace windows paths + path = path.replace("\\", "/") + if path.startswith("/"): + path = path[1:] + print("path", path, file=sys.stderr) + + info = request.args.get('info') + + if not path: + return jsonify({"error": "Missing 'path' parameter."}), 400 + + # Ensure the path is secure and resolve it within the BASE_DIR + #secure_path = secure_filename(path) + full_path = os.path.abspath(os.path.join(BASE_DIR, path)) + + if not full_path.startswith(BASE_DIR): + return jsonify({"error": "Access to the specified path is not allowed."}), 403 + + print(full_path, file=sys.stderr) + + # Check if the path exists + if not os.path.exists(full_path): + print("missing file", file=sys.stderr) + return jsonify({"contents": list()}) + + # If the path is a directory, return a JSON list of its contents + if os.path.isdir(full_path): + contents = filter(lambda x: not x.startswith("."), os.listdir(full_path)) + return jsonify({"contents": list(contents)}) + + # If the path is a file + if os.path.isfile(full_path): + if info == '1': + # Return the file size if 'info=1' is specified + file_size = os.path.getsize(full_path) + return jsonify({"size": file_size}) + else: + # Return the file as a download + try: + return send_file(full_path, as_attachment=True) + except Exception as e: + return jsonify({"error": str(e)}), 500 + + # If the path is neither a file nor a directory, return an error + return jsonify({"error": "Invalid path type."}), 400 + +if __name__ == '__main__': + app.run(debug=True) diff --git a/software.py b/software.py index 9705cc9..98400c0 100644 --- a/software.py +++ b/software.py @@ -32,7 +32,7 @@ class Software: def _load_from_yaml(self): - content = self.backend.get(self.meta_file, self.cache_dir, return_content=True, new_connection=True) + content = self.backend.get(self.meta_file, self.cache_dir, return_content=True) meta = yaml.safe_load(content) if not meta: @@ -48,10 +48,10 @@ class Software: self.run_exe = meta.get("run_exe") self.installer = meta.get("installer") - self.pictures = [ self.backend.get(pp, self.cache_dir, new_connection=True) for pp in - self.backend.list(os.path.join(self.directory, "pictures"), fullpaths=True, new_connection=True) ] + self.pictures = [ self.backend.get(pp, self.cache_dir) for pp in + self.backend.list(os.path.join(self.directory, "pictures"), fullpaths=True) ] - self.reg_files = self.backend.list(os.path.join(self.directory, "registry_files"), fullpaths=True, new_connection=True) + self.reg_files = self.backend.list(os.path.join(self.directory, "registry_files"), fullpaths=True) def get_thumbnail(self): diff --git a/statekeeper.py b/statekeeper.py new file mode 100644 index 0000000..db609f5 --- /dev/null +++ b/statekeeper.py @@ -0,0 +1,25 @@ +import requests +import os + +def add_to_download_queue(url, path): + '''The download is added to the global queue and downloaded eventually''' + _download(url, path) + +def add_to_task_queue(task): + '''Add a callback to background execution queue''' + task() + +def _download(url, path): + + response = requests.get(url + path, stream=True) + + # Check if the request was successful + if response.status_code == 200: + # Save the file locally + local_filename = os.path.join("./cache", path) + + with open(local_filename, 'wb') as f: + for chunk in response.iter_content(chunk_size=8192): # Download in chunks + f.write(chunk) + + print(f"File downloaded successfully as {local_filename}") \ No newline at end of file