wip: http backend with callbacks

..currently all callbacks are executed non-async for testing
..they may be bugs with async downloads not correctly being waited for
..there may be problems with binding changing variables correctly

client.py

@@ -185,6 +185,7 @@ def load_main():
     cache_dir_size = 0
     for software in db.find_all_metadata():
 
+        print("Software:", software)
         create_main_window_tile(software, scrollable_frame)
 
         # retrieve cache dir from any software #
@@ -240,17 +241,19 @@ def create_main_window_tile(software, parent):
 
         try:
             target_file = software.get_thumbnail()
-            print("Loading thumbnail (async):",  )
+            if not target_file:
-            img = PIL.Image.open(software.get_thumbnail())
+                return
+            print("Loading thumbnail (async):",  target_file)
+            img = PIL.Image.open(target_file)
             img = imagetools.smart_resize(img, 200, 300)
         except PIL.UnidentifiedImageError:
-            print("Failed to load thumbnail:", software.get_thumbnail())
+            print("Failed to load thumbnail:", target_file)
             img = PIL.Image.new('RGB', (200, 300))
         
-        # TODO: button reconfigure
+        button.configure(image=PIL.ImageTk.PhotoImage(img))
 
     # register the update task for the image #
-    statekeeper.add_task(callback_update_thumbnail)
+    statekeeper.add_to_task_queue(callback_update_thumbnail)
 
     # cache button and return #
     buttons.append(button)
@@ -334,7 +337,7 @@ if __name__ == "__main__":
 
         # add db backend #
         if True:
-            db = data_backend.HTTP(None, None, install_dir, remote_root_dir="./", server="http://localhost:5000", tkinter_root=app)
+            db = data_backend.HTTP(None, None, install_dir, remote_root_dir="./", server="http://localhost:5000", progress_bar_wrapper=pgw, tkinter_root=app)
         elif backend_type == "FTP/FTPS":
             db = data_backend.FTP(user, password, install_dir, server=server,
                 remote_root_dir=remote_root_dir, progress_bar_wrapper=pgw, tkinter_root=app)

client_details.py

@@ -173,12 +173,16 @@ def create_details_page(app, software, backswitch_function):
     if software.pictures:
 
         def callback_add_pictures():
+
             picture_frame = customtkinter.CTkScrollableFrame(info_frame, height=200, width=300, orientation="horizontal", fg_color="transparent")
             picture_frame.grid(column=0, row=7, sticky="we")
 
             i = 0
+            print("Software pictures in callback:", software.pictures[1:])
             for path in software.pictures[1:]:
-                img = PIL.Image.open(path)
+
+                print("Doing picture:", path)
+                img = PIL.Image.open(software.backend.get(path))
                 img = imagetools.smart_resize(img, 180, 180)
                 img = PIL.ImageTk.PhotoImage(img)
                 extra_pic_button = customtkinter.CTkButton(picture_frame, text="", image=img, command=lambda path=path: show_large_picture(app, path),

data_backend.py

@@ -97,10 +97,16 @@ class HTTP(DataBackend):
     REMOTE_PATH = "/get-path"
 
     def _get_url(self):
-        print(self.server + HTTP.REMOTE_PATH)
+        #print(self.server + HTTP.REMOTE_PATH)
         return self.server + HTTP.REMOTE_PATH
 
-    def get(self, path, cache_dir=None, return_content=False):
+    def get(self, path, cache_dir="", return_content=False):
+
+        print("Getting", path, "cache dir", cache_dir, "return content:", return_content)
+
+        if cache_dir is None:
+            print("Setting cache dir from backend default", "cur:", cache_dir, "new (default):", self.cache_dir)
+            cache_dir = self.cache_dir
 
         # check the load cache dir #
         if cache_dir:
@@ -114,32 +120,44 @@ class HTTP(DataBackend):
             fullpath = os.path.join(self.remote_root_dir, path)
 
         fullpath = fullpath.replace("\\", "/")
-        local_file = os.path.join(cache_dir, os.path.basename(path))
+        local_file = os.path.join(cache_dir, fullpath)
+        local_dir = os.path.dirname(local_file)
 
-        print("Requiring:", local_file)
+        # sanity check and create directory #
+        if not local_dir.startswith(cache_dir):
+            raise AssertionError("Local Dir does not start with cache dir:" + local_dir)
+        else:
+            os.makedirs(local_dir, exist_ok=True)
 
-        if not os.path.isfile(local_file):
+        print("Requiring:", fullpath)
+
+        if not os.path.isfile(local_file) or os.stat(local_file).st_size == 0:
 
             if return_content:
 
                 # the content is needed for the UI now and not cached, it's needs to be downloaded synchroniously #
                 # as there cannot be a meaningful UI-draw without it. #
                 r = requests.get(self._get_url(), params={ "path" : path, "as_string": True })
-
+                print("Request Content:", r.text)
                 # cache the download imediatelly #
                 with open(local_file, encoding="utf-8", mode="w") as f:
-                    f.write(r.json()["content"])
+                    f.write( r.text)
 
                 # return the content #
-                return r.json()["content"]
+                print("Content for", fullpath, ":",  r.text)
+                return r.text
 
             else:
-                statekeeper.add_to_download_queue(self._get_url(), path, first=return_content)
+                print("Async Requested for:", local_file)
+                statekeeper.add_to_download_queue(self._get_url(), path)
+                return local_file
 
         elif return_content:
+            print("Returning Cached file:", local_file)
             with open(local_file, encoding="utf-8") as fr:
                 return fr.read()
         else:
+            print("Already present:", local_file)
             return local_file
 
     def list(self, path, fullpaths=False):
@@ -155,7 +173,7 @@ class HTTP(DataBackend):
         else:
 
             r = requests.get(self._get_url(), params={ "path" : path })
-            print(r, r.status_code, r.content)
+            #print(r, r.status_code, r.content)
             paths = r.json()["contents"]
 
             if not fullpaths:
@@ -168,6 +186,7 @@ class HTTP(DataBackend):
         local_meta_file_list = []
 
         root_elements = self.list(self.remote_root_dir)
+        print("root elements:", root_elements)
         with concurrent.futures.ThreadPoolExecutor(max_workers=os.cpu_count()*5) as executor:
 
             software_dir_contents = list(executor.map(
@@ -186,6 +205,15 @@ class HTTP(DataBackend):
                 if f.endswith("meta.yaml"):
                     local_meta_file_list.append(f)
 
+
+        print("local meta:", local_meta_file_list)
+        software_list = None
         with concurrent.futures.ThreadPoolExecutor(max_workers=os.cpu_count()*5) as executor:
             software_list = executor.map(lambda meta_file: software.Software(meta_file, self, self.progress_bar_wrapper), local_meta_file_list)
-            return list(filter(lambda x: not x.invalid, software_list))
+
+        # evaluate
+        software_list = list(software_list)
+        print("Software List:", software_list)
+        print("Invalid:", list(filter(lambda x: x.invalid, software_list)))
+        print("Valid:", list(filter(lambda x: not x.invalid, software_list)))
+        return list(filter(lambda x: not x.invalid, software_list))

server/main.py

@@ -19,6 +19,7 @@ def get_path():
     path = path.replace("\\", "/")
     if path.startswith("/"):
         path = path[1:]
+
     print("path", path, file=sys.stderr)
 
     info = request.args.get('info')

software.py

@@ -26,13 +26,19 @@ class Software:
             self.invalid = False
             self._load_from_yaml()
         except ValueError as e:
+            print(e)
+            raise e
             self.invalid = True
 
+        if not progress_bar_wrapper:
+            raise AssertionError()
+
         self.progress_bar_wrapper = progress_bar_wrapper
 
     def _load_from_yaml(self):
 
         content = self.backend.get(self.meta_file, self.cache_dir, return_content=True)
+        # print("Meta-Content:", content)
 
         meta = yaml.safe_load(content)
         if not meta:
@@ -50,9 +56,14 @@ class Software:
 
         self.pictures = [ self.backend.get(pp, self.cache_dir) for pp in
                 self.backend.list(os.path.join(self.directory, "pictures"), fullpaths=True) ]
+        
+        if any([x is None for x in self.pictures]):
+            raise AssertionError("None Entries in self.pictures: " + str(self.pictures))
 
         self.reg_files = self.backend.list(os.path.join(self.directory, "registry_files"), fullpaths=True)
 
+        print("Finished Init for", self.title)
+
 
     def get_thumbnail(self):
         '''Return the thumbnail for this software'''

statekeeper.py

@@ -7,14 +7,16 @@ def add_to_download_queue(url, path):
 
 def add_to_task_queue(task):
     '''Add a callback to background execution queue'''
+    print("Executing tasks", task)
     task()
 
 def _download(url, path):
 
-    response = requests.get(url + path, stream=True)
+    response = requests.get(url + "?path=" + path, stream=True)
 
     # Check if the request was successful
     if response.status_code == 200:
+
         # Save the file locally
         local_filename = os.path.join("./cache", path)
 
@@ -22,4 +24,8 @@ def _download(url, path):
             for chunk in response.iter_content(chunk_size=8192):  # Download in chunks
                 f.write(chunk)
 
-        print(f"File downloaded successfully as {local_filename}")
+        print(f"File downloaded successfully as {local_filename}")
+    
+    else:
+
+        raise AssertionError("Non-200 Response for:", url, path, response.status_code, response.text)