feat: basic background progress bars (downloads)

data_backend.py

@@ -100,19 +100,7 @@ class HTTP(DataBackend):
         #print(self.server + HTTP.REMOTE_PATH)
         return self.server + HTTP.REMOTE_PATH
  
-    def get(self, path, cache_dir="", return_content=False, wait=False):
+    def get_local_target(self, path):
-
-        print("Getting", path, "cache dir", cache_dir, "return content:", return_content)
-
-        if cache_dir is None:
-            print("Setting cache dir from backend default", "cur:", cache_dir, "new (default):", self.cache_dir)
-            cache_dir = self.cache_dir
-
-        # check the load cache dir #
-        if cache_dir:
-            self._create_cache_dir(cache_dir)
-        elif not cache_dir and not return_content:
-            AssertionError("Need to set either cache_dir or return_content!")
 
         # prepend root dir if not given #
         fullpath = path
@@ -120,7 +108,30 @@ class HTTP(DataBackend):
             fullpath = os.path.join(self.remote_root_dir, path)
 
         fullpath = fullpath.replace("\\", "/")
-        local_file = os.path.join(cache_dir, fullpath)
+        local_file = os.path.join(self.cache_dir, fullpath)
+        print("Local Target is", local_file)
+        return local_file
+
+    def get(self, path, cache_dir=None, return_content=False, wait=False):
+
+        print("Getting", path, "cache dir", cache_dir, "return content:", return_content)
+
+        if cache_dir is None:
+            print("Setting cache dir from backend default", "cur:", cache_dir, "new (default):", self.cache_dir)
+            cache_dir = self.cache_dir
+
+        # fix cache path reuse #
+        if path.startswith(cache_dir):
+            path = path[len(cache_dir):]
+            print("Fixed path to not duble include cache dir, path:", path)
+
+        # check the load cache dir #
+        if cache_dir:
+            self._create_cache_dir(cache_dir)
+        elif not cache_dir and not return_content:
+            AssertionError("Need to set either cache_dir or return_content!")
+
+        local_file = self.get_local_target(path)
         local_dir = os.path.dirname(local_file)
 
         # sanity check and create directory #
@@ -129,7 +140,7 @@ class HTTP(DataBackend):
         else:
             os.makedirs(local_dir, exist_ok=True)
 
-        print("Requiring:", fullpath)
+        print("Requiring:", path)
 
         if not os.path.isfile(local_file) or os.stat(local_file).st_size == 0:
 
@@ -137,14 +148,24 @@ class HTTP(DataBackend):
 
                 # the content is needed for the UI now and not cached, it's needs to be downloaded synchroniously #
                 # as there cannot be a meaningful UI-draw without it. #
-                r = requests.get(self._get_url(), params={ "path" : path, "as_string": True })
+                # THIS IS THE OLD WAY
+                # r = requests.get(self._get_url(), params={ "path" : path, "as_string": True })
 
-                # cache the download imediatelly #
+                # # cache the download imediatelly #
-                with open(local_file, encoding="utf-8", mode="w") as f:
+                # with open(local_file, encoding="utf-8", mode="w") as f:
-                    f.write(r.text)
+                #     f.write(r.text)
+
+                # this is with streaming
+                chunk_size = 1024 * 1024  * 5 # 5MB
+                r = requests.get(self._get_url(), params={"path": path, "as_string": True}, stream=True)
+
+                with open(local_file, "wb") as f:
+                    for chunk in r.iter_content(chunk_size=chunk_size, decode_unicode=True):
+                        if chunk:
+                            f.write(chunk)
 
                 if return_content:                    
-                    print("Content for", fullpath, ":",  r.text)
+                    print("Content for", path, ":",  r.text)
                     return r.text
                 else:
                     return local_file

db.py

@@ -9,11 +9,20 @@ class Download(Base):
     __tablename__ = 'files'
 
     path = Column(String, primary_key=True)
+    local_path = Column(String)
+    url = Column(String)
     size = Column(Integer)
     type = Column(String)
     finished = Column(Boolean)
 
+    def __eq__(self, other):
+        return self.path == other.path
+    
+    def __hash__(self):
+        return hash(self.path)
+
 class Database:
+
     def __init__(self, db_url="sqlite:///database.db"):
         self.engine = create_engine(db_url, echo=True)
         self.session_factory = sessionmaker(bind=self.engine)

infowidget.py

@@ -14,6 +14,8 @@ import threading
 import random
 import time
 import string
+import statekeeper
+import os
 
 class ProgressBarApp:
     def __init__(self, parent):
@@ -33,8 +35,16 @@ class ProgressBarApp:
         threading.Thread(target=self.add_progress_bars, daemon=True).start()
 
     def add_progress_bars(self):
+
+        already_tracked = set()
         while self.running:
 
+            downloads = set(statekeeper.get_download())
+            new = downloads - already_tracked
+            already_tracked |= set(downloads)
+
+            for element in new:
+
                 frame = tk.Frame(self.frame)
                 frame.pack(fill=tk.X, pady=2)
 
@@ -44,28 +54,46 @@ class ProgressBarApp:
                 delete_button = tk.Button(frame, text="Delete", command=lambda f=frame: self.delete_progress(f), state=tk.DISABLED)
                 delete_button.pack(side=tk.LEFT, padx=5)
 
-            random_letter = random.choice(string.ascii_uppercase)
+                label = tk.Label(frame, text=os.path.basename(element.path))
-            label = tk.Label(frame, text=random_letter)
                 label.pack(side=tk.LEFT, padx=5)
 
                 self.progress_bars.insert(0, (progress, frame, delete_button))  # Insert at the top
                 frame.pack(fill=tk.X, pady=2, before=self.frame.winfo_children()[-1] if self.frame.winfo_children() else None)
 
-            duration = random.randint(1, 10)  # Random fill time
+                print("Starting tracker for", element.path)
-            threading.Thread(target=self.fill_progress, args=(progress, duration, frame, delete_button), daemon=True).start()
+                threading.Thread(target=self.fill_progress, args=(progress, element.path, frame, delete_button), daemon=True).start()
 
-            time.sleep(30)  # Wait before adding a new progress bar
+            time.sleep(2)  # Wait before adding a new progress bar
+
+    def fill_progress(self, progress, path, frame, delete_button):
+
+        fail_count = 0
+        while True:
 
-    def fill_progress(self, progress, duration, frame, delete_button):
-        for i in range(101):  # Fill progress bar over 'duration' seconds
-            time.sleep(duration / 100)
             if not progress.winfo_exists():  # Check if progress bar still exists
                 return
-            self.root.after(0, progress.config, {"value": i})
 
+            try:
+                percent_filled = statekeeper.get_percent_filled(path)
+            except OSError as e:
+                fail_count += 1
+                if fail_count > 6:
+                    raise e
+                else:
+                    time.sleep(1)
+                    continue
+
+            print("Percent filled:", percent_filled, path)
+            if not percent_filled or percent_filled >= 99.9:
+                self.root.after(0, progress.config, { "value" : 100 })
+                break
+            else:
+                self.root.after(0, progress.config, { "value" : percent_filled })                
+                time.sleep(0.5)
+
+        # handle finished download #
         self.root.after(0, delete_button.config, {"state": tk.NORMAL})
-
+        self.progress_bars.append((progress, frame, path, delete_button))
-        self.progress_bars.append((progress, frame, duration, delete_button))
         self.update_delete_all_button()
 
     def delete_progress(self, frame):

software.py

@@ -133,7 +133,7 @@ class Software:
             print("No main_dir:", path)
             raise AssertionError("No main_dir for this software")
         
-        statekeeper.log_begin_download(remote_file)
+        statekeeper.log_begin_download(remote_file, self.backend.get_local_target(remote_file), self.backend._get_url())
         local_file = self.backend.get(remote_file, self.cache_dir, wait=True)
         statekeeper.log_end_download(remote_file)
 

statekeeper.py

@@ -5,6 +5,9 @@ import threading
 from db import db, Download
 from sqlalchemy import or_, and_
 
+def _bytes_to_mb(size):
+    return size / (1024*1024)
+
 def add_to_download_queue(url, path):
     '''The download is added to the global queue and downloaded eventually'''
     #_download(url, path)
@@ -38,7 +41,7 @@ def _download(url, path):
 
         raise AssertionError("Non-200 Response for:", url, path, response.status_code, response.text)
 
-def log_begin_download(path):
+def log_begin_download(path, local_path, url):
 
     session = db.session()
     print("Current path", path)
@@ -49,7 +52,7 @@ def log_begin_download(path):
         raise AssertionError("ERROR: {} is already downloading.".format(path))
     else:
         print("Adding to download log:", path)
-        session.merge(Download(path=path, size=0, type="download", finished=False))
+        session.merge(Download(path=path, size=-1, type="download", local_path=local_path, url=url, finished=False))
         session.commit()
 
     db.close_session()
@@ -57,12 +60,60 @@ def log_begin_download(path):
 def log_end_download(path):
 
     session = db.session()
-    path_exists = session.query(Download).filter(Download.path==path).first()
+    obj = session.query(Download).filter(Download.path==path).first()
-    if not path_exists:
+    if not obj:
         raise AssertionError("ERROR: {} is not downloading/cannot remove.".format(path))
     else:
         print("Removing from download log:", path)
-        session.merge(Download(path=path, size=0, type="download", finished=True))
+        obj.finished = True
+        session.merge(obj)
         session.commit()
 
     db.close_session()
+
+def get_download_size(path):
+
+    session = db.session()
+    obj = session.query(Download).filter(Download.path==path).first()
+
+    if not obj :
+        print("Warning: Download-Object does no longe exist in DB. Returning -1")
+        return -1
+    elif obj.size != -1:
+        session.close()
+        return obj.size
+
+    # query size #
+    r = requests.get(obj.url, params={"path": path, "info": 1})
+    r.raise_for_status()
+    
+    size = r.json()["size"]
+    obj.size = _bytes_to_mb(size)
+    session.merge(obj)
+    session.commit()
+    session.close()
+
+    return size
+
+def get_percent_filled(path):
+
+    session = db.session()
+    obj = session.query(Download).filter(Download.path==path, Download.finished==False).first()
+    size = _bytes_to_mb(os.stat(obj.local_path).st_size)
+    total_size = get_download_size(obj.path)
+    session.close()
+    if total_size == 0:
+        return 0
+    return size / total_size * 100
+
+def get_download(path=None):
+
+    session = db.session()
+    if path:
+        MIN_SIZE_PGBAR_LIMIT = 1024*1024*100 # 100mb
+        downloads = session.query(Download).filter(Download.size>MIN_SIZE_PGBAR_LIMIT, Download.finished==False).all()
+    else:
+        downloads = session.query(Download).filter(Download.finished==False).all()
+
+    session.close()
+    return downloads