From 47f9912dc7227140266661f0435f920bcf18862d Mon Sep 17 00:00:00 2001 From: Yannik Schmidt Date: Sun, 16 Feb 2025 21:05:24 +0100 Subject: [PATCH] feat: basic background progress bars (downloads) --- data_backend.py | 63 ++++++++++++++++++++++++++++++--------------- db.py | 9 +++++++ infowidget.py | 68 ++++++++++++++++++++++++++++++++++--------------- software.py | 2 +- statekeeper.py | 63 ++++++++++++++++++++++++++++++++++++++++----- 5 files changed, 157 insertions(+), 48 deletions(-) diff --git a/data_backend.py b/data_backend.py index 2f7d461..efb72bc 100644 --- a/data_backend.py +++ b/data_backend.py @@ -99,20 +99,8 @@ class HTTP(DataBackend): def _get_url(self): #print(self.server + HTTP.REMOTE_PATH) return self.server + HTTP.REMOTE_PATH - - def get(self, path, cache_dir="", return_content=False, wait=False): - - print("Getting", path, "cache dir", cache_dir, "return content:", return_content) - - if cache_dir is None: - print("Setting cache dir from backend default", "cur:", cache_dir, "new (default):", self.cache_dir) - cache_dir = self.cache_dir - - # check the load cache dir # - if cache_dir: - self._create_cache_dir(cache_dir) - elif not cache_dir and not return_content: - AssertionError("Need to set either cache_dir or return_content!") + + def get_local_target(self, path): # prepend root dir if not given # fullpath = path @@ -120,7 +108,30 @@ class HTTP(DataBackend): fullpath = os.path.join(self.remote_root_dir, path) fullpath = fullpath.replace("\\", "/") - local_file = os.path.join(cache_dir, fullpath) + local_file = os.path.join(self.cache_dir, fullpath) + print("Local Target is", local_file) + return local_file + + def get(self, path, cache_dir=None, return_content=False, wait=False): + + print("Getting", path, "cache dir", cache_dir, "return content:", return_content) + + if cache_dir is None: + print("Setting cache dir from backend default", "cur:", cache_dir, "new (default):", self.cache_dir) + cache_dir = self.cache_dir + + # fix cache path reuse # + if path.startswith(cache_dir): + path = path[len(cache_dir):] + print("Fixed path to not duble include cache dir, path:", path) + + # check the load cache dir # + if cache_dir: + self._create_cache_dir(cache_dir) + elif not cache_dir and not return_content: + AssertionError("Need to set either cache_dir or return_content!") + + local_file = self.get_local_target(path) local_dir = os.path.dirname(local_file) # sanity check and create directory # @@ -129,7 +140,7 @@ class HTTP(DataBackend): else: os.makedirs(local_dir, exist_ok=True) - print("Requiring:", fullpath) + print("Requiring:", path) if not os.path.isfile(local_file) or os.stat(local_file).st_size == 0: @@ -137,14 +148,24 @@ class HTTP(DataBackend): # the content is needed for the UI now and not cached, it's needs to be downloaded synchroniously # # as there cannot be a meaningful UI-draw without it. # - r = requests.get(self._get_url(), params={ "path" : path, "as_string": True }) + # THIS IS THE OLD WAY + # r = requests.get(self._get_url(), params={ "path" : path, "as_string": True }) - # cache the download imediatelly # - with open(local_file, encoding="utf-8", mode="w") as f: - f.write(r.text) + # # cache the download imediatelly # + # with open(local_file, encoding="utf-8", mode="w") as f: + # f.write(r.text) + + # this is with streaming + chunk_size = 1024 * 1024 * 5 # 5MB + r = requests.get(self._get_url(), params={"path": path, "as_string": True}, stream=True) + + with open(local_file, "wb") as f: + for chunk in r.iter_content(chunk_size=chunk_size, decode_unicode=True): + if chunk: + f.write(chunk) if return_content: - print("Content for", fullpath, ":", r.text) + print("Content for", path, ":", r.text) return r.text else: return local_file diff --git a/db.py b/db.py index 1c3fff9..e736e0b 100644 --- a/db.py +++ b/db.py @@ -9,11 +9,20 @@ class Download(Base): __tablename__ = 'files' path = Column(String, primary_key=True) + local_path = Column(String) + url = Column(String) size = Column(Integer) type = Column(String) finished = Column(Boolean) + def __eq__(self, other): + return self.path == other.path + + def __hash__(self): + return hash(self.path) + class Database: + def __init__(self, db_url="sqlite:///database.db"): self.engine = create_engine(db_url, echo=True) self.session_factory = sessionmaker(bind=self.engine) diff --git a/infowidget.py b/infowidget.py index 4b3c418..b4a5ad2 100644 --- a/infowidget.py +++ b/infowidget.py @@ -14,6 +14,8 @@ import threading import random import time import string +import statekeeper +import os class ProgressBarApp: def __init__(self, parent): @@ -33,39 +35,65 @@ class ProgressBarApp: threading.Thread(target=self.add_progress_bars, daemon=True).start() def add_progress_bars(self): + + already_tracked = set() while self.running: - frame = tk.Frame(self.frame) - frame.pack(fill=tk.X, pady=2) + downloads = set(statekeeper.get_download()) + new = downloads - already_tracked + already_tracked |= set(downloads) - progress = ttk.Progressbar(frame, length=200, mode='determinate') - progress.pack(side=tk.LEFT, padx=5) + for element in new: - delete_button = tk.Button(frame, text="Delete", command=lambda f=frame: self.delete_progress(f), state=tk.DISABLED) - delete_button.pack(side=tk.LEFT, padx=5) + frame = tk.Frame(self.frame) + frame.pack(fill=tk.X, pady=2) - random_letter = random.choice(string.ascii_uppercase) - label = tk.Label(frame, text=random_letter) - label.pack(side=tk.LEFT, padx=5) + progress = ttk.Progressbar(frame, length=200, mode='determinate') + progress.pack(side=tk.LEFT, padx=5) - self.progress_bars.insert(0, (progress, frame, delete_button)) # Insert at the top - frame.pack(fill=tk.X, pady=2, before=self.frame.winfo_children()[-1] if self.frame.winfo_children() else None) + delete_button = tk.Button(frame, text="Delete", command=lambda f=frame: self.delete_progress(f), state=tk.DISABLED) + delete_button.pack(side=tk.LEFT, padx=5) - duration = random.randint(1, 10) # Random fill time - threading.Thread(target=self.fill_progress, args=(progress, duration, frame, delete_button), daemon=True).start() + label = tk.Label(frame, text=os.path.basename(element.path)) + label.pack(side=tk.LEFT, padx=5) - time.sleep(30) # Wait before adding a new progress bar + self.progress_bars.insert(0, (progress, frame, delete_button)) # Insert at the top + frame.pack(fill=tk.X, pady=2, before=self.frame.winfo_children()[-1] if self.frame.winfo_children() else None) + + print("Starting tracker for", element.path) + threading.Thread(target=self.fill_progress, args=(progress, element.path, frame, delete_button), daemon=True).start() + + time.sleep(2) # Wait before adding a new progress bar + + def fill_progress(self, progress, path, frame, delete_button): + + fail_count = 0 + while True: - def fill_progress(self, progress, duration, frame, delete_button): - for i in range(101): # Fill progress bar over 'duration' seconds - time.sleep(duration / 100) if not progress.winfo_exists(): # Check if progress bar still exists return - self.root.after(0, progress.config, {"value": i}) + try: + percent_filled = statekeeper.get_percent_filled(path) + except OSError as e: + fail_count += 1 + if fail_count > 6: + raise e + else: + time.sleep(1) + continue + + print("Percent filled:", percent_filled, path) + if not percent_filled or percent_filled >= 99.9: + self.root.after(0, progress.config, { "value" : 100 }) + break + else: + self.root.after(0, progress.config, { "value" : percent_filled }) + time.sleep(0.5) + + # handle finished download # self.root.after(0, delete_button.config, {"state": tk.NORMAL}) - - self.progress_bars.append((progress, frame, duration, delete_button)) + self.progress_bars.append((progress, frame, path, delete_button)) self.update_delete_all_button() def delete_progress(self, frame): diff --git a/software.py b/software.py index b50acc4..0566990 100644 --- a/software.py +++ b/software.py @@ -133,7 +133,7 @@ class Software: print("No main_dir:", path) raise AssertionError("No main_dir for this software") - statekeeper.log_begin_download(remote_file) + statekeeper.log_begin_download(remote_file, self.backend.get_local_target(remote_file), self.backend._get_url()) local_file = self.backend.get(remote_file, self.cache_dir, wait=True) statekeeper.log_end_download(remote_file) diff --git a/statekeeper.py b/statekeeper.py index 21587bf..9fbbe1e 100644 --- a/statekeeper.py +++ b/statekeeper.py @@ -5,6 +5,9 @@ import threading from db import db, Download from sqlalchemy import or_, and_ +def _bytes_to_mb(size): + return size / (1024*1024) + def add_to_download_queue(url, path): '''The download is added to the global queue and downloaded eventually''' #_download(url, path) @@ -38,7 +41,7 @@ def _download(url, path): raise AssertionError("Non-200 Response for:", url, path, response.status_code, response.text) -def log_begin_download(path): +def log_begin_download(path, local_path, url): session = db.session() print("Current path", path) @@ -49,7 +52,7 @@ def log_begin_download(path): raise AssertionError("ERROR: {} is already downloading.".format(path)) else: print("Adding to download log:", path) - session.merge(Download(path=path, size=0, type="download", finished=False)) + session.merge(Download(path=path, size=-1, type="download", local_path=local_path, url=url, finished=False)) session.commit() db.close_session() @@ -57,12 +60,60 @@ def log_begin_download(path): def log_end_download(path): session = db.session() - path_exists = session.query(Download).filter(Download.path==path).first() - if not path_exists: + obj = session.query(Download).filter(Download.path==path).first() + if not obj: raise AssertionError("ERROR: {} is not downloading/cannot remove.".format(path)) else: print("Removing from download log:", path) - session.merge(Download(path=path, size=0, type="download", finished=True)) + obj.finished = True + session.merge(obj) session.commit() - db.close_session() \ No newline at end of file + db.close_session() + +def get_download_size(path): + + session = db.session() + obj = session.query(Download).filter(Download.path==path).first() + + if not obj : + print("Warning: Download-Object does no longe exist in DB. Returning -1") + return -1 + elif obj.size != -1: + session.close() + return obj.size + + # query size # + r = requests.get(obj.url, params={"path": path, "info": 1}) + r.raise_for_status() + + size = r.json()["size"] + obj.size = _bytes_to_mb(size) + session.merge(obj) + session.commit() + session.close() + + return size + +def get_percent_filled(path): + + session = db.session() + obj = session.query(Download).filter(Download.path==path, Download.finished==False).first() + size = _bytes_to_mb(os.stat(obj.local_path).st_size) + total_size = get_download_size(obj.path) + session.close() + if total_size == 0: + return 0 + return size / total_size * 100 + +def get_download(path=None): + + session = db.session() + if path: + MIN_SIZE_PGBAR_LIMIT = 1024*1024*100 # 100mb + downloads = session.query(Download).filter(Download.size>MIN_SIZE_PGBAR_LIMIT, Download.finished==False).all() + else: + downloads = session.query(Download).filter(Download.finished==False).all() + + session.close() + return downloads \ No newline at end of file