sheppy/homelab_gamevault
1
0
Fork 0
mirror of https://github.com/FAUSheppy/homelab_gamevault synced 2025-12-06 06:51:36 +01:00
Code Issues Actions Packages Projects Releases Wiki Activity

feat: make software-init threadsafe & concurrent

Browse Source
This commit is contained in:
Yannik Schmidt
2024-05-31 15:49:11 +02:00
parent 9008df2f3f
commit bcc3d39e31
2 changed files with 40 additions and 31 deletions
Download Patch File Download Diff File Expand all files Collapse all files

63
data_backend.py
View File

@@ -5,6 +5,7 @@ import software
import ftplib import ftplib
import tqdm import tqdm
import ssl import ssl
import concurrent.futures
class SESSION_REUSE_FTP_TLS(ftplib.FTP_TLS): class SESSION_REUSE_FTP_TLS(ftplib.FTP_TLS):
"""Explicit FTPS, with shared TLS session""" """Explicit FTPS, with shared TLS session"""
@@ -164,7 +165,7 @@ class FTP(DataBackend):
return ftp return ftp
def get(self, path, cache_dir=None, return_content=False): def get(self, path, cache_dir=None, return_content=False, new_connection=False):
# check the load cache dir # # check the load cache dir #
if cache_dir: if cache_dir:
@@ -183,34 +184,43 @@ class FTP(DataBackend):
# print("Cachedir:", cache_dir, os.path.basename(path), local_file) # print("Cachedir:", cache_dir, os.path.basename(path), local_file)
if not os.path.isfile(local_file): if not os.path.isfile(local_file):
ftp = self._connect() ftp = self._connect(individual_connection=True)
ftp.sendcmd('TYPE I') ftp.sendcmd('TYPE I')
# load the file on remote # # load the file on remote #
total_size = ftp.size(fullpath) if not new_connection:
print(total_size) total_size = ftp.size(fullpath)
self.progress_bar_wrapper.get_pb()["maximum"] = total_size print(total_size)
self.progress_bar_wrapper.get_pb()["maximum"] = total_size
print(local_file, "not in cache, retriving..") print(local_file, "not in cache, retriving..")
with open(local_file, "w") as f: with open(local_file, "w") as f:
f.write(local_file) f.write(local_file)
with open(local_file, 'wb') as local_file_open, tqdm.tqdm( with open(local_file, 'wb') as local_file_open, tqdm.tqdm(
desc="Downloading", desc="Downloading",
total=total_size, total=total_size,
unit='B', unit='B',
unit_scale=True unit_scale=True
) as cmd_progress_bar: ) as cmd_progress_bar:
# Define a callback function to update the progress bar # # Define a callback function to update the progress bar #
def callback(data): def callback(data):
local_file_open.write(data) local_file_open.write(data)
self.root.update_idletasks() # Update the GUI if new_connection: # return if parralell
self.progress_bar_wrapper.get_pb().set( return
self.progress_bar_wrapper.get_pb().get() + len(data)/total_size) self.root.update_idletasks() # Update the GUI
cmd_progress_bar.update(len(data)) self.progress_bar_wrapper.get_pb().set(
self.progress_bar_wrapper.get_pb().get() + len(data)/total_size)
cmd_progress_bar.update(len(data))
# run with callback # # run with callback #
ftp.retrbinary('RETR ' + fullpath, callback) ftp.retrbinary('RETR ' + fullpath, callback)
else:
with open(local_file, 'wb') as fp:
ftp.retrbinary('RETR ' + fullpath, fp.write)
if new_connection:
ftp.close()
if return_content: if return_content:
with open(local_file, encoding="utf-8") as fr: with open(local_file, encoding="utf-8") as fr:
@@ -265,7 +275,6 @@ class FTP(DataBackend):
local_meta_file_list = [] local_meta_file_list = []
root_elements = self.list(self.remote_root_dir) root_elements = self.list(self.remote_root_dir)
import concurrent.futures
with concurrent.futures.ThreadPoolExecutor(max_workers=os.cpu_count()*5) as executor: with concurrent.futures.ThreadPoolExecutor(max_workers=os.cpu_count()*5) as executor:
software_dir_contents = list(executor.map( software_dir_contents = list(executor.map(
@@ -287,6 +296,6 @@ class FTP(DataBackend):
#print(meta_file_content) #print(meta_file_content)
local_meta_file_list.append(f) local_meta_file_list.append(f)
return list(filter(lambda x: not x.invalid, with concurrent.futures.ThreadPoolExecutor(max_workers=os.cpu_count()*5) as executor:
[ software.Software(meta_file, self, self.progress_bar_wrapper) software_list = executor.map(lambda meta_file: software.Software(meta_file, self, self.progress_bar_wrapper), local_meta_file_list)
for meta_file in local_meta_file_list ])) return list(filter(lambda x: not x.invalid, software_list))

8
software.py
View File

@@ -30,7 +30,7 @@ class Software:
def _load_from_yaml(self): def _load_from_yaml(self):
content = self.backend.get(self.meta_file, self.cache_dir, return_content=True) content = self.backend.get(self.meta_file, self.cache_dir, return_content=True, new_connection=True)
meta = yaml.safe_load(content) meta = yaml.safe_load(content)
if not meta: if not meta:
@@ -46,10 +46,10 @@ class Software:
self.run_exe = meta.get("run_exe") self.run_exe = meta.get("run_exe")
self.installer = meta.get("installer") self.installer = meta.get("installer")
self.pictures = [ self.backend.get(pp, self.cache_dir) for pp in self.pictures = [ self.backend.get(pp, self.cache_dir, new_connection=True) for pp in
self.backend.list(os.path.join(self.directory, "pictures"), fullpaths=True) ] self.backend.list(os.path.join(self.directory, "pictures"), fullpaths=True, new_connection=True) ]
self.reg_files = self.backend.list(os.path.join(self.directory, "registry_files"), fullpaths=True) self.reg_files = self.backend.list(os.path.join(self.directory, "registry_files"), fullpaths=True, new_connection=True)
def get_thumbnail(self): def get_thumbnail(self):