sheppy/homelab_gamevault
1
0
Fork 0
mirror of https://github.com/FAUSheppy/homelab_gamevault synced 2025-12-06 06:51:36 +01:00
Code Issues Actions Packages Projects Releases Wiki Activity

feat: cache loading for ftp

Browse Source
This commit is contained in:
Yannik Schmidt
2024-03-10 21:25:23 +01:00
parent f7da32369b
commit a770523e1f
1 changed files with 44 additions and 32 deletions
Download Patch File Download Diff File Expand all files Collapse all files

26
data_backend.py
View File

@@ -88,8 +88,10 @@ class LocalFS(DataBackend):
class FTP(DataBackend): class FTP(DataBackend):
def _connect(self): paths_listed = {}
def _connect(self):
print("Called connect")
if self.server.startswith("ftp://"): if self.server.startswith("ftp://"):
tls = False tls = False
elif self.server.startswith("ftps://"): elif self.server.startswith("ftps://"):
@@ -137,15 +139,17 @@ class FTP(DataBackend):
fullpath = os.path.join(self.remote_root_dir, path) fullpath = os.path.join(self.remote_root_dir, path)
#print(self.remote_root_dir, path, fullpath) #print(self.remote_root_dir, path, fullpath)
fullpath = fullpath.replace("\\", "/") fullpath = fullpath.replace("\\", "/")
local_file = os.path.join(cache_dir, os.path.basename(path))
if not os.path.isfile(local_file):
ftp = self._connect() ftp = self._connect()
ftp.sendcmd('TYPE I') ftp.sendcmd('TYPE I')
# load the file on remote # # load the file on remote #
total_size = ftp.size(fullpath) total_size = ftp.size(fullpath)
local_file = os.path.join(cache_dir, os.path.basename(path))
self.progress_bar["maximum"] = total_size self.progress_bar["maximum"] = total_size
print(local_file, "not in cache, retriving..")
with open(local_file, "w") as f: with open(local_file, "w") as f:
f.write(local_file) f.write(local_file)
with open(local_file, 'wb') as local_file_open, tqdm.tqdm( with open(local_file, 'wb') as local_file_open, tqdm.tqdm(
@@ -178,14 +182,22 @@ class FTP(DataBackend):
if self.remote_root_dir and not path.startswith(self.remote_root_dir): if self.remote_root_dir and not path.startswith(self.remote_root_dir):
fullpath = os.path.join(self.remote_root_dir, path) fullpath = os.path.join(self.remote_root_dir, path)
fullpath = fullpath.replace("\\", "/") fullpath = fullpath.replace("\\", "/")
print(fullpath) #print(fullpath)
# if not os.path.isdir(fullpath): # if not os.path.isdir(fullpath):
# return [] # return []
ftp = self._connect()
try: try:
# retrieve session cached paths #
if fullpath in self.paths_listed:
paths = self.paths_listed[fullpath]
#print("Retrieved paths from cache:", fullpath, paths)
else:
ftp = self._connect()
self.paths_listed.update({fullpath: []}) # in case dir does not exit
paths = ftp.nlst(fullpath) paths = ftp.nlst(fullpath)
self.paths_listed.update({fullpath: paths})
if not fullpaths: if not fullpaths:
return paths return paths
return [ os.path.join(path, filename).replace("\\", "/") for filename in paths ] return [ os.path.join(path, filename).replace("\\", "/") for filename in paths ]
@@ -205,13 +217,13 @@ class FTP(DataBackend):
root_elements = self.list(self.remote_root_dir) root_elements = self.list(self.remote_root_dir)
for s in root_elements: for s in root_elements:
print(s) #print(s)
files = self.list(s, fullpaths=True) files = self.list(s, fullpaths=True)
print(files) #print(files)
for f in files: for f in files:
if f.endswith("meta.yaml"): if f.endswith("meta.yaml"):
meta_file_content = self.get(f, cache_dir="cache", return_content=True) meta_file_content = self.get(f, cache_dir="cache", return_content=True)
print(meta_file_content) #print(meta_file_content)
local_meta_file_list.append(f) local_meta_file_list.append(f)
return [ software.Software(meta_file, self) for meta_file in local_meta_file_list ] return [ software.Software(meta_file, self) for meta_file in local_meta_file_list ]