From db7efca5bb855fb7c2907cf3a65c0c8306b65095 Mon Sep 17 00:00:00 2001 From: Yannik Schmidt Date: Wed, 27 Dec 2023 13:08:01 +0100 Subject: [PATCH] feat: fallback solution for dwd data --- .gitignore | 1 + src/main/python/fallback_csv.py | 91 ++++++++++++++++++++++++++++++ src/main/python/input_backend.py | 14 ++++- src/main/python/localization/de.py | 2 + src/main/python/main.py | 9 +-- ths_config.txt | 1 + 6 files changed, 111 insertions(+), 7 deletions(-) create mode 100644 src/main/python/fallback_csv.py diff --git a/.gitignore b/.gitignore index 8f6a37b..d76d74a 100644 --- a/.gitignore +++ b/.gitignore @@ -12,3 +12,4 @@ __py* *.log *.zip dist/ +dwd/ diff --git a/src/main/python/fallback_csv.py b/src/main/python/fallback_csv.py new file mode 100644 index 0000000..8b14115 --- /dev/null +++ b/src/main/python/fallback_csv.py @@ -0,0 +1,91 @@ +import glob +import datetime +import os + +SKIP_LINES = 13 + +def cache_content(from_time, to_time, data, dtype): + + return_string = "" + + skip_count = 0 + + for i in range(0, SKIP_LINES): + return_string += "\n" + + for d in data: + + date, temp, hum = d + + # skip outside timeframe # + if date < from_time or date > to_time: + continue + + if dtype == "lufttemperatur-aussen": + content_number = temp + elif dtype == "luftfeuchte": + content_number = hum + else: + raise ValueError("Bad dtype: {}".format(dtype)) + + date_cache_format = date.strftime("%d.%m.%Y %H:%M") + content_str = "{:1f}".format(content_number).replace(".",",") + return_string += "{};{}\n".format(date_cache_format, content_str) + + return return_string + +def generate(master_dir, from_time, to_time, cache_file, dtype): + + timeframes = [] + + if not os.path.isdir(master_dir): + os.mkdir(master_dir) + + # read files + files = glob.glob(master_dir + "/produkt_tu_stunde*.txt") + + if not files: + raise ValueError("Keine DWD_Datei in: {} gefunden. Bitte herunterladen und entpacken! https://www.dwd.de/DE/leistungen/klimadatendeutschland/klarchivstunden.html;jsessionid=C423E76B30D18F24C43F4E7E36744C8C.live21073?nn=16102") + + for fname in files: + + start = None + end = None + data = [] + + # read file + with open(fname) as f: + first_line = True + + # iterate through csv # + for line in f: + + # skip header + if first_line: + first_line = False + continue + + # read the line # + station_id, fulldate, dunno, temp, hum, dunno2 = line.split(";") + + # parse date # + date = datetime.datetime.strptime(fulldate, "%Y%m%d%H") + + # append data # + data.append((date, float(temp), float(hum))) + + # set start and end # + if not start and date: + start = date + elif date: + end = date + + # save values # + timeframes.append((start, end, data)) + + # find a fitting frame # + for start, end, data in timeframes: + if from_time >= start and to_time <= end: + return cache_content(from_time, to_time, data, dtype) + + raise ValueError("Keine Datei mit passenden Daten gefunden. Bitte Readme lesen") diff --git a/src/main/python/input_backend.py b/src/main/python/input_backend.py index 9cb3d7e..da17e04 100644 --- a/src/main/python/input_backend.py +++ b/src/main/python/input_backend.py @@ -8,6 +8,7 @@ import localization.de as de from dbfread import DBF import timeutils import codecs +import fallback_csv line_colors = ['b', 'r', 'g', 'c', 'm', 'y'] tname = CFG("temperatur_plot_name") @@ -126,8 +127,15 @@ def processExternalData(datapoints, plotNameKey, fromTime, toTime, dtype, qtText # download date if it doesn't exist # url = CFG("outside_data_url").format(dtype=dtype, fromDate=fromTimeStr, toDate=toTimeStr) r = requests.get(url) - qtTextBrowser.append(de.pg_request.format(url)) - content = r.content.decode('utf-8', "ignore") # ignore bad bytes + + # check response code # + if r.status_code != 200 or "nicht gefunden" in r.text.lower(): + qtTextBrowser.append(de.failed_to_retrieve.format("NOT FOUND")) + qtTextBrowser.append("Versuche von DWD-Datei zu laden - Dass kann einen Moment dauern") + content = fallback_csv.generate(CFG("dwd_dir"), fromTime, toTime, cacheFile, dtype) + else: + qtTextBrowser.append(de.pg_request.format(url)) + content = r.content.decode('utf-8', "ignore") # ignore bad bytes # cache data # if not os.path.isdir(cacheDir): @@ -135,7 +143,7 @@ def processExternalData(datapoints, plotNameKey, fromTime, toTime, dtype, qtText with open(fullpath, 'w') as f: f.write(content) - else: + if os.path.isfile(fullpath): # get data from cache otherwise qtTextBrowser.append(de.cache_hit.format(cacheFile)) diff --git a/src/main/python/localization/de.py b/src/main/python/localization/de.py index b0c7315..fbf3d9a 100644 --- a/src/main/python/localization/de.py +++ b/src/main/python/localization/de.py @@ -42,3 +42,5 @@ info_divergence = "INFO: Seitenverhältnisabweichung zu A4: {:.2f}%" success = "INFO: Fertig." testing_input = "Info: Eingabedatei wird geprüft..." testing_input_suc = "Info: Eingabedatei akzeptiert." + +failed_to_retrieve = "ERROR: Kann Wetterarchiv-Daten nicht herunterladen ({})" diff --git a/src/main/python/main.py b/src/main/python/main.py index bd506e3..a1f4917 100755 --- a/src/main/python/main.py +++ b/src/main/python/main.py @@ -169,20 +169,21 @@ class WidgetGallery(QDialog): # workaround for checkboxes changed # outsideDataNeeded = self.boxOTemp.isChecked() or self.boxOHumidity.isChecked() - self.datapoints = input_backend.read_in_file(self.srcFileString, + + # build dates # + try: + self.datapoints = input_backend.read_in_file(self.srcFileString, outsideData=outsideDataNeeded, plotOutsideTemp=self.boxOTemp.isChecked(), plotOutsideHum=self.boxOHumidity.isChecked(), qtTextBrowser=self.infoTextBox) - # build dates # - try: startTimeHelper = dt.datetime.strptime(self.startTimeEdit.text(),"%H:%M") endTimeHelper = dt.datetime.strptime(self.endTimeEdit.text(),"%H:%M") except ValueError as e: errorBox = QMessageBox(self) errorBox.setAttribute(PyQt5.QtCore.Qt.WA_DeleteOnClose) - errorBox.setText(self.localization.bad_time) + errorBox.setText(str(e)) errorBox.setDetailedText(str(e)) errorBox.show() self.buttonGo.setText(self.localization.button_go) diff --git a/ths_config.txt b/ths_config.txt index f87f698..c9723f0 100644 --- a/ths_config.txt +++ b/ths_config.txt @@ -105,3 +105,4 @@ fig_x_height_inches = 12100 fig_y_height_inches = 8075 default_target_dir = UseSourceDir default_source_dir = "." +dwd_dir = "./dwd/"