feat: fallback solution for dwd data

.gitignore

@@ -12,3 +12,4 @@ __py*
 *.log
 *.zip
 dist/
+dwd/

src/main/python/fallback_csv.py

@@ -0,0 +1,91 @@
+import glob
+import datetime
+import os
+
+SKIP_LINES = 13
+
+def cache_content(from_time, to_time, data, dtype):
+
+    return_string = ""
+
+    skip_count = 0
+
+    for i in range(0, SKIP_LINES):
+        return_string += "\n"
+
+    for d in data:
+
+        date, temp, hum = d
+
+        # skip outside timeframe #
+        if date < from_time or date > to_time:
+            continue
+
+        if dtype == "lufttemperatur-aussen":
+            content_number = temp
+        elif dtype == "luftfeuchte":
+            content_number = hum
+        else:
+            raise ValueError("Bad dtype: {}".format(dtype))
+
+        date_cache_format = date.strftime("%d.%m.%Y %H:%M")
+        content_str = "{:1f}".format(content_number).replace(".",",")
+        return_string += "{};{}\n".format(date_cache_format, content_str)
+
+    return return_string
+
+def generate(master_dir, from_time, to_time, cache_file, dtype):
+
+    timeframes = []
+
+    if not os.path.isdir(master_dir):
+        os.mkdir(master_dir)
+
+    # read files
+    files = glob.glob(master_dir + "/produkt_tu_stunde*.txt")
+
+    if not files:
+        raise ValueError("Keine DWD_Datei in: {} gefunden. Bitte herunterladen und entpacken! https://www.dwd.de/DE/leistungen/klimadatendeutschland/klarchivstunden.html;jsessionid=C423E76B30D18F24C43F4E7E36744C8C.live21073?nn=16102")
+
+    for fname in files:
+
+        start = None
+        end = None
+        data = []
+
+        # read file
+        with open(fname) as f:
+            first_line = True
+
+            # iterate through csv #
+            for line in f:
+
+                # skip header
+                if first_line:
+                    first_line = False
+                    continue
+
+                # read the line #
+                station_id, fulldate, dunno, temp, hum, dunno2 = line.split(";")
+
+                # parse date #
+                date = datetime.datetime.strptime(fulldate, "%Y%m%d%H")
+
+                # append data #
+                data.append((date, float(temp), float(hum)))
+
+                # set start and end #
+                if not start and date:
+                    start = date
+                elif date:
+                    end = date
+
+        # save values #
+        timeframes.append((start, end, data))
+
+    # find a fitting frame #
+    for start, end, data in timeframes:
+        if from_time >= start and to_time <= end:
+            return cache_content(from_time, to_time, data, dtype)
+
+    raise ValueError("Keine Datei mit passenden Daten gefunden. Bitte Readme lesen")

src/main/python/input_backend.py

@@ -8,6 +8,7 @@ import localization.de as de
 from dbfread import DBF
 import timeutils
 import codecs
+import fallback_csv
 
 line_colors = ['b', 'r', 'g', 'c', 'm', 'y']
 tname       = CFG("temperatur_plot_name")
@@ -126,6 +127,13 @@ def processExternalData(datapoints, plotNameKey, fromTime, toTime, dtype, qtText
         # download date if it doesn't exist #
         url = CFG("outside_data_url").format(dtype=dtype, fromDate=fromTimeStr, toDate=toTimeStr)
         r = requests.get(url)
+
+        # check response code #
+        if r.status_code != 200 or "nicht gefunden" in r.text.lower():
+            qtTextBrowser.append(de.failed_to_retrieve.format("NOT FOUND"))
+            qtTextBrowser.append("Versuche von DWD-Datei zu laden - Dass kann einen Moment dauern")
+            content = fallback_csv.generate(CFG("dwd_dir"), fromTime, toTime, cacheFile, dtype)
+        else:
             qtTextBrowser.append(de.pg_request.format(url))
             content = r.content.decode('utf-8', "ignore") # ignore bad bytes
 
@@ -135,7 +143,7 @@ def processExternalData(datapoints, plotNameKey, fromTime, toTime, dtype, qtText
         with open(fullpath, 'w') as f:
             f.write(content)
 
-    else:
+    if os.path.isfile(fullpath):
 
         # get data from cache otherwise
         qtTextBrowser.append(de.cache_hit.format(cacheFile))

src/main/python/localization/de.py

@@ -42,3 +42,5 @@ info_divergence     = "INFO: Seitenverhältnisabweichung zu A4: {:.2f}%"
 success             = "INFO: Fertig."
 testing_input       = "Info: Eingabedatei wird geprüft..."
 testing_input_suc   = "Info: Eingabedatei akzeptiert."
+
+failed_to_retrieve = "ERROR: Kann Wetterarchiv-Daten nicht herunterladen ({})"

src/main/python/main.py

@@ -169,20 +169,21 @@ class WidgetGallery(QDialog):
 
         # workaround for checkboxes changed #
         outsideDataNeeded = self.boxOTemp.isChecked() or self.boxOHumidity.isChecked()
+
+        # build dates #
+        try:
             self.datapoints = input_backend.read_in_file(self.srcFileString,
                                                 outsideData=outsideDataNeeded,
                                                 plotOutsideTemp=self.boxOTemp.isChecked(),
                                                 plotOutsideHum=self.boxOHumidity.isChecked(),
                                                 qtTextBrowser=self.infoTextBox)
 
-        # build dates #
-        try:
             startTimeHelper = dt.datetime.strptime(self.startTimeEdit.text(),"%H:%M")
             endTimeHelper   = dt.datetime.strptime(self.endTimeEdit.text(),"%H:%M")
         except ValueError as e:
             errorBox = QMessageBox(self)
             errorBox.setAttribute(PyQt5.QtCore.Qt.WA_DeleteOnClose)
-            errorBox.setText(self.localization.bad_time)
+            errorBox.setText(str(e))
             errorBox.setDetailedText(str(e))
             errorBox.show()
             self.buttonGo.setText(self.localization.button_go)

ths_config.txt

@@ -105,3 +105,4 @@ fig_x_height_inches         = 12100
 fig_y_height_inches         =  8075
 default_target_dir          = UseSourceDir
 default_source_dir          = "."
+dwd_dir = "./dwd/"