Allow usage of external temperatur datasets

.gitignore

@@ -1,4 +1,6 @@
 *.png
+*.bak
+cache/
 *.dbf
 __py*
 *.swp

frontend.py

@@ -36,7 +36,9 @@ def main_repl(datapoints,path,date1=None,date2=None,done1=False,done2=False):
 
 def selection_repl(path):
     if path != None:
-        datapoints = input_backend.read_in_file(path)
+        outsideData    = input("Außentemperatur einzeichnen? (j/n)")
+        useOutsideData = outsideData.strip().lower() in [ "j", "y" ]
+        datapoints = input_backend.read_in_file(path, outsideData=useOutsideData)
         if CFG("debug_no_interactive"):
             plot_main.plot(datapoints,path)
             return None

input_backend.py

@@ -1,6 +1,8 @@
 #!/usr/bin/python3
 from config_parse import CFG
 from datetime import datetime, timedelta
+import requests
+import os
 
 from dbfread import DBF
 import plot_timeutils
@@ -21,6 +23,14 @@ class Data:
         self.times  = []
         self.plot = plot
 
+    def getFirstTime(self):
+        '''Get time of first timestamp'''
+        return min(self.times)
+
+    def getLastTime(self):
+        '''Get time of last timestamp'''
+        return max(self.times)
+
     def get_timeframe(self, callback,date1=None,date2=None):
         out_x = []
         out_y = []
@@ -87,36 +97,100 @@ def parse_line(datapoints,line,timekey,keys,time_parser,timeformat=None):
                 datapoints[ key[1] ].data  += [ line[ key[0] ] ]
                 datapoints[ key[1] ].times += [      time      ]
 
-def read_in_file(path,backend=None):
+def processExternalData(datapoints, plotNameKey, fromTime, toTime, dtype):
-        global tname
+    '''Download and parses external data of type dtype'''
-        global hname
+
-        global dname
+    # prepare strings #
-        global opath
+    cacheDir    = CFG("cache_dir")
+    fromTimeStr = fromTime.strftime(CFG("nff_url_timeformat"))
+    toTimeStr   = toTime.strftime(CFG("nff_url_timeformat"))
+    cacheFile   = "cache_{}_{}_{}.data".format(dtype, fromTimeStr, toTimeStr)
+    fullpath   = os.path.join(cacheDir, cacheFile)
+
+    # check for cache file
+    content = None
+    if not os.path.isfile(fullpath):
+        
+        # download date if it doesn't exist #
+        url = CFG("outside_data_url").format(dtype=dtype, fromDate=fromTimeStr, toDate=toTimeStr)
+        r = requests.get(url)
+        print(url)
+        content = r.content.decode('utf-8', "ignore") # ignore bad bytes
+
+        # cache data #
+        if not os.path.isdir(cacheDir):
+            os.mkdir(cacheDir)
+        with open(fullpath, 'w') as f:
+            f.write(content)
+
+    else:
+
+        # get data from cache otherwise
+        print("INFO: Cache hit: {}".format(cacheFile))
+        with open(fullpath) as f:
+            content = f.read()
+
+    skipBecauseFirstLine = True
+    for l in content.split("\n"):
+        if not ";" in l:
+            continue
+        elif not l.strip():
+            continue
+        elif skipBecauseFirstLine:
+            skipBecauseFirstLine = False
+            continue
+
+        try:
+            timeStr, value = l.split(";")
+            timestamp = plot_timeutils.time_from_csv(timeStr, CFG("nff_input_timeformat"))
+
+            datapoints[plotNameKey].data  += [float(value.replace(",","."))]
+            datapoints[plotNameKey].times += [timestamp]
+        except ValueError as e:
+            print(l)
+            raise e
+
+
+def read_in_file(path, backend=None, outsideData=False):
+        '''Read in a file, add outside data if requested'''
 
         datapoints = dict()
+        identifiers =  [ CFG("plot_temperatur_key"),
+                         CFG("plot_humidity_key"),
+                         CFG("plot_dewcels_key"),
+                         CFG("plot_outside_temperatur_key"),
+                         CFG("plot_outside_humidity_key") ]
 
-        pt=CFG("plot_temperatur_key")
+        names  = [ CFG("temperatur_plot_name"),
-        ph=CFG("plot_humidity_key")
+                   CFG("humidity_plot_name"),
-        pd=CFG("plot_dewcels_key")
+                   CFG("dewcels_plot_name"),
+                   CFG("temperatur_outside_plot_name"),
+                   CFG("humidity_outside_plot_name") ]
 
-        ## NAME PADDING ##
+        colors = [ CFG("temperatur_color"),
-        max_name_len = max(len(tname),len(hname),len(dname))
+                   CFG("humidity_color"),
-        while len(tname) < max_name_len:
+                   CFG("dewcels_color"),
-                tname += " "
+                   CFG("temperatur_outside_color"),
-        while len(hname) < max_name_len:
+                   CFG("humidity_outside_color") ]
-                hname += " "    
-        while len(dname) < max_name_len:
-                dname += " "
 
-        datapoints.update({ pt:Data( tname,CFG("plot_temperatur") ) })
+        plotSettings = [ CFG("plot_temperatur"),
-        datapoints[pt].color = CFG("temperatur_color")      
+                         CFG("plot_humidity"),
+                         CFG("plot_dewcels"),
+                         outsideData,
+                         outsideData ]
         
-        datapoints.update({ ph:Data( hname,CFG("plot_humidity") ) })
+        assert(len(names) == len(colors) == len(identifiers) == len(plotSettings))
-        datapoints[ph].color = CFG("humidity_color")
 
-        datapoints.update({ pd:Data( dname,CFG("plot_dewcels") ) })
+        max_name_len = max([len(s) for s in names])
-        datapoints[pd].color = CFG("dewcels_color")
+        for i in range(0, len(names)):
+            while len(names[i]) < max_name_len:
+                names[i] += " "
+            datapoints.update({ identifiers[i] : Data(names[i], plotSettings[i]) })
 
+        # legacy variables...
+        pt, ph, pd, pto, pho = identifiers
+
+        # parse input file #
         if path == None:
             raise Exception("Path in plot.read_in was None")
         elif backend != None:
@@ -130,7 +204,18 @@ def read_in_file(path,backend=None):
         else:
                 raise NotImplementedError("Cannot determine filetype, cannot continue. Exit.")
         
+        # if nessesary download and process external data #
+        if outsideData:
+
+            fromTime = datapoints[CFG("plot_temperatur_key")].getFirstTime()
+            toTime   = datapoints[CFG("plot_temperatur_key")].getLastTime()
+
+            processExternalData(datapoints, pto, fromTime, toTime, CFG("dtype_temperatur"))
+            processExternalData(datapoints, pho, fromTime, toTime, CFG("dtype_humidity"))
+
+        # sanity check result #
         check_read_in(datapoints)
+
         return datapoints
 
 def dbfread(path,datapoints,pt,ph,pd):

plot_graphutils.py

@@ -27,12 +27,14 @@ def avg(array):
         return sum(array)/float(len(array))
 
 def legend_box_contents(name, y):
+    if CFG("cap_values_at_99"):
+        y = [ min([el, 99.9]) for el in y ]
     if CFG("show_min"):
-		name += " min: %.1f,"%min(y)
+        name += " min: {:4.1f},".format(min(y))
     if CFG("show_max"):
-		name += " max: %.1f,"%max(y)
+        name += " max: {:4.1f},".format(max(y))
     if CFG("show_avg"):
-		name += " Mittelwert: %.1f,"% avg(y)
+        name += " Mittelwert: {:4.1f},".format(avg(y))
     return name.rstrip(",")    
 
 def general_background_setup(tup,ymin,ymax,x):

plot_main.py

@@ -35,13 +35,15 @@ def __plot(tup,datapoints,path,date1=None,date2=None):
         ls = CFG("plot_line_style")
         tup[FIGURE],tup[AXIS] = plt.subplots(1, 1)
 
-        for g in datapoints.values():
+        for key in datapoints.keys():
+                g = datapoints[key]
+                print(key)
                 #### Check if we are supposed to plot something ####
                 if not g.plot:
                         continue
                 #### GET AND CHECK TIMEFRAMES ####
                 x,y, = g.get_timeframe(tup[CALLBACK],date1,date2)
-                if len(x) <= 0 or len(y) <= 0:
+                if not x or not y or len(x) <= 0 or len(y) <= 0:
                     print("Warning: Empty series of data '%s' (wrong start/end time?)"%g.name)
                     continue
                 else:
@@ -52,13 +54,15 @@ def __plot(tup,datapoints,path,date1=None,date2=None):
                 #### GET LINE STYLES ####
                 legend_label = plot_graphutils.legend_box_contents(g.name,y)
                 tup[AXIS].plot(unix_x, y,ls=ls,lw=lw,marker="None", label=legend_label, color=g.color)
+                legacy_x_save = x
+                lagacy_y_save = y
         
         if NO_SERIES:
                 print("Error: no data, nothing to plot. cannot continue. exit.")
                 sys.exit(1)
 
         ## GRID ##
-        plot_graphutils.general_background_setup(tup,ymin,ymax,x)
+        plot_graphutils.general_background_setup(tup, ymin, ymax, legacy_x_save)
 
         ## using unix_x relys on unix_x to be the same for all plots ##
         if path == None:

ths_config.txt

@@ -17,6 +17,8 @@ target_temperatur		= 20
 
 temperatur_plot_name	= Innenlufttemperatur
 humidity_plot_name		= rel. Luftfeuchtigkeit
+temperatur_outside_plot_name = Außenlufttemperatur
+humidity_outside_plot_name	 = rel. (a) Luftfeuchtigkeit
 dewcels_plot_name		= Taupunkt
 y_label					= Temp./r.L.
 x_label					= Datum/Uhrzeit
@@ -60,6 +62,8 @@ yaxis_start_value				= 0
 # True:  die Y-Achse beginnt auch bei xaxis_start_value wenn dadurch Werte nicht angezeit werden
 # False: wenn ein Wert im plot kleiner xaxis_start_value ist beginnt die Y-Achse beim kleinsten Wert im Plot
 yaxis_force_start_value			= False
+# cap all values at two digits to prevent formating problems
+cap_values_at_99                = True
 
 # ein höheres alpha für zu einer stärkeren Sättigung der Hintergrundfarbe (0 und es ist ganz weg)
 humidity_crit_alpha		= 0.35
@@ -75,6 +79,8 @@ acceptable_temp_color  = blue
 humidity_color			 = red
 temperatur_color		 = blue
 dewcels_color			 = green
+temperatur_outside_color = cyan
+humidity_outside_color   = orange
 
 plot_line_width = 0.5
 plot_line_style = solid
@@ -100,10 +106,17 @@ raster_minimum_hlines		    = 10
 add_hours_to_input              = 1
 add_x_labels_at_end             = 1
 
+### NFF Data URLs ###
+outside_data_url = "http://umweltdaten.nuernberg.de/csv/wetterdaten/messstation-nuernberg-flugfeld/archiv/csv-export/SUN/nuernberg-flugfeld/{dtype}/individuell/{fromDate}/{toDate}/export.csv"
+dtype_temperatur = "lufttemperatur-aussen"
+dtype_humidity   = "luftfeuchte"
+nff_url_timeformat = "%%d.%%m.%%Y"
+nff_input_timeformat = "%%d.%%m.%%Y %%H:%%M"
+
 ###### DEBUGGING ######
-no_ask_date_input           = no
+no_ask_date_input           = yes
-input_filename              = test.xls
+input_filename              = "LOG32TH_20010101_2020-06-30T131045.DBF"
-use_input_filename          = no
+use_input_filename          = yes
 debug_no_interactive        = no
 terminate_on_warning        = no
 terminate_on_missing_input_file = True
@@ -115,12 +128,14 @@ default_target_dir          = UseSourceDir
 plot_temperatur_key         = TEMP
 plot_humidity_key 	        = HUMIDITY
 plot_dewcels_key	        = TAU_P
+plot_outside_temperatur_key = O_TEMP
+plot_outside_humidity_key	= O_HUMIDITY
 always_allow_days_as_xticks = yes
 language                    = DE
 aspect_ratio                = A4
 use_gui_backend             = Agg
 enable_multicore_support 	= False
-raster_alligment_auto	 	= True
 outfile_resolution_in_dpi   = 250
 # <= what stepsize should datapoints be combined (s)
 combine_data_points         = 1
+cache_dir                   = "./cache/"