Allow usage of external temperatur datasets

2025-12-06 04:11:34 +01:00 · 2020-08-14 17:09:07 +02:00
parent 8871515497
commit 822cae7e1a
7 changed files with 158 additions and 48 deletions
--- a/.gitignore
+++ b/.gitignore
@@ -1,4 +1,6 @@
 *.png
+*.bak
+cache/
 *.dbf
 __py*
 *.swp
--- a/frontend.py
+++ b/frontend.py
@@ -36,7 +36,9 @@ def main_repl(datapoints,path,date1=None,date2=None,done1=False,done2=False):

 def selection_repl(path):
    if path != None:
-        datapoints = input_backend.read_in_file(path)
+        outsideData    = input("Außentemperatur einzeichnen? (j/n)")
+        useOutsideData = outsideData.strip().lower() in [ "j", "y" ]
+        datapoints = input_backend.read_in_file(path, outsideData=useOutsideData)
        if CFG("debug_no_interactive"):
            plot_main.plot(datapoints,path)
            return None
--- a/input_backend.py
+++ b/input_backend.py
@@ -1,6 +1,8 @@
 #!/usr/bin/python3
 from config_parse import CFG
 from datetime import datetime, timedelta
+import requests
+import os

 from dbfread import DBF
 import plot_timeutils
@@ -21,6 +23,14 @@ class Data:
        self.times  = []
        self.plot = plot

+    def getFirstTime(self):
+        '''Get time of first timestamp'''
+        return min(self.times)
+
+    def getLastTime(self):
+        '''Get time of last timestamp'''
+        return max(self.times)
+
    def get_timeframe(self, callback,date1=None,date2=None):
        out_x = []
        out_y = []
@@ -76,7 +86,7 @@ class Data:
        arr_v = [x for _,x in sorted(zip(arr_t,arr_v))]
        return (arr_t,arr_v)

-def parse_line(datapoints,line,timekey,keys,time_parser,timeformat=None):
+def parse_line(datapoints, line, timekey, keys, time_parser, timeformat=None):
        # This function expects:
        #       - datapoints { String:DataObject }
        #       - line       { String:Any        }
@@ -87,36 +97,100 @@ def parse_line(datapoints,line,timekey,keys,time_parser,timeformat=None):
                datapoints[ key[1] ].data  += [ line[ key[0] ] ]
                datapoints[ key[1] ].times += [      time      ]

-def read_in_file(path,backend=None):
-        global tname
-        global hname
-        global dname
-        global opath
+def processExternalData(datapoints, plotNameKey, fromTime, toTime, dtype):
+    '''Download and parses external data of type dtype'''
+
+    # prepare strings #
+    cacheDir    = CFG("cache_dir")
+    fromTimeStr = fromTime.strftime(CFG("nff_url_timeformat"))
+    toTimeStr   = toTime.strftime(CFG("nff_url_timeformat"))
+    cacheFile   = "cache_{}_{}_{}.data".format(dtype, fromTimeStr, toTimeStr)
+    fullpath   = os.path.join(cacheDir, cacheFile)
+
+    # check for cache file
+    content = None
+    if not os.path.isfile(fullpath):
        
+        # download date if it doesn't exist #
+        url = CFG("outside_data_url").format(dtype=dtype, fromDate=fromTimeStr, toDate=toTimeStr)
+        r = requests.get(url)
+        print(url)
+        content = r.content.decode('utf-8', "ignore") # ignore bad bytes
+
+        # cache data #
+        if not os.path.isdir(cacheDir):
+            os.mkdir(cacheDir)
+        with open(fullpath, 'w') as f:
+            f.write(content)
+
+    else:
+
+        # get data from cache otherwise
+        print("INFO: Cache hit: {}".format(cacheFile))
+        with open(fullpath) as f:
+            content = f.read()
+
+    skipBecauseFirstLine = True
+    for l in content.split("\n"):
+        if not ";" in l:
+            continue
+        elif not l.strip():
+            continue
+        elif skipBecauseFirstLine:
+            skipBecauseFirstLine = False
+            continue
+
+        try:
+            timeStr, value = l.split(";")
+            timestamp = plot_timeutils.time_from_csv(timeStr, CFG("nff_input_timeformat"))
+
+            datapoints[plotNameKey].data  += [float(value.replace(",","."))]
+            datapoints[plotNameKey].times += [timestamp]
+        except ValueError as e:
+            print(l)
+            raise e
+
+
+def read_in_file(path, backend=None, outsideData=False):
+        '''Read in a file, add outside data if requested'''
+
        datapoints = dict()
+        identifiers =  [ CFG("plot_temperatur_key"),
+                         CFG("plot_humidity_key"),
+                         CFG("plot_dewcels_key"),
+                         CFG("plot_outside_temperatur_key"),
+                         CFG("plot_outside_humidity_key") ]

-        pt=CFG("plot_temperatur_key")
-        ph=CFG("plot_humidity_key")
-        pd=CFG("plot_dewcels_key")
-        
-        ## NAME PADDING ##
-        max_name_len = max(len(tname),len(hname),len(dname))
-        while len(tname) < max_name_len:
-                tname += " "
-        while len(hname) < max_name_len:
-                hname += " "    
-        while len(dname) < max_name_len:
-                dname += " "
+        names  = [ CFG("temperatur_plot_name"),
+                   CFG("humidity_plot_name"),
+                   CFG("dewcels_plot_name"),
+                   CFG("temperatur_outside_plot_name"),
+                   CFG("humidity_outside_plot_name") ]

-        datapoints.update({ pt:Data( tname,CFG("plot_temperatur") ) })
-        datapoints[pt].color = CFG("temperatur_color")      
-        
-        datapoints.update({ ph:Data( hname,CFG("plot_humidity") ) })
-        datapoints[ph].color = CFG("humidity_color")
-        
-        datapoints.update({ pd:Data( dname,CFG("plot_dewcels") ) })
-        datapoints[pd].color = CFG("dewcels_color")
+        colors = [ CFG("temperatur_color"),
+                   CFG("humidity_color"),
+                   CFG("dewcels_color"),
+                   CFG("temperatur_outside_color"),
+                   CFG("humidity_outside_color") ]
+
+        plotSettings = [ CFG("plot_temperatur"),
+                         CFG("plot_humidity"),
+                         CFG("plot_dewcels"),
+                         outsideData,
+                         outsideData ]
        
+        assert(len(names) == len(colors) == len(identifiers) == len(plotSettings))
+
+        max_name_len = max([len(s) for s in names])
+        for i in range(0, len(names)):
+            while len(names[i]) < max_name_len:
+                names[i] += " "
+            datapoints.update({ identifiers[i] : Data(names[i], plotSettings[i]) })
+
+        # legacy variables...
+        pt, ph, pd, pto, pho = identifiers
+
+        # parse input file #
        if path == None:
            raise Exception("Path in plot.read_in was None")
        elif backend != None:
@@ -129,8 +203,19 @@ def read_in_file(path,backend=None):
                csvread_txt(path,datapoints,pt,ph,pd)
        else:
                raise NotImplementedError("Cannot determine filetype, cannot continue. Exit.")
+        
+        # if nessesary download and process external data #
+        if outsideData:

+            fromTime = datapoints[CFG("plot_temperatur_key")].getFirstTime()
+            toTime   = datapoints[CFG("plot_temperatur_key")].getLastTime()
+
+            processExternalData(datapoints, pto, fromTime, toTime, CFG("dtype_temperatur"))
+            processExternalData(datapoints, pho, fromTime, toTime, CFG("dtype_humidity"))
+
+        # sanity check result #
        check_read_in(datapoints)
+
        return datapoints

 def dbfread(path,datapoints,pt,ph,pd):
--- a/plot_graphutils.py
+++ b/plot_graphutils.py
@@ -26,14 +26,16 @@ def getlimits_y(y):
 def avg(array):
        return sum(array)/float(len(array))

-def legend_box_contents(name,y):
-	if CFG("show_min"):
-		name += " min: %.1f,"%min(y)
-	if CFG("show_max"):
-		name += " max: %.1f,"%max(y)
-	if CFG("show_avg"):
-		name += " Mittelwert: %.1f,"% avg(y)
-	return name.rstrip(",")    
+def legend_box_contents(name, y):
+    if CFG("cap_values_at_99"):
+        y = [ min([el, 99.9]) for el in y ]
+    if CFG("show_min"):
+        name += " min: {:4.1f},".format(min(y))
+    if CFG("show_max"):
+        name += " max: {:4.1f},".format(max(y))
+    if CFG("show_avg"):
+        name += " Mittelwert: {:4.1f},".format(avg(y))
+    return name.rstrip(",")    

 def general_background_setup(tup,ymin,ymax,x):

--- a/plot_main.py
+++ b/plot_main.py
@@ -35,13 +35,15 @@ def __plot(tup,datapoints,path,date1=None,date2=None):
        ls = CFG("plot_line_style")
        tup[FIGURE],tup[AXIS] = plt.subplots(1, 1)

-        for g in datapoints.values():
+        for key in datapoints.keys():
+                g = datapoints[key]
+                print(key)
                #### Check if we are supposed to plot something ####
                if not g.plot:
                        continue
                #### GET AND CHECK TIMEFRAMES ####
                x,y, = g.get_timeframe(tup[CALLBACK],date1,date2)
-                if len(x) <= 0 or len(y) <= 0:
+                if not x or not y or len(x) <= 0 or len(y) <= 0:
                    print("Warning: Empty series of data '%s' (wrong start/end time?)"%g.name)
                    continue
                else:
@@ -52,13 +54,15 @@ def __plot(tup,datapoints,path,date1=None,date2=None):
                #### GET LINE STYLES ####
                legend_label = plot_graphutils.legend_box_contents(g.name,y)
                tup[AXIS].plot(unix_x, y,ls=ls,lw=lw,marker="None", label=legend_label, color=g.color)
+                legacy_x_save = x
+                lagacy_y_save = y
        
        if NO_SERIES:
                print("Error: no data, nothing to plot. cannot continue. exit.")
                sys.exit(1)

        ## GRID ##
-        plot_graphutils.general_background_setup(tup,ymin,ymax,x)
+        plot_graphutils.general_background_setup(tup, ymin, ymax, legacy_x_save)

        ## using unix_x relys on unix_x to be the same for all plots ##
        if path == None:
--- a/plot_timeutils.py
+++ b/plot_timeutils.py
@@ -8,15 +8,15 @@ def between_dates(t,date1,date2):
        else:
            return False

-def time_from_dbf(l,timeformat):
+def time_from_dbf(l, timeformat):
        timeformat=None #dont need that here
        offset_d = datetime(1970,1,1)-datetime(1900,1,1)
        shit_epoch = l*24*60*60 #days to seconds
        unix_epoch = datetime.fromtimestamp(shit_epoch)-offset_d
        return (unix_epoch-timedelta(days=2)+timedelta(hours=CFG("add_hours_to_input"))).replace(microsecond=0)

-def time_from_csv(l,timeformat):
-        return datetime.strptime(l,timeformat)
+def time_from_csv(l, timeformat):
+        return datetime.strptime(l, timeformat)

 def unix(dt):
    return dt.timestamp()
--- a/ths_config.txt
+++ b/ths_config.txt
@@ -17,6 +17,8 @@ target_temperatur		= 20

 temperatur_plot_name	= Innenlufttemperatur
 humidity_plot_name		= rel. Luftfeuchtigkeit
+temperatur_outside_plot_name = Außenlufttemperatur
+humidity_outside_plot_name	 = rel. (a) Luftfeuchtigkeit
 dewcels_plot_name		= Taupunkt
 y_label					= Temp./r.L.
 x_label					= Datum/Uhrzeit
@@ -60,6 +62,8 @@ yaxis_start_value				= 0
 # True:  die Y-Achse beginnt auch bei xaxis_start_value wenn dadurch Werte nicht angezeit werden
 # False: wenn ein Wert im plot kleiner xaxis_start_value ist beginnt die Y-Achse beim kleinsten Wert im Plot
 yaxis_force_start_value			= False
+# cap all values at two digits to prevent formating problems
+cap_values_at_99                = True

 # ein höheres alpha für zu einer stärkeren Sättigung der Hintergrundfarbe (0 und es ist ganz weg)
 humidity_crit_alpha		= 0.35
@@ -72,9 +76,11 @@ humidity_warning_color = yellow
 acceptable_temp_color  = blue

 # Farbe der linie des graphen
-humidity_color			= red
-temperatur_color		= blue
-dewcels_color			= green
+humidity_color			 = red
+temperatur_color		 = blue
+dewcels_color			 = green
+temperatur_outside_color = cyan
+humidity_outside_color   = orange

 plot_line_width = 0.5
 plot_line_style = solid
@@ -100,10 +106,17 @@ raster_minimum_hlines		    = 10
 add_hours_to_input              = 1
 add_x_labels_at_end             = 1

+### NFF Data URLs ###
+outside_data_url = "http://umweltdaten.nuernberg.de/csv/wetterdaten/messstation-nuernberg-flugfeld/archiv/csv-export/SUN/nuernberg-flugfeld/{dtype}/individuell/{fromDate}/{toDate}/export.csv"
+dtype_temperatur = "lufttemperatur-aussen"
+dtype_humidity   = "luftfeuchte"
+nff_url_timeformat = "%%d.%%m.%%Y"
+nff_input_timeformat = "%%d.%%m.%%Y %%H:%%M"
+
 ###### DEBUGGING ######
-no_ask_date_input           = no
-input_filename              = test.xls
-use_input_filename          = no
+no_ask_date_input           = yes
+input_filename              = "LOG32TH_20010101_2020-06-30T131045.DBF"
+use_input_filename          = yes
 debug_no_interactive        = no
 terminate_on_warning        = no
 terminate_on_missing_input_file = True
@@ -115,12 +128,14 @@ default_target_dir          = UseSourceDir
 plot_temperatur_key         = TEMP
 plot_humidity_key 	        = HUMIDITY
 plot_dewcels_key	        = TAU_P
+plot_outside_temperatur_key = O_TEMP
+plot_outside_humidity_key	= O_HUMIDITY
 always_allow_days_as_xticks = yes
 language                    = DE
 aspect_ratio                = A4
 use_gui_backend             = Agg
 enable_multicore_support 	= False
-raster_alligment_auto	 	= True
 outfile_resolution_in_dpi   = 250
 # <= what stepsize should datapoints be combined (s)
 combine_data_points         = 1
+cache_dir                   = "./cache/"