diff --git a/.gitignore b/.gitignore index 5a38f37..b61119b 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,6 @@ *.png +*.bak +cache/ *.dbf __py* *.swp diff --git a/frontend.py b/frontend.py index 7f9b2ec..d871d99 100644 --- a/frontend.py +++ b/frontend.py @@ -36,7 +36,9 @@ def main_repl(datapoints,path,date1=None,date2=None,done1=False,done2=False): def selection_repl(path): if path != None: - datapoints = input_backend.read_in_file(path) + outsideData = input("Außentemperatur einzeichnen? (j/n)") + useOutsideData = outsideData.strip().lower() in [ "j", "y" ] + datapoints = input_backend.read_in_file(path, outsideData=useOutsideData) if CFG("debug_no_interactive"): plot_main.plot(datapoints,path) return None diff --git a/input_backend.py b/input_backend.py index a4b0e29..e075ec8 100644 --- a/input_backend.py +++ b/input_backend.py @@ -1,6 +1,8 @@ #!/usr/bin/python3 from config_parse import CFG from datetime import datetime, timedelta +import requests +import os from dbfread import DBF import plot_timeutils @@ -21,6 +23,14 @@ class Data: self.times = [] self.plot = plot + def getFirstTime(self): + '''Get time of first timestamp''' + return min(self.times) + + def getLastTime(self): + '''Get time of last timestamp''' + return max(self.times) + def get_timeframe(self, callback,date1=None,date2=None): out_x = [] out_y = [] @@ -76,7 +86,7 @@ class Data: arr_v = [x for _,x in sorted(zip(arr_t,arr_v))] return (arr_t,arr_v) -def parse_line(datapoints,line,timekey,keys,time_parser,timeformat=None): +def parse_line(datapoints, line, timekey, keys, time_parser, timeformat=None): # This function expects: # - datapoints { String:DataObject } # - line { String:Any } @@ -87,36 +97,100 @@ def parse_line(datapoints,line,timekey,keys,time_parser,timeformat=None): datapoints[ key[1] ].data += [ line[ key[0] ] ] datapoints[ key[1] ].times += [ time ] -def read_in_file(path,backend=None): - global tname - global hname - global dname - global opath +def processExternalData(datapoints, plotNameKey, fromTime, toTime, dtype): + '''Download and parses external data of type dtype''' + + # prepare strings # + cacheDir = CFG("cache_dir") + fromTimeStr = fromTime.strftime(CFG("nff_url_timeformat")) + toTimeStr = toTime.strftime(CFG("nff_url_timeformat")) + cacheFile = "cache_{}_{}_{}.data".format(dtype, fromTimeStr, toTimeStr) + fullpath = os.path.join(cacheDir, cacheFile) + + # check for cache file + content = None + if not os.path.isfile(fullpath): + # download date if it doesn't exist # + url = CFG("outside_data_url").format(dtype=dtype, fromDate=fromTimeStr, toDate=toTimeStr) + r = requests.get(url) + print(url) + content = r.content.decode('utf-8', "ignore") # ignore bad bytes + + # cache data # + if not os.path.isdir(cacheDir): + os.mkdir(cacheDir) + with open(fullpath, 'w') as f: + f.write(content) + + else: + + # get data from cache otherwise + print("INFO: Cache hit: {}".format(cacheFile)) + with open(fullpath) as f: + content = f.read() + + skipBecauseFirstLine = True + for l in content.split("\n"): + if not ";" in l: + continue + elif not l.strip(): + continue + elif skipBecauseFirstLine: + skipBecauseFirstLine = False + continue + + try: + timeStr, value = l.split(";") + timestamp = plot_timeutils.time_from_csv(timeStr, CFG("nff_input_timeformat")) + + datapoints[plotNameKey].data += [float(value.replace(",","."))] + datapoints[plotNameKey].times += [timestamp] + except ValueError as e: + print(l) + raise e + + +def read_in_file(path, backend=None, outsideData=False): + '''Read in a file, add outside data if requested''' + datapoints = dict() + identifiers = [ CFG("plot_temperatur_key"), + CFG("plot_humidity_key"), + CFG("plot_dewcels_key"), + CFG("plot_outside_temperatur_key"), + CFG("plot_outside_humidity_key") ] - pt=CFG("plot_temperatur_key") - ph=CFG("plot_humidity_key") - pd=CFG("plot_dewcels_key") - - ## NAME PADDING ## - max_name_len = max(len(tname),len(hname),len(dname)) - while len(tname) < max_name_len: - tname += " " - while len(hname) < max_name_len: - hname += " " - while len(dname) < max_name_len: - dname += " " + names = [ CFG("temperatur_plot_name"), + CFG("humidity_plot_name"), + CFG("dewcels_plot_name"), + CFG("temperatur_outside_plot_name"), + CFG("humidity_outside_plot_name") ] - datapoints.update({ pt:Data( tname,CFG("plot_temperatur") ) }) - datapoints[pt].color = CFG("temperatur_color") - - datapoints.update({ ph:Data( hname,CFG("plot_humidity") ) }) - datapoints[ph].color = CFG("humidity_color") - - datapoints.update({ pd:Data( dname,CFG("plot_dewcels") ) }) - datapoints[pd].color = CFG("dewcels_color") + colors = [ CFG("temperatur_color"), + CFG("humidity_color"), + CFG("dewcels_color"), + CFG("temperatur_outside_color"), + CFG("humidity_outside_color") ] + + plotSettings = [ CFG("plot_temperatur"), + CFG("plot_humidity"), + CFG("plot_dewcels"), + outsideData, + outsideData ] + assert(len(names) == len(colors) == len(identifiers) == len(plotSettings)) + + max_name_len = max([len(s) for s in names]) + for i in range(0, len(names)): + while len(names[i]) < max_name_len: + names[i] += " " + datapoints.update({ identifiers[i] : Data(names[i], plotSettings[i]) }) + + # legacy variables... + pt, ph, pd, pto, pho = identifiers + + # parse input file # if path == None: raise Exception("Path in plot.read_in was None") elif backend != None: @@ -129,8 +203,19 @@ def read_in_file(path,backend=None): csvread_txt(path,datapoints,pt,ph,pd) else: raise NotImplementedError("Cannot determine filetype, cannot continue. Exit.") + + # if nessesary download and process external data # + if outsideData: + fromTime = datapoints[CFG("plot_temperatur_key")].getFirstTime() + toTime = datapoints[CFG("plot_temperatur_key")].getLastTime() + + processExternalData(datapoints, pto, fromTime, toTime, CFG("dtype_temperatur")) + processExternalData(datapoints, pho, fromTime, toTime, CFG("dtype_humidity")) + + # sanity check result # check_read_in(datapoints) + return datapoints def dbfread(path,datapoints,pt,ph,pd): diff --git a/plot_graphutils.py b/plot_graphutils.py index 949f4bd..cc6a772 100644 --- a/plot_graphutils.py +++ b/plot_graphutils.py @@ -26,14 +26,16 @@ def getlimits_y(y): def avg(array): return sum(array)/float(len(array)) -def legend_box_contents(name,y): - if CFG("show_min"): - name += " min: %.1f,"%min(y) - if CFG("show_max"): - name += " max: %.1f,"%max(y) - if CFG("show_avg"): - name += " Mittelwert: %.1f,"% avg(y) - return name.rstrip(",") +def legend_box_contents(name, y): + if CFG("cap_values_at_99"): + y = [ min([el, 99.9]) for el in y ] + if CFG("show_min"): + name += " min: {:4.1f},".format(min(y)) + if CFG("show_max"): + name += " max: {:4.1f},".format(max(y)) + if CFG("show_avg"): + name += " Mittelwert: {:4.1f},".format(avg(y)) + return name.rstrip(",") def general_background_setup(tup,ymin,ymax,x): diff --git a/plot_main.py b/plot_main.py index 8b2f659..aec8b73 100644 --- a/plot_main.py +++ b/plot_main.py @@ -35,13 +35,15 @@ def __plot(tup,datapoints,path,date1=None,date2=None): ls = CFG("plot_line_style") tup[FIGURE],tup[AXIS] = plt.subplots(1, 1) - for g in datapoints.values(): + for key in datapoints.keys(): + g = datapoints[key] + print(key) #### Check if we are supposed to plot something #### if not g.plot: continue #### GET AND CHECK TIMEFRAMES #### x,y, = g.get_timeframe(tup[CALLBACK],date1,date2) - if len(x) <= 0 or len(y) <= 0: + if not x or not y or len(x) <= 0 or len(y) <= 0: print("Warning: Empty series of data '%s' (wrong start/end time?)"%g.name) continue else: @@ -52,13 +54,15 @@ def __plot(tup,datapoints,path,date1=None,date2=None): #### GET LINE STYLES #### legend_label = plot_graphutils.legend_box_contents(g.name,y) tup[AXIS].plot(unix_x, y,ls=ls,lw=lw,marker="None", label=legend_label, color=g.color) + legacy_x_save = x + lagacy_y_save = y if NO_SERIES: print("Error: no data, nothing to plot. cannot continue. exit.") sys.exit(1) ## GRID ## - plot_graphutils.general_background_setup(tup,ymin,ymax,x) + plot_graphutils.general_background_setup(tup, ymin, ymax, legacy_x_save) ## using unix_x relys on unix_x to be the same for all plots ## if path == None: diff --git a/plot_timeutils.py b/plot_timeutils.py index 0950cba..812f70b 100644 --- a/plot_timeutils.py +++ b/plot_timeutils.py @@ -8,15 +8,15 @@ def between_dates(t,date1,date2): else: return False -def time_from_dbf(l,timeformat): +def time_from_dbf(l, timeformat): timeformat=None #dont need that here offset_d = datetime(1970,1,1)-datetime(1900,1,1) shit_epoch = l*24*60*60 #days to seconds unix_epoch = datetime.fromtimestamp(shit_epoch)-offset_d return (unix_epoch-timedelta(days=2)+timedelta(hours=CFG("add_hours_to_input"))).replace(microsecond=0) -def time_from_csv(l,timeformat): - return datetime.strptime(l,timeformat) +def time_from_csv(l, timeformat): + return datetime.strptime(l, timeformat) def unix(dt): return dt.timestamp() diff --git a/ths_config.txt b/ths_config.txt index 5287d36..7711901 100644 --- a/ths_config.txt +++ b/ths_config.txt @@ -17,6 +17,8 @@ target_temperatur = 20 temperatur_plot_name = Innenlufttemperatur humidity_plot_name = rel. Luftfeuchtigkeit +temperatur_outside_plot_name = Außenlufttemperatur +humidity_outside_plot_name = rel. (a) Luftfeuchtigkeit dewcels_plot_name = Taupunkt y_label = Temp./r.L. x_label = Datum/Uhrzeit @@ -60,6 +62,8 @@ yaxis_start_value = 0 # True: die Y-Achse beginnt auch bei xaxis_start_value wenn dadurch Werte nicht angezeit werden # False: wenn ein Wert im plot kleiner xaxis_start_value ist beginnt die Y-Achse beim kleinsten Wert im Plot yaxis_force_start_value = False +# cap all values at two digits to prevent formating problems +cap_values_at_99 = True # ein höheres alpha für zu einer stärkeren Sättigung der Hintergrundfarbe (0 und es ist ganz weg) humidity_crit_alpha = 0.35 @@ -72,9 +76,11 @@ humidity_warning_color = yellow acceptable_temp_color = blue # Farbe der linie des graphen -humidity_color = red -temperatur_color = blue -dewcels_color = green +humidity_color = red +temperatur_color = blue +dewcels_color = green +temperatur_outside_color = cyan +humidity_outside_color = orange plot_line_width = 0.5 plot_line_style = solid @@ -100,10 +106,17 @@ raster_minimum_hlines = 10 add_hours_to_input = 1 add_x_labels_at_end = 1 +### NFF Data URLs ### +outside_data_url = "http://umweltdaten.nuernberg.de/csv/wetterdaten/messstation-nuernberg-flugfeld/archiv/csv-export/SUN/nuernberg-flugfeld/{dtype}/individuell/{fromDate}/{toDate}/export.csv" +dtype_temperatur = "lufttemperatur-aussen" +dtype_humidity = "luftfeuchte" +nff_url_timeformat = "%%d.%%m.%%Y" +nff_input_timeformat = "%%d.%%m.%%Y %%H:%%M" + ###### DEBUGGING ###### -no_ask_date_input = no -input_filename = test.xls -use_input_filename = no +no_ask_date_input = yes +input_filename = "LOG32TH_20010101_2020-06-30T131045.DBF" +use_input_filename = yes debug_no_interactive = no terminate_on_warning = no terminate_on_missing_input_file = True @@ -115,12 +128,14 @@ default_target_dir = UseSourceDir plot_temperatur_key = TEMP plot_humidity_key = HUMIDITY plot_dewcels_key = TAU_P +plot_outside_temperatur_key = O_TEMP +plot_outside_humidity_key = O_HUMIDITY always_allow_days_as_xticks = yes language = DE aspect_ratio = A4 use_gui_backend = Agg enable_multicore_support = False -raster_alligment_auto = True outfile_resolution_in_dpi = 250 # <= what stepsize should datapoints be combined (s) combine_data_points = 1 +cache_dir = "./cache/"