Implement GUI

src/main/python/config_parse.py

@@ -0,0 +1,85 @@
+import configparser
+import sys
+
+conf = None
+default_conf = None
+
+def parse_config():
+        global conf
+        global default_conf
+        
+        conf = configparser.ConfigParser()
+        conf.read("ths_config.txt")
+        default_conf = configparser.ConfigParser()
+        default_conf.read("ths_readonly_default.conf")
+        
+        if conf == None or (len(conf.sections()) == 0 and len(default_conf.sections()) == 0):
+                print("Error: Missing configuration file, cannot continue")
+                raise Exception("Missing configuration file")
+
+def get_keys(like=None):
+        ret = conf["plot"].keys()
+        if like != None:
+            ret = list(filter(lambda x:like in x,ret))
+            if len(ret) == 0:
+                print("No options that contain the string '%s'"%like)
+                return ""
+        return ret
+
+def change_cfg(key,value):
+        global conf
+        if conf == None:
+                parse_config()
+        confs = conf["plot"]
+        v = str(value)
+        key = str(key)
+        if key not in confs:
+            return False
+        else:
+            confs[key] = value
+            return True
+
+
+        
+        
+def CFG(tag):
+        global conf
+        global default_conf
+        
+        if conf == None:
+                parse_config()
+        if len(default_conf.sections()) > 0:
+                default_confs = default_conf["plot"]
+        else:
+                default_confs = None
+        confs = conf["plot"]
+        
+        if tag in confs:
+                return parse_cfg(confs[tag])
+        elif default_confs != None and tag in default_confs:
+                print("Warning: %s no found in configuration, defaulting to %s" % (str(tag),str(default_conf[tag])),sys.stderr)
+                return parse_cfg(default_confs[tag])
+        else:
+                raise Exception("Error: configuration option %s not found in configuration and no default value for it, cannot continue, exit." % str(tag))
+
+def parse_cfg(c):
+        if c == None:
+                raise Exception("Config key (%s) found but has no value. Cannot continue, exit." % str(c))
+        c = c.strip("'")
+        c = c.strip('"')
+        if c in ["yes","ja","True","Yes","Ja","true"]:
+                return True
+        if c in ["no","nein","False","No","Nein","false"]:
+                return False
+        try:
+                return int(c)
+        except ValueError:
+                pass
+        try:
+                return float(c)
+        except ValueError:
+                pass
+        return c
+                
+
+CFG("show_avg")

src/main/python/constants.py

@@ -0,0 +1,8 @@
+#!/usr/bin/python3
+from config_parse import CFG
+GLOBAL_FONT = {'family':CFG("font"),'weight':'normal','size': CFG("global_text_size")}
+BASE_PATH=CFG("default_source_path")
+SOURCE_PATH=CFG("default_target_dir")
+FIGURE=0
+AXIS=1
+CALLBACK=2

src/main/python/frontend.py

@@ -0,0 +1,77 @@
+#!/usr/bin/python3
+import sys
+import tkinter
+from   tkinter        import filedialog
+
+import plot_main
+import input_backend
+import frontend_utils as     futils
+from   config_parse   import CFG
+from   language       import LAN
+
+l = LAN[CFG("language")]
+tk = tkinter.Tk()
+tk.withdraw()
+
+def main_repl(datapoints,path,date1=None,date2=None,done1=False,done2=False):
+    ### READ IN DATES ###
+    raw1 = ""
+    raw2 = ""
+    futils.info_list(datapoints)
+    while not done1:
+        date1,done1,raw1 = futils.input_date_repl(datapoints,startdate=True)
+    while not done2:
+        date2,done2,raw2 = futils.input_date_repl(datapoints,startdate=False)
+
+    # save history here (stardate then enddate)
+    futils.save_history(raw2)
+    futils.save_history(raw1)
+
+    ### CHECK DATES ###
+    done1,done2 = futils.check_dates(path,date1,date2)
+    if not done1 or not done2:
+        main_repl(datapoints,path,date1,date2,done1,done2)
+    else:
+        plot_main.plot(datapoints,path,date1,date2)
+
+def selection_repl(path):
+    if path != None:
+        outsideData    = input("Außentemperatur einzeichnen? (j/n)")
+        useOutsideData = outsideData.strip().lower() in [ "j", "y" ]
+        datapoints = input_backend.read_in_file(path, outsideData=useOutsideData)
+        if CFG("debug_no_interactive"):
+            plot_main.plot(datapoints,path)
+            return None
+        main_repl(datapoints,path)
+    if CFG("always_restart"):
+        print("----------------------------------------")
+        tmp=input(  " -> Type 'n' or 'new' <ENTER> to restart with another file\n -> Type 'r' or 'restart'<ENTER> to use the current file again\n -> Or press just <ENTER> to exit: ") 
+        if tmp == None or tmp == "":
+            return None
+        elif tmp in ["r","restart"]:
+            return path
+        elif tmp in ["n","new"]:
+            return futils.open_file()
+        elif tmp.startswith('c'):
+            config_options(ret)
+        else:
+            sys.exit(0)
+
+def main():
+    ### PREVENT MULTICORE SUPPORT ###
+    if CFG("enable_multicore_support"):
+        raise NotImplementedError("multiprocessing not fully implemented")
+    
+    ### PROMT TO OPEN FILE ###
+    FILE_READY = False
+    path = None
+    while True:
+        if not FILE_READY:
+            path = futils.open_file()
+
+        path = selection_repl(path)
+
+        if path == None:
+            break
+        else:
+            FILE_READY = True

src/main/python/frontend_utils.py

@@ -0,0 +1,349 @@
+#!/usr/bin/python3
+import sys
+import tkinter
+import plot_main
+import config_parse
+from config_parse   import CFG
+from language       import LAN
+from datetime       import datetime,timedelta
+from plot_timeutils import between_dates
+
+l = LAN[CFG("language")]
+timeformat = "%d.%m.%y %H:%M:%S (%A)"
+def parse_date_from_user_input(s,end_of_day=False,datapoints=None):
+    today  = datetime.now()
+    day    = 0
+    month  = 0
+    year   = 0
+    hour   = 0
+    minute = 0
+    second = 0
+    
+    ## EMPTY ##
+    if s == None or s == "":
+        return (None,True)
+
+    ## TIME ##
+    if len(s.split(" ")) > 1:
+        time = s.split(" ")[1]
+        time_a = time.split(":")
+        if len(time_a) > 0:
+            hour = int(time_a[0])
+        elif end_of_day > 0:
+            hour = 23
+        if len(time_a) > 1:
+            minute = int(time_a[1])
+        elif end_of_day:
+            minute = 59
+        if len(time_a) > 2:
+            second = int(time_a[2])
+        elif end_of_day:
+            second = 59
+    elif end_of_day:
+        hour = 23
+        minute = 59
+        second = 59
+
+    ## DATE ##
+    tmp = s.split(" ")[0]
+
+    ## allow more speperators ##
+    sep = None
+    for c in ["-",".",","]:
+        if c in tmp:
+            sep = c
+            break
+    if sep == None:
+        sep = "-"
+    tmp = tmp.strip(sep)
+
+    if len(tmp.split(sep)) == 0:
+        raise ValueError("Invalid Date '%s'"%str(s))
+    else:
+        date_a = tmp.split(sep)
+        if len(date_a) > 0:
+            day = int(date_a[0])
+        if len(date_a) > 1:
+            month = int(date_a[1])
+        if len(date_a) > 2:
+            year = int(date_a[2])
+            if year < 1000:
+                year+=2000
+    
+    # remember if an explizit date was give #
+    NO_YEAR=False
+    NO_MONTH=False
+
+    if year == 0:
+        NO_YEAR=True
+        if today.month > month:
+            year = today.year
+        else:
+            year = today.year-1
+    if month == 0:
+        NO_MONTH = True
+        if today.day > day and today.year == year:
+            month = today.month
+        else:
+            month = today.month-1
+            if month < 1:
+                month = 12-month 
+
+    ret = datetime(year,month,day,hour,minute,second)
+    tmp = list(datapoints.values())[0].times
+    status = True
+    if NO_MONTH:
+        ret,status = correct_month(ret,min(tmp),max(tmp))
+    if NO_YEAR and status:
+        ret,status = correct_year(ret,min(tmp),max(tmp))
+    return (ret,status)
+
+def correct_year(date,data_start,data_end):
+        if not CFG("enable_automatic_date_correction"):
+            return (date,True)
+        elif date == None:
+            return (None,True)
+        else:
+            if between_dates(date,data_start,data_end):
+                return (date,True)
+            elif data_start.year != data_end.year:
+                print("Datensätze aus mehr als einem Jahr gefunden, Jahr muss daher angegeben werden.")
+                return (None,False)
+            else:
+                maxi = 12
+                count = 0
+                while count < maxi:
+                    if between_dates(date.replace(year=date.year-count),data_start-timedelta(days=1),data_end+timedelta(days=1)):
+                        return (date.replace(year=date.year-count),True)
+                    count += 1
+
+        return (date,True)
+
+def correct_month(date,data_start,data_end):
+        tmp_date = date.replace(year=data_end.year)
+        if not CFG("enable_automatic_date_correction"):
+            return (date,True)
+        elif date == None:
+            return (None,True)
+        else:
+            if between_dates(date,data_start,data_end):
+                return (date,True)
+            elif data_start.month != data_end.month:
+                print("Datensätze aus mehr als einem Monat gefunden, Monat muss daher angegeben werden.")
+                return (None,False)
+            else:
+                maxi = 12
+                count = maxi
+                while count >= 0:
+                    if between_dates(date.replace(month=((date.month+count)%12)+1),data_start,data_end):
+                        return (date.replace(month=date.month-count),True)
+                    count -= 1
+
+        return (date,True)
+
+history=None
+def load_history(histfile="history.log"):
+    tmp = []
+    histlength = CFG("history_file_length")
+    try:
+        with open(histfile,"r") as f:
+            for l in f:
+                tmp_line = l.strip("\n")
+                if not tmp_line in tmp:
+                    tmp += [tmp_line]
+        if len(tmp) > histlength:
+            with open(histfile,"w") as f:
+                for l in tmp[(len(tmp)-histlength-1):]:
+                    f.write("{}\n".format(l.strip("\n")))
+        tmp = tmp[(len(tmp)-histlength-1):]
+    except IOError:
+        print(">> Warnung: Eingabehistorie Datei nicht gefunden <<")
+    return tmp
+
+def save_history(string,histfile="history.log"):
+    global history
+    if not string:
+        return
+    # remove previous occourence if exists
+    if string in history:
+        history.remove(string)
+    # add
+    string = string.strip("\n")
+    history += [string]
+    # check max file length
+    with open(histfile,"a") as f:
+        f.write("{}\n".format(string.strip("\n")))
+
+def load_from_history(index):
+    global history
+    index = int(index)
+    return history[-index]
+
+def recently_used():
+    global history
+    if not history:
+        history = load_history()
+    ret = ""
+    if len(history) == 0:
+        return "  Historie leider nicht verfügbar."
+    count = 1
+    count_max = CFG("history_show")
+    count_max = len(history) if len(history) < count_max else count_max
+    double_col =  CFG("history_double_col")
+    col = 0
+    hist_reversed = history[-count_max:].copy()
+    hist_reversed.reverse()
+    #print(hist_reversed)
+    for e in hist_reversed:
+        if col == 0 and double_col:
+            ret += "  [{}] {}\t".format(count,e)
+            col = 1
+        else:
+            ret += "  [{}] {}\n".format(count,e)
+            col = 0
+        count += 1
+    return ret
+    
+
+def info_list(datapoints):
+    if len(datapoints.keys()) > 0:
+        print("Erster Datensatz:  "+min(datapoints[list(datapoints.keys())[0]].times).strftime(timeformat))
+        print("Letzer Datensatz:  "+max(datapoints[list(datapoints.keys())[0]].times).strftime(timeformat))
+        print("Anzahl Datensätze: "+str(len(datapoints[list(datapoints.keys())[0]].times)))
+        print("Kürzlich genutze Data/Zeiten: (auswählen mit 1-5 <ENTER>):\n\n{}".format(recently_used()))
+    else:
+        print("Keine Datesätze gefunden!")
+    print_sep_line(True)
+
+end_date_available = None
+def input_date_repl(datapoints,startdate=True):
+    global end_date_available
+    date = None
+
+    while True:
+        try:
+            if startdate:
+                ret = input(l["input_first_date"])
+            else:
+                ret = input(l["input_second_date"])
+        except EOFError:
+            return (date,True,None)
+        except KeyboardInterrupt:
+            sys.exit(2)
+        # change ret to value if it was an history one
+        try:
+            if int(ret) in range(1,CFG("history_show")+1):
+                try:
+                    ret = load_from_history(ret)
+                except Exception:
+                    print("Konnte historische Eingabe nicht finden.")
+                    continue
+        except:
+            pass
+        if ret in ["h","help","hilfe"]:
+            if startdate:
+                print(l["input_first_date_help"])
+            else:
+                print(l["input_second_date_help"])
+            continue
+        elif ret == "list":
+            info_list(datapoints)
+            continue
+        else:
+            try:
+                if startdate:
+                    date,ok=parse_date_from_user_input(ret,datapoints=datapoints)
+                else:
+                    date,ok=parse_date_from_user_input(ret,True,datapoints)
+                if not ok:
+                    return (None,False,ret)
+                else:
+                    return (date,True,ret)
+            except ValueError as e:
+                print(l["cannot_parse_date"] + "( was: {} )\n".format(ret))
+                return (None,False,ret)
+
+def print_sep_line(ln=False):
+    if not ln:
+        print("-----------------------------------------------")
+    else:
+        print("-----------------------------------------------",end='')
+
+
+def check_dates(path,date1,date2,options=""):
+    print_sep_line()
+    if options!="":
+        print("Config options: %s"%options)
+    print("Datei: %s"%path)
+    if date1 == None and date2 == None:
+        print("Info:  Keine Zeitbeschränkung gewählt. Alle vorhandenen Daten werden verwendet.")
+        return (True,True)
+    elif date1 == None:
+        print("Alle Werte vor %s"%date2.strftime(timeformat))
+    elif date2 == None:
+        print("Alle Werte nach %s"%date1.strftime(timeformat))
+    else:
+        print("Start: %s\nEnde:  %s"%(date1.strftime(timeformat),date2.strftime(timeformat)))
+
+    FIRST=True
+    while(True):
+        try:
+            if FIRST:
+                ret = input("Stimmt das so?\n -> <ENTER> für ja/weiter\n -> 's'<ENTER> für Startzeit ändern\n -> 'e'<ENTER> für Endzeit ändern\n -> 'b'<ENTER> für beides ändern\n -> 'exit'<ENTER> to exit\n---> ")
+            else:
+                ret = input("Versuchen sie es nochmal: ")
+        except EOFError:
+            ret = ""
+
+        if ret == 's':
+            return (False,True)
+        elif ret == 'e':
+            return (True,False)
+        elif ret == 'b':
+            return (False,False)
+        elif ret.startswith('c '):
+            tmp = config_options(ret)
+            if tmp == "":
+                pass
+            else:
+                options += "\n     "+tmp
+                check_dates(path,date1,date2,options)
+        elif ret == "":
+            return (True,True)
+        elif ret == "exit":
+            sys.exit(0)
+        FIRST=False
+
+def open_file():
+    front_end_source_path = CFG("default_source_path")
+    if CFG("use_input_filename"):
+        f = front_end_source_path + CFG("input_filename")
+        return f
+
+    path=None
+    path=tkinter.filedialog.askopenfilename(filetypes=(("DBF/XLS Files",("*.DBF","*.dbf","*.xls","*.XLS","*.txt","*.TXT")),("All Files","*.*")))
+    if path == None or path=="":
+        print("Error: No file selected!")
+        return None
+    try:
+        open(path,'r').close()
+    except IOError:
+        print("Error: Unable to open selected file, perhaps it does no longer exist or you have insufficient permissions to open it?")
+        return None
+    return path
+
+def config_options(string):
+        opt = ""
+        arg = string.split(" ")
+        if len(arg) == 2:
+            for l in config_parse.get_keys(arg[1]):
+                print(l)
+        elif len(arg) == 3:
+            if not config_parse.change_cfg(arg[1],arg[2]):
+                print("Option %s does not exist."%str(arg[1]))
+            else:
+                opt += "set %s %s"%(arg[1],arg[2])
+                print("set %s %s"%(arg[1],arg[2]))
+        else:
+            print("Ussage: c <configname> <value> / c <part_of_config_name> / c (= list all ), e to exit")
+        return opt

src/main/python/gui.py

@@ -0,0 +1,104 @@
+#!/usr/bin/python3
+
+from fbs_runtime.application_context.PyQt5 import ApplicationContext
+from PyQt5.QtCore import QDateTime, Qt, QTimer
+from PyQt5.QtWidgets import (QApplication, QCheckBox, QComboBox, QDateTimeEdit,
+        QDial, QDialog, QGridLayout, QGroupBox, QHBoxLayout, QLabel, QLineEdit,
+        QProgressBar, QPushButton, QRadioButton, QScrollBar, QSizePolicy,
+        QSlider, QSpinBox, QStyleFactory, QTableWidget, QTabWidget, QTextEdit,
+        QVBoxLayout, QWidget)
+
+import localization.de as de
+
+class WidgetGallery(QDialog):
+
+    def __init__(self, parent=None):
+        super(WidgetGallery, self).__init__(parent)
+
+        self.localization = de
+        self.srcFileString = ""
+
+        styleComboBox = QComboBox()
+        styleComboBox.addItems(QStyleFactory.keys())
+
+        self.createFileSelection()
+        self.createDateSelection()
+        self.createCheckboxArea()
+
+        mainLayout = QGridLayout()
+        mainLayout.addLayout(topLayout, 0, 0, 1, 2)
+        mainLayout.addWidget(self.fileSelectionGroup, 1, 0)
+        mainLayout.addWidget(self.dateSelectionGroupBox, 1, 1)
+        mainLayout.addWidget(self.checkboxGroup, 2, 0)
+
+        mainLayout.setRowStretch(1, 1)
+        mainLayout.setRowStretch(2, 1)
+        mainLayout.setColumnStretch(0, 1)
+        mainLayout.setColumnStretch(1, 1)
+
+        self.setLayout(mainLayout)
+
+        self.setWindowTitle(self.localization.window_title)
+
+    def createFileSelection(self):
+        '''Generate the area containing the file selectors and go button'''
+
+        self.fileSelectionGroup = QGroupBox(self.localization.file_selection)
+
+        # basic object #
+        buttonGo      = QPushButton(self.localization.button_go)
+        buttonSrcFile = QPushButton(self.localization.button_set_src_file)
+        srcFileName   = QLabel(self.srcFileString)
+
+        buttonTargetFile = QPushButton(self.target_file)
+        buttonUseSrcDir  = QCheckBox(self.localization.button_use_src_dir)
+
+        # connectors #
+        buttonGo.connect(self.run)
+        buttonSrcFile.connect(self.selectSrcFile)
+        buttonTargetFile.connect(self.selectTargetFile)
+        buttonUseSrcDir.connect(self.useSrcDir)
+
+        # layout #
+        layout = QVBoxLayout()
+
+        layout.addWidget(buttonGo)
+        layout.addWidget(buttonSrcFile)
+        layout.addWidget(srcFileName)
+
+        layout.addWidget(buttonTargetFile)
+        layout.addWidget(buttonUseSrcDir)
+
+        layout.addStretch(1)
+        self.dateSelectionGroupBox.setLayout(layout)
+
+    def createDateSelection(self):
+        '''Generate the area containing the date selectors'''
+
+        self.dateSelectionGroupBox = QGroupBox(self.localization.date_selection)
+
+        layout = QVBoxLayout()
+        layout.addWidget() # TODO
+        layout.addStretch(1)
+        self.topRightGroupBox.setLayout(layout)
+
+    def createCheckboxArea(self):
+        '''Generate area  with configuration options'''
+
+        self.checkboxGroup = QGroupBox(self.localization.options)
+
+        buttonOTemp     = QCheckBox(self.localization.button_otemp)
+        buttonOHumidity = QCheckBox(self.localization.button_ohumidity)
+
+        layout = QVBoxLayout()
+        layout.addWidget(buttonOTemp)
+        layout.addWidget(buttonOHumidity)
+        layout.addStretch(1)
+        self.topRightGroupBox.setLayout(layout)
+
+
+if __name__ == '__main__':
+    appctxt = ApplicationContext()
+    gallery = WidgetGallery()
+    gallery.show()
+    sys.exit(appctxt.app.exec_())

src/main/python/init.py

@@ -0,0 +1,6 @@
+#!/usr/bin/python3
+import frontend
+import sys
+if __name__ == "__main__":
+            frontend.main()
+            sys.exit(0)

src/main/python/input_backend.py

@@ -0,0 +1,302 @@
+#!/usr/bin/python3
+from config_parse import CFG
+from datetime import datetime, timedelta
+import requests
+import os
+
+from dbfread import DBF
+import plot_timeutils
+
+line_colors = ['b', 'r', 'g', 'c', 'm', 'y']
+tname       = CFG("temperatur_plot_name")
+hname       = CFG("humidity_plot_name")
+dname       = CFG("dewcels_plot_name")
+color_id    = 0
+
+class Data:
+    def __init__(self,name,plot=False):
+        global color_id,line_colors
+        self.name = name
+        self.color=line_colors[color_id%len(line_colors)]
+        color_id += 1
+        self.data = []
+        self.times  = []
+        self.plot = plot
+
+    def getFirstTime(self):
+        '''Get time of first timestamp'''
+        return min(self.times)
+
+    def getLastTime(self):
+        '''Get time of last timestamp'''
+        return max(self.times)
+
+    def get_timeframe(self, callback, date1=None, date2=None):
+        out_x = []
+        out_y = []
+        i = 0
+        if(len(self.times) != len(self.data)):
+            raise RuntimeError("len(timestamps) != len(data), cannot continue, this should never happen")
+        if(len(self.times) <= 2):
+            print("WARNING: No Data for %s!"%self.name)
+            return (None,None)
+        ############ AVERAGE OUT DATA #############
+        if(CFG("combine_data_points") >= (self.times[1] - self.times[0]).total_seconds()):
+            x_dp = 5
+            m_t  = 3
+            while(i+x_dp<len(self.times)):
+                # check middle time #
+                if callback(self.times[i+m_t],date1,date2):
+                    subset=0
+                    subset_data=0.0
+                    subset_time=timedelta(0)
+                    while subset < x_dp:
+                        subset_data += self.data [i+subset]
+                        subset_time += self.times[i+subset]-datetime(2000,1,1)
+                        subset += 1
+                    out_x += [ subset_time/x_dp + datetime(2000,1,1) ]
+                    out_y += [ subset_data/x_dp ]
+                i += x_dp
+        ############ AVERAGE OUT DATA ###########
+        else:
+            while(i<len(self.times)):
+                if callback(self.times[i],date1,date2):
+                    out_x += [ self.times[i] ]
+                    out_y += [ self.data[i]  ]
+                i += 1
+        return (out_x,out_y)
+
+    ## no idea on what kind of drugs I was when i wrote this function (it is somewhat ingenious though) ##
+    def _get_timeframe(self, callback,date1=None,date2=None):
+        r=dict()
+        for t,c in zip(self.times,self.data):
+            t = callback(t,date1,date2)
+            if t == None:
+                continue
+            if t in r:
+                r[t]+=[c]
+            else:
+                r.update({t:[c]})
+        arr_t = []
+        arr_v = []
+        for k,v in r.items():
+            arr_t += [k]
+            arr_v += [sum(v)/len(v)]
+        arr_t = [x for x,_ in sorted(zip(arr_t,arr_v))]
+        arr_v = [x for _,x in sorted(zip(arr_t,arr_v))]
+        return (arr_t,arr_v)
+
+def parse_line(datapoints, line, timekey, keys, time_parser, timeformat=None):
+        # This function expects:
+        #       - datapoints { String:DataObject }
+        #       - line       { String:Any        }
+        #       - timekey      String               (key for timevalue in 'line')
+        #       - keys       [ (String,String) ]    (source_key in 'line' to target_key in 'datapoints')
+        time = time_parser(line[ timekey ],timeformat)
+        for key in keys:
+                datapoints[ key[1] ].data  += [ line[ key[0] ] ]
+                datapoints[ key[1] ].times += [      time      ]
+
+def processExternalData(datapoints, plotNameKey, fromTime, toTime, dtype):
+    '''Download and parses external data of type dtype'''
+
+    # prepare strings #
+    cacheDir    = CFG("cache_dir")
+    fromTimeStr = fromTime.strftime(CFG("nff_url_timeformat"))
+    toTimeStr   = toTime.strftime(CFG("nff_url_timeformat"))
+    cacheFile   = "cache_{}_{}_{}.data".format(dtype, fromTimeStr, toTimeStr)
+    fullpath   = os.path.join(cacheDir, cacheFile)
+
+    # check for cache file
+    content = None
+    if not os.path.isfile(fullpath):
+        
+        # download date if it doesn't exist #
+        url = CFG("outside_data_url").format(dtype=dtype, fromDate=fromTimeStr, toDate=toTimeStr)
+        r = requests.get(url)
+        print(url)
+        content = r.content.decode('utf-8', "ignore") # ignore bad bytes
+
+        # cache data #
+        if not os.path.isdir(cacheDir):
+            os.mkdir(cacheDir)
+        with open(fullpath, 'w') as f:
+            f.write(content)
+
+    else:
+
+        # get data from cache otherwise
+        print("INFO: Cache hit: {}".format(cacheFile))
+        with open(fullpath) as f:
+            content = f.read()
+
+    skipBecauseFirstLine = True
+    for l in content.split("\n"):
+        if not ";" in l:
+            continue
+        elif not l.strip():
+            continue
+        elif skipBecauseFirstLine:
+            skipBecauseFirstLine = False
+            continue
+
+        try:
+            timeStr, value = l.split(";")
+            timestamp = plot_timeutils.time_from_csv(timeStr, CFG("nff_input_timeformat"))
+
+            datapoints[plotNameKey].data  += [float(value.replace(",","."))]
+            datapoints[plotNameKey].times += [timestamp]
+        except ValueError as e:
+            print(l)
+            raise e
+
+
+def read_in_file(path, backend=None, outsideData=False, plotOutsideTemp=True, plotOutsideHum=True):
+        '''Read in a file, add outside data if requested'''
+
+        datapoints = dict()
+        identifiers =  [ CFG("plot_temperatur_key"),
+                         CFG("plot_humidity_key"),
+                         CFG("plot_dewcels_key"),
+                         CFG("plot_outside_temperatur_key"),
+                         CFG("plot_outside_humidity_key") ]
+
+        names  = [ CFG("temperatur_plot_name"),
+                   CFG("humidity_plot_name"),
+                   CFG("dewcels_plot_name"),
+                   CFG("temperatur_outside_plot_name"),
+                   CFG("humidity_outside_plot_name") ]
+
+        colors = [ CFG("temperatur_color"),
+                   CFG("humidity_color"),
+                   CFG("dewcels_color"),
+                   CFG("temperatur_outside_color"),
+                   CFG("humidity_outside_color") ]
+
+        global line_colors
+        line_colors = colors
+
+        plotSettings = [ CFG("plot_temperatur"),
+                         CFG("plot_humidity"),
+                         CFG("plot_dewcels"),
+                         plotOutsideTemp,
+                         plotOutsideHum ]
+        
+        assert(len(names) == len(colors) == len(identifiers) == len(plotSettings))
+
+        max_name_len = max([len(s) for s in names])
+        for i in range(0, len(names)):
+            while len(names[i]) < max_name_len:
+                names[i] += " "
+            datapoints.update({ identifiers[i] : Data(names[i], plotSettings[i]) })
+
+        # legacy variables...
+        pt, ph, pd, pto, pho = identifiers
+
+        # parse input file #
+        if path == None:
+            raise Exception("Path in plot.read_in was None")
+        elif backend != None:
+                backend(path)
+        elif path.endswith(".DBF") or path.endswith(".dbf"):
+                dbfread(path,datapoints,pt,ph,pd)
+        elif path.endswith(".xls") or path.endswith(".XLS"):
+                csvread(path,datapoints,pt,ph,pd)
+        elif path.endswith(".txt"):
+                csvread_txt(path,datapoints,pt,ph,pd)
+        else:
+                raise NotImplementedError("Cannot determine filetype, cannot continue. Exit.")
+        
+        # if nessesary download and process external data #
+        if outsideData:
+
+            fromTime = datapoints[CFG("plot_temperatur_key")].getFirstTime()
+            toTime   = datapoints[CFG("plot_temperatur_key")].getLastTime()
+
+            processExternalData(datapoints, pto, fromTime, toTime, CFG("dtype_temperatur"))
+            processExternalData(datapoints, pho, fromTime, toTime, CFG("dtype_humidity"))
+
+        # sanity check result #
+        check_read_in(datapoints)
+
+        return datapoints
+
+def dbfread(path,datapoints,pt,ph,pd):
+        for record in DBF(path):
+            parse_line(datapoints,record,'DATETIME',[ ('TEMPCELS',pt) , ('HUMIDITY',ph) , ('DEWCELS',pd) ] ,plot_timeutils.time_from_dbf)
+
+def csvread(path,datapoints,pt,ph,pd):
+        count = 0;
+        with open(path) as f:
+            for l in f:
+                    if l.startswith(">>") or l.startswith("--") or l.startswith("NO."):
+                            count += 1
+                            continue
+                    else:
+                        row_arg = list(map(lambda s:s.replace(" ","").replace(",","."),l.split("\t")))
+                        row = {"temp":None,"hum":None,"taupunkt":None,"datetime":None}
+                        row["datetime"]     = row_arg[1]+row_arg[2]
+                        row["temp"]         = float(row_arg[3])
+                        row["hum"]          = float(row_arg[4])
+                        row["taupunkt"]     = float(row_arg[5])
+                        parse_line(datapoints,row,'datetime',[ ('temp',pt) , ('hum',ph) , ('taupunkt',pd) ],\
+                                        plot_timeutils.time_from_csv,timeformat="%d-%m-%Y%H:%M:%S")
+        print("Info: Ignored %d lines at beginning of file"%count)
+
+import codecs
+def csvread_txt(path,datapoints,pt,ph,pd):
+        count = 0;
+        f = open(path) 
+        try:
+            for l in f:
+                    if any(s in l for s in ["Logger","Datenquelle","Sensortyp","Einheit","Daten"]):
+                            count += 1
+                            continue
+                    else:
+                        row_arg = list(map(lambda s:s.replace(" ","").replace(",","."),l.split("\t")))
+                        row = {"temp":None,"hum":None,"taupunkt":None,"datetime":None}
+                        row["datetime"]     = "%s-%s-%s_%s:%s"%(row_arg[0],row_arg[1],row_arg[2],row_arg[3],row_arg[4])
+                        row["temp"]         = float(row_arg[6])
+                        row["hum"]          = float(row_arg[7])
+                        row["taupunkt"]     = 0.0
+                        parse_line(datapoints,row,'datetime',[ ('temp',pt) , ('hum',ph) , ('taupunkt',pd) ],\
+                                        plot_timeutils.time_from_csv,timeformat="%d-%m-%Y_%H:%M")
+        except (UnicodeError, IndexError):
+            count = csvread_txt_fallback(path,datapoints,pt,ph,pd)
+
+        print("Info: Ignored %d lines at beginning of the file"%count)
+        f.close()
+
+def csvread_txt_fallback(path,datapoints,pt,ph,pd):
+    '''fallback for different format and encoding of txt'''
+    count = 0
+    with codecs.open(path, "r",encoding="ISO8859_2", errors='repalce') as f:
+        for l in f:
+            if any(s in l for s in ["Logger","Datenquelle","Sensortyp","Einheit","Daten"]):
+                count += 1
+                continue
+            else:
+                date,time,temp,hum = l.replace(" ","").replace(".","-").replace(",",".").split("\t")
+                row = {"temp":None,"hum":None,"taupunkt":None,"datetime":None}
+                row["datetime"]     = "{}_{}".format(date,time[:5])
+                row["temp"]         = float(temp)
+                row["hum"]          = float(hum)
+                row["taupunkt"]     = 0.0
+                parse_line(datapoints,row,'datetime',[ ('temp',pt) , ('hum',ph) , ('taupunkt',pd) ],\
+                                plot_timeutils.time_from_csv,timeformat="%d-%m-%Y_%H:%M")
+    return count
+
+
+def check_read_in(datapoints):
+        good = False
+        for v in datapoints.values():
+            if len(v.times) != len(v.data):
+                print("more timestamps than data (or visa versa), this indicates that the file is corrupted, cannot continue")
+                good = False
+                break
+            if len(v.times) > 1:
+                good = True
+        if not good:
+            input("reading input file failed for an unknown reason, <ENTER> to exit")
+            import sys
+            sys.exit(1)

src/main/python/language.py

@@ -0,0 +1,23 @@
+end="\nDrücken sie 'STRG + C' ('STEUERUNG CANCEL') <ENTER> um das Program zu beenden"
+input_date = "Geben sie den Zeitpunkt an, an dem der Plot {} soll! \n\
+\nDatum/Uhrzeit im Format 'DD-MM-YYYY HH:MM:SS'. Wird die Uhrzeit weggelassen, \n\
+so wird 00:00:00 (Startzeit) bzw. 23:59:59 (Endzeit) angenommen.\n\
+Werden Jahr oder Monat weggelassen wird (versucht) ein passendes Datum zu wählen.\n\
+Lassen sie die Zeile leer um mit dem ersten existierenden Wert anzufangen \n\n\
+list<ENTER> um eine Übersicht über die gefunden Datenwerte zu erhalten\n\n\
+Beispiele für Formate (angenommen es ist der 12.01.2017): \n\n\
+    11 12            -> 11.01.2017 12:00:00 \n\
+    11-01            -> 11.01.2017 00:00:00 \n\
+    13               -> 13.12.2016 00:00:00 \n\
+    13-01-2017       -> 13.01.2017 00:00:00 \n\
+    13-1-2017 17:1:4 -> 13.01.2017 17:01:04 (nuller können also weggelassen werden)\n"
+hilfe=" oder h/help/hilfe <ENTER> für Hilfe\n"
+
+LAN = {"DE":{},"EN":{}}
+
+LAN["DE"]["input_first_date_help"]  = input_date.format("beginnen") + end + "\n"
+LAN["DE"]["input_second_date_help"] = input_date.format("enden")    + end + "\n"
+LAN["DE"]["input_first_date"]       = "\nStartzeit"+hilfe+"(Format: DD-MM-YY HH:MM:SS): "
+LAN["DE"]["input_second_date"]      = "\nEndzeit  "+hilfe+"(Format: DD-MM-YY HH:MM:SS): "
+LAN["DE"]["cannot_parse_date"]      = "Konnte Datum/Uhrzeit nicht verarbeiten! \n"
+LAN["DE"]["dstart_bigger_dend"]     = "Startzeit > Endzeit. MÖÖÖÖP \n"

src/main/python/localization/de.py

@@ -1,9 +1,22 @@
 window_title        = "Datenlogger Tool"
 file_selection      = "Dateiauswahl"
 button_go           = "Start"
+button_go_wait      = "Bitte Warten.."
 button_set_src_file = "Datei.."
 button_use_src_dir  = "Gleicher Ordner"
 date_selection      = "Zeitbereich"
 options             = "Optionen"
-button_otemp        = "Ausßentemperatur"
-button_ohumidity    = "Außenluftfeuchtigkeit"
+button_otemp        = "Ausßentemperatur einzeichnen"
+button_ohumidity    = "Außenluftfeuchtigkeit einzeichnen"
+output_file         = "Ausgabe"
+output_file_placeholder = "Datei auswählen.."
+src_file_dialog     = "Datei auswählen"
+src_file_extensions = "Logger Dateien (*.txt *.csv *.dbf);; Alle *.*"
+save_file_dialog    = "Zieldatei"
+wait_dialog_text    = "Datei wird gelesen.."
+error_read_in       = "Fehler beim Einlesen der Datei."
+done_text           = "Fertig"
+start_section       = "Ausführen"
+close               = "Schließen"
+open_pic            = "Bild öffnen.."
+bad_time            = "Fehlerhafte Zeitangabe!"

src/main/python/main.py

@@ -1,14 +1,22 @@
 #!/usr/bin/python3
 
 from fbs_runtime.application_context.PyQt5 import ApplicationContext
-from PyQt5.QtCore import QDateTime, Qt, QTimer
+from PyQt5.QtCore import QDateTime, Qt, QTimer, QUrl
+import PyQt5.QtCore
+import PyQt5.QtGui
 from PyQt5.QtWidgets import (QApplication, QCheckBox, QComboBox, QDateTimeEdit,
         QDial, QDialog, QGridLayout, QGroupBox, QHBoxLayout, QLabel, QLineEdit,
         QProgressBar, QPushButton, QRadioButton, QScrollBar, QSizePolicy,
         QSlider, QSpinBox, QStyleFactory, QTableWidget, QTabWidget, QTextEdit,
-        QVBoxLayout, QWidget)
+        QVBoxLayout, QWidget, QFileDialog, QDateEdit, QMessageBox)
 
 import localization.de as de
+import sys
+import datetime as dt
+
+import input_backend
+import plot_main
+import config_parse as cp
 
 class WidgetGallery(QDialog):
 
@@ -18,85 +26,250 @@ class WidgetGallery(QDialog):
         self.localization     = de
         self.srcFileString    = ""
         self.targetFileString = ""
+        self.truePath = None
 
         styleComboBox = QComboBox()
         styleComboBox.addItems(QStyleFactory.keys())
 
+        self.createStartSection()
         self.createFileSelection()
         self.createDateSelection()
         self.createCheckboxArea()
 
         mainLayout = QGridLayout()
-        mainLayout.addLayout(topLayout, 0, 0, 1, 2)
         mainLayout.addWidget(self.fileSelectionGroup, 1, 0)
-        mainLayout.addWidget(self.dateSelectionGroupBox, 1, 1)
-        mainLayout.addWidget(self.checkboxGroup, 2, 0)
-
-        mainLayout.setRowStretch(1, 1)
-        mainLayout.setRowStretch(2, 1)
-        mainLayout.setColumnStretch(0, 1)
-        mainLayout.setColumnStretch(1, 1)
+        mainLayout.addWidget(self.dateSelectionGroupBox, 2, 0)
+        mainLayout.addWidget(self.checkboxGroup, 3, 0)
+        mainLayout.addWidget(self.startSection, 4, 0)
 
         self.setLayout(mainLayout)
 
         self.setWindowTitle(self.localization.window_title)
 
+    def createStartSection(self):
+        '''Generate Aread containing the start button'''
+
+        self.startSection = QGroupBox(self.localization.start_section)
+        self.buttonGo = QPushButton(self.localization.button_go)
+        self.buttonGo.setDisabled(True)
+        self.buttonGo.clicked.connect(self.run)
+
+        layout = QVBoxLayout()
+        layout.addWidget(self.buttonGo)
+
+        self.startSection.setLayout(layout)
+
     def createFileSelection(self):
         '''Generate the area containing the file selectors and go button'''
 
         self.fileSelectionGroup = QGroupBox(self.localization.file_selection)
 
         # basic object #
-        buttonGo      = QPushButton(self.localization.button_go)
-        buttonSrcFile = QPushButton(self.localization.button_set_src_file)
-        srcFileName   = QLabel(self.srcFileString)
+        self.buttonSrcFile = QPushButton(self.localization.button_set_src_file)
+        self.srcFileName   = QLabel(self.localization.output_file)
 
-        buttonTargetFile = QPushButton(self.targetFileString)
-        buttonUseSrcDir  = QCheckBox(self.localization.button_use_src_dir)
+        self.buttonTargetFile = QPushButton(self.localization.output_file_placeholder)
+        self.boxUseSrcDir  = QCheckBox(self.localization.button_use_src_dir)
 
         # connectors #
-        buttonGo.connect(self.run)
-        buttonSrcFile.connect(self.selectSrcFile)
-        buttonTargetFile.connect(self.selectTargetFile)
-        buttonUseSrcDir.connect(self.useSrcDir)
+        self.buttonSrcFile.clicked.connect(self.selectSrcFile)
+        self.buttonTargetFile.clicked.connect(self.selectTargetFile)
+        self.boxUseSrcDir.stateChanged.connect(self.useSrcDir)
+        self.boxUseSrcDir.setChecked(True)
 
         # layout #
         layout = QVBoxLayout()
 
-        layout.addWidget(buttonGo)
-        layout.addWidget(buttonSrcFile)
-        layout.addWidget(srcFileName)
+        layout.addWidget(self.buttonSrcFile)
+        layout.addWidget(self.srcFileName)
 
-        layout.addWidget(buttonTargetFile)
-        layout.addWidget(buttonUseSrcDir)
+        layout.addWidget(self.buttonTargetFile)
+        layout.addWidget(self.boxUseSrcDir)
 
         layout.addStretch(1)
-        self.dateSelectionGroupBox.setLayout(layout)
+        self.fileSelectionGroup.setLayout(layout)
 
     def createDateSelection(self):
         '''Generate the area containing the date selectors'''
 
         self.dateSelectionGroupBox = QGroupBox(self.localization.date_selection)
 
-        layout = QVBoxLayout()
-        #layout.addWidget() # TODO
-        layout.addStretch(1)
-        self.topRightGroupBox.setLayout(layout)
+        layout = QGridLayout()
+
+        self.startDateEdit = QDateEdit(calendarPopup=True)
+        self.startDateEdit.setDisplayFormat("dd.MM.yyyy")
+        self.startDateEdit.setReadOnly(True)
+        self.startDateEdit.lineEdit().setDisabled(True)
+
+        self.endDateEdit = QDateEdit(calendarPopup=True)
+        self.endDateEdit.setDisplayFormat("dd.MM.yyyy")
+        self.endDateEdit.setReadOnly(True)
+        self.endDateEdit.lineEdit().setDisabled(True)
+
+        self.startTimeEdit = QLineEdit("00:00")
+        self.endTimeEdit   = QLineEdit("23:59")
+        self.startTimeEdit.setDisabled(True)
+        self.endTimeEdit.setDisabled(True)
+
+        layout.addWidget(self.startDateEdit, 0, 0)
+        layout.addWidget(self.startTimeEdit, 0, 1)
+
+        layout.addWidget(self.endDateEdit, 1, 0)
+        layout.addWidget(self.endTimeEdit, 1, 1)
+
+        layout.setColumnStretch(0, 1)
+        layout.setColumnStretch(1, 1)
+
+        self.dateSelectionGroupBox.setLayout(layout)
 
     def createCheckboxArea(self):
         '''Generate area  with configuration options'''
 
         self.checkboxGroup = QGroupBox(self.localization.options)
 
-        buttonOTemp     = QCheckBox(self.localization.button_otemp)
-        buttonOHumidity = QCheckBox(self.localization.button_ohumidity)
+        self.boxOTemp     = QCheckBox(self.localization.button_otemp)
+        self.boxOHumidity = QCheckBox(self.localization.button_ohumidity)
 
         layout = QVBoxLayout()
-        layout.addWidget(buttonOTemp)
-        layout.addWidget(buttonOHumidity)
+        layout.addWidget(self.boxOTemp)
+        layout.addWidget(self.boxOHumidity)
         layout.addStretch(1)
-        self.topRightGroupBox.setLayout(layout)
+        self.checkboxGroup.setLayout(layout)
 
+    def run(self):
+        '''Run generation with selected file and options'''
+
+        # set save target if nessesary #
+        self.buttonGo.setText(self.localization.button_go_wait)
+        self.buttonGo.setDisabled(True)
+        self.repaint()
+
+        if self.boxUseSrcDir.isChecked():
+            target = self.srcFileString
+            forcePath = False
+        else:
+            target = self.targetFileString
+            forcePath = True
+
+        # workaround for checkboxes changed #
+        self.datapoints = input_backend.read_in_file(self.srcFileString, 
+                                                outsideData=True,
+                                                plotOutsideTemp=self.boxOTemp.isChecked(),
+                                                plotOutsideHum=self.boxOHumidity.isChecked())
+
+        # build dates #
+        try:
+            startTimeHelper = dt.datetime.strptime(self.startTimeEdit.text(),"%H:%M")
+            endTimeHelper   = dt.datetime.strptime(self.endTimeEdit.text(),"%H:%M")
+        except ValueError as e:
+            errorBox = QMessageBox(self)
+            errorBox.setAttribute(PyQt5.QtCore.Qt.WA_DeleteOnClose)
+            errorBox.setText(self.localization.bad_time)
+            errorBox.setDetailedText(str(e))
+            errorBox.show()
+            self.buttonGo.setText(self.localization.button_go)
+            self.buttonGo.setDisabled(False)
+            return
+            
+        startTimeOffset = dt.timedelta(hours=startTimeHelper.hour, minutes=startTimeHelper.minute)
+        endTimeOffset   = dt.timedelta(hours=endTimeHelper.hour, minutes=endTimeHelper.minute)
+
+        zeroTime = dt.time(0, 0)
+        startDateTime = dt.datetime.combine(self.startDateEdit.date().toPyDate(), zeroTime)
+        startDateTime += startTimeOffset
+        endDateTime   = dt.datetime.combine(self.endDateEdit.date().toPyDate(), zeroTime)
+        endDateTime += endTimeOffset
+
+        self.truePath = plot_main.plot(self.datapoints, path=target,
+                            date1=startDateTime,
+                            date2=endDateTime,
+                            forcePath=forcePath)
+
+        self.buttonGo.setText(self.localization.button_go)
+        self.buttonGo.setDisabled(False)
+
+        doneDialog = QMessageBox(self)
+        doneDialog.setAttribute(PyQt5.QtCore.Qt.WA_DeleteOnClose)
+        doneDialog.setText(self.localization.done_text)
+        doneDialog.addButton(self.localization.open_pic, QMessageBox.YesRole)
+        doneDialog.addButton(self.localization.close, QMessageBox.NoRole)
+        doneDialog.buttonClicked.connect(self.openFile)
+        doneDialog.show()
+
+    def selectSrcFile(self):
+        '''Function to select a src-file'''
+
+        self.srcFileString = QFileDialog.getOpenFileName(self, self.localization.src_file_dialog, 
+                        "", "Data-Files (*.txt *.csv *.dbf)")[0]
+        self.srcFileName.setText(self.srcFileString)
+
+        if not self.srcFileString:
+            return
+
+        waitDialog = QMessageBox(self)
+        waitDialog.setAttribute(PyQt5.QtCore.Qt.WA_DeleteOnClose)
+        waitDialog.setText(self.localization.wait_dialog_text)
+        waitDialog.show()
+        try:
+            self.datapoints = input_backend.read_in_file(self.srcFileString, 
+                                                    outsideData=False,
+                                                    plotOutsideTemp=False,
+                                                    plotOutsideHum=False)
+        except Exception as e:
+            waitDialog.close()
+            errorBox = QMessageBox(self)
+            errorBox.setAttribute(PyQt5.QtCore.Qt.WA_DeleteOnClose)
+            errorBox.setText(self.localization.error_read_in)
+            errorBox.setDetailedText(str(e))
+            errorBox.show()
+            return
+
+        waitDialog.close()
+
+        start = self.datapoints[cp.CFG("plot_temperatur_key")].getFirstTime()
+        self.startDateEdit.setDateTime(start)
+
+        end = self.datapoints[cp.CFG("plot_temperatur_key")].getLastTime()
+        self.endDateEdit.setDateTime(end)
+
+        self.buttonGo.setDisabled(False)
+        self.endDateEdit.setReadOnly(False)
+        self.startDateEdit.setReadOnly(False)
+        self.startDateEdit.lineEdit().setDisabled(False)
+        self.endDateEdit.lineEdit().setDisabled(False)
+        self.startTimeEdit.setDisabled(False)
+        self.endTimeEdit.setDisabled(False)
+
+    def selectTargetFile(self):
+        '''Function to select a target-file'''
+        self.targetFileString = QFileDialog.getSaveFileName(self, 
+                                                self.localization.save_file_dialog)[0]
+        if not self.targetFileString:
+            return
+
+        self.buttonTargetFile.setText(self.targetFileString)
+        self.buttonGo.setDisabled(False)
+
+    def useSrcDir(self):
+        '''Function to handle use src dir checkbox'''
+        if self.boxUseSrcDir.isChecked():
+            self.buttonTargetFile.setDisabled(True)
+            if self.srcFileString:
+                self.buttonGo.setDisabled(False)
+                self.srcFileName.setText(self.srcFileString)
+        else:
+            self.buttonTargetFile.setDisabled(False)
+            if self.targetFileString:
+                self.buttonTargetFile.setText(self.targetFileString)
+            else:
+                self.buttonGo.setDisabled(True)
+
+    def openFile(self, button):
+        if button.text() == self.localization.open_pic and self.truePath:
+            print("JES")
+            PyQt5.QtGui.QDesktopServices.openUrl(QUrl.fromLocalFile(self.truePath));
+        else:
+            pass
 
 if __name__ == '__main__':
     appctxt = ApplicationContext()

src/main/python/plot.py

@@ -0,0 +1,14 @@
+#!/usr/bin/python3
+import frontend
+import sys
+if __name__ == "__main__":
+        try:
+            frontend.main()
+            #input("Done! <ENTER> to exit")
+            sys.exit(0)
+        except KeyboardInterrupt as e:
+            sys.exit(1)
+        except Exception as e:
+            print(e)
+            input("Ein Fehler ist aufgetreten, <ENTER> um zu beenden, wenn dieser Fehler unerwartet war -> Mail!")
+            sys.exit(1)

src/main/python/plot_graphutils.py

@@ -0,0 +1,233 @@
+#!/usr/bin/python3
+from config_parse import CFG
+from datetime import datetime, timedelta
+import matplotlib
+matplotlib.use(CFG("use_gui_backend"))
+import matplotlib.pyplot as plt
+import matplotlib.dates
+import matplotlib.ticker as ticker
+from constants import *
+import math
+import plot_timeutils
+matplotlib.rc('font', **GLOBAL_FONT)
+
+def getlimits_y(y):
+        ymax = max(y)+CFG("empty_space_above_plot")
+        y_min_height = CFG("yaxis_minnimum_hight")
+        if y_min_height != 0 and y_min_height > ymax:
+                ymax = y_min_height
+        y_start_val = CFG("yaxis_start_value")
+        if y_start_val < min(y) or ( CFG("yaxis_force_start_value") and not min(y) < 0):
+                ymin=y_start_val
+        else:
+                ymin=min(y)
+        return (ymin,ymax)
+
+def avg(array):
+        return sum(array)/float(len(array))
+
+def legend_box_contents(name, y):
+    if CFG("cap_values_at_99"):
+        y = [ min([el, 99.9]) for el in y ]
+    if CFG("show_min"):
+        name += " min: {:4.1f},".format(min(y))
+    if CFG("show_max"):
+        name += " max: {:4.1f},".format(max(y))
+    if CFG("show_avg"):
+        name += " Mittelwert: {:4.1f},".format(avg(y))
+    return name.rstrip(",")    
+
+def general_background_setup(tup,ymin,ymax,x):
+
+    unix_x = list(map(plot_timeutils.unix,x))
+
+    ### SET AXIS LIMITS ###
+    tup[AXIS].set_ylim([ymin,ymax])
+    tup[AXIS].set_xlim([plot_timeutils.unix(min(x)),plot_timeutils.unix(max(x))])
+
+    if CFG("draw_thresholds"):
+        hcrit=CFG("humidity_critical")
+        hwarn=CFG("humidity_warning")
+        tlow=CFG("acceptable_temp_low")
+        thigh=CFG("acceptable_temp_high")
+        tup[AXIS].axhline(y=CFG("target_temperatur"),ls=CFG("hline_line_style"),lw=CFG("hline_line_width"),color=CFG("acceptable_temp_color"))
+        tup[AXIS].axhline(y=hcrit,ls=CFG("hline_line_style"),lw=CFG("hline_line_width"),color=CFG("humidity_crit_color"))
+        tup[AXIS].axhspan(hwarn,hcrit,color=CFG("humidity_warning_color"),alpha=CFG("humidity_warning_alpha"))
+        tup[AXIS].axhspan(hcrit,ymax,color=CFG("humidity_crit_color"),alpha=CFG("humidity_crit_alpha"))
+        tup[AXIS].axhspan(tlow,thigh,color=CFG("acceptable_temp_color"),alpha=CFG("acceptable_temp_alpha"))
+    
+    #### GRID ####
+    major_xticks = gen_xticks_from_timeseries(x)
+    minor_xticks = get_minor_xticks_from_major(major_xticks)
+    if CFG("raster"):
+        grid(tup,major_xticks,ymin,ymax)
+    
+    #### XTICKS ####
+    tup[AXIS].set_xticks(major_xticks)
+    tup[AXIS].xaxis.set_major_formatter(ticker.FuncFormatter(xlabel_formater_callback))
+    tup[AXIS].xaxis.set_major_locator(ticker.FixedLocator(major_xticks, nbins=None))
+    tup[AXIS].xaxis.set_minor_locator(ticker.FixedLocator(minor_xticks, nbins=None))
+    tup[AXIS].xaxis.set_tick_params(which='minor',width=0.2,direction="out")
+    
+    tup[AXIS].yaxis.set_major_locator(ticker.MultipleLocator(CFG("y_tick_interval")))
+    tup[AXIS].yaxis.set_minor_locator(ticker.MultipleLocator(1))
+    tup[AXIS].yaxis.set_tick_params(which='minor',width=0.2,direction="out")
+
+    tup[AXIS].tick_params(axis='x',which="major",labelsize=CFG("xticks_font_size"));
+    tup[AXIS].tick_params(axis='y',which="major",labelsize=CFG("yticks_font_size"));
+                        
+    ## ROTATION XLABELS ##
+    rotation=CFG("xticks_label_degree")
+    if rotation > 0:
+        plt.xticks(rotation=rotation,ha='right')
+
+    ## AXIS LABELS
+    ylabel_box = dict(boxstyle="square",facecolor='grey', alpha=0.4, edgecolor='black',lw=0.5)
+    xlabel_box = ylabel_box
+    label_size = CFG("label_font_size")
+    spacing=0.1
+    tup[AXIS].set_ylabel(CFG("y_label"),rotation='horizontal',size=label_size,bbox=ylabel_box)
+    tup[AXIS].yaxis.set_label_coords(0.045,0.970)
+    tup[AXIS].set_xlabel(CFG("x_label"),size=label_size,bbox=xlabel_box)
+    tup[AXIS].xaxis.set_label_coords(0.945,0.03)
+    
+    ## GENERAL LEGEND ##
+    legend_handle = tup[AXIS].legend(
+                    loc=CFG("legend_location"),
+                    edgecolor="inherit",
+                    fancybox=False,
+                    borderaxespad=spacing,
+                    prop={'family': 'monospace','size':CFG("legend_font_size")}
+                    )
+    legend_handle.get_frame().set_linewidth(0.2)
+    #tup[AXIS].set_aspect(get_aspect_ratio(unix_x,ymin,ymax,major_xticks))
+
+                
+def get_aspect_ratio(ux,ymin,ymax,xticks):
+        ratio = 100
+        tmp = CFG("aspect_ratio")
+        if str(tmp) == "A4":
+            ratio = a4_aspect()
+        else:
+            ratio=tmp
+        magic_value = 3.25
+        return ratio * ( max(ux) - min(ux) ) / float(ymax - ymin + magic_value)
+
+def a4_aspect(x):
+        return ( 1/math.sqrt(2) ) * x
+
+def grid(tup,xticks,ymin,ymax):
+        lw = CFG("grid_line_width")
+        ls = CFG("grid_line_style")
+        color = CFG("grid_line_color")
+        hour_mul = 24
+        expected_vlines = len(list(filter(lambda xt: xt%3600 < 60,xticks)))
+        safety_first = 60*60 +10
+        step = xticks[1]-xticks[0] 
+        if step < (24*3600)-safety_first:
+            if expected_vlines <= 6:
+                    hour_mul = 1
+            elif expected_vlines <=12:
+                    hour_mul = 2
+            elif expected_vlines <=24:
+                    hour_mul = 4
+
+        for xt in xticks:
+                leck_mich = datetime.fromtimestamp(xt)
+                if leck_mich.hour == leck_mich.minute == leck_mich.second == 0:
+                    tup[AXIS].axvline(xt,ls="-",lw=CFG("major_line_width"),color=color)
+                else:
+                    tup[AXIS].axvline(xt,ls=ls,lw=lw,color=color)
+        ## HLINES ##
+        y_interval = CFG("raster_hline_prefered_interval")
+        cur = ymin
+        while cur < ymax:
+                cur += y_interval
+                tup[AXIS].axhline(cur,ls=ls,lw=lw,color=color)
+
+def find_step(step,x,total_xticks):
+        intervals = parse_possible_intervals()
+        start = min(x)
+        if CFG("always_allow_days_as_xticks") and step > timedelta(days=1)/2:
+                step = timedelta(days=round(step.days+1))
+                start = min(x).replace(hour=0,second=0,minute=0)
+                return (start,step)
+        
+        min_delta_step = timedelta(days=1)      # the actual step that has the lowest delta
+        min_delta      = timedelta(days=1000)   # the delta o thus step
+        for s in intervals:
+            delta = max(s,step)-min(s,step)
+            if delta < min_delta:
+                min_delta_step = s
+                min_delta      = delta
+
+        step  = min_delta_step
+        start = plot_timeutils.round_time_to_step(start,step)
+
+        warn_on_too_much_xticks(x,total_xticks,step)
+        return (start,step)
+
+def parse_possible_intervals():
+        intervals = CFG("acceptable_x_intervals")
+        parsed_intervals = []
+        for s in intervals.split(','):
+            try:
+                st = int(s[:-1])
+            except ValueError:
+                raise ValueError("'acceptable_x_intervals' muss die Form 'Zahl[s(econds),m(minutes),h(ours),d(days)]' haben!")
+            except Exception:
+                raise ValueError("invalid intervals for x_labels %s [index out of bounds], did you write something like this ',,,,' ?]"%str(intervals))
+            if s.endswith("s"):
+                if 60 % st != 0:
+                    raise ValueError("interval must fit to next bigger interval so basicly for hours 24%interval==0")
+                parsed_intervals += [timedelta(seconds=st)]
+            elif s.endswith("m"):
+                if 60 % st != 0:
+                    raise ValueError("interval must fit to next bigger interval so basicly for hours 24%interval==0")
+                parsed_intervals += [timedelta(minutes=st)]
+            elif s.endswith("h"):
+                if 24 % st != 0:
+                    raise ValueError("interval must fit to next bigger interval so basicly for hours 24%interval==0")
+                parsed_intervals += [timedelta(hours=st)]
+            elif s.endswith("d"):
+                parsed_intervals += [timedelta(days=st)]
+            else:
+                raise ValueError("invalide Zeitspezifizierer in %s (muss, s,m,h oder d sein)"%str(intervals))
+        return parsed_intervals
+
+def warn_on_too_much_xticks(x,total_xticks,step):
+        if (max(x)-min(x))/step > 2*total_xticks:
+                print("Warnung: maximales xinterval zu niedrig eine sinnvolle Anzahl an xticks zu generieren (total x_ticks: %d"%total_xticks)
+
+def get_minor_xticks_from_major(major):
+        mult = CFG("minor_xticks_per_major")
+        step = (major[1]-major[0])/mult
+        ret = []
+        for x in major:
+                if x == max(major):
+                    break
+                ret += [x+ 0*step]
+                ret += [x+ 1*step]
+                ret += [x+ 2*step]
+                ret += [x+ 3*step]
+                ret += [x+ 4*step]
+        return ret
+
+def gen_xticks_from_timeseries(x):
+        ticks=CFG("prefered_total_xticks")
+        xmin = min(x)
+        xmax = max(x)
+        delta = xmax-xmin
+        step = delta/ticks
+        cur,step = find_step(step,x,ticks)
+        xticks = []
+        xmax += step*CFG("add_x_labels_at_end")
+        while cur < xmax:
+            xticks += [plot_timeutils.unix(cur)]
+            cur+=step
+        return xticks
+
+def xlabel_formater_callback(tick_val, tick_pos):
+        dt = datetime.fromtimestamp(tick_val)
+        tformat = CFG("timeformat_x_axis").replace('$','%')
+        return dt.strftime(tformat)

src/main/python/plot_imageutils.py

@@ -0,0 +1,12 @@
+#!/usr/bin/python3
+from config_parse import CFG
+from PIL import Image
+import math
+def check_and_rotate(path):
+        img = Image.open(path)
+        div=abs(float(img.size[1])/float(img.size[0])-a4_aspect())/a4_aspect()*100
+        print("Seitenverhältnisabweichung zu A4: %.2f"%div+r'%')
+        img.rotate(CFG("image_rotation"),expand=True).save(path.strip(".png")+"_rotated.png")
+
+def a4_aspect():
+        return 1/math.sqrt(2)

src/main/python/plot_main.py

@@ -0,0 +1,97 @@
+#!/usr/bin/python3
+import sys
+from config_parse import CFG
+from constants import *
+from datetime import datetime, timedelta
+from frontend_utils import open_file
+from constants      import *
+
+import math
+import matplotlib
+matplotlib.use(CFG("use_gui_backend"))
+
+import matplotlib.pyplot as plt
+import matplotlib.dates
+import matplotlib.ticker as ticker
+
+import plot_graphutils
+import plot_imageutils
+import plot_timeutils
+
+
+def plot(datapoints, path=None, date1=None, date2=None, forcePath=False):
+        plotname = "" if CFG("name_of_plot") == "None" else CFG("name_of_plot")
+        tup = [None,None,plot_timeutils.between_dates,plotname]
+        return __plot(tup, datapoints, path, date1, date2, forcePath)
+                        
+def __plot(tup, datapoints, path, date1=None, date2=None, forcePath=False):
+        NO_SERIES  = True
+        x,y,ymin,ymax,unix_x,major_xticks = ( [] , [], -1 , -1 , [], [] )
+        lw = CFG("plot_line_width")
+        ls = CFG("plot_line_style")
+        tup[FIGURE],tup[AXIS] = plt.subplots(1, 1)
+
+        for key in datapoints.keys():
+                g = datapoints[key]
+                print(key)
+                #### Check if we are supposed to plot something ####
+                if not g.plot:
+                        continue
+                #### GET AND CHECK TIMEFRAMES ####
+                x,y, = g.get_timeframe(tup[CALLBACK],date1,date2)
+                if not x or not y or len(x) <= 0 or len(y) <= 0:
+                    print("Warning: Empty series of data '%s' (wrong start/end time?)"%g.name)
+                    continue
+                else:
+                    NO_SERIES = False
+                    unix_x = list(map(plot_timeutils.unix,x))
+                    ymin,ymax = plot_graphutils.getlimits_y(y)
+
+                #### GET LINE STYLES ####
+                legend_label = plot_graphutils.legend_box_contents(g.name,y)
+                tup[AXIS].plot(unix_x, y,ls=ls,lw=lw,marker="None", label=legend_label, color=g.color)
+                legacy_x_save = x
+                lagacy_y_save = y
+        
+        if NO_SERIES:
+                print("Error: no data, nothing to plot. cannot continue. exit.")
+                sys.exit(1)
+
+        ## GRID ##
+        plot_graphutils.general_background_setup(tup, ymin, ymax, legacy_x_save)
+
+        ## using unix_x relys on unix_x to be the same for all plots ##
+        if path == None:
+            path = open_file()
+        
+        if not forcePath:
+            pic_path = output_path(path,date1,date2)
+        else:
+            pic_path = path
+        
+
+        ## set resoltuion ##
+        DPI = CFG("outfile_resolution_in_dpi")
+        fig_x_height = CFG("fig_x_height_inches")/float(1000)
+        fig_y_height = CFG("fig_y_height_inches")/float(1000)
+        tup[FIGURE].set_size_inches(fig_x_height,fig_y_height)
+
+        ## save the figure ##
+        tup[FIGURE].savefig(pic_path,dpi=DPI,pad_inches=0.1,bbox_inches='tight',transparent=CFG("transparent_background"))
+
+        ### do operations on the finished png ###
+        plot_imageutils.check_and_rotate(pic_path)
+
+        return pic_path
+
+def output_path(path,date1,date2):
+        if date1 != None and date2 == None:
+            pic_path = path + "-nach-%s"%date1.strftime("%d.%m.%y")  + ".png"
+        elif date1 == None and date2 != None:
+            pic_path = path + "-vor-%s"%date2.strftime("%d.%m.%y")  + ".png"
+        elif date1 == None and date2 == None:
+            pic_path = path + "-alles" + ".png"
+        else:
+            pic_path = path + "-%s_to_%s"%(date1.strftime("%d.%m.%y"),date2.strftime("%d.%m.%y")) + ".png"
+        print("Output wird gespeichert nach: %s"%str(pic_path))
+        return pic_path

src/main/python/plot_timeutils.py

@@ -0,0 +1,42 @@
+#!/usr/bin/python3
+from config_parse import CFG
+from datetime import datetime, timedelta
+
+def between_dates(t,date1,date2):
+        if (date1 == None or date1 <= t) and (date2 == None or date2 > t):
+            return True
+        else:
+            return False
+
+def time_from_dbf(l, timeformat):
+        timeformat=None #dont need that here
+        offset_d = datetime(1970,1,1)-datetime(1900,1,1)
+        shit_epoch = l*24*60*60 #days to seconds
+        unix_epoch = datetime.fromtimestamp(shit_epoch)-offset_d
+        return (unix_epoch-timedelta(days=2)+timedelta(hours=CFG("add_hours_to_input"))).replace(microsecond=0)
+
+def time_from_csv(l, timeformat):
+        return datetime.strptime(l, timeformat)
+
+def unix(dt):
+    return dt.timestamp()
+
+def round_time_to_step(start,step):
+        start  += step / 2
+        discard = timedelta(days=0)
+        hround  = int(step.seconds/3600)
+        mround  = int(step.seconds/60)
+        if step >= timedelta(days=1):
+                discard = timedelta(days=start.day % step.days,hours=start.hour,minutes=start.minute,seconds=start.second)        
+        elif step >= timedelta(hours=1):
+                if hround != 0:
+                    discard = timedelta(hours=start.hour % hround,minutes=start.minute,seconds=start.second) 
+        elif step >= timedelta(minutes=1):
+                if mround != 0:
+                    discard = timedelta(minutes=start.minute % mround,seconds=start.second)
+        elif step >= timedelta(seconds=1):
+                    discard = timedelta(seconds=start.second % step.seconds)
+        else:
+                raise ValueError("Rounding time failed, this actually should be impossible. wtf. ("+str(start)+","+str(step)+","+str(discard)+")")
+        start -= discard
+        return start

src/main/python/test_cases/graphutils_test.py

@@ -0,0 +1,22 @@
+import unittest
+
+from datetime import datetime
+from datetime import timedelta
+import random
+import itertools
+
+class Graphutils_Test(unittest.TestCase):
+    def test_get_y_limits(self):
+        import plot_graphutils as gu
+        y_axis_values = [[32.3, 60.3, 35.1, 34.9, 33.0, 32.0, 32.7, 32.4, 34.0, 32.7, 33.2, 32.7, 33.2, 32.4, 34.0, \
+                        32.9, 33.4, 32.2, 30.8, 41.6, 34.7, 32.6, 35.1, 33.5, 32.5, 37.6, 32.6, 32.3, 31.3, 33.0, 34.0,\
+                        32.7, 32.7, 32.4, 32.8, 34.0, 34.1, 32.5, 33.5, 33.8, 31.0, 32.8, 34.9],[6.4, 15.3, 7.9, 7.1,\
+                        6.9, 6.4, 7.0, 6.4, 6.8, 5.8, 6.1, 6.6, 6.1, 6.8, 6.8, 5.9, 6.9, 6.3, 6.9, 12.2, 7.9, 6.5, 7.9,\
+                        6.9, 6.5, 8.9, 6.8, 6.4, 6.4, 6.0, 6.7, 6.5, 7.0, 6.4, 6.7, 7.6, 6.9, 6.5, 7.2, 6.9, 4.9, 6.7,\
+                        7.9],[24.0, 23.5, 24.4, 23.5, 24.3, 24.2, 24.5, 24.0, 23.6, 23.2, 23.2, 24.1, 23.2, 24.5, 23.6,\
+                        23.2, 24.1, 24.0, 25.4, 26.3, 24.5, 24.0, 24.4, 24.0, 24.0, 24.4, 24.4, 24.0, 24.6, 23.2, 23.5,\
+                        24.0, 24.5, 24.0, 24.1, 24.5, 23.7, 24.0, 24.4, 23.8, 23.0, 24.1, 24.5]]
+        yl_values = [(0,95),(0,95),(0,95)]
+        for y,yl in zip(y_axis_values,yl_values):
+                self.assertEqual(yl,gu.getlimits_y(y))
+    

src/main/python/test_cases/timeutils_test.py

@@ -0,0 +1,58 @@
+import unittest
+
+from datetime import datetime
+from datetime import timedelta
+import random
+import itertools
+
+class Timeutil_Test(unittest.TestCase):
+    DATES = []
+    STEPS = [ (timedelta(hours=1)),timedelta(hours=4),timedelta(days=1),timedelta(minutes=1),timedelta(minutes=3)]
+    def setUpClass():
+        random.seed("0")
+        for x in range(0,10000):
+            tmp = datetime(2018,1,1) + ( (random.random()-0.5) * timedelta(days=2*365) )
+            tmp = tmp.replace(microsecond=0)
+            Timeutil_Test.DATES += [ tmp ]
+        
+    def test_between_dates(self):
+        import plot_timeutils as tu
+        d = Timeutil_Test.DATES
+        count = 0
+        while(count < len(d)-2):
+            t  = d[count+0]
+            d1 = d[count+1]
+            d2 = d[count+2]
+            self.assertEqual(btw_wrapper(tu.between_dates(t,d1,d2),    t), d1 < t < d2, "t: "+str(t)+", d1: "+str(d1)+", d2: "+str(d2) )
+            self.assertEqual(btw_wrapper(tu.between_dates(t,d1,None),  t), d1 < t , "t: "+str(t)+", d1: "+str(d1)+", d2: "+str(d2) )
+            self.assertEqual(btw_wrapper(tu.between_dates(t,None,d1),  t), d1 > t , "t: "+str(t)+", d1: "+str(d1)+", d2: "+str(d2) )
+            self.assertEqual(btw_wrapper(tu.between_dates(t,None,None),t), True   , "t: "+str(t)+", d1: "+str(d1)+", d2: "+str(d2) )
+            count+=1
+
+
+    def test_parse_time_dbf(self):
+        import plot_timeutils as tu
+        ind = [43121.6821296,43121.6856018,43121.689074,43121.6925462,43121.6960185,43121.6994907,43121.7029629,43121.7064351,43121.7099074,43121.7133796,43121.7168518,43121.720324,43121.7237962,43121.7272685,43121.7307407,43121.7342129,43121.7376851,43121.7411574,43121.7446296,43121.7481018,43121.751574,43121.7550462,43121.7585185,43121.7619907,43121.7654629,43121.7689351,43121.7724074,43121.7758796,43121.7793518,43121.782824,43121.7862962,43121.7897685,43121.7932407,43121.7967129,43121.8001851,43121.8036574,43121.8071296,43121.8106018,43121.814074,43121.8175462,43121.8210185,43121.8244907,43121.8279629]
+        outd = ["2018-01-21 18:22:15","2018-01-21 18:27:15","2018-01-21 18:32:15","2018-01-21 18:37:15","2018-01-21 18:42:15","2018-01-21 18:47:15","2018-01-21 18:52:15","2018-01-21 18:57:15","2018-01-21 19:02:15","2018-01-21 19:07:15","2018-01-21 19:12:15","2018-01-21 19:17:15","2018-01-21 19:22:15","2018-01-21 19:27:15","2018-01-21 19:32:15","2018-01-21 19:37:15","2018-01-21 19:42:15","2018-01-21 19:47:15","2018-01-21 19:52:15","2018-01-21 19:57:15","2018-01-21 20:02:15","2018-01-21 20:07:15","2018-01-21 20:12:15","2018-01-21 20:17:15","2018-01-21 20:22:15","2018-01-21 20:27:15","2018-01-21 20:32:15","2018-01-21 20:37:15","2018-01-21 20:42:15","2018-01-21 20:47:15","2018-01-21 20:52:15","2018-01-21 20:57:15","2018-01-21 21:02:15","2018-01-21 21:07:15","2018-01-21 21:12:15","2018-01-21 21:17:15","2018-01-21 21:22:15","2018-01-21 21:27:15","2018-01-21 21:32:15","2018-01-21 21:37:15","2018-01-21 21:42:15","2018-01-21 21:47:15","2018-01-21 21:52:15"]
+        for i,o in zip(ind,outd):
+            self.assertEqual(str(tu.parse_time_dbf(i)),o)
+    
+    def test_round_time_to_step(self):
+        import plot_timeutils as tu
+        for s in Timeutil_Test.STEPS:
+            for d in Timeutil_Test.DATES:
+                rounded = tu.round_time_to_step(d,s)
+                if s < timedelta(minutes=60):
+                    self.assertEquals(rounded.minute * 60 % s.seconds ,0,'date: '+str(d)+' rounded: '+str(rounded)+' step: '+str(s))
+                elif s < timedelta(hours=24):
+                    self.assertEquals(rounded.hour*60*60  % s.seconds ,0,'date: '+str(d)+' rounded: '+str(rounded)+' step: '+str(s))
+                elif s >= timedelta(days=1):
+                    self.assertEquals(rounded.day         % s.days    ,0,'date: '+str(d)+' rounded: '+str(rounded)+' step: '+str(s))
+                else:
+                    raise AssertionError(int(s.days),0,'date: '+str(d)+' rounded: '+str(rounded)+' step: '+str(s))
+            
+
+def btw_wrapper(inp,a):
+    if inp == a:
+        return True
+    return False