sheppy/ths-datenlogger
1
0
Fork 0
mirror of https://github.com/FAUSheppy/ths-datenlogger synced 2025-12-06 04:11:34 +01:00
Code Issues Actions Packages Projects Releases Wiki Activity

refactor error output part 1

Browse Source
This commit is contained in:
Yannik Schmidt
2021-02-26 12:43:32 +01:00
parent 95f4206394
commit 03bed2a976
3 changed files with 110 additions and 83 deletions
Download Patch File Download Diff File Expand all files Collapse all files

151
src/main/python/input_backend.py
View File

@@ -7,6 +7,7 @@ import localization.de as de
from dbfread import DBF from dbfread import DBF
import timeutils import timeutils
import codecs
line_colors = ['b', 'r', 'g', 'c', 'm', 'y'] line_colors = ['b', 'r', 'g', 'c', 'm', 'y']
tname = CFG("temperatur_plot_name") tname = CFG("temperatur_plot_name")
@@ -15,7 +16,7 @@ dname = CFG("dewcels_plot_name")
color_id = 0 color_id = 0
class Data: class Data:
def __init__(self,name,plot=False): def __init__(self, name, plot=False, qtTextBrowser=None):
global color_id,line_colors global color_id,line_colors
self.name = name self.name = name
self.color=line_colors[color_id%len(line_colors)] self.color=line_colors[color_id%len(line_colors)]
@@ -23,6 +24,7 @@ class Data:
self.data = [] self.data = []
self.times = [] self.times = []
self.plot = plot self.plot = plot
self.qtTextBrowser = qtTextBrowser
def getFirstTime(self): def getFirstTime(self):
'''Get time of first timestamp''' '''Get time of first timestamp'''
@@ -36,11 +38,15 @@ class Data:
out_x = [] out_x = []
out_y = [] out_y = []
i = 0 i = 0
if(len(self.times) != len(self.data)): if(len(self.times) != len(self.data)):
raise RuntimeError("len(timestamps) != len(data), cannot continue, this should never happen") self.qtTextBrowser.append(de.special_err_1)
raise ValueError(de.special_err_1)
if(len(self.times) <= 2): if(len(self.times) <= 2):
print("WARNING: No Data for %s!"%self.name) self.qtTextBrowser.append(de.warn_no_data.format(self.name))
return (None,None) return (None,None)
############ AVERAGE OUT DATA ############# ############ AVERAGE OUT DATA #############
if(CFG("combine_data_points") >= (self.times[1] - self.times[0]).total_seconds()): if(CFG("combine_data_points") >= (self.times[1] - self.times[0]).total_seconds()):
x_dp = 5 x_dp = 5
@@ -67,7 +73,6 @@ class Data:
i += 1 i += 1
return (out_x,out_y) return (out_x,out_y)
## no idea on what kind of drugs I was when i wrote this function (it is somewhat ingenious though) ##
def _get_timeframe(self, callback,date1=None,date2=None): def _get_timeframe(self, callback,date1=None,date2=None):
r=dict() r=dict()
for t,c in zip(self.times,self.data): for t,c in zip(self.times,self.data):
@@ -89,16 +94,16 @@ class Data:
def parse_line(datapoints, line, timekey, keys, time_parser, timeformat=None): def parse_line(datapoints, line, timekey, keys, time_parser, timeformat=None):
# This function expects: # This function expects:
# - datapoints { String:DataObject } # - datapoints { String:DataObject }
# - line { String:Any } # - line { String:Any }
# - timekey String (key for timevalue in 'line') # - timekey String (key for timevalue in 'line')
# - keys [ (String,String) ] (source_key in 'line' to target_key in 'datapoints') # - keys [ (String,String) ] (source_key in 'line' to target_key in 'datapoints')
time = time_parser(line[ timekey ],timeformat) time = time_parser(line[ timekey ],timeformat)
for key in keys: for key in keys:
datapoints[ key[1] ].data += [ line[ key[0] ] ] datapoints[ key[1] ].data += [ line[ key[0] ] ]
datapoints[ key[1] ].times += [ time ] datapoints[ key[1] ].times += [ time ]
def processExternalData(datapoints, plotNameKey, fromTime, toTime, dtype): def processExternalData(datapoints, plotNameKey, fromTime, toTime, dtype, qtTextBrowser):
'''Download and parses external data of type dtype''' '''Download and parses external data of type dtype'''
# prepare strings # # prepare strings #
@@ -115,7 +120,7 @@ def processExternalData(datapoints, plotNameKey, fromTime, toTime, dtype):
# download date if it doesn't exist # # download date if it doesn't exist #
url = CFG("outside_data_url").format(dtype=dtype, fromDate=fromTimeStr, toDate=toTimeStr) url = CFG("outside_data_url").format(dtype=dtype, fromDate=fromTimeStr, toDate=toTimeStr)
r = requests.get(url) r = requests.get(url)
print(url) qtTextBrowser.append(de.pg_request.format(url))
content = r.content.decode('utf-8', "ignore") # ignore bad bytes content = r.content.decode('utf-8', "ignore") # ignore bad bytes
# cache data # # cache data #
@@ -127,12 +132,11 @@ def processExternalData(datapoints, plotNameKey, fromTime, toTime, dtype):
else: else:
# get data from cache otherwise # get data from cache otherwise
print("INFO: Cache hit: {}".format(cacheFile)) qtTextBrowser.append(de.cache_hit.format(cacheFile))
with open(fullpath) as f: with open(fullpath) as f:
content = f.read() content = f.read()
skipBecauseFirstLine = True skipBecauseFirstLine = True
error = None
for l in content.split("\n"): for l in content.split("\n"):
if not ";" in l: if not ";" in l:
continue continue
@@ -154,14 +158,13 @@ def processExternalData(datapoints, plotNameKey, fromTime, toTime, dtype):
datapoints[plotNameKey].data += [float(cleanFloat)] datapoints[plotNameKey].data += [float(cleanFloat)]
datapoints[plotNameKey].times += [timestamp] datapoints[plotNameKey].times += [timestamp]
except ValueError as e: except ValueError as e:
error = de.warning_ext_data qtTextBrowser.append(de.warning_ext_data)
# return a warning indication for qt #
return error
def read_in_file(path, backend=None, outsideData=False, plotOutsideTemp=True, plotOutsideHum=True): def read_in_file(path, backend=None, outsideData=False, plotOutsideTemp=True,
'''Read in a file, add outside data if requested''' plotOutsideHum=True, qtTextBrowser=None):
'''Read in a file, add outside data if requested, optionally give a QtTextBrowser
to output information, warnings and errors to.'''
datapoints = dict() datapoints = dict()
identifiers = [ CFG("plot_temperatur_key"), identifiers = [ CFG("plot_temperatur_key"),
@@ -197,7 +200,7 @@ def read_in_file(path, backend=None, outsideData=False, plotOutsideTemp=True, pl
for i in range(0, len(names)): for i in range(0, len(names)):
while len(names[i]) < max_name_len: while len(names[i]) < max_name_len:
names[i] += " " names[i] += " "
datapoints.update({ identifiers[i] : Data(names[i], plotSettings[i]) }) datapoints.update({ identifiers[i] : Data(names[i], plotSettings[i], qtTextBrowser) })
# legacy variables... # legacy variables...
pt, ph, pd, pto, pho = identifiers pt, ph, pd, pto, pho = identifiers
@@ -210,78 +213,84 @@ def read_in_file(path, backend=None, outsideData=False, plotOutsideTemp=True, pl
elif path.endswith(".DBF") or path.endswith(".dbf"): elif path.endswith(".DBF") or path.endswith(".dbf"):
dbfread(path,datapoints,pt,ph,pd) dbfread(path,datapoints,pt,ph,pd)
elif path.endswith(".xls") or path.endswith(".XLS"): elif path.endswith(".xls") or path.endswith(".XLS"):
csvread(path,datapoints,pt,ph,pd) csvread(path,datapoints,pt,ph,pd,qtTextBrowser)
elif path.endswith(".txt"): elif path.endswith(".txt"):
csvread_txt(path,datapoints,pt,ph,pd) csvread_txt(path,datapoints,pt,ph,pd,qtTextBrowser)
else: else:
raise NotImplementedError("Cannot determine filetype, cannot continue. Exit.") raise NotImplementedError("Cannot determine filetype, cannot continue. Exit.")
# if nessesary download and process external data # # if nessesary download and process external data #
error = None
if outsideData: if outsideData:
fromTime = datapoints[CFG("plot_temperatur_key")].getFirstTime() fromTime = datapoints[CFG("plot_temperatur_key")].getFirstTime()
toTime = datapoints[CFG("plot_temperatur_key")].getLastTime() toTime = datapoints[CFG("plot_temperatur_key")].getLastTime()
error1 = processExternalData(datapoints, pto, fromTime, toTime, CFG("dtype_temperatur")) dtypeTemp = CFG("dtype_temperatur")
error2 = processExternalData(datapoints, pho, fromTime, toTime, CFG("dtype_humidity")) dtypeHum = CFG("dtype_humidity")
# pass on warnings # processExternalData(datapoints, pto, fromTime, toTime, dtypeTemp, qtTextBrowser)
if error1: processExternalData(datapoints, pho, fromTime, toTime, dtypeHum, qtTextBrowser)
error = error1;
else:
error = error2
return (datapoints, error) return datapoints
def dbfread(path,datapoints,pt,ph,pd): def dbfread(path,datapoints,pt,ph,pd):
for record in DBF(path): for record in DBF(path):
parse_line(datapoints,record,'DATETIME',[ ('TEMPCELS',pt) , ('HUMIDITY',ph) , ('DEWCELS',pd) ] ,timeutils.time_from_dbf) parse_line(datapoints, record, 'DATETIME',
[ ('TEMPCELS',pt), ('HUMIDITY',ph), ('DEWCELS',pd) ],
timeutils.time_from_dbf)
def csvread(path,datapoints,pt,ph,pd): def csvread(path, datapoints, pt, ph, pd, qtTextBrowser):
count = 0; count = 0;
with open(path) as f: with open(path) as f:
for l in f: for l in f:
if l.startswith(">>") or l.startswith("--") or l.startswith("NO."): if l.startswith(">>") or l.startswith("--") or l.startswith("NO."):
count += 1 count += 1
continue continue
else: else:
row_arg = list(map(lambda s:s.replace(" ","").replace(",","."),l.split("\t"))) row_arg = list(map(lambda s:s.replace(" ","").replace(",","."),l.split("\t")))
row = {"temp":None,"hum":None,"taupunkt":None,"datetime":None} row = {"temp":None,"hum":None,"taupunkt":None,"datetime":None}
row["datetime"] = row_arg[1]+row_arg[2] row["datetime"] = row_arg[1]+row_arg[2]
row["temp"] = float(row_arg[3]) row["temp"] = float(row_arg[3])
row["hum"] = float(row_arg[4]) row["hum"] = float(row_arg[4])
row["taupunkt"] = float(row_arg[5]) row["taupunkt"] = float(row_arg[5])
parse_line(datapoints,row,'datetime',[ ('temp',pt) , ('hum',ph) , ('taupunkt',pd) ],\ parse_line(datapoints, row, 'datetime',
timeutils.time_from_csv,timeformat="%d-%m-%Y%H:%M:%S") [('temp',pt), ('hum',ph), ('taupunkt',pd)],
print("Info: Ignored %d lines at beginning of file"%count) timeutils.time_from_csv,
timeformat="%d-%m-%Y%H:%M:%S")
import codecs qtTextBrowser.append(de.info_ig_lines.format(count))
def csvread_txt(path,datapoints,pt,ph,pd):
count = 0; def csvread_txt(path,datapoints,pt,ph,pd,qtTextBrowser):
f = open(path) count = 0;
with open(path) as f:
try: try:
for l in f: for l in f:
if any(s in l for s in ["Logger","Datenquelle","Sensortyp","Einheit","Daten"]): if any(s in l for s in ["Logger","Datenquelle","Sensortyp","Einheit","Daten"]):
count += 1 count += 1
continue continue
else: else:
row_arg = list(map(lambda s:s.replace(" ","").replace(",","."),l.split("\t"))) row_arg = list(map(lambda s:s.replace(" ","").replace(",","."),l.split("\t")))
row = {"temp":None,"hum":None,"taupunkt":None,"datetime":None} row = {"temp":None,"hum":None,"taupunkt":None,"datetime":None}
row["datetime"] = "%s-%s-%s_%s:%s"%(row_arg[0],row_arg[1],row_arg[2],row_arg[3],row_arg[4]) row["datetime"] = "{}-{}-{}_{}:{}".format(row_arg[0],
row["temp"] = float(row_arg[6]) row_arg[1],
row["hum"] = float(row_arg[7]) row_arg[2],
row["taupunkt"] = 0.0 row_arg[3],
parse_line(datapoints,row,'datetime',[ ('temp',pt) , ('hum',ph) , ('taupunkt',pd) ],\ row_arg[4])
timeutils.time_from_csv,timeformat="%d-%m-%Y_%H:%M") row["temp"] = float(row_arg[6])
row["hum"] = float(row_arg[7])
row["taupunkt"] = 0.0
parse_line(datapoints, row, 'datetime',
[('temp',pt), ('hum',ph), ('taupunkt',pd)],
timeutils.time_from_csv,
timeformat="%d-%m-%Y_%H:%M")
except (UnicodeError, IndexError): except (UnicodeError, IndexError):
count = csvread_txt_fallback(path,datapoints,pt,ph,pd) count = csvread_txt_fallback(path,datapoints,pt,ph,pd)
print("Info: Ignored %d lines at beginning of the file"%count) qtTextBrowser.append(de.info_ig_lines.format(count))
f.close()
def csvread_txt_fallback(path,datapoints,pt,ph,pd): def csvread_txt_fallback(path,datapoints,pt,ph,pd):
'''fallback for different format and encoding of txt''' '''fallback for different format and encoding of txt'''
count = 0 count = 0
with codecs.open(path, "r",encoding="ISO8859_2", errors='replace') as f: with codecs.open(path, "r",encoding="ISO8859_2", errors='replace') as f:
for l in f: for l in f:
@@ -290,11 +299,13 @@ def csvread_txt_fallback(path,datapoints,pt,ph,pd):
continue continue
else: else:
date,time,temp,hum = l.replace(" ","").replace(".","-").replace(",",".").split("\t") date,time,temp,hum = l.replace(" ","").replace(".","-").replace(",",".").split("\t")
row = {"temp":None,"hum":None,"taupunkt":None,"datetime":None} row = { "temp":None, "hum":None, "taupunkt":None, "datetime":None }
row["datetime"] = "{}_{}".format(date,time[:5]) row["datetime"] = "{}_{}".format(date,time[:5])
row["temp"] = float(temp) row["temp"] = float(temp)
row["hum"] = float(hum) row["hum"] = float(hum)
row["taupunkt"] = 0.0 row["taupunkt"] = 0.0
parse_line(datapoints,row,'datetime',[ ('temp',pt) , ('hum',ph) , ('taupunkt',pd) ],\ parse_line(datapoints,row,'datetime',
timeutils.time_from_csv,timeformat="%d-%m-%Y_%H:%M") [('temp',pt), ('hum',ph), ('taupunkt',pd)],
timeutils.time_from_csv,
timeformat="%d-%m-%Y_%H:%M")
return count return count

8
src/main/python/localization/de.py
View File

@@ -22,4 +22,10 @@ open_pic = "Bild öffnen.."
bad_time = "Fehlerhafte Zeitangabe!" bad_time = "Fehlerhafte Zeitangabe!"
warning = "Warnung" warning = "Warnung"
button_idc = "Egal, weiter!" button_idc = "Egal, weiter!"
warning_ext_data = "Fehlerhafte oder fehlende Werte in externen Daten für angegebenen Zeitraum." warning_ext_data = "WARN: Fehlerhafte oder fehlende Werte in externen Daten für angegebenen Zeitraum."
infoOutput = "Informationen"
cache_hit = "INFO: Verwende bereits vorhandene Datei: {}"
special_err_1 = "Zeitstempelanzahl stimmt nicht mit Datensatzzahl überein."
warn_no_data = "Warnung, keine Daten für {}"
pg_request = "Downloading: {}"
info_ig_lines = "INFO: Ignoriere {} Zeilen ohne Daten am Anfang der Datei"

34
src/main/python/main.py
View File

@@ -8,7 +8,7 @@ from PyQt5.QtWidgets import (QApplication, QCheckBox, QComboBox, QDateTimeEdit,
QDial, QDialog, QGridLayout, QGroupBox, QHBoxLayout, QLabel, QLineEdit, QDial, QDialog, QGridLayout, QGroupBox, QHBoxLayout, QLabel, QLineEdit,
QProgressBar, QPushButton, QRadioButton, QScrollBar, QSizePolicy, QProgressBar, QPushButton, QRadioButton, QScrollBar, QSizePolicy,
QSlider, QSpinBox, QStyleFactory, QTableWidget, QTabWidget, QTextEdit, QSlider, QSpinBox, QStyleFactory, QTableWidget, QTabWidget, QTextEdit,
QVBoxLayout, QWidget, QFileDialog, QDateEdit, QMessageBox) QVBoxLayout, QWidget, QFileDialog, QDateEdit, QMessageBox, QTextBrowser)
import localization.de as de import localization.de as de
import sys import sys
@@ -35,12 +35,14 @@ class WidgetGallery(QDialog):
self.createFileSelection() self.createFileSelection()
self.createDateSelection() self.createDateSelection()
self.createCheckboxArea() self.createCheckboxArea()
self.createInfoOutputSection()
mainLayout = QGridLayout() mainLayout = QGridLayout()
mainLayout.addWidget(self.fileSelectionGroup, 1, 0) mainLayout.addWidget(self.fileSelectionGroup, 1, 0)
mainLayout.addWidget(self.dateSelectionGroupBox, 2, 0) mainLayout.addWidget(self.dateSelectionGroupBox, 2, 0)
mainLayout.addWidget(self.checkboxGroup, 3, 0) mainLayout.addWidget(self.checkboxGroup, 3, 0)
mainLayout.addWidget(self.startSection, 4, 0) mainLayout.addWidget(self.startSection, 4, 0)
mainLayout.addWidget(self.infoOutputSection, 5, 0)
self.setLayout(mainLayout) self.setLayout(mainLayout)
@@ -59,6 +61,17 @@ class WidgetGallery(QDialog):
self.startSection.setLayout(layout) self.startSection.setLayout(layout)
def createInfoOutputSection(self):
'''Generate Aread containing progress, error and warning outputs'''
self.infoOutputSection = QGroupBox(self.localization.infoOutput)
self.infoTextBox = QTextBrowser()
layout = QVBoxLayout()
layout.addWidget(self.infoTextBox)
self.infoOutputSection.setLayout(layout)
def createFileSelection(self): def createFileSelection(self):
'''Generate the area containing the file selectors and go button''' '''Generate the area containing the file selectors and go button'''
@@ -140,6 +153,7 @@ class WidgetGallery(QDialog):
'''Run generation with selected file and options''' '''Run generation with selected file and options'''
# set save target if nessesary # # set save target if nessesary #
self.infoTextBox.clear()
self.buttonGo.setText(self.localization.button_go_wait) self.buttonGo.setText(self.localization.button_go_wait)
self.buttonGo.setDisabled(True) self.buttonGo.setDisabled(True)
self.repaint() self.repaint()
@@ -153,17 +167,11 @@ class WidgetGallery(QDialog):
# workaround for checkboxes changed # # workaround for checkboxes changed #
outsideDataNeeded = self.boxOTemp.isChecked() or self.boxOHumidity.isChecked() outsideDataNeeded = self.boxOTemp.isChecked() or self.boxOHumidity.isChecked()
self.datapoints, error = input_backend.read_in_file(self.srcFileString, self.datapoints = input_backend.read_in_file(self.srcFileString,
outsideData=outsideDataNeeded, outsideData=outsideDataNeeded,
plotOutsideTemp=self.boxOTemp.isChecked(), plotOutsideTemp=self.boxOTemp.isChecked(),
plotOutsideHum=self.boxOHumidity.isChecked()) plotOutsideHum=self.boxOHumidity.isChecked(),
qtTextBrowser=self.infoTextBox)
if error:
errorBox = QMessageBox(self)
errorBox.setAttribute(PyQt5.QtCore.Qt.WA_DeleteOnClose)
errorBox.setText(self.localization.warning)
errorBox.setDetailedText(error)
errorBox.show()
# build dates # # build dates #
try: try:
@@ -218,11 +226,13 @@ class WidgetGallery(QDialog):
waitDialog.setAttribute(PyQt5.QtCore.Qt.WA_DeleteOnClose) waitDialog.setAttribute(PyQt5.QtCore.Qt.WA_DeleteOnClose)
waitDialog.setText(self.localization.wait_dialog_text) waitDialog.setText(self.localization.wait_dialog_text)
waitDialog.show() waitDialog.show()
try: try:
self.datapoints, error = input_backend.read_in_file(self.srcFileString, self.datapoints = input_backend.read_in_file(self.srcFileString,
outsideData=False, outsideData=False,
plotOutsideTemp=False, plotOutsideTemp=False,
plotOutsideHum=False) plotOutsideHum=False,
qtTextBrowser=self.infoTextBox)
except Exception as e: except Exception as e:
waitDialog.close() waitDialog.close()
errorBox = QMessageBox(self) errorBox = QMessageBox(self)