From 62ecf6dd60e28545b0e7a5517973a69d243aa78b Mon Sep 17 00:00:00 2001 From: Yannik Schmidt Date: Wed, 6 Feb 2019 20:57:04 +0100 Subject: [PATCH] Implement serialized caching --- FileReader.py | 36 +++++++++++++++++++++++++----- Player.py | 12 +++++++++- Round.py | 52 +++++++++++++++++++++++++++++++++++++++++--- insurgencyParsing.py | 40 ++++++++++++++++++++++++++++++---- startInsurgency.py | 9 ++++++-- 5 files changed, 133 insertions(+), 16 deletions(-) diff --git a/FileReader.py b/FileReader.py index b646aa8..c48d6a2 100644 --- a/FileReader.py +++ b/FileReader.py @@ -3,29 +3,53 @@ import time import threading import insurgencyParsing as iparse -def readfile(filename, start_at_end, exit_on_eof, parsingBackend, cpus=1): +from datetime import datetime + +DATE_LENGTH = 15 + +def readfile(filename, start_at_end, exit_on_eof, parsingBackend, startAtTime, cacheFile, cpus=1): f = open(filename) if start_at_end: f.seek(0,2) + + if startAtTime: + while True: + line = f.readline() + try: + dt = datetime.strptime(line[:DATE_LENGTH], "%b %d %H:%M:%S") + if not dt: + break + if dt > startAtTime: + break + except IndexError: + pass + except ValueError: + pass try: if cpus > 1: raise NotImplementedError("Multiprocessing not implemeted yet") else: if callable(parsingBackend): - parsingBackend(f, exit_on_eof, start_at_end) + parsingBackend(f, exit_on_eof, start_at_end, cacheFile) else: - parsingBackend.parse(f, exit_on_eof, start_at_end) + parsingBackend.parse(f, exit_on_eof, start_at_end, cacheFile) except TypeError: raise RuntimeError("parsingBackend musst be callable or have .parse() callable") f.close() -def readfiles(filenames, start_at_end=False, nofollow=False,parsingBackend=iparse, oneThread=False): +def readfiles(filenames, start_at_end, nofollow, oneThread, cacheFile, parsingBackend=iparse): + if cacheFile: + startAtTime = parsingBackend.loadCache(cacheFile) + print(startAtTime) + else: + startAtTime = None for f in filenames: if oneThread: - readfile(f, start_at_end, nofollow, parsingBackend) + readfile(f, start_at_end, nofollow, parsingBackend, startAtTime, cacheFile) else: threading.Thread(target=readfile,args=\ - (f, start_at_end, nofollow, parsingBackend,)).start() + ( f, start_at_end, nofollow, \ + parsingBackend, startAtTime, cacheFile, )).start() diff --git a/Player.py b/Player.py index 5d4ccb9..97f11a7 100644 --- a/Player.py +++ b/Player.py @@ -33,7 +33,7 @@ class DummyPlayer(Player): super().__init__(steamid, name) class PlayerInRound(Player): - def __init__(self, steamid, name, team, timestamp): + def __init__(self, steamid, name, team, timestamp=None): self.name = name self.cur_name = name self.steamid = steamid @@ -50,6 +50,16 @@ class PlayerInRound(Player): self.timestamp = timestamp def __str__(self): return "TMP-Player: {}, ID: {}, active: {}".format(self.cur_name,self.steamid,self.active_time) + def serialize(self): + return "{}0x42{}0x42{}0x42{}".format(self.name, self.steamid, \ + self.team, self.active_time.seconds) + + def deserialize(string): + name, steamid, team, active_time = string.split("0x42") + active_time = datetime.timedelta(seconds=int(active_time)) + + team = int(team) + return PlayerInRound(steamid, name, team, active_time) class PlayerForDatabase(Player): def __init__(self,steamid,name,rating,player=None): diff --git a/Round.py b/Round.py index f03c639..8c6404d 100644 --- a/Round.py +++ b/Round.py @@ -1,4 +1,7 @@ -from datetime import timedelta +from datetime import timedelta, datetime +import threading +from Player import PlayerInRound +import TrueSkillWrapper as TS ## A comment on why the login-offset is nessesary ## ## - losing teams tend to have players leaving and joining more rapidly @@ -9,14 +12,24 @@ from datetime import timedelta loginoffset = timedelta(seconds=60) class Round: - def __init__(self,winner_team,loser_team,_map,duration,starttime,winner_side=None): + writeLock = threading.RLock() + def __init__(self, winner_team, loser_team, _map, duration,\ + starttime, winner_side=None, cache=None): self.winners = winner_team self.losers = loser_team self.winner_side = winner_side self.map = _map self.duration = duration self.start = starttime - self.add_fake_players() + #self.add_fake_players() + if cache: + Round.writeLock.acquire() + with open(cache, "a") as f: + f.write(self.serialize()) + f.write("\n") + f.flush() + Round.writeLock.release() + def normalized_playtimes(self): '''returns a dict-Object with {key=(teamid,player):value=player_time_played/total_time_of_round}''' @@ -102,3 +115,36 @@ class Round: return max(w1,w2)/min(w1,w2) + def serialize(self): + # full = winners|losers|winner_side|map|duration|start + # winners/losers = p1,p2,p3 ... + winners = "" + losers = "" + for p in self.winners: + winners += "," + p.serialize() + for p in self.losers: + losers += "," + p.serialize() + + teams = "{}|{}".format(winners, losers) + startTimeStr = self.start.strftime("%b %d %H:%M:%S") + + ret = "{}|{}|{}|{}|{}".format(teams, self.winner_side, \ + self.map, self.duration.seconds, startTimeStr) + return ret + + def deserialize(string): + string = string.strip("\n") + winnersStr, losersStr, winner_side, _map, duration, startTimeStr = string.split("|") + winners = dict() + losers = dict() + for pStr in winnersStr.split(","): + if pStr == "": + continue + winners.update({PlayerInRound.deserialize(pStr):TS.new_rating()}) + for pStr in losersStr.split(","): + if pStr == "": + continue + losers.update({PlayerInRound.deserialize(pStr):TS.new_rating()}) + startTime = datetime.strptime(startTimeStr, "%b %d %H:%M:%S") + duration = timedelta(seconds=int(duration)) + return Round(winners, losers, _map, duration, startTime, winner_side) diff --git a/insurgencyParsing.py b/insurgencyParsing.py index d51e2c9..9df8e22 100644 --- a/insurgencyParsing.py +++ b/insurgencyParsing.py @@ -20,13 +20,44 @@ def get_key(dic,key): tmp = list(dic) return tmp[tmp.index(key)] -def parse(f, exit_on_eof=True, start_at_end=False): +def loadCache(cacheFile): + rounds = [] + if not cacheFile: + return None + with open(cacheFile, "r") as f: + for line in f: + rounds += [Round.Round.deserialize(line)] + + # return if file was empty # + if len(rounds) == 0: + return None + + # put them in right order # + rounds = sorted(rounds, key=lambda r: r.start) + + # parse Rounds # + for r in rounds: + try: + SB.sync_from_database(r.winners) + SB.sync_from_database(r.losers) + TS.evaluate_round(r) + except Warning: + pass + + # find newest # + lastRoundByDate = rounds[-1].start + return lastRoundByDate + + + +def parse(f, exit_on_eof=True, start_at_end=False, cacheFile=None): last_round_end = None seek_start = True round_lines = [] last_line_was_winner = False lineCount = 0 startTime = datetime.now() + while True: old_line_nr = f.tell() line = f.readline() @@ -68,7 +99,7 @@ def parse(f, exit_on_eof=True, start_at_end=False): # parse and evaluate round # if evalRound: - nextRound = parseRoundFromLines(round_lines) + nextRound = parseRoundFromLines(round_lines, cacheFile) round_lines = [] evalRound = False if nextRound: @@ -78,7 +109,7 @@ def parse(f, exit_on_eof=True, start_at_end=False): pass -def parseRoundFromLines(r): +def parseRoundFromLines(r, cacheFile=None): # get an event series # es = EventSeries() @@ -123,7 +154,8 @@ def parseRoundFromLines(r): except Warning as e: TS.dirty_rounds += 1 return None - return Round.Round(winners,losers,es.get_map(),es.get_duration(),es.get_starttime()) + return Round.Round(winners, losers, es.get_map(), es.get_duration(), \ + es.get_starttime(), cache=cacheFile) def create_event(etype,line,timestamp): TEAMCHANGE = ["teamchange"] diff --git a/startInsurgency.py b/startInsurgency.py index 731fbd2..961252e 100755 --- a/startInsurgency.py +++ b/startInsurgency.py @@ -20,13 +20,18 @@ parser.add_argument('--no-follow','-nf',dest='nofollow', action='store_const',\ parser.add_argument('--one-thread', dest='oneThread', action='store_const',\ const=True, default=False, \ help="run everything in main thread (implies no-follow)") +parser.add_argument('--cache-file', dest='cacheFile',\ + help="A cache file which makes restarting the system fast") if __name__ == "__main__": args = parser.parse_args() + if args.cacheFile: + open(args.cacheFile, "a").close() FileReader.readfiles( args.files ,\ start_at_end=args.start_at_end,\ - nofollow=args.nofollow, - oneThread=args.oneThread) + nofollow=args.nofollow, \ + oneThread=args.oneThread, \ + cacheFile=args.cacheFile) if args.oneThread: for l in StorrageBackend.dumpRatings().split("\n"): print(l)