Implement serialized caching

FileReader.py

@@ -3,29 +3,53 @@ import time
 import threading
 import insurgencyParsing as iparse
 
-def readfile(filename, start_at_end, exit_on_eof, parsingBackend, cpus=1):
+from datetime import datetime
+
+DATE_LENGTH = 15
+
+def readfile(filename, start_at_end, exit_on_eof, parsingBackend, startAtTime, cacheFile, cpus=1):
 
     f = open(filename)
     if start_at_end:
         f.seek(0,2)
+
+    if startAtTime:
+        while True:
+            line = f.readline()
+            try:
+                dt = datetime.strptime(line[:DATE_LENGTH], "%b %d %H:%M:%S")
+                if not dt:
+                    break
+                if dt > startAtTime:
+                    break
+            except IndexError:
+                pass
+            except ValueError:
+                pass
     
     try:
         if cpus > 1:
             raise NotImplementedError("Multiprocessing not implemeted yet")
         else:
             if callable(parsingBackend):
-                parsingBackend(f, exit_on_eof, start_at_end)
+                parsingBackend(f, exit_on_eof, start_at_end, cacheFile)
             else:
-                parsingBackend.parse(f, exit_on_eof, start_at_end)
+                parsingBackend.parse(f, exit_on_eof, start_at_end, cacheFile)
     except TypeError:
         raise RuntimeError("parsingBackend musst be callable or have .parse() callable")
     
     f.close()
 
-def readfiles(filenames, start_at_end=False, nofollow=False,parsingBackend=iparse, oneThread=False):
+def readfiles(filenames, start_at_end, nofollow, oneThread, cacheFile, parsingBackend=iparse):
+        if cacheFile:
+            startAtTime = parsingBackend.loadCache(cacheFile)
+            print(startAtTime)
+        else:
+            startAtTime = None
         for f in filenames:
             if oneThread:
-                readfile(f, start_at_end, nofollow, parsingBackend)
+                readfile(f, start_at_end, nofollow, parsingBackend, startAtTime, cacheFile)
             else:
                 threading.Thread(target=readfile,args=\
-                            (f, start_at_end, nofollow, parsingBackend,)).start()
+                            ( f, start_at_end, nofollow, \
+                              parsingBackend, startAtTime, cacheFile, )).start()

Player.py

@@ -33,7 +33,7 @@ class DummyPlayer(Player):
             super().__init__(steamid, name)
 
 class PlayerInRound(Player):
-        def __init__(self, steamid, name, team, timestamp):
+        def __init__(self, steamid, name, team, timestamp=None):
             self.name = name
             self.cur_name = name
             self.steamid = steamid
@@ -50,6 +50,16 @@ class PlayerInRound(Player):
             self.timestamp = timestamp
         def __str__(self):
             return "TMP-Player: {}, ID: {}, active: {}".format(self.cur_name,self.steamid,self.active_time)
+        def serialize(self):
+            return "{}0x42{}0x42{}0x42{}".format(self.name, self.steamid, \
+                            self.team, self.active_time.seconds)
+
+        def deserialize(string):
+            name, steamid, team, active_time = string.split("0x42")
+            active_time = datetime.timedelta(seconds=int(active_time))
+
+            team = int(team)
+            return PlayerInRound(steamid, name, team, active_time)
 
 class PlayerForDatabase(Player):
         def __init__(self,steamid,name,rating,player=None):

Round.py

@@ -1,4 +1,7 @@
-from datetime import timedelta
+from datetime import timedelta, datetime
+import threading
+from Player import PlayerInRound
+import TrueSkillWrapper as TS
 
 ## A comment on why the login-offset is nessesary ##
 ## - losing teams tend to have players leaving and joining more rapidly
@@ -9,14 +12,24 @@ from datetime import timedelta
 loginoffset = timedelta(seconds=60)
 
 class Round:
-    def __init__(self,winner_team,loser_team,_map,duration,starttime,winner_side=None):
+    writeLock = threading.RLock()
+    def __init__(self, winner_team, loser_team, _map, duration,\
+                    starttime, winner_side=None, cache=None):
             self.winners     = winner_team
             self.losers      = loser_team
             self.winner_side = winner_side
             self.map         = _map
             self.duration    = duration
             self.start       = starttime
-            self.add_fake_players()
+            #self.add_fake_players()
+            if cache:
+                Round.writeLock.acquire()
+                with open(cache, "a") as f:
+                    f.write(self.serialize())
+                    f.write("\n")
+                    f.flush()
+                Round.writeLock.release()
+
 
     def normalized_playtimes(self):
         '''returns a dict-Object with {key=(teamid,player):value=player_time_played/total_time_of_round}'''
@@ -102,3 +115,36 @@ class Round:
 
         return max(w1,w2)/min(w1,w2)
 
+    def serialize(self):
+        # full = winners|losers|winner_side|map|duration|start
+        # winners/losers = p1,p2,p3 ...
+        winners = ""
+        losers = ""
+        for p in self.winners:
+            winners += "," + p.serialize()
+        for p in self.losers:
+            losers += "," + p.serialize()
+        
+        teams = "{}|{}".format(winners, losers)
+        startTimeStr = self.start.strftime("%b %d %H:%M:%S")
+
+        ret = "{}|{}|{}|{}|{}".format(teams, self.winner_side, \
+                                        self.map, self.duration.seconds, startTimeStr)
+        return ret
+
+    def deserialize(string):
+        string = string.strip("\n")
+        winnersStr, losersStr, winner_side, _map, duration, startTimeStr = string.split("|")
+        winners = dict()
+        losers  = dict()
+        for pStr in winnersStr.split(","):
+            if pStr == "":
+                continue
+            winners.update({PlayerInRound.deserialize(pStr):TS.new_rating()})
+        for pStr in losersStr.split(","):
+            if pStr == "":
+                continue
+            losers.update({PlayerInRound.deserialize(pStr):TS.new_rating()})
+        startTime = datetime.strptime(startTimeStr, "%b %d %H:%M:%S")
+        duration = timedelta(seconds=int(duration))
+        return Round(winners, losers, _map, duration, startTime, winner_side)

insurgencyParsing.py

@@ -20,13 +20,44 @@ def get_key(dic,key):
     tmp = list(dic)
     return tmp[tmp.index(key)]
 
-def parse(f, exit_on_eof=True, start_at_end=False):
+def loadCache(cacheFile):
+    rounds = []
+    if not cacheFile:
+        return None
+    with open(cacheFile, "r") as f:
+        for line in f:
+            rounds += [Round.Round.deserialize(line)]
+
+    # return if file was empty #
+    if len(rounds) == 0:
+        return None
+
+    # put them in right order #
+    rounds = sorted(rounds, key=lambda r: r.start)
+
+    # parse Rounds #
+    for r in rounds:
+        try:
+            SB.sync_from_database(r.winners)
+            SB.sync_from_database(r.losers)
+            TS.evaluate_round(r)
+        except Warning:
+            pass
+
+    # find newest #
+    lastRoundByDate = rounds[-1].start
+    return lastRoundByDate
+
+        
+
+def parse(f, exit_on_eof=True, start_at_end=False, cacheFile=None):
     last_round_end = None
     seek_start = True
     round_lines = []
     last_line_was_winner = False
     lineCount = 0
     startTime = datetime.now()
+
     while True:
         old_line_nr = f.tell()
         line = f.readline()
@@ -68,7 +99,7 @@ def parse(f, exit_on_eof=True, start_at_end=False):
 
         # parse and evaluate round #
         if evalRound:
-            nextRound = parseRoundFromLines(round_lines)
+            nextRound = parseRoundFromLines(round_lines, cacheFile)
             round_lines = []
             evalRound = False
             if nextRound:
@@ -78,7 +109,7 @@ def parse(f, exit_on_eof=True, start_at_end=False):
                     pass
 
 
-def parseRoundFromLines(r):
+def parseRoundFromLines(r, cacheFile=None):
 
     # get an event series #
     es = EventSeries()
@@ -123,7 +154,8 @@ def parseRoundFromLines(r):
     except Warning as e:
         TS.dirty_rounds += 1
         return None
-    return Round.Round(winners,losers,es.get_map(),es.get_duration(),es.get_starttime())
+    return Round.Round(winners, losers, es.get_map(), es.get_duration(), \
+                    es.get_starttime(), cache=cacheFile)
 
 def create_event(etype,line,timestamp):
     TEAMCHANGE      = ["teamchange"]

startInsurgency.py

@@ -20,13 +20,18 @@ parser.add_argument('--no-follow','-nf',dest='nofollow', action='store_const',\
 parser.add_argument('--one-thread', dest='oneThread', action='store_const',\
                 const=True, default=False, \
                 help="run everything in main thread (implies no-follow)")
+parser.add_argument('--cache-file', dest='cacheFile',\
+                help="A cache file which makes restarting the system fast")
 
 if __name__ == "__main__":
         args = parser.parse_args()
+        if args.cacheFile:
+            open(args.cacheFile, "a").close()
         FileReader.readfiles( args.files ,\
                               start_at_end=args.start_at_end,\
-                              nofollow=args.nofollow,
-                              oneThread=args.oneThread)
+                              nofollow=args.nofollow, \
+                              oneThread=args.oneThread, \
+                              cacheFile=args.cacheFile)
         if args.oneThread:
             for l in StorrageBackend.dumpRatings().split("\n"):
                 print(l)