skillbird/FileReader.py

import TrueSkillWrapper as TS
import time
import threading
import insurgencyParsing as iparse

DATE_LENGTH = 15

def readfile(filename, start_at_end, exit_on_eof, parsingBackend, startAtTime, cacheFile, cpus=1):

    f = open(filename)
    if start_at_end:
        f.seek(0,2)

    if startAtTime:
        while True:
            line = f.readline()
            try:
                dt = parsingBackend.parseDate(line)
                if not dt:
                    break
                if dt > startAtTime:
                    break
            except IndexError:
                pass
            except ValueError:
                pass
    
    try:
        if cpus > 1:
            raise NotImplementedError("Multiprocessing not implemeted yet")
        else:
            if callable(parsingBackend):
                parsingBackend(f, exit_on_eof, start_at_end, cacheFile)
            else:
                parsingBackend.parse(f, exit_on_eof, start_at_end, cacheFile)
    except TypeError:
        raise RuntimeError("parsingBackend musst be callable or have .parse() callable")
    
    f.close()

def readfiles(filenames, start_at_end, nofollow, oneThread, cacheFile, parsingBackend=iparse):
        if cacheFile:
            startAtTime = parsingBackend.loadCache(cacheFile)
            print(startAtTime)
        else:
            startAtTime = None
        for f in filenames:
            if oneThread:
                readfile(f, start_at_end, nofollow, parsingBackend, startAtTime, cacheFile)
            else:
                threading.Thread(target=readfile,args=\
                            ( f, start_at_end, nofollow, \
                              parsingBackend, startAtTime, cacheFile, )).start()