""" Generic routines for manipulating log files. 5 Dec 2008, Lev """ # Incomplete Changelog # # 5 Dec 2008 # Add 'around' method # # 16 Dec 2008 # Modify 'subset' method to handle correctly empty logs import numpy import os import os.path import time import datetime import matplotlib.dates import flib class TimeLog(object): """ A base class for various logs. Data is stored in a 2-dimensional array 'self.data': First column ([:,0]) is time (sec, since 00:00 1 January 1970 GMT), the rest is log-specific 'self.device' contains the name of the logged device. The purpose is to separate logs of the same type, e.g. paroscientific and melting curve thermometry logs. Two logs can be concatenated with a '&' sign, one of the logs can be None - in this case the other is returned as a result of concatenation. The child classes should provide the following functionality: 1. Methods to access custom 'self.data' fields. 2. A 'fieldcount()' static method that returns number of columns in 'self.data' array. 3. A 'load(filename)' static method that should load a log from a file and return an object of the appropriate child class. """ def __init__(self, log = None, device = None): """ TimeLog constructor. 'log' is an N x fieldcount() array, containing the log. If the argument is None, the empty log is created. 'device' is the name of the device that was used for the log. Columns have the following meaning: 1 [:,0] Time (sec, since 00:00 1 January 1970 GMT) ... rest is log-specific """ if log is None: log = numpy.zeros((0,self.fieldcount())) else: log = numpy.array(log) if len(log.shape) != 2 or log.shape[1] != self.fieldcount(): return ValueError self.data = log self.device = device def __len__(self): return len(self.data) def t(self): "time, seconds since 00:00 1 January 1970 GMT"; return self.data[:,0] def plottime(self): "time for pylab.plot_date function"; return matplotlib.dates.epoch2num(self.t()) def delay_before(self): """ Return intervals between pulses, first element is NaN - the delay before the first pulse, so """ return numpy.concatenate(([numpy.NaN], self.t()[1:] - self.t()[:-1])) def isempty(self): "return True if there are no entries in the log, False otherwise"; return len(self) < 1 def copy(self): return type(self)(self.data.copy(), self.device) def compatible(self, other): "Return True if the two TimeLogs are compatible (of the same type and for the same device), otherwise False" # if not ((type(self) is type(other)) and (self.data.shape[1] == other.data.shape[1]) and (self.device == other.device)): # print '%s != %s' % (self, other) return (type(self) is type(other)) and (self.data.shape[1] == other.data.shape[1]) and (self.device == other.device) def __and__(self, other): if other is None: return self if not self.compatible(other): raise ValueError # type(self)(...) creates an object of the same type as self, # which is supposed to be a class inherited from TimeLog return type(self)(numpy.concatenate((self.data, other.data)), self.device) # this will be called only if the first one is not a TimeLog, # None is the only non-Timelog instance that can be concatenated with a TimeLog def __rand__(self, other): if other is None: return self else: raise ValueError def sort(self): """ Sort the log accending the timestamps """ if self.isempty(): return self.data = self.data[self.data[:,0].argsort(),:] def subset(self, indices): """ Return a subset of the log consisting of entries listed in 'indices' """ return type(self)(self.data[indices] if len(self) > 0 else numpy.zeros((0,self.fieldcount())), self.device) def period(self, start = None, end = None): """ Return a subset of the log between the two dates, in C and UNIX format or free-form strings. If the start date is not specified, the earliest is assumed. If the end date is not specified, the latest is assumed. """ if self.isempty(): return self if start is None: start = min(self.t()) else: start = flib.tm(start) if end is None: end = max(self.t()) else: end = flib.tm(end) return self.subset((self.t() >= start) & (self.t() <= end)) def noperiod(self, start, end): """ Return a subset of the log outside the specified period, in C and UNIX format or free-form strings. """ if self.isempty(): return self return self.subset((self.t() <= flib.tm(start)) | (self.t() >= flib.tm(end))) def around(self, start, end = None): """ Return a subset of the log encompassing a period of time defined by 'start'-'end' or a moment in time defined by 'start', if 'end' is not specified. """ start = flib.tm(start) end = flib.tm(end) if end is not None else start t = self.t() if any(t <= start): start = numpy.max(t[t <= start]) if any(t >= end): end = numpy.min(t[t >= end]) return self.subset((t >= start) & (t <= end)) def __repr__(self): if self.device is None: return '%s %s' % (type(self).__name__, self.datespan()) else: return '%s %s %s' % (self.device, type(self).__name__, self.datespan()) def datespan(self): """ Return string representation of the date range of the log """ if self.isempty(): return '(empty)' else: return flib.datespan(min(self.t()), max(self.t())) def justbefore(self, t): """ Return a single point log with the regord at the moment 't', or just before it. Time is accepted in any format recognised by 'flib.tm()'. """ tt = self.t() return self.subset(tt == max(tt[tt <= flib.tm(t)])) def justafter(self, t): """ Return a single point log with the regord at the moment 't', or just before it. Time is accepted in any format recognised by 'flib.tm()'. """ tt = self.t() return self.subset(tt == min(tt[tt >= flib.tm(t)])) def steady(self, condition, before, after, allow_change_by = None): """ Return a boolean array of the length of the log with True in those cells only where 'condition' (a 1D array of the same length as the log) is the same as in every point within 'before' seconds before the cell in question and 'after' seconds after the it (time is the only information retained from the log itself). If 'allow_change_by' is 'None', the condition is considered steady if it does not change, otherwise changes not greater than the value of this argument are allowed. Both 'condition' and 'allow_change_by' are expected to be numerical then. """ if allow_change_by is None: same = lambda a, b: a == b else: same = lambda a, b: abs(b - a) <= allow_change_by if len(condition) != len(self): raise ValueError('"condition" is wrong size') mask = numpy.ones([len(self)], dtype=bool) t = self.t() for i in xrange(len(self)): for j in xrange(i-1, -1, -1): if t[j] < t[i] - before: break if not same(condition[j], condition[i]): mask[i] = False break if mask[i]: for j in xrange(i+1, len(self), 1): if t[j] > t[i] + after: break if not same(condition[j], condition[i]): mask[i] = False break return mask @classmethod def loaddir(cls, dirname, start = None, end = None, skipErrors = False): """ Load log files from a given directory. The function walkes through a directory and treats files named 'YYYYMMDD.dat' as log files. If either boundary time (start, end) is specified, only files with relevant dates are read. Selected files are read using the load() static method of a child class and appended to the log using '&'. If no files were read, an empty log is created using the child class default constructor. The resulting log is returned. """ # create 8-character strings to bound filenames in the directory to if start is not None: startfn = flib.dt2filename(flib.tm(start)) else: startfn = "00000000" # the minimum 8-character-digit string if end is not None: endfn = flib.dt2filename(flib.tm(end)) else: endfn = "99999999" # the maximum 8-character-digit string log = None for filename in os.listdir(dirname): day = None if len(filename) > 8 and filename[:8] >= startfn and filename[:8] <= endfn: if skipErrors: try: day = cls.load(os.path.join(dirname, filename)) except Exception: pass else: day = cls.load(os.path.join(dirname, filename)) if day is not None: log &= day if log is None: return cls() log.sort() return log.period(start) def average(self, Nwindow = 10): """ Average 'Nwindow' point long sections of the log to reduce size and scatter """ l = self.copy() l.data = flib.average(self.data, Nwindow) return l # these methods must be defined in the child classes: # # @staticmethod # def fieldcount(): "return number of columns in the self.data"; return 1 # # @staticmethod # def load(filename): return NotImplemented from plmlog import PLMLog from mctlog import MCTLog from parolog import ParoLog from pidlog import PIDLog from kelvinoxlog import KelvinoxLog from oneKpotlog import OneKPotLog from LDlog import LDLog from tritonlog import TritonLog