""" scopelog.py A tool for parsing "Tlog.dat" created by scope programs and maintaining a cache of log files. Classes: 'TraceInfo', 'ScopeLog', 'ScopeLogCache' Functions: 'load', 'getinfo', 'getinfos', 'loadbinary', 'loadbinaries' Please note, that 'ScopeLog' is not derived from 'TimeLog' 'ScopeLog' is not multi-thread safe!!! Version: 26 May 2008 """ # Changelog # # 4 Dec 2009 (Lev) # initial version # # 22 Jan 2009 (Lev) # add static function getinfos() # # 26 May 2009 (Lev) # remove an entry from the cache if a folder of a log file was removed # # 28 May 2009 (Lev) # Fix a bug that prevented loading a log file from "log.dat" # (this, rather than "logT.dat", was used in the PXI scope program early 2008). # Properly support quoted filenames (even with tabs within them), # as in logs created by the new PXI scope program. # import numpy, os.path, logging, glob, time import flib, aver, nmr class TraceInfo(object): """ A record about a trace in the log Contains: dir - directory where both the log and the trace reside filename - trace file name Nav - number of averages smplrate - sampling rate [1/s] vrange - scope input range [+-V] delay - scope delay [s] starttime - time [sec since C Epoch] when the aquisition started finaltime - time [sec since C Epoch] when the aquisition finished T_MCTNS_s - starting MCTNS temperature [mK, Greywall scale] T_MCTNS_f - final MCTNS temperature [mK, Greywall scale] T_MCTNS - mean MCTNS temperature [mK, Greywall scale] - Mean object """ def __init__(self, path, Nav, smplrate, delay, vrange, starttime, finaltime, T_MCTNS_s, T_MCTNS_f): self.path = path self.Nav = Nav self.smplrate = smplrate self.delay = delay self.vrange = vrange self.starttime = starttime self.finaltime = finaltime self.T_MCTNS_s = T_MCTNS_s self.T_MCTNS_f = T_MCTNS_f def basename(self): "trace filename without path"; return os.path.basename(self.path) def dirname(self): "name of directory containing the trace"; return os.path.dirname(self.path) def timespan(self): "return a string with timespan when the signal was taken (UTC)"; return flib.timespan2str(self.starttime, self.finaltime) def recordlist(self): "return a list containing 'self' - see 'AveragedTraceInfo' for details"; return [self] def __repr__(self): return "%s: %s" % (self.basename(), self.timespan()) def safeNav(self): return self.Nav if self.Nav > 0 else 1 class AveragedTraceInfo(TraceInfo): """ A combined record about a set of averaged traces """ def __init__(self, records, pathtemplate, indices): """ records - a set of TraceLog's of inidividual traces pathtemplate - filename template indices - indices used for making filenames out of template """ smplrates = flib.finites([i.smplrate for i in records]) delays = flib.finites([i.delay for i in records]) vranges = flib.finites([i.vrange for i in records]) if len(smplrates) > 0: if max(smplrates) != min(smplrates): raise ValueError('Can not average traces with different sampling rates') else: smplrate = smplrates[0] else: smplrate = numpy.NaN if len(delays) > 0: if max(delays) != min(delays): raise ValueError('Can not average traces with different aquisition delays') else: delay = delays[0] else: delay = numpy.NaN vrange = vranges[0] if len(vranges) > 0 and max(vranges) == min(vranges) else numpy.NaN Nav = numpy.sum([i.safeNav() for i in records]) starttimes = numpy.array([i.starttime for i in records]) finaltimes = numpy.array([i.finaltime for i in records]) TT_MCTNS_s = numpy.array([i.T_MCTNS_s for i in records]) TT_MCTNS_f = numpy.array([i.T_MCTNS_f for i in records]) starttime = min(starttimes) finaltime = max(finaltimes) T_MCTNS_s = TT_MCTNS_s[starttimes == starttime][0] T_MCTNS_f = TT_MCTNS_f[finaltimes == finaltime][0] if indices is not None: fmt = '%' + pathtemplate.split('%')[1] for i in range(len(fmt)): if fmt[i].isalpha(): break fmt = fmt[:i+1] path = pathtemplate.replace(fmt, '*') + ', *=' + flib.a2str([fmt % i for i in indices], maxlen=2, sep=',') else: fmt = None path = ', '.join(pathtemplate) TraceInfo.__init__(self, path, Nav, smplrate, delay, vrange, starttime, finaltime, T_MCTNS_s, T_MCTNS_f) self.records = records self.pathtemplate = pathtemplate self.indices = indices self.fmt = fmt def basename(self): "trace filename without common path and indices"; parts = self.path.split('*') parts[0] = os.path.basename(parts[0]) return '*'.join(parts) def dirname(self): "name of a common directory for the traces"; return os.path.dirname(self.path.split('*')[0]) def recordlist(self): "return a list of individual records contained in this set"; return self.records class ScopeLog(dict): """ A representation of a scope log file. Contains a dictionary of entries for each trace and a timestamp of the log file. Please note, that 'ScopeLog' is not derived from 'TimeLog'!!! """ LogFileNames = ['logT.dat', 'log.dat'] def __init__(self, logfile, list=[]): dict.__init__(self, list) self.order = [] self.logfile = logfile self.logfile_modified = os.path.getmtime(logfile) @staticmethod def load(path): """ Load a scope log file. If 'path' points to a directory, look for a logfile named 'logT.dat' or 'log.dat' in it, in this order. Otherwise load a logfile from a file pointed by 'path' itself. Return a dictionary with absolute trace filenames as keys and 'TraceInfo' objects as values. """ path = os.path.abspath(path) if os.path.isdir(path): for logfile in ScopeLog.LogFileNames: logfile = os.path.join(path, logfile) if os.path.isfile(logfile): break else: raise ValueError("No scope log file found in '%s'" % path) else: logfile = path path = os.path.dirname(logfile) scopelog = ScopeLog(logfile) handle = open(logfile, 'r') try: lineno = 0 for line in handle: lineno += 1 line = line.strip() if len(line) < 1 or not line[0].isdigit(): continue # on April 2009 quotes were introduced around filename. if line.count('"') == 2: tokens = [] pre, filename, post = line.split('"') tokens += pre.split('\t')[:-1] tokens += [filename] tokens += post.split('\t')[1:] else: tokens = line.split('\t') try: if len(tokens) == 10: # File Format 1: # (1) Date string # (2) Time string # (3) Time trace file name # (4) # averages acquired in trace # (5) Universal time (sec, initial) # (6) Universal time (sec, final) # (7) T_NSMCT (mK, initial) # (8) T_NSMCT (mK, final) # (9) PLM-4 M0 (final) # (10) PLM-4 Xmit pulse length record = TraceInfo(os.path.join(path, tokens[2]), int(tokens[3]), numpy.NaN, numpy.NaN, numpy.NaN, flib.labview2tm(float(tokens[4])), flib.labview2tm(float(tokens[5])), float(tokens[6]), float(tokens[7])) elif len(tokens) == 18: # File Format 2: # (1) Date string # (2) Time string # (3) Time trace file name # (4) No. of averages taken in trace # (5) Max. amplitude (V) # (6) f_peak (Hz) # (7) Initial T_NSMCT (mK) # (8) Final T_NSMCT (mK) # (9) Mean T_NSMCT (mK) # (10) PLM-4 M0 # (11) Mean "universal time" (sec) # (12) PLM-4 Xmit pulse length # (13) PXI 5922 acquisition delay (sec) # (14) Sampling rate (Hz) # (15) No. points acquired in trace # (16) PXI 5922 voltage range (+- x Vpp) # (17) Univ. time at start of trace (sec) # (18) Univ. time at end of trace (sec) #dir, filename, Nav, smplrate, delay, range, starttime, finaltime, T_MCTNS record = TraceInfo(os.path.join(path, tokens[2]), int(tokens[3]), float(tokens[13]), float(tokens[12]), float(tokens[15]), flib.labview2tm(float(tokens[16])), flib.labview2tm(float(tokens[17])), float(tokens[6]), float(tokens[7])) else: raise ValueError('%s: uknown log file format' % logfile) if scopelog.has_key(record.path): logging.warn("'%s' is featured twice in the logfile '%s'" % (record.basename(), logfile)) scopelog[record.path] = record scopelog.order.append(record.path) except Exception, e: logging.warn("Can't process line %d in '%s': %s" % (lineno, logfile, e)) finally: handle.close() return scopelog def isoutdated(self): "Return True is a logfile on disk is newer than this object" return os.path.getmtime(self.logfile) > self.logfile_modified def __repr__(self): return "scope log '%s' %d entries" % (self.logfile, len(self)) def dirname(self): return os.path.dirname(self.logfile) class ScopeLogCache(list): """ A cache for scope logs. The most recently demanded logs are kept in memory and a log is only read from the hard drive and parsed if it is not in the cache or if a cached version is outdated. Logs are kept in a list ordered from the most to the least recently demanded and if a critical number is exceeded, the log from the tail of the list is purged when adding a new one. """ def __init__(self, cachesize = 3): """ Create a cache. Maximum size can be specified. """ if cachesize < 1: raise ValueError('Cache size should be positive') self.cachesize = cachesize def setmaxsize(self, cachesize): if cachesize < 1: raise ValueError('Cache size should be positive') self.cachesize = cachesize while len(self) > cachesize: self.pop() def getinfo(self, trace): """ Return a 'TraceInfo' about a trace pointed by filename if available. Otherwise None is returned. """ trace = os.path.abspath(trace) dir = os.path.dirname(trace) for log in self: if not os.path.exists(log.dirname()) or not os.path.exists(log.logfile): # cache contains an entry that has been removed from the file system # purge it self.remove(log) continue if self.samefile(log.dirname(), dir): # remove the log from the list. # if no update is need, # it will be added to the head of the list later. self.remove(log) break else: log = None if log and log.isoutdated(): del log log = None if log is None: # no log was in the cache or a log in the cache was outdated. # try to load a log. try: log = ScopeLog.load(dir) except ValueError: # could not locate a log for the trace return None if len(self) >= self.cachesize: self.pop() self.insert(0, log) if trace not in log: return None else: return log[trace] @staticmethod def samefile(file1, file2): if 'samefile' in os.path.__dict__: return os.path.samefile(file1, file2) else: return os.path.abspath(file1).lower() == os.path.abspath(file2).lower() # a static cache __logcache__ = ScopeLogCache() def getinfo(trace): """ Return information about a trace refered by its path as a 'TraceInfo' object or 'None' if no appropriate log file was found. This function is a frontend to a static 'ScopeLogCache'. """ return __logcache__.getinfo(trace) def getinfos(filename, indices): """ Return information about a traces refered by path as a 'AveragedTraceInfo' object or 'None' if no appropriate log file was found for any of the traces. """ filenames = [filename % i for i in indices] if len(filenames) < 1: raise ValueError('No traces are going to be averaged') infos = [] for fn in filenames: info = getinfo(fn) if info is None: del infos return None infos.append(info) return AveragedTraceInfo(infos, filename, indices) def setcachesize(size): "Modify static log cache size"; __logcache__.setmaxsize(size) def getcachesize(): "Return static log cache size"; return __logcache__.cachesize def loadbinary(filename, smplrate = None): """ Loads signal from a binary file and fill all log information. 'smplrate' argument is used if sampling rate could not be inferred from the log (1st version). It should be specified in samples per second. """ if not os.path.exists(filename): raise ValueError("%s: no such file" % filename) info = getinfo(filename) if smplrate is None: if info is None: raise ValueError("Not scope log found for '%s'. Please specify sampling rate explicitely" % filename) if not (info.smplrate > 0): raise ValueError("Sampling rate was not logged for '%s'. Please specify it explicitely" % filename) if info is not None and info.smplrate > 0: smplrate = info.smplrate trace = nmr.Trace.loadbinary(filename, smplrate) if info is None: info = TraceInfo(os.path.abspath(filename), numpy.NaN, smplrate, numpy.NaN, numpy.NaN, os.path.getctime(filename), os.path.getmtime(filename), numpy.NaN, numpy.NaN) trace.info = info return trace def loadbinaries(filename, indices = None, smplrate = None): """ Loads signals from binary files, average them and fill all log information. If 'indices' is specified, 'filename' is a template for the signal file name, otherwise it is a list of files. 'smplrate' argument is used if sampling rate could not be inferred from the log (1st version). It should be specified in samples per second. """ if indices is not None: filenames = [filename % i for i in indices] else: filenames = filename if len(filenames) < 1: raise ValueError('No traces are going to be averaged') average = 0 N = 0 infos = [] for fn in filenames: trace = loadbinary(fn, smplrate) Nav = trace.info.safeNav() average += Nav * trace N += Nav infos.append(trace.info) average /= N average.info = AveragedTraceInfo(infos, filename, indices) return average def writelogline(trace, info, Npoints): trace = os.path.abspath(trace) dir = os.path.dirname(trace) for logfile in ScopeLog.LogFileNames[-1::-1]: logfile = os.path.join(dir, logfile) if os.path.exists(logfile): break if not os.path.exists(logfile): out = open(logfile, 'w') out.write("""# (1) Date string # (2) Time string # (3) Time trace file name # (4) No. of averages taken in trace # (5) Max. amplitude (V) # (6) f_peak (Hz) # (7) Initial T_NSMCT (mK) # (8) Final T_NSMCT (mK) # (9) Mean T_NSMCT (mK) # (10) PLM-4 M0 # (11) Mean "universal time" (sec) # (12) PLM-4 Xmit pulse length # (13) PXI 5922 acquisition delay (sec) # (14) Sampling rate (Hz) # (15) No. points acquired in trace # (16) channel 0 voltage range (Vpp) # (17) Univ. time at start of trace (sec) # (18) Univ. time at end of trace (sec) """) else: out = open(logfile, 'a') out.write('\t'.join([ # (1) Date string # (2) Time string time.strftime('%d/%m/%Y\t%H:%M:%S', time.gmtime(info.finaltime)), # (3) Time trace file name '"' + os.path.basename(trace) + '"', # (4) No. of averages taken in trace '%g' % info.safeNav(), # (5) Max. amplitude (V) # (6) f_peak (Hz) # (7) Initial T_NSMCT (mK) # (8) Final T_NSMCT (mK) # (9) Mean T_NSMCT (mK) # (10) PLM-4 M0 "0.0000000", "0.0000000", "0.00000", "0.00000", "0.00000", "0", # (11) Mean "universal time" (sec) "%.3f" % (0.5*(flib.tm2labview(info.starttime) + flib.tm2labview(info.finaltime))), # (12) PLM-4 Xmit pulse length "0", # (13) PXI 5922 acquisition delay (sec) "%.5e" % info.delay, # (14) Sampling rate (Hz) "%.5e" % info.smplrate, # (15) No. points acquired in trace "%d" % Npoints, # (16) channel 0 voltage range (Vpp) "%.5e" % info.vrange, # (17) Univ. time at start of trace (sec) "%.3f" % flib.tm2labview(info.starttime), # (18) Univ. time at end of trace (sec) "%.3f" % flib.tm2labview(info.finaltime), ]) + '\n') out.close()