From f37617bfa6b6183eeae5a3805dfa3c78b4ad12ce Mon Sep 17 00:00:00 2001 From: gregor Date: Fri, 30 Jan 2015 13:11:50 +0100 Subject: [PATCH 01/35] relocated find times test into separate test file and use a tmp diretory tree --- tests/test_findermethods.py | 33 +++++++++++++++++++++++++++++++++ tests/test_imports.py | 10 ---------- 2 files changed, 33 insertions(+), 10 deletions(-) create mode 100644 tests/test_findermethods.py diff --git a/tests/test_findermethods.py b/tests/test_findermethods.py new file mode 100644 index 0000000..9433731 --- /dev/null +++ b/tests/test_findermethods.py @@ -0,0 +1,33 @@ +import pytest +import os +import shutil + +from Owls import io + +@pytest.fixture +def create_directory_tree(tmpdir): + folders = range(10) + folders.extend([str(float(_)) for _ in folders]) + folders.extend(["1.23e-09"]) + ignored = ["system", "constant", "0-bck", "0.old"] + for folder in folders: + tmpdir.mkdir(str(folder)) + for folder in ignored: + tmpdir.mkdir(folder) + return tmpdir.dirname, folders, ignored + +def test_findtimes(create_directory_tree): + base, folders, ignored = create_directory_tree + def contains_all(files, exp): + for e in exp: + if str(e) in files: + continue + else: + print "Missing ", e + return False + return True + + found = io.find_times(base + "/test_findtimes0/") + assert contains_all( + found, + folders) diff --git a/tests/test_imports.py b/tests/test_imports.py index 1fa3d83..a866270 100644 --- a/tests/test_imports.py +++ b/tests/test_imports.py @@ -17,16 +17,6 @@ def test_imports(): # import Owls as ow # ow.read_sets(folder=basepath) -def test_findtimes(): - """ are all times and times in sets found """ - from Owls import io - def contains_all(res): - times = [str(_*50) for _ in range(21)] - return all([time in res for time in times]) - - assert contains_all(io.find_times(fold=basepath)) - assert contains_all(io.find_times(fold=setspath)) - def test_findDataFiles(): """ are all files in the the sets and times folder are found """ from Owls import io From bb166352d89adff92957a2fed18cc314ab64107b Mon Sep 17 00:00:00 2001 From: gregor Date: Mon, 2 Feb 2015 11:34:26 +0100 Subject: [PATCH 02/35] test finding datafiles with specified folder list --- tests/test_findermethods.py | 42 ++++++++++++++++++++++++++++--------- 1 file changed, 32 insertions(+), 10 deletions(-) diff --git a/tests/test_findermethods.py b/tests/test_findermethods.py index 9433731..4ab98b4 100644 --- a/tests/test_findermethods.py +++ b/tests/test_findermethods.py @@ -6,28 +6,50 @@ @pytest.fixture def create_directory_tree(tmpdir): + files = ["U", "UMean", "U_0", "foo.xy"] folders = range(10) folders.extend([str(float(_)) for _ in folders]) folders.extend(["1.23e-09"]) - ignored = ["system", "constant", "0-bck", "0.old"] + ignored = ["system", "constant", "0-bck", "0.old", "sets"] for folder in folders: - tmpdir.mkdir(str(folder)) + time_fold = tmpdir.mkdir(str(folder)) + for f in files: + handle = time_fold.join(f) + handle.write("foo") for folder in ignored: tmpdir.mkdir(folder) - return tmpdir.dirname, folders, ignored + folders_str = [str(f) for f in folders] + return tmpdir.dirname, folders_str, ignored, files def test_findtimes(create_directory_tree): - base, folders, ignored = create_directory_tree - def contains_all(files, exp): + """ based on the create_directory_tree fixture + a list of folder to be found and ignored is + passed, so we test if finds all in folders + and none of ignored + """ + from operator import truth + from operator import contains + from operator import not_ + + base, folders, ignored, _ = create_directory_tree + def _all(files, exp, negate=truth): for e in exp: - if str(e) in files: + if negate(contains(files,e)): continue else: - print "Missing ", e + print "Failed on ", e return False return True found = io.find_times(base + "/test_findtimes0/") - assert contains_all( - found, - folders) + assert _all(found, folders) + assert _all(found, ignored, negate=not_) + +def test_findDataFiles(create_directory_tree): + """ test if all files in the times folder are found """ + path, folders, ignored, files = create_directory_tree + search_folds = [path + "/test_findDataFiles0/" + fold + "/" for fold in folders] + found = io.find_datafiles(fold=search_folds) + for search_fold in search_folds: + for file_ in files: + assert search_fold + file_ in found[search_fold] From 0aab05cac0870fe255b23eb125efc0ca20e72e08 Mon Sep 17 00:00:00 2001 From: gregor Date: Mon, 2 Feb 2015 11:36:03 +0100 Subject: [PATCH 03/35] removed trailing ws, add more documentation to find_datafiles --- Owls/io.py | 76 ++++++++++++++++++++++++++++++------------------------ 1 file changed, 42 insertions(+), 34 deletions(-) diff --git a/Owls/io.py b/Owls/io.py index 7a33b49..606fc2a 100644 --- a/Owls/io.py +++ b/Owls/io.py @@ -59,12 +59,20 @@ def find_datafiles( subfolder="{}", ): """ Find all datafiles in each time folder, - - Returns a dictionary of lists containing data + + Returns a dictionary of lists containing data files for every found time step - - Subfolders: specify wheter to search in cwd or in a specific subfolder + + fold: list of time folders to look for data files + if False time folders in cwd will be taken + filelist: A list of file names which are accepted, + if false all files will be returned + + subfolders: specify wheter to search in cwd or in a specific subfolder accepting a search pattern + Returns: + Ordered dict with times as key and + list of found files """ search_folder = (subfolder if not subfolder.startswith('./{}') else "./{}/") try: @@ -78,9 +86,9 @@ def find_datafiles( return dict() def _get_datafiles_from_dir(path=False, fn_filter=False): - """ Return file names of Foam files from cwd if no path - is specified explicitly. - If no filter list is given the complete list of files will be returned + """ Return file names of Foam files from cwd if no path + is specified explicitly. + If no filter list is given the complete list of files will be returned else only files matching that list """ search_dir = (path if path else os.getcwd() + "/") @@ -137,14 +145,14 @@ def dataframe_to_foam(fullname, ftype, dataframe, boundaries): f.write("\n// ************************************************************************* //") class Origins(): - """ Class to manage fields to file relation and store hashes + """ Class to manage fields to file relation and store hashes dct = {'hash':34jd 0.0:{'hash':234s #time 'centreline':{'hash':94143e #loc 'U':filename,3424} } - } + } """ from collections import defaultdict def __init__(self): @@ -153,7 +161,7 @@ def __init__(self): @classmethod def from_dict(cls, dct): pass - + def to_dict(self): pass @@ -214,13 +222,13 @@ class ProgressBar(): """ A class providing progress bars """ def __init__(self, n_tot, bins=10): - #FEATURE: Add timings + #FEATURE: Add timings self.tot = float(n_tot) self.count = 0.0 self.cur = 0.0 def next(self): - self.count += 1.0 + self.count += 1.0 if self.count/self.tot > self.cur: print "#", self.cur += 0.1 @@ -237,11 +245,11 @@ def import_foam_folder( """ returns a Dataframe for every file in fileList """ #import StringIO from pandas import concat - + fileList = find_datafiles(subfolder=search_format, filelist=file_names) if not fileList: print "no files found" - return + return p_bar = ProgressBar(n_tot=sum([len(l) for l in fileList.itervalues()])) df = DataFrame() #df.index = MultiIndex.from_tuples(zip([],[]),names=['Loc',0]) @@ -259,7 +267,7 @@ def import_foam_folder( loc = x.index.values[-1][0] if df_tmp.empty: df_tmp = x - else: + else: try: # use combine first for all df at existing Loc or # if not Loc is specified (Eul or Lag fields) @@ -286,7 +294,7 @@ def import_foam_folder( """ Time Loc Pos U V -1000 radVel+10 0.1 +1000 radVel+10 0.1 2 radVel+20 0.1 0.2 @@ -294,11 +302,11 @@ def import_foam_folder( def foam_to_csv(fn, ): - """ helper function for d3.js data conversion + """ helper function for d3.js data conversion prints data directly to std:out """ import re - try: + try: with open(fn) as f: content = f.readlines() start, num_entries = if_header_skip(content) @@ -326,7 +334,7 @@ def read_boundary_names(fn): pass def read_data_file(fn, skiplines=1, maxlines=False): - """ A function to read any foam data files returning data and + """ A function to read any foam data files returning data and index after header """ @@ -350,7 +358,7 @@ def read_data_file(fn, skiplines=1, maxlines=False): df = DataFrame(data=data, columns=names) if loc: df['Loc'] = loc - else: + else: df['Loc'] = range(len(df)) df.set_index('Loc', append=True, inplace=True) df.index.names=['Id','Loc'] @@ -366,7 +374,7 @@ def read_data_file(fn, skiplines=1, maxlines=False): df = DataFrame(data=data, columns=[field]) df['Loc'] = "Field" df.set_index('Loc', append=True, inplace=True) - df.index.names=['Id','Loc'] + df.index.names=['Id','Loc'] df = df.reorder_levels(['Loc','Id']) hashes={field: int(hashlib.md5(str(data)).hexdigest(),16)} return field, df, hashes @@ -384,9 +392,9 @@ def hash_series(series): def evaluate_names(fullfilename, num_entries): - """ Infere field names and Loc from given filename + """ Infere field names and Loc from given filename - Example: + Example: U -> Field, [u,v,w] centreLine_U.xy -> centreLine, [Pos,u,v,w] """ @@ -420,16 +428,16 @@ def req_file(file_name, requested): def import_logs(folder, keys): """ keys = {"ExectionTime": ["ExecTime", "ClockTime"]} - - return a DataFrame - + + return a DataFrame + Loc, Time KeyName1 Keyname2 - 1 0.1 - + 1 0.1 + 0.2 2 - + """ def find_start(log): """ Fast forward through file till 'Starting time loop' """ @@ -439,14 +447,14 @@ def find_start(log): def extract(line, keys): - """ + """ returns key and values as list "ExecutionTime":[0,1] """ import re for key, col_names in keys.iteritems(): - if re.search(key, line): - return col_names, map(float,filter(lambda x: + if re.search(key, line): + return col_names, map(float,filter(lambda x: x, re.findall("[0-9]+[.]?[0-9]*[e]?[\-]?[0-9]*", line))) return None, None @@ -454,7 +462,7 @@ def extract(line, keys): logs = [fold + "/" + log for log in files if 'log' in log] p_bar = ProgressBar(n_tot = len(logs)) # Lets make sure that we find Timesteps in the log - keys.update({"^Time = ": ['Time']}) + keys.update({"^Time = ": ['Time']}) for log_number, log_name in enumerate(logs): with open(log_name) as log: @@ -470,7 +478,7 @@ def extract(line, keys): # a new time step has begun # flush datadict and concat to df # Very slow but, so far the solution - # to keep subiterations attached to correct time + # to keep subiterations attached to correct time # FIXME: still needs handling of different length dictionaries df = concat([df,DataFrame(dataDict)]) dataDict = defaultdict(list) From 4672bee964994e0e51b3c94867e7176002b601de Mon Sep 17 00:00:00 2001 From: gregor Date: Mon, 2 Feb 2015 11:42:00 +0100 Subject: [PATCH 04/35] removed trailing FIXME label --- Owls/io.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Owls/io.py b/Owls/io.py index 606fc2a..612953c 100644 --- a/Owls/io.py +++ b/Owls/io.py @@ -79,7 +79,7 @@ def find_datafiles( times = (fold if fold else find_times(search_folder.format(""))) od = OrderedDict() return OrderedDict( - [(time, _get_datafiles_from_dir(subfolder.format(time), filelist)) #FIXME use ordered dict here + [(time, _get_datafiles_from_dir(subfolder.format(time), filelist)) for time in times] ) except: From a8221421333f534842a6f4e271fcacff4b755d25 Mon Sep 17 00:00:00 2001 From: gregor Date: Thu, 5 Feb 2015 09:40:25 +0100 Subject: [PATCH 05/35] replaced search pattern by regex, removed os.chdir(), fixed trailing ws --- Owls/frames.py | 52 ++++++++++++++++++++++++++------------------------ 1 file changed, 27 insertions(+), 25 deletions(-) diff --git a/Owls/frames.py b/Owls/frames.py index e65ccd7..52df1af 100644 --- a/Owls/frames.py +++ b/Owls/frames.py @@ -21,27 +21,27 @@ def items_from_dict(dict, func, **kwargs): return Cases([func(folder=folder,name=name, symb=symb, **kwargs) for name, (folder,symb) in dict.iteritems()]) -def read_sets(folder, name="None", search="./sets/{}/", **kwargs): +def read_sets(folder, name="None", search="(postProcessing)*sets/" + io.FPNUMBER, **kwargs): return FoamFrame(folder=folder, search_files=False, search_pattern=search, name=name, show_func="plot", **kwargs) def read_lag(folder, files, skiplines=1, - name="None", cloud="coalCloud1", **kwargs + name="None", cloud="[A-Za-z]*Cloud1", **kwargs ): return FoamFrame(folder=folder, search_files=files, - search_pattern= "./{}/" + "lagrangian/{}/".format(cloud), + search_pattern=io.FPNUMBER + "/lagrangian/{}".format(cloud), name=name, skiplines=skiplines, show_func="scatter", **kwargs) def read_eul(folder, files, skiplines=1, name="None", **kwargs): return FoamFrame(folder=folder, search_files=files, - search_pattern="./{}/", name=name, + search_pattern=io.FPNUMBER, name=name, skiplines=skiplines, show_func="scatter", **kwargs) def read_exp(folder, name="None", **kwargs): #FIXME make it read exp/*dat directly without 0 folder return FoamFrame(folder=folder, search_files=False, - search_pattern="./{}/", name=name, show_func="scatter", **kwargs) + search_pattern=io.FPNUMBER, name=name, show_func="scatter", **kwargs) def read_log(folder, keys, log_name='*log*', plot_properties=False): @@ -106,7 +106,7 @@ def multi_merge(*args, **kwargs): # select by regex # now see if we have a match selector = kwargs.get('by', "[A-Za-z0-9_\-]") - # skip if search is empty + # skip if search is empty if (not re.search(selector, name) or not re.search(selector, name_)): continue @@ -151,10 +151,10 @@ def select(self, case): def filter(self, selector): """ select a specific item """ if type(selector) == list: - return MultiItem({name:case for name,case in self.cases if + return MultiItem({name:case for name,case in self.cases if name in selector}) else: - return MultiItem({name:case for name,case in self.cases if + return MultiItem({name:case for name,case in self.cases if func(name)}) def iteritems(self): @@ -164,21 +164,21 @@ def iteritems(self): def by(self, overlay=True): """ recursiv grouping function - + Examples: - + mi.by(overlay=True) -> { cat1_1:{cat2_1:FoamFrame1, cat2_2:FoamFrame2, ... } cat1_2:{cat2_1:FoamFrame3, ... } } - + m1.by(overlay=False) -> { (cat1_1,cat2_1): FoamFrame1, (cat1_1,cat2_2): FoamFrame2, ... } - + needs .show() to check if self.data is recursive """ pass @@ -290,8 +290,8 @@ class FoamFrame(DataFrame): { "rad_pos": lambda field -> position "centreLine": [] lambda field -> i of [] - - } + + } example: lambda field: re.search('[0-9]*\.[0-9]*').group()[0] @@ -313,10 +313,11 @@ def __init__(self, *args, **kwargs): files = kwargs.get('search_files', None) properties = kwargs.get('properties', None) lines = kwargs.get('maxlines', 0) - search = kwargs.get('search_pattern', "{}") + search = kwargs.get('search_pattern', io.FPNUMBER) folder = kwargs.get('folder', None) plot_properties = kwargs.get('plot_properties', None) show_func = kwargs.get('show_func', None) + validate = kwargs.get('validate', True) keys = [ 'skiplines', @@ -336,20 +337,20 @@ def __init__(self, *args, **kwargs): except: pass - #TODO explain what happens here + #TODO explain what happens here if folder == None: #super(FoamFrame, self).__init__(*args, **kwargs) DataFrame.__init__(self, *args, **kwargs) else: - os.chdir(folder) #FIXME necessary for read in? if case_data_base.has_key(folder): print "re-importing", else: print "importing", print name + ": ", origins, data = io.import_foam_folder( - search_format=search, - file_names=files, + path=folder, + search=search, + files=files, skiplines=skip, maxlines=lines, ) @@ -363,7 +364,8 @@ def __init__(self, *args, **kwargs): data.index.levels[0], symb, show_func) - self.validate_origins(folder, origins) + if validate: + self.validate_origins(folder, origins) # register to database case_data_base.sync() @@ -449,7 +451,7 @@ def latest(self): # pass # # def get_hashes(self): - # """ returns hashes of current selection based + # """ returns hashes of current selection based # on the data read from disk """ # pass @@ -613,15 +615,15 @@ def vars(self): # print "%s Warning: requested field %s not in data base" %(self.name, field) # print e # return Series() - + # def __str__(self): # return """Foam case object # Data Fields: {} - # Total number of items {} + # Total number of items {} # Data root: {}""".format( # str([_ for _ in self.vars]), "unknown", self.folder) - + # def reread(self): - # """ re-read foam data """ + # """ re-read foam data """ # self.origins, self.data = self._read_data() From 2b5f0356faeaea9e9ebfc8282e49544edce20ca7 Mon Sep 17 00:00:00 2001 From: gregor Date: Thu, 5 Feb 2015 09:41:20 +0100 Subject: [PATCH 06/35] implemented regex based datafolder search --- Owls/io.py | 73 ++++++++++++++++++++++++++++++++++-------------------- 1 file changed, 46 insertions(+), 27 deletions(-) diff --git a/Owls/io.py b/Owls/io.py index 612953c..8965630 100644 --- a/Owls/io.py +++ b/Owls/io.py @@ -18,8 +18,8 @@ from collections import defaultdict from collections import OrderedDict from IPython.display import display, clear_output -from frames import * -from plot import * + +FPNUMBER = "[-+]?[0-9]*\.?[0-9]+([eE][-+]?[0-9]+)?" FOAM_HEADER = """ /*--------------------------------*- C++ -*----------------------------------*\\ @@ -54,9 +54,9 @@ def match(d, event): return d[reg_exp_key] def find_datafiles( - fold=False, - filelist=False, - subfolder="{}", + path=False, + files=False, + search=FPNUMBER, ): """ Find all datafiles in each time folder, @@ -66,24 +66,29 @@ def find_datafiles( fold: list of time folders to look for data files if False time folders in cwd will be taken filelist: A list of file names which are accepted, - if false all files will be returned - - subfolders: specify wheter to search in cwd or in a specific subfolder - accepting a search pattern + if false all files will be returned + subfolder: specify wheter to search in cwd or in a specific subfolder + accepting a search pattern Returns: Ordered dict with times as key and list of found files """ - search_folder = (subfolder if not subfolder.startswith('./{}') else "./{}/") - try: - times = (fold if fold else find_times(search_folder.format(""))) - od = OrderedDict() - return OrderedDict( - [(time, _get_datafiles_from_dir(subfolder.format(time), filelist)) - for time in times] - ) - except: - return dict() + data_folders = find_datafolders(search, path) + return OrderedDict( + [(time, _get_datafiles_from_dir(time, files)) + for time in data_folders] + ) + +def find_datafolders(regex, path=False): + """ Find data folders according to regex + replaces old find_times function + Returns sorted list of times as strings """ + search_folder = (path if path else os.getcwd()) + complete_regex = search_folder + regex + "$" + folders = [fold for fold,_,_ in os.walk(search_folder) + if re.match(complete_regex,fold)] + folders.sort() + return folders def _get_datafiles_from_dir(path=False, fn_filter=False): """ Return file names of Foam files from cwd if no path @@ -91,13 +96,14 @@ def _get_datafiles_from_dir(path=False, fn_filter=False): If no filter list is given the complete list of files will be returned else only files matching that list """ - search_dir = (path if path else os.getcwd() + "/") - cur_dir = os.walk(search_dir) + path = (path if path else os.getcwd() + "/") + path = (path + "/" if not path.endswith("/") else path) + cur_dir = os.walk(path) root, dirs, files = next(cur_dir) if fn_filter: - l = [search_dir + f for f in files if f in fn_filter] + l = [path + f for f in files if f in fn_filter] else: - l = [search_dir + f for f in files if not f.startswith('.')] + l = [path + f for f in files if not f.startswith('.')] l.sort() return l @@ -236,9 +242,19 @@ def next(self): def done(self): print "[done]", +def strip_time(path, base): + """ try to extract time from path """ + wo_base = path.replace(base,'') + match = re.match(wo_base, FPNUMBER) + if match: + return float(match.group()[0]) + else: + return 0.0 + def import_foam_folder( - search_format, - file_names, + path, + search, + files, skiplines=1, maxlines=0, ): @@ -246,7 +262,9 @@ def import_foam_folder( #import StringIO from pandas import concat - fileList = find_datafiles(subfolder=search_format, filelist=file_names) + if not path.endswith('/'): + path = path + '/' + fileList = find_datafiles(path, search=search, files=files) if not fileList: print "no files found" return @@ -256,6 +274,7 @@ def import_foam_folder( from collections import defaultdict origins = Origins() for time, files in fileList.iteritems(): #FIXME dont iterate twice + time = strip_time(time, path) df_tmp = DataFrame() for fn in files: #ret = read_table(StringIO.StringIO(foam_to_csv(fn))) @@ -280,6 +299,7 @@ def import_foam_folder( except Exception as e: print x print e + field_names = ([field_names] if not type(field_names) == list else field_names) for field in field_names: origins.insert(time,loc,field,fn,hashes[field]) df_tmp['Time'] = float(time) @@ -305,7 +325,6 @@ def foam_to_csv(fn, ): """ helper function for d3.js data conversion prints data directly to std:out """ - import re try: with open(fn) as f: content = f.readlines() From 157bc6e09bf3b7160636c73e846b6f9788a2b222 Mon Sep 17 00:00:00 2001 From: gregor Date: Thu, 5 Feb 2015 09:42:15 +0100 Subject: [PATCH 07/35] test finder methods, test new find_datafolders --- tests/test_findermethods.py | 89 ++++++++++++++++++++++++++++--------- 1 file changed, 67 insertions(+), 22 deletions(-) diff --git a/tests/test_findermethods.py b/tests/test_findermethods.py index 4ab98b4..7e8bb28 100644 --- a/tests/test_findermethods.py +++ b/tests/test_findermethods.py @@ -6,50 +6,95 @@ @pytest.fixture def create_directory_tree(tmpdir): + def create_files(fold, files): + for f in files: + handle = fold.join(f) + handle.write("foo") + + def create_times(basedir, times): + for time in times: + time_fold = basedir.mkdir(str(time)) + yield time_fold + files = ["U", "UMean", "U_0", "foo.xy"] folders = range(10) folders.extend([str(float(_)) for _ in folders]) - folders.extend(["1.23e-09"]) - ignored = ["system", "constant", "0-bck", "0.old", "sets"] - for folder in folders: - time_fold = tmpdir.mkdir(str(folder)) - for f in files: - handle = time_fold.join(f) - handle.write("foo") + folders.extend(["1.23e-09"]) + ignored = ["system", "constant", "0-bck", "0.old"] + for folder in create_times(tmpdir, folders): + particle_fold = folder.mkdir("lagrangian").mkdir("particleCloud1") + create_files(folder, files) + create_files(particle_fold, files) for folder in ignored: tmpdir.mkdir(folder) + for sets_folder in create_times(tmpdir.mkdir("sets"), folders): + create_files(sets_folder, files) folders_str = [str(f) for f in folders] + ignored.append("sets") return tmpdir.dirname, folders_str, ignored, files -def test_findtimes(create_directory_tree): - """ based on the create_directory_tree fixture +def test_finddatafolders(create_directory_tree): + """ based on the create_directory_tree fixture a list of folder to be found and ignored is passed, so we test if finds all in folders - and none of ignored + and none of ignored """ from operator import truth from operator import contains from operator import not_ + test_dir = "/test_finddatafolders0/" + base, folders, ignored, _ = create_directory_tree - def _all(files, exp, negate=truth): + def _all(base, files, exp, negate=truth, expansion=""): for e in exp: - if negate(contains(files,e)): + abs_e = base + e + expansion + # print abs_e, files + if negate(contains(files, abs_e)): continue else: print "Failed on ", e return False return True - - found = io.find_times(base + "/test_findtimes0/") - assert _all(found, folders) - assert _all(found, ignored, negate=not_) + + eulerian = io.find_datafolders( + regex=io.FPNUMBER, + path=base + test_dir + ) + assert _all(base+test_dir, eulerian, folders) + assert _all(base+test_dir, eulerian, ignored, negate=not_) + + lagrangian = io.find_datafolders( + regex=io.FPNUMBER + "/lagrangian/[\w]*Cloud1", + path=base + test_dir + ) + assert _all(base+test_dir, eulerian, ignored, negate=not_) + assert _all(base+test_dir, lagrangian, ignored, negate=not_, + expansion="/lagrangian/particleCloud1") + + sets = io.find_datafolders( + regex= "sets/" + io.FPNUMBER, + path=base + test_dir + ) + assert _all(base+test_dir + "sets/", sets, folders) + assert _all(base+test_dir + "sets/", sets, ignored, negate=not_) def test_findDataFiles(create_directory_tree): """ test if all files in the times folder are found """ - path, folders, ignored, files = create_directory_tree - search_folds = [path + "/test_findDataFiles0/" + fold + "/" for fold in folders] - found = io.find_datafiles(fold=search_folds) - for search_fold in search_folds: - for file_ in files: - assert search_fold + file_ in found[search_fold] + base, folders, ignored, files = create_directory_tree + test_dir = base + "/test_findDataFiles0/" + + def _test(testdir, folders, files, result, extra=""): + for search_fold in folders: + for file_ in files: + target = testdir + "{}{}/{}".format(search_fold, extra, file_) + assert target in result[testdir + search_fold + extra] + + eulerian = io.find_datafiles(path=test_dir) + _test(test_dir, folders, files, eulerian) + + sets = io.find_datafiles(path=test_dir, search="sets/" + io.FPNUMBER) + _test(test_dir + "sets/", folders, files, sets) + + lagrangian = io.find_datafiles(path=test_dir, search=io.FPNUMBER + "/lagrangian/[\w]*Cloud[0-9]?") + _test(test_dir, folders, files, lagrangian, extra="/lagrangian/particleCloud1") From 755f4aad44d7a8fe0693ac5b8714141c95dbaa87 Mon Sep 17 00:00:00 2001 From: gregor Date: Thu, 5 Feb 2015 09:43:31 +0100 Subject: [PATCH 08/35] refactored tests, still needs more work --- tests/test_imports.py | 36 +++++++++++++----------------------- tests/test_integration.py | 14 ++++++++++++++ 2 files changed, 27 insertions(+), 23 deletions(-) create mode 100644 tests/test_integration.py diff --git a/tests/test_imports.py b/tests/test_imports.py index a866270..3569dac 100644 --- a/tests/test_imports.py +++ b/tests/test_imports.py @@ -1,32 +1,22 @@ import os -basepath = os.getcwd() + "/examples/buoyantCavity" -setspath = basepath + "/sets" - - -setsfiles = ['y0.1_T.xy', 'y0.1_U.xy', 'y0.2_T.xy', 'y0.2_U.xy', 'y0.3_T.xy', 'y0.3_U.xy', 'y0.4_T.xy', 'y0.4_U.xy', 'y0.5_T.xy', 'y0.5_U.xy', 'y0.6_T.xy', 'y0.6_U.xy', 'y0.7_T.xy', 'y0.7_U.xy', 'y0.8_T.xy', 'y0.8_U.xy', 'y0.9_T.xy', 'y0.9_U.xy'] - -basefiles = ['T', 'U', 'alphat', 'k', 'mut', 'omega', 'p', 'p_rgh', 'phi'] +exec_path = "/home/go/documents/code/Owls" def test_imports(): """ are the main modules importable """ from Owls import io from Owls import frames -# def test_find_times(): -# import Owls as ow -# ow.read_sets(folder=basepath) - -def test_findDataFiles(): - """ are all files in the the sets and times folder are found """ +def test_importFoamFolder(): from Owls import io - read_folder = lambda x=False,filt=False: [path.replace(x,'') for path in io._get_datafiles_from_dir(x, filt)] - setsfolder = setspath + "/1000/" - basefolder = basepath + "/1000/" - datafilessets = read_folder(setsfolder) - datafilesbase = read_folder(basefolder) - assert setsfiles == datafilessets - assert basefiles == datafilesbase - for fn in basefiles: - # print read_folder(basefolder,fn) - assert [fn] == read_folder(basefolder,[fn]) + path = exec_path + "/examples/buoyantCavity/" + euls = io.import_foam_folder( + path=path, + search=io.FPNUMBER, + files=['T','U'], + ) + sets = io.import_foam_folder( + path=path, + search="sets/" + io.FPNUMBER, + files=False, + ) diff --git a/tests/test_integration.py b/tests/test_integration.py new file mode 100644 index 0000000..49c4b15 --- /dev/null +++ b/tests/test_integration.py @@ -0,0 +1,14 @@ +import pytest + +import os +import Owls as ow + +exec_path = os.getcwd() + +def test_readEul(): + path = exec_path + "/examples/buoyantCavity" + ow.read_eul(folder=path, files=["T","U"], validate=False) + +def test_readSets(): + path = exec_path + "/examples/buoyantCavity" + ow.read_sets(folder=path, validate=False) From e466a6b64543066c510987bf140c54f535319bbb Mon Sep 17 00:00:00 2001 From: gregor Date: Sun, 8 Feb 2015 10:11:44 +0100 Subject: [PATCH 09/35] try except hashing --- Owls/frames.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/Owls/frames.py b/Owls/frames.py index e65ccd7..6706c02 100644 --- a/Owls/frames.py +++ b/Owls/frames.py @@ -392,7 +392,10 @@ def validate_origins(self, folder, origins): loc_name, loc_hash = loc field_name, field_hash = field filename, item_hash = item - orig_hash = case_data_base[folder][time_name][loc_name][field_name][1] + try: + orig_hash = case_data_base[folder][time_name][loc_name][field_name][1] + except: + orig_hash = item_hash if (item_hash != orig_hash): print "" print "corrupted fields:" From e9b21a475b9b5018a6ce2c269479052d303fbc3b Mon Sep 17 00:00:00 2001 From: gregor Date: Sun, 8 Feb 2015 10:21:29 +0100 Subject: [PATCH 10/35] add preHooks execution --- Owls/__init__.py | 1 + Owls/foam.py | 58 ++++++++++++++++++++++++++++++++++++++++++++++++ Owls/frames.py | 8 +++++-- Owls/io.py | 7 +++--- 4 files changed, 69 insertions(+), 5 deletions(-) create mode 100644 Owls/foam.py diff --git a/Owls/__init__.py b/Owls/__init__.py index 63b3ff7..faa6225 100644 --- a/Owls/__init__.py +++ b/Owls/__init__.py @@ -8,6 +8,7 @@ from .version import __version__ from io import * from plot import * +from foam import * from frames import * print "Owls Version: " + __version__ diff --git a/Owls/foam.py b/Owls/foam.py new file mode 100644 index 0000000..ed0f553 --- /dev/null +++ b/Owls/foam.py @@ -0,0 +1,58 @@ +""" +Launchers for openfoam utilities + +""" +import os +import subprocess + +def execute(cmd): + return subprocess.check_call(cmd, shell=True) == 0 + +def execute_in_path(path, cmd, func=execute): + print(os.getcwd()) + old_dir = os.getcwd() + os.chdir(path) + try: + ret = func(cmd) + except: + ret = False + os.chdir(old_dir) + return ret + +class genericHook(object): + + def __init__(self, path, cmd): + self.path = path + self.cmd = cmd + + def execute(self)reconstructPa + return execute_in_path(self.path, self.cmd) + +class cellCentres(genericHook): + + def __init__(self, path): + genericHook.__init__(self, path, "writeCellCentres") + +class decompose(genericHook): + + def __init__(self, path, latest=False): + cmd = "decomposePar" + if latest: + cmd += " -latestTime" + genericHook.__init__(self, path, cmd) + +class reconstruct(genericHook): + + def __init__(self, path, latest=False): + cmd = "reconstructPar" + if latest: + cmd += " -latestTime" + genericHook.__init__(self, path, cmd) + +class sample(genericHook): + + def __init__(self, path, latest=False): + cmd = "sample" + if latest: + cmd += " -latestTime" + genericHook.__init__(self, path, cmd) diff --git a/Owls/frames.py b/Owls/frames.py index 52df1af..b9b194f 100644 --- a/Owls/frames.py +++ b/Owls/frames.py @@ -32,10 +32,10 @@ def read_lag(folder, files, skiplines=1, search_pattern=io.FPNUMBER + "/lagrangian/{}".format(cloud), name=name, skiplines=skiplines, show_func="scatter", **kwargs) -def read_eul(folder, files, skiplines=1, name="None", **kwargs): +def read_eul(folder, files, skiplines=1, name="None", preHooks=None, **kwargs): return FoamFrame(folder=folder, search_files=files, search_pattern=io.FPNUMBER, name=name, - skiplines=skiplines, show_func="scatter", + skiplines=skiplines, show_func="scatter", preHooks=preHooks, **kwargs) def read_exp(folder, name="None", **kwargs): @@ -318,6 +318,7 @@ def __init__(self, *args, **kwargs): plot_properties = kwargs.get('plot_properties', None) show_func = kwargs.get('show_func', None) validate = kwargs.get('validate', True) + preHooks = kwargs.get('preHooks', None) keys = [ 'skiplines', @@ -342,6 +343,9 @@ def __init__(self, *args, **kwargs): #super(FoamFrame, self).__init__(*args, **kwargs) DataFrame.__init__(self, *args, **kwargs) else: + if preHooks: + for hook in preHooks: + hook.execute() if case_data_base.has_key(folder): print "re-importing", else: diff --git a/Owls/io.py b/Owls/io.py index 8965630..4fbf710 100644 --- a/Owls/io.py +++ b/Owls/io.py @@ -245,9 +245,10 @@ def done(self): def strip_time(path, base): """ try to extract time from path """ wo_base = path.replace(base,'') - match = re.match(wo_base, FPNUMBER) + match = re.search(FPNUMBER, wo_base) if match: - return float(match.group()[0]) + time = float(match.group()) + return time else: return 0.0 @@ -302,7 +303,7 @@ def import_foam_folder( field_names = ([field_names] if not type(field_names) == list else field_names) for field in field_names: origins.insert(time,loc,field,fn,hashes[field]) - df_tmp['Time'] = float(time) + df_tmp['Time'] = time if df.empty: df = df_tmp else: From c3cd19a652a0175a522afc6d98001f1e42f446f2 Mon Sep 17 00:00:00 2001 From: gregor Date: Sun, 15 Feb 2015 21:15:10 +0100 Subject: [PATCH 11/35] rename MultiItem to MultiFrame, moved code to MultiFrame.py --- Owls/MultiFrame.py | 105 ++++++++++++++++++++++++++++++++++++++++ Owls/frames.py | 116 +++------------------------------------------ 2 files changed, 112 insertions(+), 109 deletions(-) create mode 100644 Owls/MultiFrame.py diff --git a/Owls/MultiFrame.py b/Owls/MultiFrame.py new file mode 100644 index 0000000..07d05df --- /dev/null +++ b/Owls/MultiFrame.py @@ -0,0 +1,105 @@ +class MultiFrame(): + """ Class for storage of multiple case items + or faceted data from FoamFrame + """ + #TODO: implememt multi-facetting + # e.g. (cases.by_index('Loc') <- returns a MultiFrame + # .by_case(overlay=True) <- MultiFrame method + # .show('T') + #TODO: implement __repr__ method + def __init__(self, cases=None): + if type(cases) == list: + self.cases = OrderedDict([(case.name,case) for case in cases]) + elif type(cases) == OrderedDict: + self.cases=cases + else: + self.cases={} + + def __getitem__(self, field): + return [serie[field] for serie in self.cases.itervalues()] + + def names(self): + return [name for name in self.cases] + + def select(self, case): + """ select a specific item """ + return self.cases[case] + + def filter(self, selector): + """ select a specific item """ + if type(selector) == list: + return MultiFrame({name:case for name,case in self.cases if + name in selector}) + else: + return MultiFrame({name:case for name,case in self.cases if + func(name)}) + + def iteritems(self): + for name,case in self.cases.iteritems(): + yield name,case + + def by(self, overlay=True): + """ + recursiv grouping function + + Examples: + + mi.by(overlay=True) -> { cat1_1:{cat2_1:FoamFrame1, + cat2_2:FoamFrame2, + ... } + cat1_2:{cat2_1:FoamFrame3, + ... } + } + + m1.by(overlay=False) -> { (cat1_1,cat2_1): FoamFrame1, + (cat1_1,cat2_2): FoamFrame2, + ... + } + + needs .show() to check if self.data is recursive + """ + pass + + def scatter(self, y, x='Pos', z=False, overlay=False, **kwargs): + import bokeh.plotting as bk + return self._draw(x, y, z=z, overlay=overlay, + inst_func="scatter", **kwargs) + + def plot(self, y, x='Pos', z=False, overlay=False, **kwargs): + return self._draw(x, y, z=z, overlay=overlay, + inst_func="plot", **kwargs) + + def show(self, y, x='Pos', z=False, overlay=False, **kwargs): + return self._draw(x, y, z=z, overlay=overlay, + inst_func="show", **kwargs) + + def _draw(self, x, y, z, overlay, inst_func, **kwargs): + import bokeh.plotting as bk + import numpy as np + def greatest_divisor(number): + if number == 1: + return 1 + for i in reversed(range(number)): + if number % i == 0: + return i + else: + return 1 + + if not overlay: + rows=[] + for name, instance in self.cases.iteritems(): + bk.figure() + rows.append( + getattr(instance, inst_func) + (x=x, y=y, title=str(name), **kwargs) #FIXME num cars + ) + rows = np.array(rows).reshape(greatest_divisor(len(rows)),-1).tolist() + return bk.GridPlot(children=rows, title="Scatter") + else: + bk.hold() + colors = plt.next_color() + for name, instance in self.cases.iteritems(): + color = next(colors) + getattr(instance, inst_func)(x=x, y=y, title="", color=color, legend=name, **kwargs) + bk.hold(False) + return bk.curplot() diff --git a/Owls/frames.py b/Owls/frames.py index b9b194f..6d3c691 100644 --- a/Owls/frames.py +++ b/Owls/frames.py @@ -1,6 +1,5 @@ import os import re -import io import shelve import plot as plt from collections import OrderedDict @@ -9,6 +8,10 @@ from pandas import DataFrame from pandas import concat +from MultiFrame import MultiFrame +import io + + Series.__repr__ = (lambda x: ("Hash: {}\nTimes: {}\nLoc: {}\nValues: {}".format( io.hash_series(x), list(set(x.keys().get_level_values('Time'))), # avoid back and forth conversion @@ -121,111 +124,6 @@ def multi_merge(*args, **kwargs): plots.append(merge(*sub_plots, x=x, y=y, title=name, colors=colors)) return plots -class MultiItem(): - """ Class for storage of multiple case items - or faceted data from FoamFrame - """ - #TODO: implememt multi-facetting - # e.g. (cases.by_index('Loc') <- returns a MultiItem - # .by_case(overlay=True) <- MultiItem method - # .show('T') - #TODO: implement __repr__ method - def __init__(self, cases=None): - if type(cases) == list: - self.cases = OrderedDict([(case.name,case) for case in cases]) - elif type(cases) == OrderedDict: - self.cases=cases - else: - self.cases={} - - def __getitem__(self, field): - return [serie[field] for serie in self.cases.itervalues()] - - def names(self): - return [name for name in self.cases] - - def select(self, case): - """ select a specific item """ - return self.cases[case] - - def filter(self, selector): - """ select a specific item """ - if type(selector) == list: - return MultiItem({name:case for name,case in self.cases if - name in selector}) - else: - return MultiItem({name:case for name,case in self.cases if - func(name)}) - - def iteritems(self): - for name,case in self.cases.iteritems(): - yield name,case - - def by(self, overlay=True): - """ - recursiv grouping function - - Examples: - - mi.by(overlay=True) -> { cat1_1:{cat2_1:FoamFrame1, - cat2_2:FoamFrame2, - ... } - cat1_2:{cat2_1:FoamFrame3, - ... } - } - - m1.by(overlay=False) -> { (cat1_1,cat2_1): FoamFrame1, - (cat1_1,cat2_2): FoamFrame2, - ... - } - - needs .show() to check if self.data is recursive - """ - pass - - def scatter(self, y, x='Pos', z=False, overlay=False, **kwargs): - import bokeh.plotting as bk - return self._draw(x, y, z=z, overlay=overlay, - inst_func="scatter", **kwargs) - - def plot(self, y, x='Pos', z=False, overlay=False, **kwargs): - return self._draw(x, y, z=z, overlay=overlay, - inst_func="plot", **kwargs) - - def show(self, y, x='Pos', z=False, overlay=False, **kwargs): - return self._draw(x, y, z=z, overlay=overlay, - inst_func="show", **kwargs) - - def _draw(self, x, y, z, overlay, inst_func, **kwargs): - import bokeh.plotting as bk - import numpy as np - def greatest_divisor(number): - if number == 1: - return 1 - for i in reversed(range(number)): - if number % i == 0: - return i - else: - return 1 - - if not overlay: - rows=[] - for name, instance in self.cases.iteritems(): - bk.figure() - rows.append( - getattr(instance, inst_func) - (x=x, y=y, title=name, **kwargs) #FIXME num cars - ) - rows = np.array(rows).reshape(greatest_divisor(len(rows)),-1).tolist() - return bk.GridPlot(children=rows, title="Scatter") - else: - bk.hold() - colors = plt.next_color() - for name, instance in self.cases.iteritems(): - color = next(colors) - getattr(instance, inst_func)(x=x, y=y, title="", color=color, legend=name, **kwargs) - bk.hold(False) - return bk.curplot() class PlotProperties(): @@ -557,7 +455,7 @@ def filter(self, name, index=None, field=None): .filter(name='Loc', index=lambda x: 0.2 Date: Wed, 18 Feb 2015 10:07:44 +0100 Subject: [PATCH 12/35] FEAT skiptimes, init empty PlotProps as default --- Owls/{frames.py => FoamFrame.py} | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) rename Owls/{frames.py => FoamFrame.py} (99%) diff --git a/Owls/frames.py b/Owls/FoamFrame.py similarity index 99% rename from Owls/frames.py rename to Owls/FoamFrame.py index 6d3c691..307203b 100644 --- a/Owls/frames.py +++ b/Owls/FoamFrame.py @@ -156,7 +156,6 @@ def __init__(self, origins, name, self.show_func=show_func - class FoamFrame(DataFrame): """ Data reprensentation of OpenFOAM field (eulerian and lagrangian) and set files. Instantiated through read methods, e.g: @@ -206,6 +205,7 @@ class FoamFrame(DataFrame): def __init__(self, *args, **kwargs): skip = kwargs.get('skiplines', 1) + times = kwargs.get('skiptimes', 1) name = kwargs.get('name', 'None') symb = kwargs.get('symb', 'o') files = kwargs.get('search_files', None) @@ -213,7 +213,7 @@ def __init__(self, *args, **kwargs): lines = kwargs.get('maxlines', 0) search = kwargs.get('search_pattern', io.FPNUMBER) folder = kwargs.get('folder', None) - plot_properties = kwargs.get('plot_properties', None) + plot_properties = kwargs.get('plot_properties', PlotProperties()) show_func = kwargs.get('show_func', None) validate = kwargs.get('validate', True) preHooks = kwargs.get('preHooks', None) @@ -255,6 +255,7 @@ def __init__(self, *args, **kwargs): files=files, skiplines=skip, maxlines=lines, + skiptimes=times, ) DataFrame.__init__(self, data) self.properties = Props( From 9cbafde88740e70033f253f86fa0a2d02b13e041 Mon Sep 17 00:00:00 2001 From: gregor Date: Wed, 18 Feb 2015 10:08:50 +0100 Subject: [PATCH 13/35] fixed OrderedDict import --- Owls/MultiFrame.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Owls/MultiFrame.py b/Owls/MultiFrame.py index 07d05df..51bcf26 100644 --- a/Owls/MultiFrame.py +++ b/Owls/MultiFrame.py @@ -1,3 +1,5 @@ +from collections import OrderedDict + class MultiFrame(): """ Class for storage of multiple case items or faceted data from FoamFrame From 208ecde942f7843df5f517ce08ec98b797c1a655 Mon Sep 17 00:00:00 2001 From: gregor Date: Wed, 18 Feb 2015 10:09:54 +0100 Subject: [PATCH 14/35] fixed function definition typo --- Owls/foam.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Owls/foam.py b/Owls/foam.py index ed0f553..aeacf57 100644 --- a/Owls/foam.py +++ b/Owls/foam.py @@ -25,7 +25,7 @@ def __init__(self, path, cmd): self.path = path self.cmd = cmd - def execute(self)reconstructPa + def execute(self): return execute_in_path(self.path, self.cmd) class cellCentres(genericHook): From 918d3cfe9bba364fadb0f2610b0cfba3a36a71c7 Mon Sep 17 00:00:00 2001 From: gregor Date: Wed, 18 Feb 2015 10:10:43 +0100 Subject: [PATCH 15/35] add skiptimes --- Owls/io.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Owls/io.py b/Owls/io.py index 4fbf710..2380c35 100644 --- a/Owls/io.py +++ b/Owls/io.py @@ -258,6 +258,7 @@ def import_foam_folder( files, skiplines=1, maxlines=0, + skiptimes=1, ): """ returns a Dataframe for every file in fileList """ #import StringIO @@ -274,7 +275,8 @@ def import_foam_folder( #df.index = MultiIndex.from_tuples(zip([],[]),names=['Loc',0]) from collections import defaultdict origins = Origins() - for time, files in fileList.iteritems(): #FIXME dont iterate twice + els = list(fileList.iteritems())[::skiptimes] + for time, files in els: time = strip_time(time, path) df_tmp = DataFrame() for fn in files: From e0a6129aaaacc517df5a8067329e8851a5b445ca Mon Sep 17 00:00:00 2001 From: gregor Date: Wed, 18 Feb 2015 15:23:20 +0100 Subject: [PATCH 16/35] moved merge and multi merge to Owls.plot -> untested, Feat: read decomposed cases - untested --- Owls/FoamFrame.py | 85 +++++++++-------------------------------------- 1 file changed, 15 insertions(+), 70 deletions(-) diff --git a/Owls/FoamFrame.py b/Owls/FoamFrame.py index 307203b..8787f6a 100644 --- a/Owls/FoamFrame.py +++ b/Owls/FoamFrame.py @@ -26,18 +26,25 @@ def items_from_dict(dict, func, **kwargs): def read_sets(folder, name="None", search="(postProcessing)*sets/" + io.FPNUMBER, **kwargs): return FoamFrame(folder=folder, search_files=False, - search_pattern=search, name=name, show_func="plot", **kwargs) + search_pattern=search, name=name, + show_func="plot", preHooks=None, **kwargs) def read_lag(folder, files, skiplines=1, - name="None", cloud="[A-Za-z]*Cloud1", **kwargs + name="None", cloud="[A-Za-z]*Cloud1", + preHooks=None, decomposed=False, **kwargs ): + search = io.FPNUMBER + "/lagrangian/" + cloud, + search = (search if not decomposed else "processor[0-9]?/" + search) return FoamFrame(folder=folder, search_files=files, - search_pattern=io.FPNUMBER + "/lagrangian/{}".format(cloud), - name=name, skiplines=skiplines, show_func="scatter", **kwargs) + search_pattern=search, name=name, skiplines=skiplines, + show_func="scatter", **kwargs) -def read_eul(folder, files, skiplines=1, name="None", preHooks=None, **kwargs): +def read_eul(folder, files, skiplines=1, name="None", decomposed=False, + preHooks=None, **kwargs): + search = io.FPNUMBER + search = (search if not decomposed else "processor[0-9]?/" + search) return FoamFrame(folder=folder, search_files=files, - search_pattern=io.FPNUMBER, name=name, + search_pattern=search, name=name, skiplines=skiplines, show_func="scatter", preHooks=preHooks, **kwargs) @@ -61,70 +68,6 @@ def read_log(folder, keys, log_name='*log*', plot_properties=False): ) return ff -def merge(*args, **kwargs): - import bokeh.plotting as bk - bk.figure() - bk.hold() - y = kwargs.get('y',None) - x = kwargs.get('x','Pos') - try: - kwargs.pop('y') - kwargs.pop('x') - except: - pass - y = (y if type(y) == list else [y]*len(args)) #FIXME do the same for x - for yi,p in zip(y,args): - p.show(x=x, y=yi, color=next(kwargs["colors"]), **kwargs) - return bk.curplot() - -def multi_merge(*args, **kwargs): - """ call merge for all args - - Examples: mm=multi_merge( - sets1.latest.by_index('Loc'), - sets2.latest.by_index('Loc'), - by='[0-9]+', - x='Pos', - y='vMean' - order=[x-10,x+25]) - - """ - import bokeh.plotting as bk - y = kwargs.get('y',None) - x = kwargs.get('x','Pos') - plots=[] - c = args[0] - # go through all items to be plotted - items = ( - ((name,data) for name, data in c.iteritems() if name in kwargs['order']) - if kwargs.get('order',False) else c.iteritems() - ) - for name, data in items: - sub_plots=[data] - colors = plt.next_color() - for c_ in args[1:]: - # and through all sets to be plotted - for name_, plot_ in c_.iteritems(): - if not kwargs.get('order', False): - # select by regex - # now see if we have a match - selector = kwargs.get('by', "[A-Za-z0-9_\-]") - # skip if search is empty - if (not re.search(selector, name) or - not re.search(selector, name_)): - continue - # append to subplot if same schema - if (re.search(selector, name).group() - == re.search(selector, name_).group()): - sub_plots.append(plot_) - else: - #select by name in order list - if name_ == name: - sub_plots.append(plot_) - plots.append(merge(*sub_plots, x=x, y=y, title=name, colors=colors)) - return plots - - class PlotProperties(): def __init__(self): @@ -220,6 +163,8 @@ def __init__(self, *args, **kwargs): keys = [ 'skiplines', + 'skiptimes', + 'preHooks', 'name', 'symb', 'search_files', From f9058ca2f292a7d02d890a8be8541745aa8dd900 Mon Sep 17 00:00:00 2001 From: gregor Date: Wed, 18 Feb 2015 15:23:59 +0100 Subject: [PATCH 17/35] changed striptimes to handle processor folders --- Owls/io.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/Owls/io.py b/Owls/io.py index 2380c35..fe10bd8 100644 --- a/Owls/io.py +++ b/Owls/io.py @@ -244,8 +244,9 @@ def done(self): def strip_time(path, base): """ try to extract time from path """ - wo_base = path.replace(base,'') - match = re.search(FPNUMBER, wo_base) + wo_base = path.replace(base, '') + wo_proc = re.sub('processor[0-9]?', '', wo_base) + match = re.search(FPNUMBER, wo_proc) if match: time = float(match.group()) return time From a03166ea9ebbf7de6a77cdc536bd6ab8ae7cf1a1 Mon Sep 17 00:00:00 2001 From: gregor Date: Wed, 18 Feb 2015 15:24:25 +0100 Subject: [PATCH 18/35] included merge and multimerge --- Owls/plot.py | 63 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 63 insertions(+) diff --git a/Owls/plot.py b/Owls/plot.py index f00ac19..b14153b 100644 --- a/Owls/plot.py +++ b/Owls/plot.py @@ -15,6 +15,69 @@ "red", "silver", "teal", "yellow"] } +def merge(*args, **kwargs): + import bokeh.plotting as bk + bk.figure() + bk.hold() + y = kwargs.get('y',None) + x = kwargs.get('x','Pos') + try: + kwargs.pop('y') + kwargs.pop('x') + except: + pass + y = (y if type(y) == list else [y]*len(args)) #FIXME do the same for x + for yi,p in zip(y,args): + p.show(x=x, y=yi, color=next(kwargs["colors"]), **kwargs) + return bk.curplot() + +def multi_merge(*args, **kwargs): + """ call merge for all args + + Examples: mm=multi_merge( + sets1.latest.by_index('Loc'), + sets2.latest.by_index('Loc'), + by='[0-9]+', + x='Pos', + y='vMean' + order=[x-10,x+25]) + + """ + import bokeh.plotting as bk + y = kwargs.get('y',None) + x = kwargs.get('x','Pos') + plots=[] + c = args[0] + # go through all items to be plotted + items = ( + ((name,data) for name, data in c.iteritems() if name in kwargs['order']) + if kwargs.get('order',False) else c.iteritems() + ) + for name, data in items: + sub_plots=[data] + colors = plt.next_color() + for c_ in args[1:]: + # and through all sets to be plotted + for name_, plot_ in c_.iteritems(): + if not kwargs.get('order', False): + # select by regex + # now see if we have a match + selector = kwargs.get('by', "[A-Za-z0-9_\-]") + # skip if search is empty + if (not re.search(selector, name) or + not re.search(selector, name_)): + continue + # append to subplot if same schema + if (re.search(selector, name).group() + == re.search(selector, name_).group()): + sub_plots.append(plot_) + else: + #select by name in order list + if name_ == name: + sub_plots.append(plot_) + plots.append(merge(*sub_plots, x=x, y=y, title=name, colors=colors)) + return plots + def next_color(): for col in config['color_cycle']: yield col From 6e8f726c092247dff1bb9059d17e68a9b6fcb7f1 Mon Sep 17 00:00:00 2001 From: gregor Date: Wed, 18 Feb 2015 15:25:16 +0100 Subject: [PATCH 19/35] updated tests --- tests/test_findermethods.py | 36 ++++++++++++++++++++++++++++++------ tests/test_integration.py | 23 +++++++++++++++++++---- 2 files changed, 49 insertions(+), 10 deletions(-) diff --git a/tests/test_findermethods.py b/tests/test_findermethods.py index 7e8bb28..3475371 100644 --- a/tests/test_findermethods.py +++ b/tests/test_findermethods.py @@ -7,26 +7,32 @@ @pytest.fixture def create_directory_tree(tmpdir): def create_files(fold, files): + mock_data = "\n".join(["1 2", "3 4", "5 6"]) for f in files: handle = fold.join(f) - handle.write("foo") + handle.write(mock_data) def create_times(basedir, times): for time in times: time_fold = basedir.mkdir(str(time)) yield time_fold - files = ["U", "UMean", "U_0", "foo.xy"] + files = ["U", "UMean", "U_0", "foo.xy", "T"] folders = range(10) folders.extend([str(float(_)) for _ in folders]) folders.extend(["1.23e-09"]) ignored = ["system", "constant", "0-bck", "0.old"] + for prc_nr in range(4): + proc_dir = tmpdir.mkdir("processor" + str(prc_nr)) + for folder in create_times(proc_dir, folders): + create_files(folder, files) + for folder in create_times(tmpdir, folders): - particle_fold = folder.mkdir("lagrangian").mkdir("particleCloud1") - create_files(folder, files) - create_files(particle_fold, files) + particle_fold = folder.mkdir("lagrangian").mkdir("particleCloud1") + create_files(folder, files) + create_files(particle_fold, files) for folder in ignored: - tmpdir.mkdir(folder) + tmpdir.mkdir(folder) for sets_folder in create_times(tmpdir.mkdir("sets"), folders): create_files(sets_folder, files) folders_str = [str(f) for f in folders] @@ -79,6 +85,11 @@ def _all(base, files, exp, negate=truth, expansion=""): assert _all(base+test_dir + "sets/", sets, folders) assert _all(base+test_dir + "sets/", sets, ignored, negate=not_) + eulerian_decomp = io.find_datafolders( + regex="processor[0-9]\/" + io.FPNUMBER, + path=base + test_dir + ) + def test_findDataFiles(create_directory_tree): """ test if all files in the times folder are found """ base, folders, ignored, files = create_directory_tree @@ -91,6 +102,7 @@ def _test(testdir, folders, files, result, extra=""): assert target in result[testdir + search_fold + extra] eulerian = io.find_datafiles(path=test_dir) + els_eulerian = len(eulerian) _test(test_dir, folders, files, eulerian) sets = io.find_datafiles(path=test_dir, search="sets/" + io.FPNUMBER) @@ -98,3 +110,15 @@ def _test(testdir, folders, files, result, extra=""): lagrangian = io.find_datafiles(path=test_dir, search=io.FPNUMBER + "/lagrangian/[\w]*Cloud[0-9]?") _test(test_dir, folders, files, lagrangian, extra="/lagrangian/particleCloud1") + + eulerian_decomp = io.find_datafiles(path=test_dir, search="processor[0-9]\/" + io.FPNUMBER) + els_eulerian_decomp = len(eulerian_decomp) + + assert els_eulerian * 4 == els_eulerian_decomp + + eulerian_decomp = io.import_foam_folder( + path=test_dir, + search="processor[0-9]\/" + io.FPNUMBER, + files=['T']) + print eulerian_decomp + diff --git a/tests/test_integration.py b/tests/test_integration.py index 49c4b15..eb8a6e5 100644 --- a/tests/test_integration.py +++ b/tests/test_integration.py @@ -5,10 +5,25 @@ exec_path = os.getcwd() -def test_readEul(): +@pytest.fixture(scope="session") +def readEul(): path = exec_path + "/examples/buoyantCavity" - ow.read_eul(folder=path, files=["T","U"], validate=False) + return ow.read_eul(folder=path, files=["T","U"], validate=False) -def test_readSets(): +@pytest.fixture(scope="session") +def readSets(): path = exec_path + "/examples/buoyantCavity" - ow.read_sets(folder=path, validate=False) + return ow.read_sets(folder=path, validate=False) + +def test_eul(readEul): + ff = readEul + assert ff.times + assert type(ff.latest) + assert type(ff.filter('T', field=lambda x: x>293.0)) + +def test_sets(readSets): + ff = readSets + assert ff.times + assert type(ff.latest) + assert type(ff.filter('T', field=lambda x: x>293.0)) + assert ff.by_index('Loc') From 73d7b1741cd4f09affc9b61c55975af0af9f15b0 Mon Sep 17 00:00:00 2001 From: gregor Date: Wed, 18 Feb 2015 15:25:56 +0100 Subject: [PATCH 20/35] changed import structure --- Owls/__init__.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Owls/__init__.py b/Owls/__init__.py index faa6225..6b5cae3 100644 --- a/Owls/__init__.py +++ b/Owls/__init__.py @@ -9,6 +9,7 @@ from io import * from plot import * from foam import * -from frames import * +from FoamFrame import * +from MultiFrame import * print "Owls Version: " + __version__ From eac44e989d67fd49a9d7d655e439338de09ec377 Mon Sep 17 00:00:00 2001 From: gregor Date: Wed, 18 Feb 2015 16:16:43 +0100 Subject: [PATCH 21/35] fixed imports --- tests/test_imports.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/tests/test_imports.py b/tests/test_imports.py index 3569dac..9b57be9 100644 --- a/tests/test_imports.py +++ b/tests/test_imports.py @@ -5,7 +5,11 @@ def test_imports(): """ are the main modules importable """ from Owls import io - from Owls import frames + from Owls import FoamFrame + from Owls import MultiFrame + from Owls import foam + from Owls import plot + def test_importFoamFolder(): from Owls import io From 83b300a37804b891089b9150a846c05a60f9600f Mon Sep 17 00:00:00 2001 From: gregor Date: Tue, 24 Feb 2015 14:13:31 +0100 Subject: [PATCH 22/35] Fixed read_exp wrapper, pass empty search to init --- Owls/FoamFrame.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/Owls/FoamFrame.py b/Owls/FoamFrame.py index 8787f6a..12f21a7 100644 --- a/Owls/FoamFrame.py +++ b/Owls/FoamFrame.py @@ -48,10 +48,9 @@ def read_eul(folder, files, skiplines=1, name="None", decomposed=False, skiplines=skiplines, show_func="scatter", preHooks=preHooks, **kwargs) -def read_exp(folder, name="None", **kwargs): - #FIXME make it read exp/*dat directly without 0 folder +def read_exp(folder, name="None", search="", **kwargs): return FoamFrame(folder=folder, search_files=False, - search_pattern=io.FPNUMBER, name=name, show_func="scatter", **kwargs) + search_pattern=search, name=name, show_func="scatter", **kwargs) def read_log(folder, keys, log_name='*log*', plot_properties=False): From be5ef290bd300480cd8fd3dc65315c903a060ce2 Mon Sep 17 00:00:00 2001 From: gregor Date: Tue, 24 Feb 2015 14:14:08 +0100 Subject: [PATCH 23/35] test read_exp --- tests/test_integration.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/tests/test_integration.py b/tests/test_integration.py index eb8a6e5..1f7b1af 100644 --- a/tests/test_integration.py +++ b/tests/test_integration.py @@ -15,6 +15,11 @@ def readSets(): path = exec_path + "/examples/buoyantCavity" return ow.read_sets(folder=path, validate=False) +@pytest.fixture(scope="session") +def readExp(): + path = exec_path + "/examples/buoyantCavity/sets/100" + return ow.read_sets(folder=path, validate=False, search="") + def test_eul(readEul): ff = readEul assert ff.times @@ -27,3 +32,6 @@ def test_sets(readSets): assert type(ff.latest) assert type(ff.filter('T', field=lambda x: x>293.0)) assert ff.by_index('Loc') + +def test_sets(readExp): + ff = readSets From 93653a2172a58abcf0d1534dc940f98dc5259f34 Mon Sep 17 00:00:00 2001 From: gregor Date: Tue, 24 Feb 2015 14:30:52 +0100 Subject: [PATCH 24/35] updated readExp test --- tests/test_integration.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/tests/test_integration.py b/tests/test_integration.py index 1f7b1af..8afbdfa 100644 --- a/tests/test_integration.py +++ b/tests/test_integration.py @@ -18,7 +18,7 @@ def readSets(): @pytest.fixture(scope="session") def readExp(): path = exec_path + "/examples/buoyantCavity/sets/100" - return ow.read_sets(folder=path, validate=False, search="") + return ow.read_exp(folder=path, validate=False, search="") def test_eul(readEul): ff = readEul @@ -33,5 +33,7 @@ def test_sets(readSets): assert type(ff.filter('T', field=lambda x: x>293.0)) assert ff.by_index('Loc') -def test_sets(readExp): - ff = readSets +def test_exp(readExp): + ff = readExp + assert ff + assert ff.times From ab0bf7dd6b169c2ba1121793fab0ab610df423ce Mon Sep 17 00:00:00 2001 From: gregor Date: Tue, 24 Feb 2015 14:32:54 +0100 Subject: [PATCH 25/35] updated readExp test --- tests/test_integration.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/test_integration.py b/tests/test_integration.py index 8afbdfa..74d532a 100644 --- a/tests/test_integration.py +++ b/tests/test_integration.py @@ -35,5 +35,4 @@ def test_sets(readSets): def test_exp(readExp): ff = readExp - assert ff assert ff.times From 5b50cc88878c3ad515279a8b7951bebc81e271a9 Mon Sep 17 00:00:00 2001 From: gregor Date: Wed, 25 Feb 2015 17:31:54 +0100 Subject: [PATCH 26/35] added plot ranges --- Owls/FoamFrame.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/Owls/FoamFrame.py b/Owls/FoamFrame.py index c8f1b21..eddcc64 100644 --- a/Owls/FoamFrame.py +++ b/Owls/FoamFrame.py @@ -374,8 +374,25 @@ def _label(axis, field): label = self.properties.plot_properties.select(field, axis + '_label', "None") return label + def _range(axis, field): + from bokeh.objects import Range1d + p_range_args = kwargs.get(axis + '_label', False) + if p_range_args: + self.properties.plot_properties.insert(field, {axis + '_range': p_range}) + else: + p_range = self.properties.plot_properties.select(field, axis + '_range') + if not p_range: + return False + else: + return Range1d(start=p_range[0], end=p_range[1]) + + bk.xaxis().axis_label = _label('x', x) + if _range('x', x): + ret.x_range = _range('x', x) bk.yaxis().axis_label = _label('y', y[0]) #TODO can this make sense for multiplots? + if _range('y', y[0]): + ret.y_range = _range('y', y[0]) return ret def scatter(self, y, x='Pos', z=False, title="", **kwargs): From b410215a0c4daff6e81ec9c50f556b067cffe0d7 Mon Sep 17 00:00:00 2001 From: gregor Date: Wed, 25 Feb 2015 17:32:31 +0100 Subject: [PATCH 27/35] fixed color switcher --- Owls/MultiFrame.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Owls/MultiFrame.py b/Owls/MultiFrame.py index 51bcf26..0387992 100644 --- a/Owls/MultiFrame.py +++ b/Owls/MultiFrame.py @@ -1,4 +1,5 @@ from collections import OrderedDict +import plot class MultiFrame(): """ Class for storage of multiple case items @@ -99,7 +100,7 @@ def greatest_divisor(number): return bk.GridPlot(children=rows, title="Scatter") else: bk.hold() - colors = plt.next_color() + colors = plot.next_color() for name, instance in self.cases.iteritems(): color = next(colors) getattr(instance, inst_func)(x=x, y=y, title="", color=color, legend=name, **kwargs) From 4df861c512e178b24fb063e93edd1c8c63b13d11 Mon Sep 17 00:00:00 2001 From: gregor Date: Wed, 25 Feb 2015 17:36:20 +0100 Subject: [PATCH 28/35] fixed color switcher --- Owls/plot.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Owls/plot.py b/Owls/plot.py index b14153b..3933f60 100644 --- a/Owls/plot.py +++ b/Owls/plot.py @@ -1,6 +1,7 @@ import matplotlib.pyplot as plt import matplotlib as mpl import numpy +import re import os import shutil import hashlib @@ -55,7 +56,7 @@ def multi_merge(*args, **kwargs): ) for name, data in items: sub_plots=[data] - colors = plt.next_color() + colors = next_color() for c_ in args[1:]: # and through all sets to be plotted for name_, plot_ in c_.iteritems(): From 7d0ab419456efe75ee5a0898da5673399e5e310d Mon Sep 17 00:00:00 2001 From: gregor Date: Wed, 25 Feb 2015 17:36:46 +0100 Subject: [PATCH 29/35] multimerge integration test --- tests/test_integration.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/test_integration.py b/tests/test_integration.py index 74d532a..3867cda 100644 --- a/tests/test_integration.py +++ b/tests/test_integration.py @@ -36,3 +36,7 @@ def test_sets(readSets): def test_exp(readExp): ff = readExp assert ff.times + ow.multi_merge( + ff.latest.by_index('Loc'), + ff.latest.by_index('Loc'), + x='Pos', y='T') From 7c2ea7038b73f0d197ce86bbb3795fca0bc86ea2 Mon Sep 17 00:00:00 2001 From: gregor Date: Thu, 26 Feb 2015 11:27:52 +0100 Subject: [PATCH 30/35] infinit color cycle --- Owls/plot.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Owls/plot.py b/Owls/plot.py index 3933f60..8bdee9c 100644 --- a/Owls/plot.py +++ b/Owls/plot.py @@ -80,7 +80,8 @@ def multi_merge(*args, **kwargs): return plots def next_color(): - for col in config['color_cycle']: + from itertools import cycle + for col in cycle(config['color_cycle']): yield col def rolling_mean(x,y,z): From ba3735949a38341f725bc5fa147d9b752ec63956 Mon Sep 17 00:00:00 2001 From: gregor Date: Thu, 26 Feb 2015 11:38:32 +0100 Subject: [PATCH 31/35] deactivated experimental database --- Owls/FoamFrame.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/Owls/FoamFrame.py b/Owls/FoamFrame.py index eddcc64..10bbdfa 100644 --- a/Owls/FoamFrame.py +++ b/Owls/FoamFrame.py @@ -17,8 +17,10 @@ list(set(x.keys().get_level_values('Time'))), # avoid back and forth conversion list(set(x.keys().get_level_values('Loc'))), x.values))) #TODO monkey patch to use hashes +Database = False -case_data_base = shelve.open(os.path.expanduser('~') + "/.owls/db") +if Database: + case_data_base = shelve.open(os.path.expanduser('~') + "/.owls/db") def items_from_dict(dict, func, **kwargs): return Cases([func(folder=folder,name=name, symb=symb, **kwargs) @@ -188,7 +190,7 @@ def __init__(self, *args, **kwargs): if preHooks: for hook in preHooks: hook.execute() - if case_data_base.has_key(folder): + if case_data_base.has_key(folder) and Database: print "re-importing", else: print "importing", @@ -211,10 +213,11 @@ def __init__(self, *args, **kwargs): data.index.levels[0], symb, show_func) - if validate: + if validate and Database: self.validate_origins(folder, origins) # register to database - case_data_base.sync() + if Database: + case_data_base.sync() def validate_origins(self, folder, origins): origins.update_hashes() @@ -252,6 +255,7 @@ def validate_origins(self, folder, origins): case_data_base[folder] = origins.dct else: case_data_base[folder] = origins.dct + def add(self, data, label): """ Add a given Series From c90883a5eb9fa787f24a0892098bb8b3b19d79e6 Mon Sep 17 00:00:00 2001 From: gregor Date: Thu, 26 Feb 2015 11:47:24 +0100 Subject: [PATCH 32/35] provide mock database --- Owls/FoamFrame.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Owls/FoamFrame.py b/Owls/FoamFrame.py index 10bbdfa..e854e6d 100644 --- a/Owls/FoamFrame.py +++ b/Owls/FoamFrame.py @@ -21,6 +21,8 @@ if Database: case_data_base = shelve.open(os.path.expanduser('~') + "/.owls/db") +else: + case_data_base = dict() def items_from_dict(dict, func, **kwargs): return Cases([func(folder=folder,name=name, symb=symb, **kwargs) From c1ad1c961f3b1324907a9792481d937ec7759149 Mon Sep 17 00:00:00 2001 From: gregor Date: Thu, 26 Feb 2015 11:58:52 +0100 Subject: [PATCH 33/35] specified bokeh version, since new bokeh version is not backward compatible --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index dff00a2..fffadce 100644 --- a/setup.py +++ b/setup.py @@ -19,7 +19,7 @@ 'pyzmq', 'tornado', 'jinja2', - 'bokeh', + 'bokeh=0.7.1', ], 'name': 'Owls' } From 8a1a461194fd8c9d5e30712d48f97341480f87b2 Mon Sep 17 00:00:00 2001 From: gregor Date: Thu, 26 Feb 2015 12:40:43 +0100 Subject: [PATCH 34/35] fixed bokeh version --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index fffadce..0784b54 100644 --- a/setup.py +++ b/setup.py @@ -19,7 +19,7 @@ 'pyzmq', 'tornado', 'jinja2', - 'bokeh=0.7.1', + 'bokeh==0.7.1', ], 'name': 'Owls' } From 4ae4a9e478efaa9eb8a68ca2843fc0a1ee6aaedd Mon Sep 17 00:00:00 2001 From: gregor Date: Thu, 26 Feb 2015 12:50:07 +0100 Subject: [PATCH 35/35] lets stick to bokeh 0.6.1 for now --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 0784b54..8d97b6e 100644 --- a/setup.py +++ b/setup.py @@ -19,7 +19,7 @@ 'pyzmq', 'tornado', 'jinja2', - 'bokeh==0.7.1', + 'bokeh==0.6.1', ], 'name': 'Owls' }