Commit 3e87996a authored by Paul McCarthy's avatar Paul McCarthy 🚵
Browse files

TEST: test subject inclusion in parallel run

parent ae7a3def
......@@ -508,6 +508,7 @@ def test_loadFile_parallel():
exp = pd.DataFrame(data[:, 1:],
index=data[:, 0],
columns=colnames[1:])
exp.index.name = 'eid'
# loaded in one go
got1 = importing.loadFile(
......@@ -542,11 +543,42 @@ def test_loadFile_parallel():
nrows=9987,
pool=pool)
def eq(df):
def eq(df, exp):
assert np.all(df.eq(exp))
assert np.all(df.columns == exp.columns)
assert np.all(df.index == exp.index)
assert np.all(df.columns == exp.columns)
assert np.all(df.index == exp.index)
assert df.index.name == exp.index.name
eq(got1)
eq(got2)
eq(got3)
eq(got1, exp)
eq(got2, exp)
eq(got3, exp)
# loaded in chunks in
# multiple processes,
# with subject inclusion
# criteria
include = np.random.choice(data[:, 0], 40000)
exclude = np.random.choice(data[:, 0], 5000)
exprs = ['v1 > 25 && v2 < 75', 'v20 >= 10']
with mp.Pool(8) as pool:
got = importing.loadFile(
'data.txt',
vartable,
True,
dialect,
cols,
cols,
subjects=include,
subjectExprs=exprs,
exclude=exclude,
nrows=6987,
pool=pool)
mask = np.zeros(nsubjs, dtype=np.bool)
mask[include - 1] = 1
emask = (data[:, 1] > 25) & (data[:, 2] < 75)
emask = emask | (data[:, 20] >= 10)
mask = mask & emask
mask[exclude - 1] = 0
eq(got, exp[mask])
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment