Commit 7cddef08 authored by Paul McCarthy's avatar Paul McCarthy 🚵
Browse files

TEST: Fix binariseCategorical tests, add test to make sure

processData handles metadata
parent ad8034eb
......@@ -272,7 +272,6 @@ def test_processData_variable_types():
assert called_on['all_independent_except'] == [[1], [2], [3]]
@clear_plugins
def test_processData_returnValues():
......@@ -311,28 +310,101 @@ def test_processData_returnValues():
remcols.append(col)
return remcols, newseries, newvids
@custom.processor()
def add_and_remove_meta(dtable, vids):
remcols = []
newseries = []
newvids = []
newmeta = []
for v in vids:
col = dtable.columns(v)[0]
data = dtable[:, col.name]
newseries.append(pd.Series(data + 10,
name='{}-0.0'.format(v * 10)))
newvids.append(v * 10)
remcols.append(col)
newmeta.append(v * 10)
return remcols, newseries, newvids, newmeta
procfile = tw.dedent("""
Variable\tProcess
1:3\tremove
4:6\tadd
7:9\tadd_and_remove
10:12\tnothing
13:15\tadd_and_remove_meta
""").strip()
with tempdir():
open('processing.tsv', 'wt').write(procfile)
gen_test_data(12, 50, 'data.tsv')
gen_test_data(15, 50, 'data.tsv')
proctable = loadtables.loadProcessingTable('processing.tsv')
vartable, _, cattable = gen_tables(range(1, 13))[:3]
vartable, _, cattable = gen_tables(range(1, 16))[:3]
dtable, _ = importing.importData('data.tsv',
vartable,
proctable,
cattable)
processing.processData(dtable)
gotcols = [c.name for c in dtable.allColumns[1:]]
expcols = ['{}-0.0'.format(v)
for v in [4, 5, 6, 10, 11, 12, 40, 50, 60, 70, 80, 90]]
for v in [4, 5, 6,
10, 11, 12,
40, 50, 60,
70, 80, 90,
130, 140, 150]]
assert sorted(expcols) == sorted(gotcols)
@clear_plugins
def test_processData_metadata():
@custom.metaproc()
def modmeta(val):
return val * 10
@custom.processor()
def process(dtable, vids):
newseries = []
newvids = []
newmeta = []
for v in vids:
col = dtable.columns(v)[0]
data = dtable[:, col.name]
newseries.append(pd.Series(data + 10,
name='{}-0.0'.format(v * 10)))
newvids.append(v * 10)
newmeta.append(v * 10)
return [], newseries, newvids, newmeta
procfile = tw.dedent("""
Variable\tProcess
1\tprocess
2\tprocess(metaproc='modmeta')
""").strip()
with tempdir():
open('processing.tsv', 'wt').write(procfile)
gen_test_data(2, 50, 'data.tsv')
proctable = loadtables.loadProcessingTable('processing.tsv')
vartable, _, cattable = gen_tables([1, 2])[:3]
dtable, _ = importing.importData('data.tsv',
vartable,
proctable,
cattable)
processing.processData(dtable)
gotcols = [c.name for c in dtable.allColumns[1:]]
expcols = ['{}-0.0'.format(v) for v in [1, 2, 10, 20]]
assert sorted(expcols) == sorted(gotcols)
col1, col2, col10, col20 = dtable.allColumns[1:]
assert col1 .metadata is None
assert col2 .metadata is None
assert col10.metadata == 10
assert col20.metadata == 200
......@@ -75,7 +75,7 @@ def test_binariseCateorical():
vartable, proctable, cattable = gen_tables(range(1, 13))[:3]
dt, _ = importing.importData('data.txt', vartable, proctable, cattable)
remove, add, addvids = pfns.binariseCategorical(
remove, add, addvids, meta = pfns.binariseCategorical(
dt, vids,
acrossVisits=False,
acrossInstances=False,
......@@ -98,8 +98,9 @@ def test_binariseCateorical():
assert addvids[i] == int(cols[didx].split('-')[0])
assert add[i].name == '{}.{}'.format(cols[didx], int(u))
assert (add[i] == exp).all()
assert meta[i] == u
remove, add, addvids = pfns.binariseCategorical(
remove, add, addvids, meta = pfns.binariseCategorical(
dt, vids,
acrossVisits=True,
acrossInstances=False,
......@@ -122,9 +123,10 @@ def test_binariseCateorical():
assert addvids[i] == vid
assert got.name == '{}.{}.{}'.format(vid, instance, int(u))
assert (got == exp).all()
assert meta[i] == u
assert names == [r.name for r in remove]
remove, add, addvids = pfns.binariseCategorical(
remove, add, addvids, meta = pfns.binariseCategorical(
dt, vids,
acrossVisits=False,
acrossInstances=True,
......@@ -147,9 +149,10 @@ def test_binariseCateorical():
assert addvids[i] == vid
assert got.name == '{}.{}.{}'.format(vid, visit, int(u))
assert (got == exp).all()
assert meta[i] == u
assert names == [r.name for r in remove]
remove, add, addvids = pfns.binariseCategorical(
remove, add, addvids, meta = pfns.binariseCategorical(
dt, vids,
acrossVisits=True,
acrossInstances=True,
......@@ -171,6 +174,7 @@ def test_binariseCateorical():
assert addvids[i] == vid
assert got.name == '{}.{}'.format(vid, int(u))
assert (got == exp).all()
assert meta[i] == u
assert names == [r.name for r in remove]
......@@ -186,14 +190,17 @@ def test_binariseCateorical_no_replace():
vartable, proctable, cattable = gen_tables(vids)[:3]
dt, _ = importing.importData('data.txt', vartable, proctable, cattable)
add, addvids = pfns.binariseCategorical(
remove, add, addvids, meta = pfns.binariseCategorical(
dt, [1], replace=False, nameFormat='{vid}.{value}')
uniq = np.unique(data[:, 1])
assert len(remove) == 0
assert len(add) == len(uniq)
assert len(addvids) == len(uniq)
assert sorted(uniq) == sorted(meta)
assert sorted([c.name for c in add]) == \
sorted(['1.{}'.format(u) for u in uniq])
assert np.all([v == 1 for v in addvids])
......@@ -213,7 +220,7 @@ def test_binariseCategorical_nonnumeric():
vartable, proctable, cattable = gen_tables([1])[:3]
dt, _ = importing.importData('data.txt', vartable, proctable, cattable)
remove, add, addvids = pfns.binariseCategorical(
remove, add, addvids, meta = pfns.binariseCategorical(
dt, [1],
acrossVisits=True,
acrossInstances=True,
......@@ -226,6 +233,7 @@ def test_binariseCategorical_nonnumeric():
assert '1-0.0' in remnames and '1-1.0' in remnames
assert len(addvids) == len(add)
assert all([v == 1 for v in addvids])
assert sorted(unique) == sorted(meta)
for u in unique:
name = '1.{}'.format(u)
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment