TEST: Fix binariseCategorical tests, add test to make sure

processData handles metadata

ukbparse/tests/test_processing.py

@@ -272,7 +272,6 @@ def test_processData_variable_types():
         assert called_on['all_independent_except'] == [[1], [2], [3]]
 
 
-
 @clear_plugins
 def test_processData_returnValues():
 
@@ -311,28 +310,101 @@ def test_processData_returnValues():
             remcols.append(col)
         return remcols, newseries, newvids
 
+    @custom.processor()
+    def add_and_remove_meta(dtable, vids):
+        remcols   = []
+        newseries = []
+        newvids   = []
+        newmeta   = []
+        for v in vids:
+            col = dtable.columns(v)[0]
+            data = dtable[:, col.name]
+            newseries.append(pd.Series(data + 10,
+                                       name='{}-0.0'.format(v * 10)))
+            newvids.append(v * 10)
+            remcols.append(col)
+            newmeta.append(v * 10)
+        return remcols, newseries, newvids, newmeta
+
     procfile = tw.dedent("""
     Variable\tProcess
     1:3\tremove
     4:6\tadd
     7:9\tadd_and_remove
     10:12\tnothing
+    13:15\tadd_and_remove_meta
     """).strip()
 
     with tempdir():
         open('processing.tsv', 'wt').write(procfile)
-        gen_test_data(12, 50, 'data.tsv')
+        gen_test_data(15, 50, 'data.tsv')
 
         proctable = loadtables.loadProcessingTable('processing.tsv')
-        vartable, _, cattable = gen_tables(range(1, 13))[:3]
+        vartable, _, cattable = gen_tables(range(1, 16))[:3]
         dtable, _ = importing.importData('data.tsv',
                                          vartable,
                                          proctable,
                                          cattable)
+
         processing.processData(dtable)
 
         gotcols = [c.name for c in dtable.allColumns[1:]]
         expcols = ['{}-0.0'.format(v)
-                   for v in [4, 5, 6, 10, 11, 12, 40, 50, 60, 70, 80, 90]]
+                   for v in [4, 5, 6,
+                             10, 11, 12,
+                             40, 50, 60,
+                             70, 80, 90,
+                             130, 140, 150]]
 
         assert sorted(expcols) == sorted(gotcols)
+
+
+@clear_plugins
+def test_processData_metadata():
+
+    @custom.metaproc()
+    def modmeta(val):
+        return val * 10
+
+    @custom.processor()
+    def process(dtable, vids):
+        newseries = []
+        newvids   = []
+        newmeta   = []
+        for v in vids:
+            col = dtable.columns(v)[0]
+            data = dtable[:, col.name]
+            newseries.append(pd.Series(data + 10,
+                                       name='{}-0.0'.format(v * 10)))
+            newvids.append(v * 10)
+            newmeta.append(v * 10)
+        return [], newseries, newvids, newmeta
+
+    procfile = tw.dedent("""
+    Variable\tProcess
+    1\tprocess
+    2\tprocess(metaproc='modmeta')
+    """).strip()
+
+    with tempdir():
+        open('processing.tsv', 'wt').write(procfile)
+        gen_test_data(2, 50, 'data.tsv')
+
+        proctable = loadtables.loadProcessingTable('processing.tsv')
+        vartable, _, cattable = gen_tables([1, 2])[:3]
+        dtable, _ = importing.importData('data.tsv',
+                                         vartable,
+                                         proctable,
+                                         cattable)
+        processing.processData(dtable)
+        gotcols = [c.name for c in dtable.allColumns[1:]]
+        expcols = ['{}-0.0'.format(v) for v in [1, 2, 10, 20]]
+
+        assert sorted(expcols) == sorted(gotcols)
+
+        col1, col2, col10, col20 = dtable.allColumns[1:]
+
+        assert col1 .metadata is None
+        assert col2 .metadata is None
+        assert col10.metadata == 10
+        assert col20.metadata == 200

ukbparse/tests/test_processing_functions.py

@@ -75,7 +75,7 @@ def test_binariseCateorical():
         vartable, proctable, cattable = gen_tables(range(1, 13))[:3]
         dt, _ = importing.importData('data.txt', vartable, proctable, cattable)
 
-        remove, add, addvids = pfns.binariseCategorical(
+        remove, add, addvids, meta = pfns.binariseCategorical(
             dt, vids,
             acrossVisits=False,
             acrossInstances=False,
@@ -98,8 +98,9 @@ def test_binariseCateorical():
                 assert addvids[i] == int(cols[didx].split('-')[0])
                 assert add[i].name == '{}.{}'.format(cols[didx], int(u))
                 assert (add[i] == exp).all()
+                assert meta[i] == u
 
-        remove, add, addvids = pfns.binariseCategorical(
+        remove, add, addvids, meta = pfns.binariseCategorical(
             dt, vids,
             acrossVisits=True,
             acrossInstances=False,
@@ -122,9 +123,10 @@ def test_binariseCateorical():
                     assert addvids[i] == vid
                     assert got.name == '{}.{}.{}'.format(vid, instance, int(u))
                     assert (got == exp).all()
+                    assert meta[i] == u
         assert names == [r.name for r in remove]
 
-        remove, add, addvids = pfns.binariseCategorical(
+        remove, add, addvids, meta = pfns.binariseCategorical(
             dt, vids,
             acrossVisits=False,
             acrossInstances=True,
@@ -147,9 +149,10 @@ def test_binariseCateorical():
                     assert addvids[i] == vid
                     assert got.name == '{}.{}.{}'.format(vid, visit, int(u))
                     assert (got == exp).all()
+                    assert meta[i] == u
         assert names == [r.name for r in remove]
 
-        remove, add, addvids = pfns.binariseCategorical(
+        remove, add, addvids, meta = pfns.binariseCategorical(
             dt, vids,
             acrossVisits=True,
             acrossInstances=True,
@@ -171,6 +174,7 @@ def test_binariseCateorical():
                 assert addvids[i] == vid
                 assert got.name == '{}.{}'.format(vid, int(u))
                 assert (got == exp).all()
+                assert meta[i] == u
         assert names == [r.name for r in remove]
 
 
@@ -186,14 +190,17 @@ def test_binariseCateorical_no_replace():
         vartable, proctable, cattable = gen_tables(vids)[:3]
         dt, _ = importing.importData('data.txt', vartable, proctable, cattable)
 
-        add, addvids = pfns.binariseCategorical(
+        remove, add, addvids, meta = pfns.binariseCategorical(
             dt, [1], replace=False, nameFormat='{vid}.{value}')
 
         uniq = np.unique(data[:, 1])
 
+        assert len(remove)  == 0
         assert len(add)     == len(uniq)
         assert len(addvids) == len(uniq)
 
+        assert sorted(uniq) == sorted(meta)
+
         assert sorted([c.name for c in add]) == \
                sorted(['1.{}'.format(u) for u in uniq])
         assert np.all([v == 1 for v in addvids])
@@ -213,7 +220,7 @@ def test_binariseCategorical_nonnumeric():
         vartable, proctable, cattable = gen_tables([1])[:3]
         dt, _ = importing.importData('data.txt', vartable, proctable, cattable)
 
-        remove, add, addvids = pfns.binariseCategorical(
+        remove, add, addvids, meta = pfns.binariseCategorical(
             dt, [1],
             acrossVisits=True,
             acrossInstances=True,
@@ -226,6 +233,7 @@ def test_binariseCategorical_nonnumeric():
         assert '1-0.0' in remnames and '1-1.0' in remnames
         assert len(addvids) == len(add)
         assert all([v == 1 for v in addvids])
+        assert sorted(unique) == sorted(meta)
 
         for u in unique:
             name = '1.{}'.format(u)