BF: New series VIDs returned by processing functions were not being parsed

correctly. DataTable.addColumns made more flexible - can now pass in either
None, or a list containing either vids or None.

ukbparse/cleaning_functions.py

@@ -21,7 +21,7 @@ The following cleaning functions are available:
    makeNa
 
 
-All cleanin functions (with two exceptions - :func:`remove` and
+All cleaning functions (with two exceptions - :func:`remove` and
 :func:`keepVisits`, explained below) will be passed the following as their
 first positional arguments, followed by any arguments specified in the
 variable table:

ukbparse/datatable.py

@@ -357,13 +357,16 @@ class DataTable(object):
                      automatically assigned.
         """
 
+        if vids is None:
+            vids = [None] * len(series)
+
         for s in series:
             if s.name in self.__data.columns:
                 raise ValueError(
                     'A column with name {} already exists - remove '
                     'it, or assign to it directly'.format(s.name))
 
-        if vids is not None and len(vids) != len(series):
+        if len(vids) != len(series):
             raise ValueError('length of vids does not match series')
 
         if self.__isstore:
@@ -373,14 +376,16 @@ class DataTable(object):
             startidx = len(self.__data.columns)
             idxs     = range(startidx, startidx + len(series))
 
-            # vid not provided - auto-generate
-            # a vid for each column
-            if vids is None:
-                startvid = max(max(self.variables) + 1, AUTO_VARIABLE_ID)
-                vids     = range(startvid, startvid + len(series))
+            # if vids are not provided, auto-generate
+            # a vid for each column starting from here.
+            startvid = max(max(self.variables) + 1, AUTO_VARIABLE_ID)
 
             for s, idx, vid in zip(series, idxs, vids):
 
+                if vid is None:
+                    vid      = startvid
+                    startvid = startvid + 1
+
                 col = Column(None, s.name, idx, vid, 0, 0)
                 self.__data[s.name] = s
 

ukbparse/processing.py

@@ -120,6 +120,10 @@ def runProcess(proc, dtable, vids):
     add     = []
     addvids = []
 
+    def genvids(result, vi, si):
+        if result[vi] is None: return [None] * len(result[si])
+        else:                  return result[vi]
+
     for result in results:
         if result is None:
             continue
@@ -131,15 +135,15 @@ def runProcess(proc, dtable, vids):
 
             # series/vids to add
             if len(result) == 2:
-                add    .extend(result[0])
-                addvids.extend(result[1])
+                add    .extend(        result[0])
+                addvids.extend(genvids(result, 1, 0))
 
             # columns to remove, and
             # series/vids to add
             elif len(result) == 3:
-                remove .extend(result[0])
-                add    .extend(result[1])
-                addvids.extend(result[2])
+                remove .extend(        result[0])
+                add    .extend(        result[1])
+                addvids.extend(genvids(result, 2, 1))
             else:
                 raise error