Commit 61e2a6e9 authored by Paul McCarthy's avatar Paul McCarthy 🚵
Browse files

Merge branch 'bf/childval_keyerror' into 'master'

Bf/childval keyerror

See merge request fsl/ukbparse!105
parents 2fb4c9e7 2a40bebd
......@@ -2,6 +2,21 @@
======================
0.14.6 (Saturday 16th March 2019)
---------------------------------
Fixed
^^^^^
* Fixed a ``KeyError`` which was occurring during the child-value replacement
stage for input files which did not have column names of the form
``[variable]-[visit].[instance]``.
* Fixed some issues introduced by behavioural changes in the
``pandas.HDFStore`` class.
0.14.5 (Thursday 17th January 2019)
-----------------------------------
......
......@@ -176,7 +176,7 @@ def _runChildValues(dtable, exprs, cvals, vid):
# variable, and for each parent
# variable that correspond to
# this variable+visit+instance
colname = util.generateColumnName(vid, visit, instance)
colname = dtable.columns(vid, visit, instance)[0].name
pvars = list(set(it.chain(*[e.variables for e in expr])))
# get the true column names
......
......@@ -10,7 +10,7 @@ which contain data on imaged subjects.
import functools as ft
import datetime
import datetime as dt
import calendar
import pandas as pd
......@@ -50,19 +50,20 @@ def load_FMRIBImaging(infile):
year = int(date[ :4])
month = int(date[4:6])
day = int(date[6:8])
return datetime.date(year, month, day)
return dt.date(year, month, day)
def parse_acq_time(time):
hour = int(time[ :2])
minute = int(time[2:4])
second = int(time[4:6])
micro = int(time[7:])
return datetime.time(hour, minute, second, micro)
return dt.time(hour, minute, second, micro,
dt.timezone(dt.timedelta(0)))
def combine_datetime(date, time):
date = date.to_pydatetime().date()
time = time.to_pydatetime().time()
return datetime.datetime.combine(date, time)
time = time.to_pydatetime().timetz()
return dt.datetime.combine(date, time)
names = ['eid',
'acq_date',
......@@ -82,7 +83,9 @@ def load_FMRIBImaging(infile):
converters=converters,
delim_whitespace=True)
print('origtime', df['acq_time'])
df['acq_time'] = df['acq_date'].combine(df['acq_time'], combine_datetime)
print('combined', df['acq_time'])
df.drop('acq_date', axis=1, inplace=True)
return df
......
......@@ -198,7 +198,7 @@ class HDFStoreCollection(object):
for the file with the given ``idx``.
"""
if mode in ('w', 'a', 'r+') and self.__fileLocks is not None:
if self.__fileLocks is not None:
lock = self.__fileLocks[idx]
else:
lock = mp.Lock()
......@@ -206,7 +206,6 @@ class HDFStoreCollection(object):
with lock:
with pd.HDFStore(self.__storeFiles[idx],
mode=mode,
format='table',
*args, **kwargs) as f:
yield f
......@@ -490,7 +489,11 @@ class StoreSlicer(object):
for idx, colblock in colblocks.items():
with store.storeFile(idx, mode='r') as s:
for col in colblock:
series = s.select(col)
try:
series = s.select(col)
except Exception:
series = None
if series is None:
series = self.__unstorable.get(col, None)
results.append(series.loc[rows])
......
#!/usr/bin/env python
#
# test_nonstandard_columns.py -
#
# Author: Paul McCarthy <pauldmccarthy@gmail.com>
#
import shlex
import textwrap as tw
import pandas as pd
import numpy as np
import ukbparse.main as main
from . import (patch_logging,
clear_plugins,
tempdir,
gen_tables,
gen_DataTable,
gen_test_data,
gen_DataTableFromDataFrame)
@patch_logging
def test_nonstandard_columns():
indata = tw.dedent("""
f.eid,f.1.0.0,f.2.0.0,f.3.0.0
1,4,7,3
2,1,9,2
3,,3,5
""").strip()
tests = [
('-nb -nv 2 7',
"""
f.eid,f.1.0.0,f.2.0.0,f.3.0.0
1,4,,3
2,1,9,2
3,,3,5
"""),
('-nb -re 3 2,3 10,20',
"""
f.eid,f.1.0.0,f.2.0.0,f.3.0.0
1,4,7,20
2,1,9,10
3,,3,5
"""),
('-nb -cv 1 "v2 == 3" 25',
"""
f.eid,f.1.0.0,f.2.0.0,f.3.0.0
1,4,7,3
2,1,9,2
3,25,3,5
"""),
]
for cmd, expout in tests:
with tempdir():
expout = tw.dedent(expout).strip()
with open('in.csv', 'wt') as f: f.write(indata)
with open('exp.csv', 'wt') as f: f.write(expout)
main.main(shlex.split(cmd + ' out.csv in.csv'))
got = pd.read_csv('out.csv', index_col=0, delimiter='\t')
exp = pd.read_csv('exp.csv', index_col=0)
assert np.all(exp.columns == got.columns)
assert np.all(np.isnan(exp.loc[:, :]) == np.isnan(got.loc[:, :]))
assert np.all(exp.loc[:, :].notna() == got.loc[:, :].notna())
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment