Skip to content
GitLab
Menu
Projects
Groups
Snippets
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in
Toggle navigation
Menu
Open sidebar
FSL
ukbparse
Commits
7cddef08
Commit
7cddef08
authored
May 09, 2019
by
Paul McCarthy
🚵
Browse files
TEST: Fix binariseCategorical tests, add test to make sure
processData handles metadata
parent
ad8034eb
Changes
2
Hide whitespace changes
Inline
Side-by-side
ukbparse/tests/test_processing.py
View file @
7cddef08
...
...
@@ -272,7 +272,6 @@ def test_processData_variable_types():
assert
called_on
[
'all_independent_except'
]
==
[[
1
],
[
2
],
[
3
]]
@
clear_plugins
def
test_processData_returnValues
():
...
...
@@ -311,28 +310,101 @@ def test_processData_returnValues():
remcols
.
append
(
col
)
return
remcols
,
newseries
,
newvids
@
custom
.
processor
()
def
add_and_remove_meta
(
dtable
,
vids
):
remcols
=
[]
newseries
=
[]
newvids
=
[]
newmeta
=
[]
for
v
in
vids
:
col
=
dtable
.
columns
(
v
)[
0
]
data
=
dtable
[:,
col
.
name
]
newseries
.
append
(
pd
.
Series
(
data
+
10
,
name
=
'{}-0.0'
.
format
(
v
*
10
)))
newvids
.
append
(
v
*
10
)
remcols
.
append
(
col
)
newmeta
.
append
(
v
*
10
)
return
remcols
,
newseries
,
newvids
,
newmeta
procfile
=
tw
.
dedent
(
"""
Variable
\t
Process
1:3
\t
remove
4:6
\t
add
7:9
\t
add_and_remove
10:12
\t
nothing
13:15
\t
add_and_remove_meta
"""
).
strip
()
with
tempdir
():
open
(
'processing.tsv'
,
'wt'
).
write
(
procfile
)
gen_test_data
(
1
2
,
50
,
'data.tsv'
)
gen_test_data
(
1
5
,
50
,
'data.tsv'
)
proctable
=
loadtables
.
loadProcessingTable
(
'processing.tsv'
)
vartable
,
_
,
cattable
=
gen_tables
(
range
(
1
,
1
3
))[:
3
]
vartable
,
_
,
cattable
=
gen_tables
(
range
(
1
,
1
6
))[:
3
]
dtable
,
_
=
importing
.
importData
(
'data.tsv'
,
vartable
,
proctable
,
cattable
)
processing
.
processData
(
dtable
)
gotcols
=
[
c
.
name
for
c
in
dtable
.
allColumns
[
1
:]]
expcols
=
[
'{}-0.0'
.
format
(
v
)
for
v
in
[
4
,
5
,
6
,
10
,
11
,
12
,
40
,
50
,
60
,
70
,
80
,
90
]]
for
v
in
[
4
,
5
,
6
,
10
,
11
,
12
,
40
,
50
,
60
,
70
,
80
,
90
,
130
,
140
,
150
]]
assert
sorted
(
expcols
)
==
sorted
(
gotcols
)
@
clear_plugins
def
test_processData_metadata
():
@
custom
.
metaproc
()
def
modmeta
(
val
):
return
val
*
10
@
custom
.
processor
()
def
process
(
dtable
,
vids
):
newseries
=
[]
newvids
=
[]
newmeta
=
[]
for
v
in
vids
:
col
=
dtable
.
columns
(
v
)[
0
]
data
=
dtable
[:,
col
.
name
]
newseries
.
append
(
pd
.
Series
(
data
+
10
,
name
=
'{}-0.0'
.
format
(
v
*
10
)))
newvids
.
append
(
v
*
10
)
newmeta
.
append
(
v
*
10
)
return
[],
newseries
,
newvids
,
newmeta
procfile
=
tw
.
dedent
(
"""
Variable
\t
Process
1
\t
process
2
\t
process(metaproc='modmeta')
"""
).
strip
()
with
tempdir
():
open
(
'processing.tsv'
,
'wt'
).
write
(
procfile
)
gen_test_data
(
2
,
50
,
'data.tsv'
)
proctable
=
loadtables
.
loadProcessingTable
(
'processing.tsv'
)
vartable
,
_
,
cattable
=
gen_tables
([
1
,
2
])[:
3
]
dtable
,
_
=
importing
.
importData
(
'data.tsv'
,
vartable
,
proctable
,
cattable
)
processing
.
processData
(
dtable
)
gotcols
=
[
c
.
name
for
c
in
dtable
.
allColumns
[
1
:]]
expcols
=
[
'{}-0.0'
.
format
(
v
)
for
v
in
[
1
,
2
,
10
,
20
]]
assert
sorted
(
expcols
)
==
sorted
(
gotcols
)
col1
,
col2
,
col10
,
col20
=
dtable
.
allColumns
[
1
:]
assert
col1
.
metadata
is
None
assert
col2
.
metadata
is
None
assert
col10
.
metadata
==
10
assert
col20
.
metadata
==
200
ukbparse/tests/test_processing_functions.py
View file @
7cddef08
...
...
@@ -75,7 +75,7 @@ def test_binariseCateorical():
vartable
,
proctable
,
cattable
=
gen_tables
(
range
(
1
,
13
))[:
3
]
dt
,
_
=
importing
.
importData
(
'data.txt'
,
vartable
,
proctable
,
cattable
)
remove
,
add
,
addvids
=
pfns
.
binariseCategorical
(
remove
,
add
,
addvids
,
meta
=
pfns
.
binariseCategorical
(
dt
,
vids
,
acrossVisits
=
False
,
acrossInstances
=
False
,
...
...
@@ -98,8 +98,9 @@ def test_binariseCateorical():
assert
addvids
[
i
]
==
int
(
cols
[
didx
].
split
(
'-'
)[
0
])
assert
add
[
i
].
name
==
'{}.{}'
.
format
(
cols
[
didx
],
int
(
u
))
assert
(
add
[
i
]
==
exp
).
all
()
assert
meta
[
i
]
==
u
remove
,
add
,
addvids
=
pfns
.
binariseCategorical
(
remove
,
add
,
addvids
,
meta
=
pfns
.
binariseCategorical
(
dt
,
vids
,
acrossVisits
=
True
,
acrossInstances
=
False
,
...
...
@@ -122,9 +123,10 @@ def test_binariseCateorical():
assert
addvids
[
i
]
==
vid
assert
got
.
name
==
'{}.{}.{}'
.
format
(
vid
,
instance
,
int
(
u
))
assert
(
got
==
exp
).
all
()
assert
meta
[
i
]
==
u
assert
names
==
[
r
.
name
for
r
in
remove
]
remove
,
add
,
addvids
=
pfns
.
binariseCategorical
(
remove
,
add
,
addvids
,
meta
=
pfns
.
binariseCategorical
(
dt
,
vids
,
acrossVisits
=
False
,
acrossInstances
=
True
,
...
...
@@ -147,9 +149,10 @@ def test_binariseCateorical():
assert
addvids
[
i
]
==
vid
assert
got
.
name
==
'{}.{}.{}'
.
format
(
vid
,
visit
,
int
(
u
))
assert
(
got
==
exp
).
all
()
assert
meta
[
i
]
==
u
assert
names
==
[
r
.
name
for
r
in
remove
]
remove
,
add
,
addvids
=
pfns
.
binariseCategorical
(
remove
,
add
,
addvids
,
meta
=
pfns
.
binariseCategorical
(
dt
,
vids
,
acrossVisits
=
True
,
acrossInstances
=
True
,
...
...
@@ -171,6 +174,7 @@ def test_binariseCateorical():
assert
addvids
[
i
]
==
vid
assert
got
.
name
==
'{}.{}'
.
format
(
vid
,
int
(
u
))
assert
(
got
==
exp
).
all
()
assert
meta
[
i
]
==
u
assert
names
==
[
r
.
name
for
r
in
remove
]
...
...
@@ -186,14 +190,17 @@ def test_binariseCateorical_no_replace():
vartable
,
proctable
,
cattable
=
gen_tables
(
vids
)[:
3
]
dt
,
_
=
importing
.
importData
(
'data.txt'
,
vartable
,
proctable
,
cattable
)
add
,
addvids
=
pfns
.
binariseCategorical
(
remove
,
add
,
addvids
,
meta
=
pfns
.
binariseCategorical
(
dt
,
[
1
],
replace
=
False
,
nameFormat
=
'{vid}.{value}'
)
uniq
=
np
.
unique
(
data
[:,
1
])
assert
len
(
remove
)
==
0
assert
len
(
add
)
==
len
(
uniq
)
assert
len
(
addvids
)
==
len
(
uniq
)
assert
sorted
(
uniq
)
==
sorted
(
meta
)
assert
sorted
([
c
.
name
for
c
in
add
])
==
\
sorted
([
'1.{}'
.
format
(
u
)
for
u
in
uniq
])
assert
np
.
all
([
v
==
1
for
v
in
addvids
])
...
...
@@ -213,7 +220,7 @@ def test_binariseCategorical_nonnumeric():
vartable
,
proctable
,
cattable
=
gen_tables
([
1
])[:
3
]
dt
,
_
=
importing
.
importData
(
'data.txt'
,
vartable
,
proctable
,
cattable
)
remove
,
add
,
addvids
=
pfns
.
binariseCategorical
(
remove
,
add
,
addvids
,
meta
=
pfns
.
binariseCategorical
(
dt
,
[
1
],
acrossVisits
=
True
,
acrossInstances
=
True
,
...
...
@@ -226,6 +233,7 @@ def test_binariseCategorical_nonnumeric():
assert
'1-0.0'
in
remnames
and
'1-1.0'
in
remnames
assert
len
(
addvids
)
==
len
(
add
)
assert
all
([
v
==
1
for
v
in
addvids
])
assert
sorted
(
unique
)
==
sorted
(
meta
)
for
u
in
unique
:
name
=
'1.{}'
.
format
(
u
)
...
...
Write
Preview
Supports
Markdown
0%
Try again
or
attach a new file
.
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment