Skip to content

Commit ee684eb

Browse files
manifest-ruleseffigies
manifest-rules
authored and
effigies
committed
Backport gh-1298: Support "flat" ASCII-encoded GIFTI DataArrays
TEST: Unit test for loading ASCII-encoded "flat" GIFTI data array. Currently failing RF: Make sure that ASCII-encoded DataArrays are returned with expected shape RF: Consistently apply data type, shape and index order in GIFTI data blocks TEST: Expect data arrays to be the advertised shapes
1 parent c1ac829 commit ee684eb

File tree

4 files changed

+130
-42
lines changed

4 files changed

+130
-42
lines changed

‎nibabel/gifti/gifti.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -745,7 +745,7 @@ def agg_data(self, intent_code=None):
745745
>>> triangles_2 = surf_img.agg_data('triangle')
746746
>>> triangles_3 = surf_img.agg_data(1009) # Numeric code for pointset
747747
>>> print(np.array2string(triangles))
748-
[0 1 2]
748+
[[0 1 2]]
749749
>>> np.array_equal(triangles, triangles_2)
750750
True
751751
>>> np.array_equal(triangles, triangles_3)

‎nibabel/gifti/parse_gifti_fast.py‎

Lines changed: 29 additions & 37 deletions
Original file line numberDiff line numberDiff line change
@@ -68,17 +68,21 @@ def read_data_block(darray, fname, data, mmap):
6868
if mmap is True:
6969
mmap = 'c'
7070
enclabel = gifti_encoding_codes.label[darray.encoding]
71-
dtype = data_type_codes.type[darray.datatype]
7271

72+
if enclabel not in ('ASCII', 'B64BIN', 'B64GZ', 'External'):
73+
raise GiftiParseError(f'Unknown encoding {darray.encoding}')
74+
75+
# Encode the endianness in the dtype
76+
byteorder = gifti_endian_codes.byteorder[darray.endian]
77+
dtype = data_type_codes.dtype[darray.datatype].newbyteorder(byteorder)
78+
79+
shape = tuple(darray.dims)
80+
order = array_index_order_codes.npcode[darray.ind_ord]
81+
82+
# GIFTI_ENCODING_ASCII
7383
if enclabel == 'ASCII':
74-
# GIFTI_ENCODING_ASCII
75-
c = StringIO(data)
76-
da = np.loadtxt(c, dtype=dtype)
77-
return da # independent of the endianness
78-
elif enclabel not in ('B64BIN', 'B64GZ', 'External'):
79-
return 0
80-
81-
# GIFTI_ENCODING_EXTBIN
84+
return np.loadtxt(StringIO(data), dtype=dtype, ndmin=1).reshape(shape, order=order)
85+
8286
# We assume that the external data file is raw uncompressed binary, with
8387
# the data type/endianness/ordering specified by the other DataArray
8488
# attributes
@@ -94,53 +98,41 @@ def read_data_block(darray, fname, data, mmap):
9498
newarr = None
9599
if mmap:
96100
try:
97-
newarr = np.memmap(
101+
return np.memmap(
98102
ext_fname,
99103
dtype=dtype,
100104
mode=mmap,
101105
offset=darray.ext_offset,
102-
shape=tuple(darray.dims),
106+
shape=shape,
107+
order=order,
103108
)
104109
# If the memmap fails, we ignore the error and load the data into
105110
# memory below
106111
except (AttributeError, TypeError, ValueError):
107112
pass
108113
# mmap=False or np.memmap failed
109114
if newarr is None:
110-
# We can replace this with a call to np.fromfile in numpy>=1.17,
111-
# as an "offset" parameter was added in that version.
112-
with open(ext_fname, 'rb') as f:
113-
f.seek(darray.ext_offset)
114-
nbytes = np.prod(darray.dims) * dtype().itemsize
115-
buff = f.read(nbytes)
116-
newarr = np.frombuffer(buff, dtype=dtype)
115+
return np.fromfile(
116+
ext_fname,
117+
dtype=dtype,
118+
count=np.prod(darray.dims),
119+
offset=darray.ext_offset,
120+
).reshape(shape, order=order)
117121

118122
# Numpy arrays created from bytes objects are read-only.
119123
# Neither b64decode nor decompress will return bytearrays, and there
120124
# are not equivalents to fobj.readinto to allow us to pass them, so
121125
# there is not a simple way to avoid making copies.
122126
# If this becomes a problem, we should write a decoding interface with
123127
# a tunable chunk size.
128+
dec = base64.b64decode(data.encode('ascii'))
129+
if enclabel == 'B64BIN':
130+
buff = bytearray(dec)
124131
else:
125-
dec = base64.b64decode(data.encode('ascii'))
126-
if enclabel == 'B64BIN':
127-
# GIFTI_ENCODING_B64BIN
128-
buff = bytearray(dec)
129-
else:
130-
# GIFTI_ENCODING_B64GZ
131-
buff = bytearray(zlib.decompress(dec))
132-
del dec
133-
newarr = np.frombuffer(buff, dtype=dtype)
134-
135-
sh = tuple(darray.dims)
136-
if len(newarr.shape) != len(sh):
137-
newarr = newarr.reshape(sh, order=array_index_order_codes.npcode[darray.ind_ord])
138-
139-
# check if we need to byteswap
140-
required_byteorder = gifti_endian_codes.byteorder[darray.endian]
141-
if required_byteorder in ('big', 'little') and required_byteorder != sys.byteorder:
142-
newarr = newarr.byteswap()
143-
return newarr
132+
# GIFTI_ENCODING_B64GZ
133+
buff = bytearray(zlib.decompress(dec))
134+
del dec
135+
return np.frombuffer(buff, dtype=dtype).reshape(shape, order=order)
144136

145137

146138
def _str2int(in_str):

‎nibabel/gifti/tests/data/ascii_flat_data.gii‎

Lines changed: 76 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,76 @@
1+
<?xml version="1.0" encoding="UTF-8"?>
2+
<!DOCTYPE GIFTI SYSTEM "http://www.nitrc.org/frs/download.php/115/gifti.dtd">
3+
<GIFTI Version="1.0" NumberOfDataArrays="2">
4+
<MetaData>
5+
<MD>
6+
<Name><![CDATA[Caret-Version]]></Name>
7+
<Value><![CDATA[5.512]]></Value>
8+
</MD>
9+
<MD>
10+
<Name><![CDATA[date]]></Name>
11+
<Value><![CDATA[Thu Dec 27 14:27:43 2007]]></Value>
12+
</MD>
13+
<MD>
14+
<Name><![CDATA[encoding]]></Name>
15+
<Value><![CDATA[XML]]></Value>
16+
</MD>
17+
</MetaData>
18+
<LabelTable/>
19+
<DataArray Intent="NIFTI_INTENT_POINTSET"
20+
DataType="NIFTI_TYPE_FLOAT32"
21+
ArrayIndexingOrder="RowMajorOrder"
22+
Dimensionality="2"
23+
Dim0="10"
24+
Dim1="3"
25+
Encoding="ASCII"
26+
Endian="LittleEndian"
27+
ExternalFileName=""
28+
ExternalFileOffset="">
29+
<MetaData>
30+
<MD>
31+
<Name><![CDATA[AnatomicalStructurePrimary]]></Name>
32+
<Value><![CDATA[CortexLeft]]></Value>
33+
</MD>
34+
<MD>
35+
<Name><![CDATA[AnatomicalStructureSecondary]]></Name>
36+
<Value><![CDATA[Pial]]></Value>
37+
</MD>
38+
<MD>
39+
<Name><![CDATA[GeometricType]]></Name>
40+
<Value><![CDATA[Anatomical]]></Value>
41+
</MD>
42+
<MD>
43+
<Name><![CDATA[UniqueID]]></Name>
44+
<Value><![CDATA[{70e032e9-4123-47ee-965d-5b29107cbd83}]]></Value>
45+
</MD>
46+
</MetaData>
47+
<CoordinateSystemTransformMatrix>
48+
<DataSpace><![CDATA[NIFTI_XFORM_TALAIRACH]]></DataSpace>
49+
<TransformedSpace><![CDATA[NIFTI_XFORM_TALAIRACH]]></TransformedSpace>
50+
<MatrixData>1.000000 0.000000 0.000000 0.000000 0.000000 1.000000 0.000000 0.000000 0.000000 0.000000 1.000000 0.000000 0.000000 0.000000 0.000000 1.000000</MatrixData>
51+
</CoordinateSystemTransformMatrix>
52+
<Data>155.17539978 135.58103943 98.30715179 140.33973694 190.0491333 73.24776459 157.3598938 196.97969055 83.65809631 171.46174622 137.43661499 78.4709549 148.54592896 97.06752777 65.96373749 123.45701599 111.46841431 66.3571167 135.30892944 202.28720093 36.38148499 178.28155518 162.59469604 37.75128937 178.11087036 115.28820038 57.17986679 142.81582642 82.82115173 31.02205276</Data>
53+
</DataArray>
54+
<DataArray Intent="NIFTI_INTENT_TRIANGLE"
55+
DataType="NIFTI_TYPE_INT32"
56+
ArrayIndexingOrder="RowMajorOrder"
57+
Dimensionality="2"
58+
Dim0="10"
59+
Dim1="3"
60+
Encoding="ASCII"
61+
Endian="LittleEndian"
62+
ExternalFileName=""
63+
ExternalFileOffset="">
64+
<MetaData>
65+
<MD>
66+
<Name><![CDATA[TopologicalType]]></Name>
67+
<Value><![CDATA[CLOSED]]></Value>
68+
</MD>
69+
<MD>
70+
<Name><![CDATA[UniqueID]]></Name>
71+
<Value><![CDATA[{747d8015-455b-43ad-82ac-dcfb7606004a}]]></Value>
72+
</MD>
73+
</MetaData>
74+
<Data>6402 17923 25602 14085 25602 17923 25602 14085 4483 17923 1602 14085 4483 25603 25602 25604 25602 25603 25602 25604 6402 25603 3525 25604 1123 17922 12168 25604 12168 17922 </Data>
75+
</DataArray>
76+
</GIFTI>

‎nibabel/gifti/tests/test_parse_gifti_fast.py‎

Lines changed: 24 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -39,9 +39,19 @@
3939
DATA_FILE5 = pjoin(IO_DATA_PATH, 'base64bin.gii')
4040
DATA_FILE6 = pjoin(IO_DATA_PATH, 'rh.aparc.annot.gii')
4141
DATA_FILE7 = pjoin(IO_DATA_PATH, 'external.gii')
42-
43-
datafiles = [DATA_FILE1, DATA_FILE2, DATA_FILE3, DATA_FILE4, DATA_FILE5, DATA_FILE6, DATA_FILE7]
44-
numDA = [2, 1, 1, 1, 2, 1, 2]
42+
DATA_FILE8 = pjoin(IO_DATA_PATH, 'ascii_flat_data.gii')
43+
44+
datafiles = [
45+
DATA_FILE1,
46+
DATA_FILE2,
47+
DATA_FILE3,
48+
DATA_FILE4,
49+
DATA_FILE5,
50+
DATA_FILE6,
51+
DATA_FILE7,
52+
DATA_FILE8,
53+
]
54+
numDA = [2, 1, 1, 1, 2, 1, 2, 2]
4555

4656
DATA_FILE1_darr1 = np.array(
4757
[
@@ -50,7 +60,7 @@
5060
[-17.614349, -65.401642, 21.071466],
5161
]
5262
)
53-
DATA_FILE1_darr2 = np.array([0, 1, 2])
63+
DATA_FILE1_darr2 = np.array([[0, 1, 2]])
5464

5565
DATA_FILE2_darr1 = np.array(
5666
[
@@ -152,6 +162,10 @@
152162
dtype=np.int32,
153163
)
154164

165+
DATA_FILE8_darr1 = np.copy(DATA_FILE5_darr1)
166+
167+
DATA_FILE8_darr2 = np.copy(DATA_FILE5_darr2)
168+
155169

156170
def assert_default_types(loaded):
157171
default = loaded.__class__()
@@ -448,3 +462,9 @@ def test_load_compressed():
448462
img7 = load(fn)
449463
assert_array_almost_equal(img7.darrays[0].data, DATA_FILE7_darr1)
450464
assert_array_almost_equal(img7.darrays[1].data, DATA_FILE7_darr2)
465+
466+
467+
def test_load_flat_ascii_data():
468+
img = load(DATA_FILE8)
469+
assert_array_almost_equal(img.darrays[0].data, DATA_FILE8_darr1)
470+
assert_array_almost_equal(img.darrays[1].data, DATA_FILE8_darr2)

0 commit comments

Comments
 (0)