From dcc249903b7957cffca61f6ff82761d615f273ce Mon Sep 17 00:00:00 2001 From: Zach Lindsey Date: Wed, 13 Mar 2024 11:16:04 -0500 Subject: [PATCH 1/3] use csvreader for parsing csv files --- nipype/interfaces/utility/csv.py | 15 ++++---- nipype/interfaces/utility/tests/test_csv.py | 41 +++++++++++++++++++++ 2 files changed, 48 insertions(+), 8 deletions(-) diff --git a/nipype/interfaces/utility/csv.py b/nipype/interfaces/utility/csv.py index 3bfc46203d..da09e425fb 100644 --- a/nipype/interfaces/utility/csv.py +++ b/nipype/interfaces/utility/csv.py @@ -2,6 +2,7 @@ # vi: set ft=python sts=4 ts=4 sw=4 et: """CSV Handling utilities """ +import csv from ..base import traits, TraitedSpec, DynamicTraitedSpec, File, BaseInterface from ..io import add_traits @@ -13,6 +14,7 @@ class CSVReaderInputSpec(DynamicTraitedSpec, TraitedSpec): header = traits.Bool( False, usedefault=True, desc="True if the first line is a column header" ) + delimiter = traits.String(",", usedefault=True, desc="Delimiter to use.") class CSVReader(BaseInterface): @@ -52,14 +54,11 @@ def _append_entry(self, outputs, entry): outputs[key].append(value) return outputs - def _parse_line(self, line): - line = line.replace("\n", "") - entry = [x.strip() for x in line.split(",")] - return entry - def _get_outfields(self): with open(self.inputs.in_file) as fid: - entry = self._parse_line(fid.readline()) + reader = csv.reader(fid, delimiter=self.inputs.delimiter) + + entry = next(reader) if self.inputs.header: self._outfields = tuple(entry) else: @@ -82,10 +81,10 @@ def _list_outputs(self): for key in self._outfields: outputs[key] = [] # initialize outfields with open(self.inputs.in_file) as fid: - for line in fid.readlines(): + reader = csv.reader(fid, delimiter=self.inputs.delimiter) + for entry in reader: if self.inputs.header and isHeader: # skip header line isHeader = False continue - entry = self._parse_line(line) outputs = self._append_entry(outputs, entry) return outputs diff --git a/nipype/interfaces/utility/tests/test_csv.py b/nipype/interfaces/utility/tests/test_csv.py index 923af9d837..3d4dcabe95 100644 --- a/nipype/interfaces/utility/tests/test_csv.py +++ b/nipype/interfaces/utility/tests/test_csv.py @@ -26,3 +26,44 @@ def test_csvReader(tmpdir): assert out.outputs.column_0 == ["foo", "bar", "baz"] assert out.outputs.column_1 == ["hello", "world", "goodbye"] assert out.outputs.column_2 == ["300.1", "5", "0.3"] + + +def test_csvReader_quoted(tmpdir): + header = "files,labels,erosion\n" + lines = ["foo,\"hello, world\",300.1\n"] + + name = tmpdir.join("testfile.csv").strpath + with open(name, "w") as fid: + reader = utility.CSVReader() + fid.writelines(lines) + fid.flush() + reader.inputs.in_file = name + out = reader.run() + + assert out.outputs.column_0 == ["foo"] + assert out.outputs.column_1 == ["hello, world"] + assert out.outputs.column_2 == ["300.1"] + + +def test_csvReader_tabs(tmpdir): + header = "files\tlabels\terosion\n" + lines = ["foo\thello\t300.1\n", "bar\tworld\t5\n", "baz\tgoodbye\t0.3\n"] + for x in range(2): + name = tmpdir.join("testfile.csv").strpath + with open(name, "w") as fid: + reader = utility.CSVReader(delimiter="\t") + if x % 2 == 0: + fid.write(header) + reader.inputs.header = True + fid.writelines(lines) + fid.flush() + reader.inputs.in_file = name + out = reader.run() + if x % 2 == 0: + assert out.outputs.files == ["foo", "bar", "baz"] + assert out.outputs.labels == ["hello", "world", "goodbye"] + assert out.outputs.erosion == ["300.1", "5", "0.3"] + else: + assert out.outputs.column_0 == ["foo", "bar", "baz"] + assert out.outputs.column_1 == ["hello", "world", "goodbye"] + assert out.outputs.column_2 == ["300.1", "5", "0.3"] From 5e84919fe1efd30524bf6c4a3b79d14d9be4a8de Mon Sep 17 00:00:00 2001 From: Zach Lindsey Date: Wed, 13 Mar 2024 13:52:35 -0500 Subject: [PATCH 2/3] add csvreader autotest --- nipype/interfaces/utility/tests/test_auto_CSVReader.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/nipype/interfaces/utility/tests/test_auto_CSVReader.py b/nipype/interfaces/utility/tests/test_auto_CSVReader.py index be24c59eb4..a96a4d11bf 100644 --- a/nipype/interfaces/utility/tests/test_auto_CSVReader.py +++ b/nipype/interfaces/utility/tests/test_auto_CSVReader.py @@ -4,6 +4,9 @@ def test_CSVReader_inputs(): input_map = dict( + delimiter=dict( + usedefault=True, + ), header=dict( usedefault=True, ), From f746c3408fe7e658bbc9cb0084c27540dc6d3a64 Mon Sep 17 00:00:00 2001 From: Chris Markiewicz Date: Sun, 17 Mar 2024 10:40:31 -0400 Subject: [PATCH 3/3] Update nipype/interfaces/utility/tests/test_csv.py --- nipype/interfaces/utility/tests/test_csv.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nipype/interfaces/utility/tests/test_csv.py b/nipype/interfaces/utility/tests/test_csv.py index 3d4dcabe95..8a8f06e509 100644 --- a/nipype/interfaces/utility/tests/test_csv.py +++ b/nipype/interfaces/utility/tests/test_csv.py @@ -30,7 +30,7 @@ def test_csvReader(tmpdir): def test_csvReader_quoted(tmpdir): header = "files,labels,erosion\n" - lines = ["foo,\"hello, world\",300.1\n"] + lines = ['foo,"hello, world",300.1\n'] name = tmpdir.join("testfile.csv").strpath with open(name, "w") as fid: