Skip to content

Commit

Permalink
Fix handling of embedded commas in CSV fields. For ravel highperforma…
Browse files Browse the repository at this point in the history
highperformancecoder committed Nov 15, 2023
1 parent aa2819d commit 467cbc7
Showing 3 changed files with 13 additions and 15 deletions.
10 changes: 2 additions & 8 deletions engine/CSVParser.cc
Original file line number Diff line number Diff line change
@@ -540,15 +540,9 @@ namespace minsky
chomp(line);
while (r)
{
// count the number of quote characters after last separator. If odd, then line is not terminated correctly
auto n=line.rfind(spec.separator);
if (n==string::npos)
n=0;
else
++n;
int quoteCount=0;
for (; n<line.size(); ++n)
if (line[n]==spec.quote)
for (auto i: line)
if (i==spec.quote)
++quoteCount;
if (quoteCount%2==0) break; // data line correctly terminated
string buf;
4 changes: 4 additions & 0 deletions test/00/godleyTableWindow.sh
Original file line number Diff line number Diff line change
@@ -26,6 +26,10 @@ pass()
exit 0
}

# test disabled, as it is no longer relevant in Minsky 3.x
# TODO: to be refactored for 3.x, or removed entirely
pass

trap "fail" 1 2 3 15
cat >input.tcl <<EOF
source $here/test/assert.tcl
14 changes: 7 additions & 7 deletions test/testCSVParser.cc
Original file line number Diff line number Diff line change
@@ -241,15 +241,15 @@ SUITE(CSVParser)
TEST_FIXTURE(DataSpec,embeddedNewline)
{
string input="A comment\n"
";;foobar\n" // horizontal dim name
"foo;bar;A;B;C\n"
"A;'A\nB';1.2;1.3;1.4\n"
"A;B;1;'2\n.0';3\n"
"B;AB;3;2;1\n";
",,foobar\n" // horizontal dim name
"foo,bar,A,B,C\n"
"A,'A\nB',1.2,1.3,'1,000'\n"
"A,B,1,'2\n.0',3\n"
"B,AB,3,2,1\n";

istringstream is(input);

separator=';';
separator=',';
quote='\'';
setDataArea(3,2);
numCols=5;
@@ -276,7 +276,7 @@ SUITE(CSVParser)
CHECK_EQUAL("C", str(v.hypercube().xvectors[2][2]));
CHECK(v.hypercube().dims()==v.tensorInit.hypercube().dims());
CHECK_EQUAL(12, v.tensorInit.size());
CHECK_ARRAY_CLOSE(vector<double>({1.2,3,1,-1,1.3,2,2,-1,1.4,1,3,-1}),
CHECK_ARRAY_CLOSE(vector<double>({1.2,3,1,-1,1.3,2,2,-1,1000,1,3,-1}),
v.tensorInit, 12, 1e-4);
}

0 comments on commit 467cbc7

Please sign in to comment.