This is an automated email from the ASF dual-hosted git repository.
virajjasani pushed a commit to branch 5.3
in repository https://gitbox.apache.org/repos/asf/phoenix.git
The following commit(s) were added to refs/heads/5.3 by this push:
new 56cf07cd7a PHOENIX-7267 CsvBulkLoadTool fails job due to a bad record
with "(startline 1) EOF reached before encapsulated token finished" (#2399)
56cf07cd7a is described below
commit 56cf07cd7a549a29b83e7198a335ebd7e9e1dc37
Author: Xavier Fernandis <[email protected]>
AuthorDate: Sat Jun 13 04:06:20 2026 +0530
PHOENIX-7267 CsvBulkLoadTool fails job due to a bad record with "(startline
1) EOF reached before encapsulated token finished" (#2399)
---
.../org/apache/phoenix/util/CSVCommonsLoader.java | 19 ++--
phoenix-core-server/pom.xml | 4 +
.../phoenix/mapreduce/CsvToKeyValueMapper.java | 13 ++-
.../mapreduce/FormatToBytesWritableMapper.java | 11 ++-
.../apache/phoenix/end2end/CSVCommonsLoaderIT.java | 31 +++---
.../phoenix/mapreduce/CsvToKeyValueMapperTest.java | 104 +++++++++++++++++++++
.../phoenix/util/csv/CsvUpsertExecutorTest.java | 4 +-
.../pherf/result/impl/CSVFileResultHandler.java | 10 +-
.../phoenix/pherf/util/GoogleChartGenerator.java | 38 ++++----
pom.xml | 2 +-
10 files changed, 185 insertions(+), 51 deletions(-)
diff --git
a/phoenix-core-client/src/main/java/org/apache/phoenix/util/CSVCommonsLoader.java
b/phoenix-core-client/src/main/java/org/apache/phoenix/util/CSVCommonsLoader.java
index 3ec34088f6..89300c6860 100644
---
a/phoenix-core-client/src/main/java/org/apache/phoenix/util/CSVCommonsLoader.java
+++
b/phoenix-core-client/src/main/java/org/apache/phoenix/util/CSVCommonsLoader.java
@@ -105,12 +105,12 @@ public class CSVCommonsLoader {
* @return CSVFormat based on constructor settings.
*/
private CSVFormat buildFormat() {
- CSVFormat format =
-
CSVFormat.DEFAULT.withIgnoreEmptyLines(true).withDelimiter(asControlCharacter(fieldDelimiter))
- .withQuote(asControlCharacter(quoteCharacter));
+ CSVFormat.Builder builder =
CSVFormat.DEFAULT.builder().setIgnoreEmptyLines(true)
+ .setDelimiter(asControlCharacter(fieldDelimiter))
+ .setQuote(asControlCharacter(quoteCharacter));
if (escapeCharacter != null) {
- format = format.withEscape(asControlCharacter(escapeCharacter));
+ builder.setEscape(asControlCharacter(escapeCharacter));
}
switch (headerSource) {
@@ -119,17 +119,17 @@ public class CSVCommonsLoader {
break;
case IN_LINE:
// an empty string array triggers csv loader to grab the first line as
the header
- format = format.withHeader(new String[0]);
+ builder.setHeader(new String[0]);
break;
case SUPPLIED_BY_USER:
// a populated string array supplied by the user
- format = format.withHeader(columns.toArray(new
String[columns.size()]));
+ builder.setHeader(columns.toArray(new String[columns.size()]));
break;
default:
throw new RuntimeException("Header source was unable to be inferred.");
}
- return format;
+ return builder.get();
}
/**
@@ -146,12 +146,13 @@ public class CSVCommonsLoader {
* constructor determines the format for the CSV files.
*/
public void upsert(String fileName) throws Exception {
- CSVParser parser = CSVParser.parse(new File(fileName), Charsets.UTF_8,
format);
+ CSVParser parser =
CSVParser.builder().setFormat(format).setCharset(Charsets.UTF_8)
+ .setFile(new File(fileName)).get();
upsert(parser);
}
public void upsert(Reader reader) throws Exception {
- CSVParser parser = new CSVParser(reader, format);
+ CSVParser parser =
CSVParser.builder().setFormat(format).setReader(reader).get();
upsert(parser);
}
diff --git a/phoenix-core-server/pom.xml b/phoenix-core-server/pom.xml
index 92ee57d742..499522b7f5 100644
--- a/phoenix-core-server/pom.xml
+++ b/phoenix-core-server/pom.xml
@@ -30,6 +30,10 @@
<groupId>org.apache.phoenix</groupId>
<artifactId>phoenix-core-client</artifactId>
</dependency>
+ <dependency>
+ <groupId>commons-io</groupId>
+ <artifactId>commons-io</artifactId>
+ </dependency>
<!-- shaded thirdparty dependencies -->
<dependency>
diff --git
a/phoenix-core-server/src/main/java/org/apache/phoenix/mapreduce/CsvToKeyValueMapper.java
b/phoenix-core-server/src/main/java/org/apache/phoenix/mapreduce/CsvToKeyValueMapper.java
index 68e694e33f..c116b109d6 100644
---
a/phoenix-core-server/src/main/java/org/apache/phoenix/mapreduce/CsvToKeyValueMapper.java
+++
b/phoenix-core-server/src/main/java/org/apache/phoenix/mapreduce/CsvToKeyValueMapper.java
@@ -19,6 +19,7 @@ package org.apache.phoenix.mapreduce;
import java.io.IOException;
import java.io.StringReader;
+import java.io.UncheckedIOException;
import java.util.List;
import org.apache.commons.csv.CSVFormat;
import org.apache.commons.csv.CSVParser;
@@ -89,8 +90,8 @@ public class CsvToKeyValueMapper extends
FormatToBytesWritableMapper<CSVRecord>
private final CSVFormat csvFormat;
CsvLineParser(Character fieldDelimiter, Character quote, Character escape)
{
- this.csvFormat =
CSVFormat.DEFAULT.withIgnoreEmptyLines(true).withDelimiter(fieldDelimiter)
- .withEscape(escape).withQuote(quote);
+ this.csvFormat = CSVFormat.DEFAULT.builder().setIgnoreEmptyLines(true)
+ .setDelimiter(fieldDelimiter).setEscape(escape).setQuote(quote).get();
}
@Override
@@ -98,8 +99,12 @@ public class CsvToKeyValueMapper extends
FormatToBytesWritableMapper<CSVRecord>
// TODO Creating a new parser for each line seems terribly inefficient
but
// there's no public way to parse single lines via commons-csv. We
should update
// it to create a LineParser class like this one.
- CSVParser csvParser = new CSVParser(new StringReader(input), csvFormat);
- return Iterables.getFirst(csvParser, null);
+ try (CSVParser csvParser =
+ CSVParser.builder().setFormat(csvFormat).setReader(new
StringReader(input)).get()) {
+ return Iterables.getFirst(csvParser, null);
+ } catch (UncheckedIOException e) {
+ throw new IOException(e.getMessage(), e.getCause());
+ }
}
}
}
diff --git
a/phoenix-core-server/src/main/java/org/apache/phoenix/mapreduce/FormatToBytesWritableMapper.java
b/phoenix-core-server/src/main/java/org/apache/phoenix/mapreduce/FormatToBytesWritableMapper.java
index d99f76371b..2e0c0daa49 100644
---
a/phoenix-core-server/src/main/java/org/apache/phoenix/mapreduce/FormatToBytesWritableMapper.java
+++
b/phoenix-core-server/src/main/java/org/apache/phoenix/mapreduce/FormatToBytesWritableMapper.java
@@ -110,6 +110,7 @@ public abstract class FormatToBytesWritableMapper<RECORD>
protected List<String> tableNames;
protected List<String> logicalNames;
protected MapperUpsertListener<RECORD> upsertListener;
+ protected boolean ignoreInvalidRows;
/*
* lookup table for column index. Index in the List matches to the index in
tableNames List
@@ -147,8 +148,8 @@ public abstract class FormatToBytesWritableMapper<RECORD>
throw new RuntimeException(e);
}
- upsertListener =
- new MapperUpsertListener<RECORD>(context,
conf.getBoolean(IGNORE_INVALID_ROW_CONFKEY, true));
+ ignoreInvalidRows = conf.getBoolean(IGNORE_INVALID_ROW_CONFKEY, true);
+ upsertListener = new MapperUpsertListener<RECORD>(context,
ignoreInvalidRows);
upsertExecutor = buildUpsertExecutor(conf);
preUpdateProcessor = PhoenixConfigurationUtil.loadPreUpsertProcessor(conf);
}
@@ -164,7 +165,13 @@ public abstract class FormatToBytesWritableMapper<RECORD>
try {
record = getLineParser().parse(value.toString());
} catch (IOException e) {
+ // When --ignore-errors (-g) is not set, a malformed record (e.g.
unterminated quote)
+ // fails the mapper and hence the entire job. When --ignore-errors is
set, the bad
+ // record is skipped and counted via the "Parser errors" counter.
context.getCounter(COUNTER_GROUP_NAME, "Parser errors").increment(1L);
+ if (!ignoreInvalidRows) {
+ throw new IOException("Error parsing input: " + e.getMessage(), e);
+ }
return;
}
diff --git
a/phoenix-core/src/it/java/org/apache/phoenix/end2end/CSVCommonsLoaderIT.java
b/phoenix-core/src/it/java/org/apache/phoenix/end2end/CSVCommonsLoaderIT.java
index 2579f31c7a..2b477a691a 100644
---
a/phoenix-core/src/it/java/org/apache/phoenix/end2end/CSVCommonsLoaderIT.java
+++
b/phoenix-core/src/it/java/org/apache/phoenix/end2end/CSVCommonsLoaderIT.java
@@ -117,7 +117,8 @@ public class CSVCommonsLoaderIT extends
ParallelStatsDisabledIT {
PreparedStatement statement =
conn.prepareStatement("SELECT SYMBOL, COMPANY FROM " + stockTableName);
ResultSet phoenixResultSet = statement.executeQuery();
- parser = new CSVParser(new StringReader(STOCK_CSV_VALUES_WITH_HEADER),
csvUtil.getFormat());
+ parser = CSVParser.builder().setFormat(csvUtil.getFormat())
+ .setReader(new StringReader(STOCK_CSV_VALUES_WITH_HEADER)).get();
for (CSVRecord record : parser) {
assertTrue(phoenixResultSet.next());
int i = 0;
@@ -161,7 +162,8 @@ public class CSVCommonsLoaderIT extends
ParallelStatsDisabledIT {
PreparedStatement statement =
tenantConn.prepareStatement("SELECT SYMBOL, COMPANY FROM " +
stockTableMultiName);
ResultSet phoenixResultSet = statement.executeQuery();
- parser = new CSVParser(new StringReader(STOCK_CSV_VALUES_WITH_HEADER),
csvUtil.getFormat());
+ parser = CSVParser.builder().setFormat(csvUtil.getFormat())
+ .setReader(new StringReader(STOCK_CSV_VALUES_WITH_HEADER)).get();
for (CSVRecord record : parser) {
assertTrue(phoenixResultSet.next());
int i = 0;
@@ -200,7 +202,8 @@ public class CSVCommonsLoaderIT extends
ParallelStatsDisabledIT {
PreparedStatement statement =
conn.prepareStatement("SELECT SYMBOL, COMPANY FROM " + stockTableName);
ResultSet phoenixResultSet = statement.executeQuery();
- parser = new CSVParser(new StringReader(STOCK_TDV_VALUES_WITH_HEADER),
csvUtil.getFormat());
+ parser = CSVParser.builder().setFormat(csvUtil.getFormat())
+ .setReader(new StringReader(STOCK_TDV_VALUES_WITH_HEADER)).get();
for (CSVRecord record : parser) {
assertTrue(phoenixResultSet.next());
int i = 0;
@@ -239,8 +242,8 @@ public class CSVCommonsLoaderIT extends
ParallelStatsDisabledIT {
PreparedStatement statement =
conn.prepareStatement("SELECT SYMBOL, COMPANY FROM " + stockTableName);
ResultSet phoenixResultSet = statement.executeQuery();
- parser =
- new CSVParser(new StringReader(STOCK_CSV_VALUES_WITH_DELIMITER),
csvUtil.getFormat());
+ parser = CSVParser.builder().setFormat(csvUtil.getFormat())
+ .setReader(new StringReader(STOCK_CSV_VALUES_WITH_DELIMITER)).get();
for (CSVRecord record : parser) {
assertTrue(phoenixResultSet.next());
int i = 0;
@@ -279,7 +282,8 @@ public class CSVCommonsLoaderIT extends
ParallelStatsDisabledIT {
PreparedStatement statement =
conn.prepareStatement("SELECT SYMBOL, COMPANY FROM " + stockTableName);
ResultSet phoenixResultSet = statement.executeQuery();
- parser = new CSVParser(new StringReader(STOCK_CSV_VALUES),
csvUtil.getFormat());
+ parser = CSVParser.builder().setFormat(csvUtil.getFormat())
+ .setReader(new StringReader(STOCK_CSV_VALUES)).get();
for (CSVRecord record : parser) {
assertTrue(phoenixResultSet.next());
int i = 0;
@@ -317,7 +321,8 @@ public class CSVCommonsLoaderIT extends
ParallelStatsDisabledIT {
PreparedStatement statement =
conn.prepareStatement("SELECT SYMBOL, COMPANY FROM " + stockTableName);
ResultSet phoenixResultSet = statement.executeQuery();
- parser = new CSVParser(new StringReader(STOCK_CSV_VALUES),
csvUtil.getFormat());
+ parser = CSVParser.builder().setFormat(csvUtil.getFormat())
+ .setReader(new StringReader(STOCK_CSV_VALUES)).get();
for (CSVRecord record : parser) {
assertTrue(phoenixResultSet.next());
int i = 0;
@@ -356,7 +361,8 @@ public class CSVCommonsLoaderIT extends
ParallelStatsDisabledIT {
PreparedStatement statement =
conn.prepareStatement("SELECT SYMBOL, COMPANY FROM " + stockTableName);
ResultSet phoenixResultSet = statement.executeQuery();
- parser = new CSVParser(new StringReader(STOCK_CSV_VALUES),
csvUtil.getFormat());
+ parser = CSVParser.builder().setFormat(csvUtil.getFormat())
+ .setReader(new StringReader(STOCK_CSV_VALUES)).get();
for (CSVRecord record : parser) {
assertTrue(phoenixResultSet.next());
assertEquals(record.get(0), phoenixResultSet.getString(1));
@@ -483,7 +489,8 @@ public class CSVCommonsLoaderIT extends
ParallelStatsDisabledIT {
"SELECT CKEY, CVARCHAR, CCHAR, CINTEGER, CDECIMAL, CUNSIGNED_INT,
CBOOLEAN, CBIGINT, CUNSIGNED_LONG, CTIME, CDATE FROM "
+ DATATYPE_TABLE);
ResultSet phoenixResultSet = statement.executeQuery();
- parser = new CSVParser(new StringReader(DATATYPES_CSV_VALUES),
csvUtil.getFormat());
+ parser = CSVParser.builder().setFormat(csvUtil.getFormat())
+ .setReader(new StringReader(DATATYPES_CSV_VALUES)).get();
for (CSVRecord record : parser) {
assertTrue(phoenixResultSet.next());
@@ -531,8 +538,8 @@ public class CSVCommonsLoaderIT extends
ParallelStatsDisabledIT {
PreparedStatement statement =
conn.prepareStatement("SELECT MYKEY, MYVALUE FROM " +
ENCAPSULATED_CHARS_TABLE);
ResultSet phoenixResultSet = statement.executeQuery();
- parser = new CSVParser(new
StringReader(CSV_VALUES_ENCAPSULATED_CONTROL_CHARS_WITH_HEADER),
- csvUtil.getFormat());
+ parser = CSVParser.builder().setFormat(csvUtil.getFormat())
+ .setReader(new
StringReader(CSV_VALUES_ENCAPSULATED_CONTROL_CHARS_WITH_HEADER)).get();
for (CSVRecord record : parser) {
assertTrue(phoenixResultSet.next());
int i = 0;
@@ -568,7 +575,7 @@ public class CSVCommonsLoaderIT extends
ParallelStatsDisabledIT {
fail();
} catch (RuntimeException e) {
assertTrue(e.getMessage(),
- e.getMessage().contains("invalid char between encapsulated token and
delimiter"));
+ e.getMessage().contains("Invalid character between encapsulated
token and delimiter"));
}
} finally {
if (parser != null) parser.close();
diff --git
a/phoenix-core/src/test/java/org/apache/phoenix/mapreduce/CsvToKeyValueMapperTest.java
b/phoenix-core/src/test/java/org/apache/phoenix/mapreduce/CsvToKeyValueMapperTest.java
index 242166ceb1..4247361ea3 100644
---
a/phoenix-core/src/test/java/org/apache/phoenix/mapreduce/CsvToKeyValueMapperTest.java
+++
b/phoenix-core/src/test/java/org/apache/phoenix/mapreduce/CsvToKeyValueMapperTest.java
@@ -18,9 +18,15 @@
package org.apache.phoenix.mapreduce;
import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNull;
import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
import java.io.IOException;
+import java.io.StringReader;
+import java.io.UncheckedIOException;
+import org.apache.commons.csv.CSVFormat;
+import org.apache.commons.csv.CSVParser;
import org.apache.commons.csv.CSVRecord;
import org.junit.Test;
@@ -49,4 +55,102 @@ public class CsvToKeyValueMapperTest {
assertTrue(parsed.isConsistent());
assertEquals(1, parsed.getRecordNumber());
}
+
+ @Test(expected = IOException.class)
+ public void testCsvLineParserUnterminatedQuoteThrowsIOException() throws
IOException {
+ CsvToKeyValueMapper.CsvLineParser lineParser =
+ new CsvToKeyValueMapper.CsvLineParser(',', '"', '\\');
+ // Unterminated quoted field should throw IOException
+ lineParser.parse("1,\"unterminated quote,3");
+ }
+
+ @Test(expected = IOException.class)
+ public void testCsvLineParserEOFInEncapsulatedToken() throws IOException {
+ CsvToKeyValueMapper.CsvLineParser lineParser =
+ new CsvToKeyValueMapper.CsvLineParser(',', '"', '\\');
+ // This simulates the exact error: "EOF reached before encapsulated token
finished"
+ lineParser.parse("3,\"Charlie,Sales");
+ }
+
+ @Test
+ public void testCsvLineParserEmptyLineReturnsNull() throws IOException {
+ CsvToKeyValueMapper.CsvLineParser lineParser =
+ new CsvToKeyValueMapper.CsvLineParser(',', '"', '\\');
+ CSVRecord parsed = lineParser.parse("");
+ assertNull(parsed);
+ }
+
+ @Test
+ public void testCsvLineParserValidRecordAfterBadRecord() throws IOException {
+ CsvToKeyValueMapper.CsvLineParser lineParser =
+ new CsvToKeyValueMapper.CsvLineParser(',', '"', '\\');
+
+ // Bad record throws
+ try {
+ lineParser.parse("3,\"Charlie,Sales");
+ fail("Expected IOException for unterminated quote");
+ } catch (IOException e) {
+ // expected
+ }
+
+ // Valid record still parses fine (parser is stateless per line)
+ CSVRecord parsed = lineParser.parse("4,Diana,Finance");
+ assertEquals("4", parsed.get(0));
+ assertEquals("Diana", parsed.get(1));
+ assertEquals("Finance", parsed.get(2));
+ }
+
+ @Test
+ public void testCsvLineParserEmbeddedCommaInQuotedField() throws IOException
{
+ CsvToKeyValueMapper.CsvLineParser lineParser =
+ new CsvToKeyValueMapper.CsvLineParser(',', '"', '\\');
+ // Valid: comma inside quoted field
+ CSVRecord parsed = lineParser.parse("1,\"Sales, Marketing\",HR");
+ assertEquals("1", parsed.get(0));
+ assertEquals("Sales, Marketing", parsed.get(1));
+ assertEquals("HR", parsed.get(2));
+ }
+
+ @Test
+ public void testCsvLineParserOnlyDelimiters() throws IOException {
+ CsvToKeyValueMapper.CsvLineParser lineParser =
+ new CsvToKeyValueMapper.CsvLineParser(',', '"', '\\');
+ // Valid: three empty fields
+ CSVRecord parsed = lineParser.parse(",,");
+ assertEquals("", parsed.get(0));
+ assertEquals("", parsed.get(1));
+ assertEquals("", parsed.get(2));
+ }
+
+ @Test(expected = IOException.class)
+ public void testCsvLineParserSingleDanglingQuote() throws IOException {
+ CsvToKeyValueMapper.CsvLineParser lineParser =
+ new CsvToKeyValueMapper.CsvLineParser(',', '"', '\\');
+ // Single dangling quote - unterminated encapsulated token
+ lineParser.parse("\"");
+ }
+
+ @Test(expected = IOException.class)
+ public void testCsvLineParserInvalidCharAfterQuotedField() throws
IOException {
+ CsvToKeyValueMapper.CsvLineParser lineParser =
+ new CsvToKeyValueMapper.CsvLineParser(',', '"', '\\');
+ // Invalid char between closing quote and delimiter: "Eve,"Bad
+ lineParser.parse("5,\"Eve,\"Bad quote handling");
+ }
+
+ @Test
+ public void testCsvParserIteratorThrowsUncheckedIOException() throws
IOException {
+ // Proves that the CSVParser iterator throws UncheckedIOException on
malformed input.
+ // This is WHY CsvLineParser.parse() needs to catch UncheckedIOException.
+ CSVFormat format =
CSVFormat.DEFAULT.builder().setIgnoreEmptyLines(true).get();
+ try (CSVParser parser = CSVParser.builder().setFormat(format)
+ .setReader(new StringReader("1,\"unterminated quote")).get()) {
+ for (CSVRecord record : parser) {
+ fail("Should have thrown UncheckedIOException");
+ }
+ } catch (UncheckedIOException e) {
+ // This is what the iterator throws — confirming Uncheck.get() wraps
IOException
+ assertTrue(e.getCause() instanceof IOException);
+ }
+ }
}
diff --git
a/phoenix-core/src/test/java/org/apache/phoenix/util/csv/CsvUpsertExecutorTest.java
b/phoenix-core/src/test/java/org/apache/phoenix/util/csv/CsvUpsertExecutorTest.java
index eed02eb2cb..1044ae96f2 100644
---
a/phoenix-core/src/test/java/org/apache/phoenix/util/csv/CsvUpsertExecutorTest.java
+++
b/phoenix-core/src/test/java/org/apache/phoenix/util/csv/CsvUpsertExecutorTest.java
@@ -18,6 +18,7 @@
package org.apache.phoenix.util.csv;
import java.io.IOException;
+import java.io.StringReader;
import java.sql.Connection;
import java.sql.SQLException;
import java.util.List;
@@ -60,7 +61,8 @@ public class CsvUpsertExecutorTest extends
AbstractUpsertExecutorTest<CSVRecord,
}
}
String inputRecord = Joiner.on(',').join(columnValues);
- return Iterables.getFirst(CSVParser.parse(inputRecord, CSVFormat.DEFAULT),
null);
+ return Iterables.getFirst(CSVParser.builder().setFormat(CSVFormat.DEFAULT)
+ .setReader(new StringReader(inputRecord)).get(), null);
}
@Before
diff --git
a/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/result/impl/CSVFileResultHandler.java
b/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/result/impl/CSVFileResultHandler.java
index 2928c3f727..e2a04b0157 100644
---
a/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/result/impl/CSVFileResultHandler.java
+++
b/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/result/impl/CSVFileResultHandler.java
@@ -23,6 +23,8 @@ import java.io.PrintWriter;
import java.nio.charset.Charset;
import java.util.ArrayList;
import java.util.List;
+import java.util.stream.Collectors;
+import java.util.stream.StreamSupport;
import org.apache.commons.csv.CSVFormat;
import org.apache.commons.csv.CSVParser;
import org.apache.commons.csv.CSVPrinter;
@@ -47,7 +49,8 @@ public class CSVFileResultHandler extends CSVResultHandler {
public synchronized List<Result> read() throws IOException {
util.ensureBaseResultDirExists();
File file = new File(resultFileName);
- try (CSVParser parser = CSVParser.parse(file, Charset.defaultCharset(),
CSVFormat.DEFAULT)) {
+ try (CSVParser parser = CSVParser.builder().setFormat(CSVFormat.DEFAULT)
+ .setCharset(Charset.defaultCharset()).setFile(file).get()) {
List<CSVRecord> records = parser.getRecords();
List<Result> results = new ArrayList<>();
String header = null;
@@ -55,11 +58,12 @@ public class CSVFileResultHandler extends CSVResultHandler {
// First record is the CSV Header
if (record.getRecordNumber() == 1) {
- header = record.toString();
+ header = StreamSupport.stream(record.spliterator(), false)
+ .collect(Collectors.joining(PherfConstants.RESULT_FILE_DELIMETER));
continue;
}
List<ResultValue> resultValues = new ArrayList<>();
- for (String val :
record.toString().split(PherfConstants.RESULT_FILE_DELIMETER)) {
+ for (String val : record) {
resultValues.add(new ResultValue(val));
}
Result result = new Result(resultFileDetails, header, resultValues);
diff --git
a/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/util/GoogleChartGenerator.java
b/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/util/GoogleChartGenerator.java
index 5960bb070c..d6cac7d359 100644
---
a/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/util/GoogleChartGenerator.java
+++
b/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/util/GoogleChartGenerator.java
@@ -91,27 +91,27 @@ public class GoogleChartGenerator {
String resultFileName = resultDir + PherfConstants.PATH_SEPARATOR +
PherfConstants.RESULT_PREFIX
+ label + ResultFileDetails.CSV_AGGREGATE_PERFORMANCE.getExtension();
- FileReader in = new FileReader(resultFileName);
- final CSVParser parser = new CSVParser(in, CSVFormat.DEFAULT.withHeader());
-
- for (CSVRecord record : parser) {
- String group = record.get("QUERY_GROUP");
- String query = record.get("QUERY");
- String explain = record.get("EXPLAIN_PLAN");
- String tenantId = record.get("TENANT_ID");
- long avgTime = Long.parseLong(record.get("AVG_TIME_MS"));
- long minTime = Long.parseLong(record.get("AVG_MIN_TIME_MS"));
- long numRuns = Long.parseLong(record.get("RUN_COUNT"));
- long rowCount = Long.parseLong(record.get("RESULT_ROW_COUNT"));
- Node node = new Node(minTime, avgTime, numRuns, explain, query,
tenantId, label, rowCount);
-
- if (datanodes.containsKey(group)) {
- datanodes.get(group).getDataSet().put(label, node);
- } else {
- datanodes.put(group, new DataNode(label, node));
+ try (CSVParser parser =
+
CSVParser.builder().setFormat(CSVFormat.DEFAULT.builder().setHeader().get())
+ .setReader(new FileReader(resultFileName)).get()) {
+ for (CSVRecord record : parser) {
+ String group = record.get("QUERY_GROUP");
+ String query = record.get("QUERY");
+ String explain = record.get("EXPLAIN_PLAN");
+ String tenantId = record.get("TENANT_ID");
+ long avgTime = Long.parseLong(record.get("AVG_TIME_MS"));
+ long minTime = Long.parseLong(record.get("AVG_MIN_TIME_MS"));
+ long numRuns = Long.parseLong(record.get("RUN_COUNT"));
+ long rowCount = Long.parseLong(record.get("RESULT_ROW_COUNT"));
+ Node node = new Node(minTime, avgTime, numRuns, explain, query,
tenantId, label, rowCount);
+
+ if (datanodes.containsKey(group)) {
+ datanodes.get(group).getDataSet().put(label, node);
+ } else {
+ datanodes.put(group, new DataNode(label, node));
+ }
}
}
- parser.close();
}
/**
diff --git a/pom.xml b/pom.xml
index cac9a90d55..957b373067 100644
--- a/pom.xml
+++ b/pom.xml
@@ -112,7 +112,7 @@
<protoc.version>2.5.0</protoc.version>
<commons-io.version>2.18.0</commons-io.version>
<commons-lang3.version>3.18.0</commons-lang3.version>
- <commons-csv.version>1.0</commons-csv.version>
+ <commons-csv.version>1.14.1</commons-csv.version>
<commons-compress.version>1.26.0</commons-compress.version>
<sqlline.version>1.9.0</sqlline.version>
<jcip-annotations.version>1.0-1</jcip-annotations.version>
