This is an automated email from the ASF dual-hosted git repository.
nehapawar pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/pinot.git
The following commit(s) were added to refs/heads/master by this push:
new cf68e22 [7187] Validate CSV Header For Configured Delimiter (#7237)
cf68e22 is described below
commit cf68e22cd153d9f4d82db960cb89d73e18df1a8c
Author: suddendust <[email protected]>
AuthorDate: Thu Aug 19 00:39:08 2021 +0530
[7187] Validate CSV Header For Configured Delimiter (#7237)
Relevant discussion in the issue 7187. Currently, it was assumed that the
configured delimiter in the ingestion job spec also works for the header. This
PR simply adds an explicit check that throws an exception if the header does
not contain the delimiter.
---
.../plugin/inputformat/csv/CSVRecordReader.java | 20 +++++++
.../inputformat/csv/CSVRecordReaderTest.java | 70 ++++++++++++++++++++++
2 files changed, 90 insertions(+)
diff --git
a/pinot-plugins/pinot-input-format/pinot-csv/src/main/java/org/apache/pinot/plugin/inputformat/csv/CSVRecordReader.java
b/pinot-plugins/pinot-input-format/pinot-csv/src/main/java/org/apache/pinot/plugin/inputformat/csv/CSVRecordReader.java
index fc14272..01a394a 100644
---
a/pinot-plugins/pinot-input-format/pinot-csv/src/main/java/org/apache/pinot/plugin/inputformat/csv/CSVRecordReader.java
+++
b/pinot-plugins/pinot-input-format/pinot-csv/src/main/java/org/apache/pinot/plugin/inputformat/csv/CSVRecordReader.java
@@ -85,6 +85,8 @@ public class CSVRecordReader implements RecordReader {
if (csvHeader == null) {
format = format.withHeader();
} else {
+ //validate header for the delimiter before splitting
+ validateHeaderForDelimiter(delimiter, csvHeader, format);
format = format.withHeader(StringUtils.split(csvHeader, delimiter));
}
Character commentMarker = config.getCommentMarker();
@@ -103,6 +105,24 @@ public class CSVRecordReader implements RecordReader {
_recordExtractor.init(fieldsToRead, recordExtractorConfig);
}
+ private void validateHeaderForDelimiter(char delimiter, String csvHeader,
CSVFormat format)
+ throws IOException {
+ CSVParser parser =
format.parse(RecordReaderUtils.getBufferedReader(_dataFile));
+ Iterator<CSVRecord> iterator = parser.iterator();
+ if (iterator.hasNext() && recordHasMultipleValues(iterator.next()) &&
delimiterNotPresentInHeader(delimiter,
+ csvHeader)) {
+ throw new IllegalArgumentException("Configured header does not contain
the configured delimiter");
+ }
+ }
+
+ private boolean recordHasMultipleValues(CSVRecord record) {
+ return record.size() > 1;
+ }
+
+ private boolean delimiterNotPresentInHeader(char delimiter, String
csvHeader) {
+ return !StringUtils.contains(csvHeader, delimiter);
+ }
+
private void init()
throws IOException {
_parser = _format.parse(RecordReaderUtils.getBufferedReader(_dataFile));
diff --git
a/pinot-plugins/pinot-input-format/pinot-csv/src/test/java/org/apache/pinot/plugin/inputformat/csv/CSVRecordReaderTest.java
b/pinot-plugins/pinot-input-format/pinot-csv/src/test/java/org/apache/pinot/plugin/inputformat/csv/CSVRecordReaderTest.java
index 9cd98d3..7fa6b9b 100644
---
a/pinot-plugins/pinot-input-format/pinot-csv/src/test/java/org/apache/pinot/plugin/inputformat/csv/CSVRecordReaderTest.java
+++
b/pinot-plugins/pinot-input-format/pinot-csv/src/test/java/org/apache/pinot/plugin/inputformat/csv/CSVRecordReaderTest.java
@@ -20,6 +20,7 @@ package org.apache.pinot.plugin.inputformat.csv;
import java.io.File;
import java.io.FileWriter;
+import java.io.IOException;
import java.util.List;
import java.util.Map;
import org.apache.commons.csv.CSVFormat;
@@ -32,6 +33,7 @@ import org.apache.pinot.spi.data.readers.GenericRow;
import org.apache.pinot.spi.data.readers.PrimaryKey;
import org.apache.pinot.spi.data.readers.RecordReader;
import org.testng.Assert;
+import org.testng.annotations.Test;
public class CSVRecordReaderTest extends AbstractRecordReaderTest {
@@ -100,4 +102,72 @@ public class CSVRecordReaderTest extends
AbstractRecordReaderTest {
}
Assert.assertFalse(recordReader.hasNext());
}
+
+ @Test
+ public void testInvalidDelimiterInHeader() {
+ //setup
+ CSVRecordReaderConfig csvRecordReaderConfig = new CSVRecordReaderConfig();
+ csvRecordReaderConfig.setMultiValueDelimiter(CSV_MULTI_VALUE_DELIMITER);
+
csvRecordReaderConfig.setHeader("col1;col2;col3;col4;col5;col6;col7;col8;col9;col10");
+ csvRecordReaderConfig.setDelimiter(',');
+ CSVRecordReader csvRecordReader = new CSVRecordReader();
+
+ //execute and assert
+ Assert.assertThrows(IllegalArgumentException.class,
+ () -> csvRecordReader.init(_dataFile, null, csvRecordReaderConfig));
+ }
+
+ @Test
+ public void testValidDelimiterInHeader()
+ throws IOException {
+ //setup
+ CSVRecordReaderConfig csvRecordReaderConfig = new CSVRecordReaderConfig();
+ csvRecordReaderConfig.setMultiValueDelimiter(CSV_MULTI_VALUE_DELIMITER);
+
csvRecordReaderConfig.setHeader("col1,col2,col3,col4,col5,col6,col7,col8,col9,col10");
+ csvRecordReaderConfig.setDelimiter(',');
+ CSVRecordReader csvRecordReader = new CSVRecordReader();
+
+ //read all fields
+ //execute and assert
+ csvRecordReader.init(_dataFile, null, csvRecordReaderConfig);
+ Assert.assertTrue(csvRecordReader.hasNext());
+ }
+
+ /**
+ * When CSV records contain a single value, then no exception should be
throw while initialising.
+ * This test requires a different setup from the rest of the tests as it
requires a single-column
+ * CSV. Therefore, we re-write already generated records into a new file,
but only the first
+ * column.
+ *
+ * @throws IOException
+ */
+ @Test
+ public void testHeaderDelimiterSingleColumn()
+ throws IOException {
+ //setup
+
+ //create a single value CSV
+ Schema pinotSchema = getPinotSchema();
+ //write only the first column in the schema
+ String column = pinotSchema.getColumnNames().toArray(new String[0])[0];
+ //use a different file name so that other tests aren't affected
+ File file = new File(_tempDir, "data1.csv");
+ try (FileWriter fileWriter = new FileWriter(file);
+ CSVPrinter csvPrinter = new CSVPrinter(fileWriter,
CSVFormat.DEFAULT.withHeader(column))) {
+ for (Map<String, Object> r : _records) {
+ Object[] record = new Object[1];
+ record[0] = r.get(column);
+ csvPrinter.printRecord(record);
+ }
+ }
+
+ CSVRecordReaderConfig csvRecordReaderConfig = new CSVRecordReaderConfig();
+ csvRecordReaderConfig.setMultiValueDelimiter(CSV_MULTI_VALUE_DELIMITER);
+ csvRecordReaderConfig.setHeader("col1");
+ CSVRecordReader csvRecordReader = new CSVRecordReader();
+
+ //execute and assert
+ csvRecordReader.init(file, null, csvRecordReaderConfig);
+ Assert.assertTrue(csvRecordReader.hasNext());
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
