ysn2233 commented on a change in pull request #7991: [FLINK-11928][Connectors /
FileSystem]Create a Compression string writer for flink-connectors-filesystem
URL: https://github.com/apache/flink/pull/7991#discussion_r265978795
##########
File path:
flink-connectors/flink-connector-filesystem/src/main/java/org/apache/flink/streaming/connectors/fs/CompressionStringWriter.java
##########
@@ -0,0 +1,71 @@
+package org.apache.flink.streaming.connectors.fs;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.compress.CodecPool;
+import org.apache.hadoop.io.compress.CompressionCodec;
+import org.apache.hadoop.io.compress.CompressionCodecFactory;
+import org.apache.hadoop.io.compress.CompressionOutputStream;
+import org.apache.hadoop.io.compress.Compressor;
+
+import java.io.IOException;
+
+/**
+ * A base class that compresses the input element and write them to the
filesystem. Default serialization is to
+ * write events separates by newline.
+ * Extends the class and override write() to make custom writing
+ */
+public class CompressionStringWriter<T> extends StreamWriterBase<T> implements
Writer<T>{
+
+ private String codecName;
+
+ private transient CompressionOutputStream compressedOutputStream;
+
+ public CompressionStringWriter(String codecName) {
+ this.codecName = codecName;
+ }
+
+ protected CompressionStringWriter(CompressionStringWriter<T> other) {
+ super(other);
+ this.codecName = other.codecName;
+ }
+
+ @Override
+ public void open(FileSystem fs, Path path) throws IOException {
+ super.open(fs, path);
+ Configuration conf = fs.getConf();
+ if (!codecName.equals("None")) {
Review comment:
I have fixed some problem you mentioned. And I tested the writer on both
hdfs and local file system which works fine
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
With regards,
Apache Git Services
