This is an automated email from the ASF dual-hosted git repository.
caiconghui pushed a commit to branch libhdfs3
in repository https://gitbox.apache.org/repos/asf/doris-thirdparty.git
The following commit(s) were added to refs/heads/libhdfs3 by this push:
new dad0008 [enhancement](libhdfs3) Support crc32 checksum (#7)
dad0008 is described below
commit dad00080bdf065ee1a1b493f56e5a8024c961044
Author: GoGoWen <82132356+gogo...@users.noreply.github.com>
AuthorDate: Tue Aug 9 20:30:13 2022 +0800
[enhancement](libhdfs3) Support crc32 checksum (#7)
---
src/client/LocalBlockReader.cpp | 5 +--
src/client/RemoteBlockReader.cpp | 5 +--
src/common/Crc32.h | 74 ++++++++++++++++++++++++++++++++++++++++
3 files changed, 80 insertions(+), 4 deletions(-)
diff --git a/src/client/LocalBlockReader.cpp b/src/client/LocalBlockReader.cpp
index 2455ad1..4080b72 100644
--- a/src/client/LocalBlockReader.cpp
+++ b/src/client/LocalBlockReader.cpp
@@ -20,6 +20,7 @@
* limitations under the License.
*/
#include "BigEndian.h"
+#include "Crc32.h"
#include "datatransfer.pb.h"
#include "Exception.h"
#include "ExceptionInternal.h"
@@ -76,8 +77,8 @@ LocalBlockReader::LocalBlockReader(const
shared_ptr<ReadShortCircuitInfo>& info,
break;
case ChecksumTypeProto::CHECKSUM_CRC32:
- THROW(HdfsIOException,
- "LocalBlockReader does not support CRC32 checksum.");
+ checksum = std::make_shared<Crc32>();
+ checksumSize = sizeof(int32_t);
break;
case ChecksumTypeProto::CHECKSUM_CRC32C:
diff --git a/src/client/RemoteBlockReader.cpp b/src/client/RemoteBlockReader.cpp
index d7faf6e..508dfdd 100644
--- a/src/client/RemoteBlockReader.cpp
+++ b/src/client/RemoteBlockReader.cpp
@@ -20,6 +20,7 @@
* limitations under the License.
*/
#include "BigEndian.h"
+#include "Crc32.h"
#include "DataTransferProtocolSender.h"
#include "Exception.h"
#include "ExceptionInternal.h"
@@ -146,8 +147,8 @@ void RemoteBlockReader::checkResponse() {
break;
case ChecksumTypeProto::CHECKSUM_CRC32:
- THROW(HdfsIOException, "RemoteBlockReader does not support CRC32
checksum, Block: %s, from Datanode: %s",
- binfo.toString().c_str(), datanode.formatAddress().c_str());
+ checksum = std::make_shared<Crc32>();
+ checksumSize = sizeof(int32_t);
break;
case ChecksumTypeProto::CHECKSUM_CRC32C:
diff --git a/src/common/Crc32.h b/src/common/Crc32.h
new file mode 100644
index 0000000..2d5cfec
--- /dev/null
+++ b/src/common/Crc32.h
@@ -0,0 +1,74 @@
+/********************************************************************
+ * 2014 -
+ * open source under Apache License Version 2.0
+ ********************************************************************/
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#ifndef _HDFS_LIBHDFS3_COMMON_CRC32_H_
+#define _HDFS_LIBHDFS3_COMMON_CRC32_H_
+
+#include "Checksum.h"
+
+#include <boost/crc.hpp>
+
+namespace Hdfs {
+namespace Internal {
+
+/**
+ * Calculate CRC with boost::crc_32_type.
+ */
+class Crc32: public Checksum {
+public:
+ /**
+ * Constructor.
+ */
+ Crc32() {
+ }
+
+ uint32_t getValue() {
+ return crc.checksum();
+ }
+
+ /**
+ * @ref Checksum#reset()
+ */
+ void reset() {
+ crc.reset();
+ }
+
+ /**
+ * @ref Checksum#update(const void *, int)
+ */
+ void update(const void * b, int len) {
+ crc.process_bytes((const char*) b, len);
+ }
+
+ /**
+ * Destory an Crc32 instance.
+ */
+ ~Crc32() {
+ }
+
+private:
+ boost::crc_32_type crc;
+};
+
+}
+}
+
+#endif /* _HDFS_LIBHDFS3_COMMON_CRC32_H_ */
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org
