jnioche commented on code in PR #2110:
URL: https://github.com/apache/stormcrawler/pull/2110#discussion_r3917340126
##########
external/warc/src/main/java/org/apache/stormcrawler/warc/WARCRecordFormat.java:
##########
@@ -129,19 +129,120 @@ static String sanitizeWarcFieldValue(String value) {
Pattern.compile("(?i)(?:Content-(?:Encoding|Length)|Transfer-Encoding)");
protected static final String X_HIDE_HEADER = "X-Crawler-";
+ /**
+ * Configuration key setting the algorithm used to compute the
WARC-Payload-Digest and
+ * WARC-Block-Digest fields. Supported values are {@value
#DIGEST_ALGORITHM_SHA1} (the default)
+ * and {@value #DIGEST_ALGORITHM_SHA256}.
+ *
+ * <p>Note: SHA-1 is the convention across the WARC ecosystem and
downstream tooling (CDX
+ * indexes, revisit record handling) may expect it. Change the default
deliberately, not
+ * casually.
+ */
+ public static final String DIGEST_ALGORITHM_PARAM =
"warc.digest.algorithm";
+
+ public static final String DIGEST_ALGORITHM_SHA1 = "sha1";
+
+ public static final String DIGEST_ALGORITHM_SHA256 = "sha256";
+
private static final Base32 base32 = new Base32();
- private static final String digestNoContent = getDigestSha1(new byte[0]);
protected final String protocolMDprefix;
+ /** JCA name of the message digest algorithm, e.g. "SHA-1". */
+ private final String digestJCAName;
+
+ /** Algorithm prefix of the WARC digest fields, e.g. "sha1:". */
+ private final String digestPrefix;
+
+ private final String digestNoContent;
+
public WARCRecordFormat(String protocolMDprefix) {
+ this(protocolMDprefix, DIGEST_ALGORITHM_SHA1);
+ }
+
+ public WARCRecordFormat(String protocolMDprefix, String digestAlgorithm) {
this.protocolMDprefix = protocolMDprefix;
+ this.digestJCAName = getDigestJCAName(digestAlgorithm);
+ this.digestPrefix =
digestJCAName.toLowerCase(Locale.ROOT).replace("-", "") + ":";
+ this.digestNoContent = getDigest(new byte[0]);
}
+ /**
+ * Resolve the configured digest algorithm to the JCA name of the message
digest. The value is
+ * matched case-insensitively and an optional hyphen is ignored, i.e.
"sha256",
+ * "SHA-256" etc. are all accepted.
+ *
+ * @throws IllegalArgumentException if the value is not a supported
algorithm
+ */
+ private static String getDigestJCAName(String digestAlgorithm) {
+ if (digestAlgorithm == null) {
+ return "SHA-1";
+ }
+ return switch
(digestAlgorithm.trim().toLowerCase(Locale.ROOT).replace("-", "")) {
+ case DIGEST_ALGORITHM_SHA1 -> "SHA-1";
+ case DIGEST_ALGORITHM_SHA256 -> "SHA-256";
+ default ->
+ throw new IllegalArgumentException(
+ "Unsupported value ["
+ + digestAlgorithm
+ + "] for "
+ + DIGEST_ALGORITHM_PARAM
+ + ", supported algorithms: "
+ + DIGEST_ALGORITHM_SHA1
+ + ", "
+ + DIGEST_ALGORITHM_SHA256);
+ };
+ }
+
+ /**
+ * Compute the digest of the given bytes with the configured algorithm.
+ *
+ * @return digest in the form
"<algorithm>:<base32>", e.g.
+ * "sha1:..."
+ */
+ public String getDigest(byte[] bytes) {
+ MessageDigest md = DigestUtils.getDigest(digestJCAName);
+ return digestPrefix + base32Unpadded(md.digest(bytes));
+ }
+
+ /**
+ * Compute the digest of the concatenation of the two given byte arrays
with the configured
+ * algorithm.
+ *
+ * @return digest in the form
"<algorithm>:<base32>", e.g.
+ * "sha1:..."
+ */
+ public String getDigest(byte[] bytes1, byte[] bytes2) {
+ MessageDigest md = DigestUtils.getDigest(digestJCAName);
+ md.update(bytes1);
+ return digestPrefix + base32Unpadded(md.digest(bytes2));
+ }
+
+ /**
+ * Base32-encode a digest value without the trailing "=" padding
characters: the WARC
+ * digest fields define the digest value as a token, which does not allow
the padding character
+ * (cf. ISO 28500 WARC 1.1, WARC-Block-Digest / WARC-Payload-Digest).
SHA-1 digests are
+ * unaffected (32 characters without padding), while e.g. SHA-256 digests
would end in
+ * "====".
+ */
+ private static String base32Unpadded(byte[] digest) {
+ return StringUtils.stripEnd(base32.encodeAsString(digest), "=");
+ }
+
+ /**
+ * @deprecated use {@link #getDigest(byte[])} instead; the algorithm is
set by {@link
+ * #DIGEST_ALGORITHM_PARAM} and no longer fixed to SHA-1
+ */
+ @Deprecated
public static String getDigestSha1(byte[] bytes) {
return "sha1:" + base32.encodeAsString(DigestUtils.sha1(bytes));
}
+ /**
+ * @deprecated use {@link #getDigest(byte[], byte[])} instead; the
algorithm is set by {@link
+ * #DIGEST_ALGORITHM_PARAM} and no longer fixed to SHA-1
+ */
+ @Deprecated
public static String getDigestSha1(byte[] bytes1, byte[] bytes2) {
Review Comment:
> We are going to do a new major release anyway, so if this is deprecated it
can also be removed imho.
>
> what do others think? @jnioche ?
perfect time to remove it indeed
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
