This is an automated email from the ASF dual-hosted git repository. leerho pushed a commit to branch Fix_endian_issues in repository https://gitbox.apache.org/repos/asf/datasketches-java.git
commit 0d0939522c509cf974bb0b17493dfbb1a3197307 Author: Lee Rhodes <[email protected]> AuthorDate: Sun Aug 3 13:04:51 2025 -0700 Made detection of BigEndian consistent throughout. Added finals (automatic) where possible. Added parentheses (automatic) to reduce ambiguity Fixed some missing license headers. --- pom.xml | 5 +- .../BigEndianNativeOrderNotSupportedException.java | 35 +++++++++ .../org/apache/datasketches/common/Family.java | 14 ++-- .../java/org/apache/datasketches/common/Util.java | 11 ++- .../org/apache/datasketches/cpc/PreambleUtil.java | 14 +--- .../org/apache/datasketches/hll/PreambleUtil.java | 12 +-- .../datasketches/quantiles/PreambleUtil.java | 11 +-- .../apache/datasketches/sampling/PreambleUtil.java | 14 ---- .../theta/DirectQuickSelectSketch.java | 2 - .../apache/datasketches/theta/HeapAlphaSketch.java | 34 ++++----- .../datasketches/theta/HeapQuickSelectSketch.java | 2 +- .../datasketches/theta/IntersectionImpl.java | 2 +- .../apache/datasketches/theta/PreambleUtil.java | 15 +--- .../datasketches/theta/SingleItemSketch.java | 6 +- .../apache/datasketches/theta/UpdateSketch.java | 8 +- .../apache/datasketches/tuple/CompactSketch.java | 19 ++--- .../datasketches/tuple/QuickSelectSketch.java | 23 ++---- .../tuple/arrayofdoubles/ArrayOfDoublesSketch.java | 2 +- .../DirectArrayOfDoublesCompactSketch.java | 21 +----- .../DirectArrayOfDoublesQuickSelectSketch.java | 14 +--- .../HeapArrayOfDoublesCompactSketch.java | 12 +-- .../HeapArrayOfDoublesQuickSelectSketch.java | 9 +-- .../datasketches/count/CountMinSketchTest.java | 7 +- .../datasketches/hash/MurmurHash3FFM21bTest.java | 2 - .../kll/KllMemorySegmentRequestApp.java | 19 +++++ .../ClassicQuantilesMemorySegmentRequestApp.java | 19 +++++ .../theta/DirectQuickSelectSketchTest.java | 9 +-- .../theta/ReadOnlyMemorySegmentTest.java | 86 ++++++++++------------ .../apache/datasketches/theta/UnionImplTest.java | 13 ---- tools/testng.xml | 20 +++++ 30 files changed, 214 insertions(+), 246 deletions(-) diff --git a/pom.xml b/pom.xml index d1de2064d..8990e1eb0 100644 --- a/pom.xml +++ b/pom.xml @@ -96,10 +96,9 @@ under the License. <!-- System-wide properties --> <maven.version>3.9.10</maven.version> <java.version>24</java.version> - <jvm-arguments>-Xmx10g -Duser.language=en -Duser.country=US -Dfile.encoding=UTF-8</jvm-arguments> + <jvm-arguments>-Xmx4g -Duser.language=en -Duser.country=US -Dfile.encoding=UTF-8</jvm-arguments> <maven.compiler.source>${java.version}</maven.compiler.source> <maven.compiler.target>${java.version}</maven.compiler.target> - <argLine>${jvm-arguments}</argLine> <charset.encoding>UTF-8</charset.encoding> <project.build.sourceEncoding>${charset.encoding}</project.build.sourceEncoding> <project.build.resourceEncoding>${charset.encoding}</project.build.resourceEncoding> @@ -159,7 +158,7 @@ under the License. <version>${maven-compiler-plugin.version}</version> <configuration> <compilerArgs> - <arg>${jvm-arguments}</arg> + <arg></arg> </compilerArgs> </configuration> </plugin> diff --git a/src/main/java/org/apache/datasketches/common/BigEndianNativeOrderNotSupportedException.java b/src/main/java/org/apache/datasketches/common/BigEndianNativeOrderNotSupportedException.java new file mode 100644 index 000000000..67dcd34d7 --- /dev/null +++ b/src/main/java/org/apache/datasketches/common/BigEndianNativeOrderNotSupportedException.java @@ -0,0 +1,35 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.datasketches.common; + +/** + * The DataSketches Library is not supported on Big Endian machines. + */ +public class BigEndianNativeOrderNotSupportedException extends SketchesException { + private static final long serialVersionUID = 1L; + + /** + * Constructs a new runtime exception with the message: + * "The DataSketches Library is not supported on Big Endian machines." + */ + public BigEndianNativeOrderNotSupportedException() { + super("The DataSketches Library is not supported on Big Endian machines."); + } +} diff --git a/src/main/java/org/apache/datasketches/common/Family.java b/src/main/java/org/apache/datasketches/common/Family.java index 829a95960..0ca3d7cb8 100644 --- a/src/main/java/org/apache/datasketches/common/Family.java +++ b/src/main/java/org/apache/datasketches/common/Family.java @@ -19,6 +19,7 @@ package org.apache.datasketches.common; +import java.nio.ByteOrder; import java.util.HashMap; import java.util.Locale; import java.util.Map; @@ -165,19 +166,22 @@ public enum Family { private static final Map<Integer, Family> lookupID = new HashMap<>(); private static final Map<String, Family> lookupFamName = new HashMap<>(); - private int id_; - private String famName_; - private int minPreLongs_; - private int maxPreLongs_; + private final int id_; + private final String famName_; + private final int minPreLongs_; + private final int maxPreLongs_; static { for (final Family f : values()) { lookupID.put(f.getID(), f); lookupFamName.put(f.getFamilyName().toUpperCase(Locale.US), f); } + if (ByteOrder.nativeOrder() != ByteOrder.LITTLE_ENDIAN) { + throw new BigEndianNativeOrderNotSupportedException(); + } } - private Family(final int id, final String famName, final int minPreLongs, final int maxPreLongs) { + Family(final int id, final String famName, final int minPreLongs, final int maxPreLongs) { id_ = id; famName_ = famName.toUpperCase(Locale.US); minPreLongs_ = minPreLongs; diff --git a/src/main/java/org/apache/datasketches/common/Util.java b/src/main/java/org/apache/datasketches/common/Util.java index 2c934522c..c36e15937 100644 --- a/src/main/java/org/apache/datasketches/common/Util.java +++ b/src/main/java/org/apache/datasketches/common/Util.java @@ -28,6 +28,7 @@ import static java.lang.foreign.ValueLayout.JAVA_BYTE; import static org.apache.datasketches.hash.MurmurHash3.hash; import java.lang.foreign.MemorySegment; +import java.nio.ByteOrder; import java.util.Comparator; /** @@ -38,6 +39,12 @@ import java.util.Comparator; @SuppressWarnings("unchecked") public final class Util { + static { + if (ByteOrder.nativeOrder() != ByteOrder.LITTLE_ENDIAN) { + throw new BigEndianNativeOrderNotSupportedException(); + } + } + /** * The java line separator character as a String. */ @@ -812,7 +819,7 @@ public final class Util { } /** - * Is item1 Less-Than item2 + * Is item1 Less-Than item2? * @param <T> the type * @param item1 item one * @param item2 item two @@ -824,7 +831,7 @@ public final class Util { } /** - * Is item1 Less-Than-Or-Equal-To item2 + * Is item1 Less-Than-Or-Equal-To item2? * @param <T> the type * @param item1 item one * @param item2 item two diff --git a/src/main/java/org/apache/datasketches/cpc/PreambleUtil.java b/src/main/java/org/apache/datasketches/cpc/PreambleUtil.java index 713e72ded..80960f69a 100644 --- a/src/main/java/org/apache/datasketches/cpc/PreambleUtil.java +++ b/src/main/java/org/apache/datasketches/cpc/PreambleUtil.java @@ -31,7 +31,6 @@ import static org.apache.datasketches.cpc.RuntimeAsserts.rtAssert; import static org.apache.datasketches.cpc.RuntimeAsserts.rtAssertEquals; import java.lang.foreign.MemorySegment; -import java.nio.ByteOrder; import java.util.Objects; import org.apache.datasketches.common.Family; @@ -142,12 +141,6 @@ final class PreambleUtil { private PreambleUtil() {} - static { - if (ByteOrder.nativeOrder() == ByteOrder.BIG_ENDIAN) { - throw new SketchesStateException("This sketch will not work on Big Endian CPUs."); - } - } - private static final String fmt = "%10d%10x"; /** @@ -156,7 +149,7 @@ final class PreambleUtil { static final byte SER_VER = 1; //Flag bit masks, Byte 5 - static final int BIG_ENDIAN_FLAG_MASK = 1; //Reserved. + static final int RESERVED_FLAG_MASK = 1; //Reserved. static final int COMPRESSED_FLAG_MASK = 2; static final int HIP_FLAG_MASK = 4; static final int SUP_VAL_FLAG_MASK = 8; //num Suprising Values > 0 @@ -584,7 +577,6 @@ final class PreambleUtil { //Flags of the Flags byte final String flagsStr = zeroPad(Integer.toBinaryString(flags), 8) + ", " + (flags); - final boolean bigEndian = (flags & BIG_ENDIAN_FLAG_MASK) > 0; final boolean compressed = (flags & COMPRESSED_FLAG_MASK) > 0; final boolean hasHip = (flags & HIP_FLAG_MASK) > 0; final boolean hasSV = (flags & SUP_VAL_FLAG_MASK) > 0; @@ -593,8 +585,6 @@ final class PreambleUtil { final int formatOrdinal = (flags >>> 2) & 0x7; final Format format = Format.ordinalToFormat(formatOrdinal); - final String nativeOrderStr = ByteOrder.nativeOrder().toString(); - long numCoupons = 0; long numSv = 0; long winOffset = 0; @@ -616,8 +606,6 @@ final class PreambleUtil { sb.append("Byte 3: lgK : ").append(lgK).append(LS); sb.append("Byte 4: First Interesting Col : ").append(fiCol).append(LS); sb.append("Byte 5: Flags : ").append(flagsStr).append(LS); - sb.append(" BIG_ENDIAN_STORAGE : ").append(bigEndian).append(LS); - sb.append(" (Native Byte Order) : ").append(nativeOrderStr).append(LS); sb.append(" Compressed : ").append(compressed).append(LS); sb.append(" Has HIP : ").append(hasHip).append(LS); sb.append(" Has Surprising Values : ").append(hasSV).append(LS); diff --git a/src/main/java/org/apache/datasketches/hll/PreambleUtil.java b/src/main/java/org/apache/datasketches/hll/PreambleUtil.java index d824dea59..a43a6f121 100644 --- a/src/main/java/org/apache/datasketches/hll/PreambleUtil.java +++ b/src/main/java/org/apache/datasketches/hll/PreambleUtil.java @@ -32,7 +32,6 @@ import static org.apache.datasketches.hll.HllUtil.RESIZE_DENOM; import static org.apache.datasketches.hll.HllUtil.RESIZE_NUMER; import java.lang.foreign.MemorySegment; -import java.nio.ByteOrder; import org.apache.datasketches.common.Family; @@ -133,7 +132,7 @@ final class PreambleUtil { static int HLL_BYTE_ARR_START = 40; //Flag bit masks - static final int BIG_ENDIAN_FLAG_MASK = 1; //Set but not read. Reserved. + static final int RESERVED_FLAG_MASK = 1; //Set to 0 but not read. static final int READ_ONLY_FLAG_MASK = 2; //Set but not read. Reserved. static final int EMPTY_FLAG_MASK = 4; static final int COMPACT_FLAG_MASK = 8; @@ -150,8 +149,6 @@ final class PreambleUtil { static final int LIST_PREINTS = 2; static final int HASH_SET_PREINTS = 3; static final int HLL_PREINTS = 10; - static final boolean NATIVE_ORDER_IS_BIG_ENDIAN = - (ByteOrder.nativeOrder() == ByteOrder.BIG_ENDIAN); static String toString(final byte[] byteArr) { final MemorySegment seg = MemorySegment.ofArray(byteArr); @@ -168,8 +165,6 @@ final class PreambleUtil { final int flags = seg.get(JAVA_BYTE, FLAGS_BYTE); //Flags final String flagsStr = zeroPad(Integer.toBinaryString(flags), 8) + ", " + (flags); - final boolean bigEndian = (flags & BIG_ENDIAN_FLAG_MASK) > 0; - final String nativeOrder = ByteOrder.nativeOrder().toString(); final boolean compact = (flags & COMPACT_FLAG_MASK) > 0; final boolean oooFlag = (flags & OUT_OF_ORDER_FLAG_MASK) > 0; final boolean readOnly = (flags & READ_ONLY_FLAG_MASK) > 0; @@ -219,8 +214,6 @@ final class PreambleUtil { } //expand byte 5: Flags sb.append("Byte 5: Flags: : ").append(flagsStr).append(LS); - sb.append(" BIG_ENDIAN_STORAGE : ").append(bigEndian).append(LS); - sb.append(" (Native Byte Order) : ").append(nativeOrder).append(LS); sb.append(" READ_ONLY : ").append(readOnly).append(LS); sb.append(" EMPTY : ").append(empty).append(LS); sb.append(" COMPACT : ").append(compact).append(LS); @@ -286,8 +279,7 @@ final class PreambleUtil { } static int extractLgArr(final MemorySegment seg) { - final int lgArr = seg.get(JAVA_BYTE, LG_ARR_BYTE) & 0XFF; - return lgArr; + return seg.get(JAVA_BYTE, LG_ARR_BYTE) & 0XFF; } static void insertLgArr(final MemorySegment wseg, final int lgArr) { diff --git a/src/main/java/org/apache/datasketches/quantiles/PreambleUtil.java b/src/main/java/org/apache/datasketches/quantiles/PreambleUtil.java index bb4cccee3..fdaa5bc19 100644 --- a/src/main/java/org/apache/datasketches/quantiles/PreambleUtil.java +++ b/src/main/java/org/apache/datasketches/quantiles/PreambleUtil.java @@ -28,7 +28,6 @@ import static org.apache.datasketches.common.Util.LS; import static org.apache.datasketches.quantiles.ClassicUtil.computeRetainedItems; import java.lang.foreign.MemorySegment; -import java.nio.ByteOrder; //@formatter:off @@ -92,15 +91,12 @@ final class PreambleUtil { static final int COMBINED_BUFFER = 32; //to 39 (Only for DoublesSketch) // flag bit masks - static final int BIG_ENDIAN_FLAG_MASK = 1; + static final int RESERVED_FLAG_MASK = 1; static final int READ_ONLY_FLAG_MASK = 2; static final int EMPTY_FLAG_MASK = 4; static final int COMPACT_FLAG_MASK = 8; static final int ORDERED_FLAG_MASK = 16; - static final boolean NATIVE_ORDER_IS_BIG_ENDIAN = - (ByteOrder.nativeOrder() == ByteOrder.BIG_ENDIAN); - /** * Default K for about 1.7% normalized rank accuracy */ @@ -143,8 +139,6 @@ final class PreambleUtil { final int familyID = extractFamilyID(srcSeg); final String famName = idToFamily(familyID).toString(); final int flags = extractFlags(srcSeg); - final boolean bigEndian = (flags & BIG_ENDIAN_FLAG_MASK) > 0; - final String nativeOrder = ByteOrder.nativeOrder().toString(); final boolean readOnly = (flags & READ_ONLY_FLAG_MASK) > 0; final boolean empty = (flags & EMPTY_FLAG_MASK) > 0; final boolean compact = (flags & COMPACT_FLAG_MASK) > 0; @@ -166,8 +160,7 @@ final class PreambleUtil { sb.append("Byte 1: Serialization Version: ").append(serVer).append(LS); sb.append("Byte 2: Family : ").append(famName).append(LS); sb.append("Byte 3: Flags Field : ").append(String.format("%02o", flags)).append(LS); - sb.append(" BIG ENDIAN : ").append(bigEndian).append(LS); - sb.append(" (Native Byte Order) : ").append(nativeOrder).append(LS); + sb.append(" RESERVED : ").append(LS); sb.append(" READ ONLY : ").append(readOnly).append(LS); sb.append(" EMPTY : ").append(empty).append(LS); sb.append(" COMPACT : ").append(compact).append(LS); diff --git a/src/main/java/org/apache/datasketches/sampling/PreambleUtil.java b/src/main/java/org/apache/datasketches/sampling/PreambleUtil.java index f2697b59d..52d93362e 100644 --- a/src/main/java/org/apache/datasketches/sampling/PreambleUtil.java +++ b/src/main/java/org/apache/datasketches/sampling/PreambleUtil.java @@ -28,7 +28,6 @@ import static org.apache.datasketches.common.Util.LS; import static org.apache.datasketches.common.Util.zeroPad; import java.lang.foreign.MemorySegment; -import java.nio.ByteOrder; import java.util.Locale; import org.apache.datasketches.common.Family; @@ -210,8 +209,6 @@ final class PreambleUtil { static final int EBPPS_RHO_DOUBLE = 32; // flag bit masks - //static final int BIG_ENDIAN_FLAG_MASK = 1; - //static final int READ_ONLY_FLAG_MASK = 2; static final int EMPTY_FLAG_MASK = 4; static final int HAS_PARTIAL_ITEM_MASK = 8; // EBPPS only static final int GADGET_FLAG_MASK = 128; @@ -221,9 +218,6 @@ final class PreambleUtil { static final int VAROPT_SER_VER = 2; static final int EBPPS_SER_VER = 1; - static final boolean NATIVE_ORDER_IS_BIG_ENDIAN = - (ByteOrder.nativeOrder() == ByteOrder.BIG_ENDIAN); - // STRINGS /** @@ -274,8 +268,6 @@ final class PreambleUtil { // Flags final int flags = extractFlags(seg); final String flagsStr = zeroPad(Integer.toBinaryString(flags), 8) + ", " + (flags); - //final boolean bigEndian = (flags & BIG_ENDIAN_FLAG_MASK) > 0; - //final String nativeOrder = ByteOrder.nativeOrder().toString(); //final boolean readOnly = (flags & READ_ONLY_FLAG_MASK) > 0; final boolean isEmpty = (flags & EMPTY_FLAG_MASK) > 0; final boolean isGadget = (flags & GADGET_FLAG_MASK) > 0; @@ -304,8 +296,6 @@ final class PreambleUtil { .append("Byte 1: Serialization Version: ").append(serVer).append(LS) .append("Byte 2: Family : ").append(family.toString()).append(LS) .append("Byte 3: Flags Field : ").append(flagsStr).append(LS) - //.append(" BIG_ENDIAN_STORAGE : ").append(bigEndian).append(LS) - //.append(" (Native Byte Order) : ").append(nativeOrder).append(LS) //.append(" READ_ONLY : ").append(readOnly).append(LS) .append(" EMPTY : ").append(isEmpty).append(LS); if (family == Family.VAROPT) { @@ -344,8 +334,6 @@ final class PreambleUtil { // Flags final int flags = extractFlags(seg); final String flagsStr = zeroPad(Integer.toBinaryString(flags), 8) + ", " + (flags); - //final boolean bigEndian = (flags & BIG_ENDIAN_FLAG_MASK) > 0; - //final String nativeOrder = ByteOrder.nativeOrder().toString(); //final boolean readOnly = (flags & READ_ONLY_FLAG_MASK) > 0; final boolean isEmpty = (flags & EMPTY_FLAG_MASK) > 0; @@ -366,8 +354,6 @@ final class PreambleUtil { + "Byte 1: Serialization Version : " + serVer + LS + "Byte 2: Family : " + family.toString() + LS + "Byte 3: Flags Field : " + flagsStr + LS - //+ " BIG_ENDIAN_STORAGE : " + bigEndian + LS - //+ " (Native Byte Order) : " + nativeOrder + LS //+ " READ_ONLY : " + readOnly + LS + " EMPTY : " + isEmpty + LS + "Bytes 4-7: Max Sketch Size (maxK): " + k + LS diff --git a/src/main/java/org/apache/datasketches/theta/DirectQuickSelectSketch.java b/src/main/java/org/apache/datasketches/theta/DirectQuickSelectSketch.java index 7356d4d5f..22e37ca6f 100644 --- a/src/main/java/org/apache/datasketches/theta/DirectQuickSelectSketch.java +++ b/src/main/java/org/apache/datasketches/theta/DirectQuickSelectSketch.java @@ -151,7 +151,6 @@ class DirectQuickSelectSketch extends DirectQuickSelectSketchR { insertFamilyID(dstSeg, family.getID()); //byte 2 insertLgNomLongs(dstSeg, lgNomLongs); //byte 3 insertLgArrLongs(dstSeg, lgArrLongs); //byte 4 - //flags: bigEndian = readOnly = compact = ordered = false; empty = true : 00100 = 4 insertFlags(dstSeg, EMPTY_FLAG_MASK); //byte 5 insertSeedHash(dstSeg, Util.computeSeedHash(seed)); //bytes 6,7 insertCurCount(dstSeg, 0); //bytes 8-11 @@ -250,7 +249,6 @@ class DirectQuickSelectSketch extends DirectQuickSelectSketchR { final int preambleLongs = wseg_.get(JAVA_BYTE, PREAMBLE_LONGS_BYTE) & 0X3F; final int preBytes = preambleLongs << 3; wseg_.asSlice(preBytes, arrLongs * 8L).fill((byte)0); - //flags: bigEndian = readOnly = compact = ordered = false; empty = true. wseg_.set(JAVA_BYTE, FLAGS_BYTE, (byte) EMPTY_FLAG_MASK); wseg_.set(JAVA_INT_UNALIGNED, RETAINED_ENTRIES_INT, 0); final float p = wseg_.get(JAVA_FLOAT_UNALIGNED, P_FLOAT); diff --git a/src/main/java/org/apache/datasketches/theta/HeapAlphaSketch.java b/src/main/java/org/apache/datasketches/theta/HeapAlphaSketch.java index b4b7848c5..8a35631ab 100644 --- a/src/main/java/org/apache/datasketches/theta/HeapAlphaSketch.java +++ b/src/main/java/org/apache/datasketches/theta/HeapAlphaSketch.java @@ -107,7 +107,7 @@ final class HeapAlphaSketch extends HeapUpdateSketch { has.hashTableThreshold_ = setHashTableThreshold(lgNomLongs, lgArrLongs); has.curCount_ = 0; has.thetaLong_ = (long)(p * LONG_MAX_VALUE_AS_DOUBLE); - has.empty_ = true; //other flags: bigEndian = readOnly = compact = ordered = false; + has.empty_ = true; //other flags: reserved = readOnly = compact = ordered = false; has.cache_ = new long[1 << lgArrLongs]; return has; } @@ -200,8 +200,7 @@ final class HeapAlphaSketch extends HeapUpdateSketch { public int getRetainedEntries(final boolean valid) { if (curCount_ > 0) { if (valid && isDirty()) { - final int curCount = HashOperations.countPart(getCache(), getLgArrLongs(), getThetaLong()); - return curCount; + return HashOperations.countPart(getCache(), getLgArrLongs(), getThetaLong()); } } return curCount_; @@ -262,7 +261,7 @@ final class HeapAlphaSketch extends HeapUpdateSketch { } @Override - public final void reset() { + public void reset() { final int lgArrLongs = ThetaUtil.startingSubMultiple(lgNomLongs_ + 1, getResizeFactor().lg(), ThetaUtil.MIN_LG_ARR_LONGS); if (lgArrLongs == lgArrLongs_) { @@ -342,13 +341,10 @@ final class HeapAlphaSketch extends HeapUpdateSketch { //Decrement theta, make dirty, don't bother check size, already not-empty. thetaLong_ = (long) (thetaLong_ * alpha_); dirty_ = true; //now may have dirty values - } - else { - //inserts (not entries!) <= k. It may not be at tgt size. - //Check size, don't decrement theta. cnt already ++, empty_ already false; - if (isOutOfSpace(curCount_)) { - resizeClean(); //not dirty, not at tgt size. - } + } else //inserts (not entries!) <= k. It may not be at tgt size. + //Check size, don't decrement theta. count already ++, empty_ already false; + if (isOutOfSpace(curCount_)) { + resizeClean(); //not dirty, not at tgt size. } } else { //r > 0: sketch mode and not dirty (e.g., after a rebuild). @@ -373,7 +369,7 @@ final class HeapAlphaSketch extends HeapUpdateSketch { * @param hash must not be 0. If not a duplicate, it will be inserted into the hash array * @return <a href="{@docRoot}/resources/dictionary.html#updateReturnState">See Update Return State</a> */ - final UpdateReturnState enhancedHashInsert(final long[] hashTable, final long hash) { + UpdateReturnState enhancedHashInsert(final long[] hashTable, final long hash) { final int arrayMask = (1 << lgArrLongs_) - 1; // arrayLongs -1 // make odd and independent of curProbe: final int stride = (2 * (int) ((hash >>> lgArrLongs_) & STRIDE_MASK)) + 1; @@ -440,7 +436,7 @@ final class HeapAlphaSketch extends HeapUpdateSketch { //At tgt size or greater //Checks for rare lockup condition // Used by hashUpdate(), rebuild() - private final void rebuildDirty() { + private void rebuildDirty() { final int curCountBefore = curCount_; forceRebuildDirtyCache(); //changes curCount_ only if (curCountBefore == curCount_) { @@ -452,7 +448,7 @@ final class HeapAlphaSketch extends HeapUpdateSketch { //curCount > hashTableThreshold //Checks for rare lockup condition // Used by hashUpdate() - private final void resizeClean() { + private void resizeClean() { //must resize, but are we at tgt size? final int lgTgtLongs = lgNomLongs_ + 1; if (lgTgtLongs > lgArrLongs_) { @@ -470,7 +466,7 @@ final class HeapAlphaSketch extends HeapUpdateSketch { //Force resize. Changes lgArrLongs_ only. Theta doesn't change, count doesn't change. // Used by rebuildDirty(), resizeClean() - private final void forceResizeCleanCache(final int lgResizeFactor) { + private void forceResizeCleanCache(final int lgResizeFactor) { assert (!dirty_); // Should never be dirty before a resize. lgArrLongs_ += lgResizeFactor; // new tgt size final long[] tgtArr = new long[1 << lgArrLongs_]; @@ -483,7 +479,7 @@ final class HeapAlphaSketch extends HeapUpdateSketch { //Cache stays the same size. Must be dirty. Theta doesn't change, count will change. // Used by rebuildDirtyAtTgtSize() - private final void forceRebuildDirtyCache() { + private void forceRebuildDirtyCache() { final long[] tgtArr = new long[1 << lgArrLongs_]; curCount_ = HashOperations.hashArrayInsert(cache_, tgtArr, lgArrLongs_, thetaLong_); cache_ = tgtArr; @@ -515,7 +511,7 @@ final class HeapAlphaSketch extends HeapUpdateSketch { * @return the variance. */ // @formatter:on - private static final double getVariance(final double k, final double p, final double alpha, + private static double getVariance(final double k, final double p, final double alpha, final double theta, final int count) { final double kPlus1 = k + 1.0; final double y = 1.0 / p; @@ -551,7 +547,7 @@ final class HeapAlphaSketch extends HeapUpdateSketch { * @param p <a href="{@docRoot}/resources/dictionary.html#p">See Sampling Probability, <i>p</i></a>. * @return R. */ - private static final int getR(final double theta, final double alpha, final double p) { + private static int getR(final double theta, final double alpha, final double p) { final double split1 = (p * (alpha + 1.0)) / 2.0; if (theta > split1) { return 0; } if (theta > (alpha * split1)) { return 1; } @@ -565,7 +561,7 @@ final class HeapAlphaSketch extends HeapUpdateSketch { * @param lgArrLongs <a href="{@docRoot}/resources/dictionary.html#lgArrLongs">See lgArrLongs</a>. * @return the hash table threshold */ - private static final int setHashTableThreshold(final int lgNomLongs, final int lgArrLongs) { + private static int setHashTableThreshold(final int lgNomLongs, final int lgArrLongs) { final double fraction = (lgArrLongs <= lgNomLongs) ? ThetaUtil.RESIZE_THRESHOLD : ThetaUtil.REBUILD_THRESHOLD; return (int) Math.floor(fraction * (1 << lgArrLongs)); } diff --git a/src/main/java/org/apache/datasketches/theta/HeapQuickSelectSketch.java b/src/main/java/org/apache/datasketches/theta/HeapQuickSelectSketch.java index 46e469004..5d8af6bfb 100644 --- a/src/main/java/org/apache/datasketches/theta/HeapQuickSelectSketch.java +++ b/src/main/java/org/apache/datasketches/theta/HeapQuickSelectSketch.java @@ -96,7 +96,7 @@ class HeapQuickSelectSketch extends HeapUpdateSketch { hashTableThreshold_ = getHashTableThreshold(lgNomLongs, lgArrLongs_); curCount_ = 0; thetaLong_ = (long)(p * LONG_MAX_VALUE_AS_DOUBLE); - empty_ = true; //other flags: bigEndian = readOnly = compact = ordered = false; + empty_ = true; //other flags: reserved = readOnly = compact = ordered = false; cache_ = new long[1 << lgArrLongs_]; } diff --git a/src/main/java/org/apache/datasketches/theta/IntersectionImpl.java b/src/main/java/org/apache/datasketches/theta/IntersectionImpl.java index 6db8f6da1..ebb4a6215 100644 --- a/src/main/java/org/apache/datasketches/theta/IntersectionImpl.java +++ b/src/main/java/org/apache/datasketches/theta/IntersectionImpl.java @@ -154,7 +154,7 @@ final class IntersectionImpl extends Intersection { insertFamilyID(dstSeg, Family.INTERSECTION.getID()); //lgNomLongs not used by Intersection //lgArrLongs set by hardReset - //flags are already 0: bigEndian = readOnly = compact = ordered = empty = false; + //flags are already 0: reserved = readOnly = compact = ordered = empty = false; //seedHash loaded and checked in IntersectionImpl constructor //Pre1 //CurCount set by hardReset diff --git a/src/main/java/org/apache/datasketches/theta/PreambleUtil.java b/src/main/java/org/apache/datasketches/theta/PreambleUtil.java index ed368bd8b..26e263852 100644 --- a/src/main/java/org/apache/datasketches/theta/PreambleUtil.java +++ b/src/main/java/org/apache/datasketches/theta/PreambleUtil.java @@ -28,7 +28,6 @@ import static org.apache.datasketches.common.Util.LS; import static org.apache.datasketches.common.Util.zeroPad; import java.lang.foreign.MemorySegment; -import java.nio.ByteOrder; import org.apache.datasketches.common.Family; import org.apache.datasketches.common.ResizeFactor; @@ -175,7 +174,7 @@ final class PreambleUtil { static final int UNION_THETA_LONG = 24; //8-byte aligned, only used by Union // flag bit masks - static final int BIG_ENDIAN_FLAG_MASK = 1; //SerVer 1, 2, 3 + static final int RESERVED_FLAG_MASK = 1; //SerVer 1, 2, 3. Now Reserved, no longer used. static final int READ_ONLY_FLAG_MASK = 2; //Set but not read. Reserved. SerVer 1, 2, 3 static final int EMPTY_FLAG_MASK = 4; //SerVer 2, 3 static final int COMPACT_FLAG_MASK = 8; //SerVer 2 was NO_REBUILD_FLAG_MASK, 3 @@ -197,9 +196,6 @@ final class PreambleUtil { static final int NUM_ENTRIES_BYTES_BYTE_V4 = 4; // number of bytes used for the number of entries static final int THETA_LONG_V4 = 8; //8-byte aligned - static final boolean NATIVE_ORDER_IS_BIG_ENDIAN = - (ByteOrder.nativeOrder() == ByteOrder.BIG_ENDIAN); - /** * Computes the number of bytes required for an updatable sketch using a hash-table cache. * This does not apply for compact sketches. @@ -207,7 +203,7 @@ final class PreambleUtil { * @param preambleLongs current preamble size * @return the size in bytes */ - static final int getSegBytes(final int lgArrLongs, final int preambleLongs) { + static int getSegBytes(final int lgArrLongs, final int preambleLongs) { return (8 << lgArrLongs) + (preambleLongs << 3); } @@ -248,8 +244,6 @@ final class PreambleUtil { final int flags = extractFlags(seg); final String flagsStr = (flags) + ", 0x" + (Integer.toHexString(flags)) + ", " + zeroPad(Integer.toBinaryString(flags), 8); - final String nativeOrder = ByteOrder.nativeOrder().toString(); - final boolean bigEndian = (flags & BIG_ENDIAN_FLAG_MASK) > 0; final boolean readOnly = (flags & READ_ONLY_FLAG_MASK) > 0; final boolean empty = (flags & EMPTY_FLAG_MASK) > 0; final boolean compact = (flags & COMPACT_FLAG_MASK) > 0; @@ -290,7 +284,6 @@ final class PreambleUtil { final StringBuilder sb = new StringBuilder(); sb.append(LS); sb.append("### SKETCH PREAMBLE SUMMARY:").append(LS); - sb.append("Native Byte Order : ").append(nativeOrder).append(LS); sb.append("Byte 0: Preamble Longs : ").append(preLongs).append(LS); sb.append("Byte 0: ResizeFactor : ").append(rfId + ", " + rf.toString()).append(LS); sb.append("Byte 1: Serialization Version: ").append(serVer).append(LS); @@ -299,7 +292,7 @@ final class PreambleUtil { sb.append("Byte 4: LgArrLongs : ").append(lgArrLongs).append(LS); sb.append("Byte 5: Flags Field : ").append(flagsStr).append(LS); sb.append(" Bit Flag Name : State:").append(LS); - sb.append(" 0 BIG_ENDIAN_STORAGE : ").append(bigEndian).append(LS); + sb.append(" 0 RESERVED : ").append(LS); sb.append(" 1 READ_ONLY : ").append(readOnly).append(LS); sb.append(" 2 EMPTY : ").append(empty).append(LS); sb.append(" 3 COMPACT : ").append(compact).append(LS); @@ -514,7 +507,7 @@ final class PreambleUtil { return preLongs; } - static final short checkSegmentSeedHash(final MemorySegment seg, final long seed) { + static short checkSegmentSeedHash(final MemorySegment seg, final long seed) { final short seedHashSeg = (short) extractSeedHash(seg); Util.checkSeedHashes(seedHashSeg, Util.computeSeedHash(seed)); //throws if bad seedHash return seedHashSeg; diff --git a/src/main/java/org/apache/datasketches/theta/SingleItemSketch.java b/src/main/java/org/apache/datasketches/theta/SingleItemSketch.java index c6ed7ec52..abf8df391 100644 --- a/src/main/java/org/apache/datasketches/theta/SingleItemSketch.java +++ b/src/main/java/org/apache/datasketches/theta/SingleItemSketch.java @@ -384,14 +384,14 @@ final class SingleItemSketch extends CompactSketch { return (short) (pre0_ >>> 48); } - static final boolean otherCheckForSingleItem(final MemorySegment seg) { + static boolean otherCheckForSingleItem(final MemorySegment seg) { return otherCheckForSingleItem(extractPreLongs(seg), extractSerVer(seg), extractFamilyID(seg), extractFlags(seg) ); } - static final boolean otherCheckForSingleItem(final int preLongs, final int serVer, + static boolean otherCheckForSingleItem(final int preLongs, final int serVer, final int famId, final int flags) { - // Flags byte: SI=X, Ordered=T, Compact=T, Empty=F, ReadOnly=T, BigEndian=F = X11010 = 0x1A. + // Flags byte: SI=X, Ordered=T, Compact=T, Empty=F, ReadOnly=T, Reserved=F = X11010 = 0x1A. // Flags mask will be 0x1F. // SingleItem flag may not be set due to a historical bug, so we can't depend on it for now. // However, if the above flags are correct, preLongs == 1, SerVer >= 3, FamilyID == 3, diff --git a/src/main/java/org/apache/datasketches/theta/UpdateSketch.java b/src/main/java/org/apache/datasketches/theta/UpdateSketch.java index d60ab0037..7db1988e9 100644 --- a/src/main/java/org/apache/datasketches/theta/UpdateSketch.java +++ b/src/main/java/org/apache/datasketches/theta/UpdateSketch.java @@ -25,7 +25,6 @@ import static org.apache.datasketches.common.Util.LONG_MAX_VALUE_AS_DOUBLE; import static org.apache.datasketches.common.Util.checkBounds; import static org.apache.datasketches.hash.MurmurHash3.hash; import static org.apache.datasketches.theta.CompactOperations.componentsToCompact; -import static org.apache.datasketches.theta.PreambleUtil.BIG_ENDIAN_FLAG_MASK; import static org.apache.datasketches.theta.PreambleUtil.COMPACT_FLAG_MASK; import static org.apache.datasketches.theta.PreambleUtil.FAMILY_BYTE; import static org.apache.datasketches.theta.PreambleUtil.ORDERED_FLAG_MASK; @@ -433,11 +432,10 @@ public abstract class UpdateSketch extends Sketch { //Check flags final int flags = extractFlags(srcSeg); //byte 5 - final int flagsMask = - ORDERED_FLAG_MASK | COMPACT_FLAG_MASK | READ_ONLY_FLAG_MASK | BIG_ENDIAN_FLAG_MASK; - if ((flags & flagsMask) > 0) { + final int badFlagsMask = ORDERED_FLAG_MASK | COMPACT_FLAG_MASK | READ_ONLY_FLAG_MASK; + if ((flags & badFlagsMask) > 0) { throw new SketchesArgumentException( - "Possible corruption: Input srcSeg cannot be: big-endian, compact, ordered, nor read-only"); + "Possible corruption: Input srcSeg cannot be: compact, ordered, nor read-only"); } //Check seed hashes diff --git a/src/main/java/org/apache/datasketches/tuple/CompactSketch.java b/src/main/java/org/apache/datasketches/tuple/CompactSketch.java index 20eac81fb..b05fd9c15 100644 --- a/src/main/java/org/apache/datasketches/tuple/CompactSketch.java +++ b/src/main/java/org/apache/datasketches/tuple/CompactSketch.java @@ -26,7 +26,6 @@ import static org.apache.datasketches.thetacommon.HashOperations.count; import java.lang.foreign.MemorySegment; import java.lang.reflect.Array; -import java.nio.ByteOrder; import org.apache.datasketches.common.ByteArrayUtil; import org.apache.datasketches.common.Family; @@ -52,7 +51,7 @@ public final class CompactSketch<S extends Summary> extends Sketch<S> { private enum FlagsLegacy { IS_BIG_ENDIAN, IS_EMPTY, HAS_ENTRIES, IS_THETA_INCLUDED } - private enum Flags { IS_BIG_ENDIAN, IS_READ_ONLY, IS_EMPTY, IS_COMPACT, IS_ORDERED } + private enum Flags { IS_RESERVED, IS_READ_ONLY, IS_EMPTY, IS_COMPACT, IS_ORDERED } /** * Create a CompactSketch from correct components @@ -90,19 +89,15 @@ public final class CompactSketch<S extends Summary> extends Sketch<S> { .validateType(seg.get(JAVA_BYTE, offset++), SerializerDeserializer.SketchType.CompactSketch); if (version <= serialVersionUIDLegacy) { // legacy serial format final byte flags = seg.get(JAVA_BYTE, offset++); - final boolean isBigEndian = (flags & 1 << FlagsLegacy.IS_BIG_ENDIAN.ordinal()) > 0; - if (isBigEndian ^ ByteOrder.nativeOrder().equals(ByteOrder.BIG_ENDIAN)) { - throw new SketchesArgumentException("Byte order mismatch"); - } - empty_ = (flags & 1 << FlagsLegacy.IS_EMPTY.ordinal()) > 0; - final boolean isThetaIncluded = (flags & 1 << FlagsLegacy.IS_THETA_INCLUDED.ordinal()) > 0; + empty_ = (flags & (1 << FlagsLegacy.IS_EMPTY.ordinal())) > 0; + final boolean isThetaIncluded = (flags & (1 << FlagsLegacy.IS_THETA_INCLUDED.ordinal())) > 0; if (isThetaIncluded) { thetaLong_ = seg.get(JAVA_LONG_UNALIGNED, offset); offset += Long.BYTES; } else { thetaLong_ = Long.MAX_VALUE; } - final boolean hasEntries = (flags & 1 << FlagsLegacy.HAS_ENTRIES.ordinal()) > 0; + final boolean hasEntries = (flags & (1 << FlagsLegacy.HAS_ENTRIES.ordinal())) > 0; if (hasEntries) { int classNameLength = 0; if (version == serialVersionWithSummaryClassNameUID) { @@ -130,7 +125,7 @@ public final class CompactSketch<S extends Summary> extends Sketch<S> { offset++; //skip unused byte final byte flags = seg.get(JAVA_BYTE, offset++); offset += 2; //skip 2 unused bytes - empty_ = (flags & 1 << Flags.IS_EMPTY.ordinal()) > 0; + empty_ = (flags & (1 << Flags.IS_EMPTY.ordinal())) > 0; thetaLong_ = Long.MAX_VALUE; int count = 0; if (!empty_) { @@ -201,7 +196,7 @@ public final class CompactSketch<S extends Summary> extends Sketch<S> { @Override public byte[] toByteArray() { final int count = getRetainedEntries(); - final boolean isSingleItem = count == 1 && !isEstimationMode(); + final boolean isSingleItem = (count == 1) && !isEstimationMode(); final int preambleLongs = isEmpty() || isSingleItem ? 1 : isEstimationMode() ? 3 : 2; int summariesSizeBytes = 0; @@ -213,7 +208,7 @@ public final class CompactSketch<S extends Summary> extends Sketch<S> { } } - final int sizeBytes = Long.BYTES * preambleLongs + Long.BYTES * count + summariesSizeBytes; + final int sizeBytes = (Long.BYTES * preambleLongs) + (Long.BYTES * count) + summariesSizeBytes; final byte[] bytes = new byte[sizeBytes]; int offset = 0; bytes[offset++] = (byte) preambleLongs; diff --git a/src/main/java/org/apache/datasketches/tuple/QuickSelectSketch.java b/src/main/java/org/apache/datasketches/tuple/QuickSelectSketch.java index 708486df1..c264366a4 100644 --- a/src/main/java/org/apache/datasketches/tuple/QuickSelectSketch.java +++ b/src/main/java/org/apache/datasketches/tuple/QuickSelectSketch.java @@ -30,7 +30,6 @@ import static org.apache.datasketches.thetacommon.HashOperations.count; import java.lang.foreign.MemorySegment; import java.lang.reflect.Array; -import java.nio.ByteOrder; import java.util.Objects; import org.apache.datasketches.common.ByteArrayUtil; @@ -49,7 +48,7 @@ import org.apache.datasketches.thetacommon.ThetaUtil; class QuickSelectSketch<S extends Summary> extends Sketch<S> { private static final byte serialVersionUID = 2; - private enum Flags { IS_BIG_ENDIAN, IS_IN_SAMPLING_MODE, IS_EMPTY, HAS_ENTRIES, IS_THETA_INCLUDED } + private enum Flags { IS_RESERVED, IS_IN_SAMPLING_MODE, IS_EMPTY, HAS_ENTRIES, IS_THETA_INCLUDED } private static final int DEFAULT_LG_RESIZE_FACTOR = ResizeFactor.X8.lg(); private final int nomEntries_; @@ -245,22 +244,18 @@ class QuickSelectSketch<S extends Summary> extends Sketch<S> { SerializerDeserializer.validateType(seg.get(JAVA_BYTE, offset++), //byte 3 SerializerDeserializer.SketchType.QuickSelectSketch); final byte flags = seg.get(JAVA_BYTE, offset++); //byte 4 - final boolean isBigEndian = (flags & 1 << Flags.IS_BIG_ENDIAN.ordinal()) > 0; - if (isBigEndian ^ ByteOrder.nativeOrder().equals(ByteOrder.BIG_ENDIAN)) { - throw new SketchesArgumentException("Endian byte order mismatch"); - } myNomEntries = 1 << seg.get(JAVA_BYTE, offset++); //byte 5 myLgCurrentCapacity = seg.get(JAVA_BYTE, offset++); //byte 6 myLgResizeFactor = seg.get(JAVA_BYTE, offset++); //byte 7 checkBounds(0, preambleLongs * 8L, seg.byteSize()); - final boolean isInSamplingMode = (flags & 1 << Flags.IS_IN_SAMPLING_MODE.ordinal()) > 0; + final boolean isInSamplingMode = (flags & (1 << Flags.IS_IN_SAMPLING_MODE.ordinal())) > 0; mySamplingProbability = isInSamplingMode ? seg.get(JAVA_FLOAT_UNALIGNED, offset) : 1f; //bytes 8 - 11 if (isInSamplingMode) { offset += Float.BYTES; } - final boolean isThetaIncluded = (flags & 1 << Flags.IS_THETA_INCLUDED.ordinal()) > 0; + final boolean isThetaIncluded = (flags & (1 << Flags.IS_THETA_INCLUDED.ordinal())) > 0; if (isThetaIncluded) { myThetaLong = seg.get(JAVA_LONG_UNALIGNED, offset); offset += Long.BYTES; @@ -292,7 +287,7 @@ class QuickSelectSketch<S extends Summary> extends Sketch<S> { myRetEntries++; myEmpty = false; } - myEmpty = (flags & 1 << Flags.IS_EMPTY.ordinal()) > 0; + myEmpty = (flags & (1 << Flags.IS_EMPTY.ordinal())) > 0; myRebuildThreshold = setRebuildThreshold(myHashTable, myNomEntries); return myThetaLong; } @@ -460,17 +455,15 @@ class QuickSelectSketch<S extends Summary> extends Sketch<S> { if (retEntries_ > 0) { sizeBytes += Integer.BYTES; // count } - sizeBytes += Long.BYTES * retEntries_ + summariesBytesLength; + sizeBytes += (Long.BYTES * retEntries_) + summariesBytesLength; final byte[] bytes = new byte[sizeBytes]; int offset = 0; bytes[offset++] = PREAMBLE_LONGS; bytes[offset++] = serialVersionUID; bytes[offset++] = (byte) Family.TUPLE.getID(); bytes[offset++] = (byte) SerializerDeserializer.SketchType.QuickSelectSketch.ordinal(); - final boolean isBigEndian = ByteOrder.nativeOrder().equals(ByteOrder.BIG_ENDIAN); bytes[offset++] = (byte) ( - (isBigEndian ? 1 << Flags.IS_BIG_ENDIAN.ordinal() : 0) - | (isInSamplingMode() ? 1 << Flags.IS_IN_SAMPLING_MODE.ordinal() : 0) + (isInSamplingMode() ? 1 << Flags.IS_IN_SAMPLING_MODE.ordinal() : 0) | (empty_ ? 1 << Flags.IS_EMPTY.ordinal() : 0) | (retEntries_ > 0 ? 1 << Flags.HAS_ENTRIES.ordinal() : 0) | (isThetaIncluded ? 1 << Flags.IS_THETA_INCLUDED.ordinal() : 0) @@ -512,7 +505,7 @@ class QuickSelectSketch<S extends Summary> extends Sketch<S> { @SuppressWarnings("unchecked") void merge(final long hash, final S summary, final SummarySetOperations<S> summarySetOps) { empty_ = false; - if (hash > 0 && hash < thetaLong_) { + if ((hash > 0) && (hash < thetaLong_)) { final int index = findOrInsert(hash); if (index < 0) { insertSummary(~index, (S)summary.copy()); //did not find, so insert @@ -590,7 +583,7 @@ class QuickSelectSketch<S extends Summary> extends Sketch<S> { lgCurrentCapacity_ = Integer.numberOfTrailingZeros(newSize); retEntries_ = 0; for (int i = 0; i < oldHashTable.length; i++) { - if (oldSummaryTable[i] != null && oldHashTable[i] < thetaLong_) { + if ((oldSummaryTable[i] != null) && (oldHashTable[i] < thetaLong_)) { insert(oldHashTable[i], oldSummaryTable[i]); } } diff --git a/src/main/java/org/apache/datasketches/tuple/arrayofdoubles/ArrayOfDoublesSketch.java b/src/main/java/org/apache/datasketches/tuple/arrayofdoubles/ArrayOfDoublesSketch.java index c901f81fd..b86f5f624 100644 --- a/src/main/java/org/apache/datasketches/tuple/arrayofdoubles/ArrayOfDoublesSketch.java +++ b/src/main/java/org/apache/datasketches/tuple/arrayofdoubles/ArrayOfDoublesSketch.java @@ -39,7 +39,7 @@ public abstract class ArrayOfDoublesSketch { // So a sketch can be non-empty, and have no entries. // For example, as a result of a sampling, when some data was presented to the sketch, but no // entries were retained. - static enum Flags { IS_BIG_ENDIAN, IS_IN_SAMPLING_MODE, IS_EMPTY, HAS_ENTRIES } + enum Flags { IS_RESERVED, IS_IN_SAMPLING_MODE, IS_EMPTY, HAS_ENTRIES } static final int SIZE_OF_KEY_BYTES = Long.BYTES; static final int SIZE_OF_VALUE_BYTES = Double.BYTES; diff --git a/src/main/java/org/apache/datasketches/tuple/arrayofdoubles/DirectArrayOfDoublesCompactSketch.java b/src/main/java/org/apache/datasketches/tuple/arrayofdoubles/DirectArrayOfDoublesCompactSketch.java index 083b5e06a..44bb4360f 100644 --- a/src/main/java/org/apache/datasketches/tuple/arrayofdoubles/DirectArrayOfDoublesCompactSketch.java +++ b/src/main/java/org/apache/datasketches/tuple/arrayofdoubles/DirectArrayOfDoublesCompactSketch.java @@ -26,7 +26,6 @@ import static java.lang.foreign.ValueLayout.JAVA_LONG_UNALIGNED; import static java.lang.foreign.ValueLayout.JAVA_SHORT_UNALIGNED; import java.lang.foreign.MemorySegment; -import java.nio.ByteOrder; import org.apache.datasketches.common.Family; import org.apache.datasketches.common.SketchesArgumentException; @@ -43,7 +42,7 @@ import org.apache.datasketches.tuple.SerializerDeserializer; final class DirectArrayOfDoublesCompactSketch extends ArrayOfDoublesCompactSketch { // this value exists only on heap, never serialized - private MemorySegment seg_; + private final MemorySegment seg_; /** * Converts the given UpdatableArrayOfDoublesSketch to this compact form. @@ -72,12 +71,10 @@ final class DirectArrayOfDoublesCompactSketch extends ArrayOfDoublesCompactSketc dstSeg.set(JAVA_BYTE, FAMILY_ID_BYTE, (byte) Family.TUPLE.getID()); dstSeg.set(JAVA_BYTE, SKETCH_TYPE_BYTE, (byte) SerializerDeserializer.SketchType.ArrayOfDoublesCompactSketch.ordinal()); - final boolean isBigEndian = ByteOrder.nativeOrder().equals(ByteOrder.BIG_ENDIAN); isEmpty_ = sketch.isEmpty(); final int count = sketch.getRetainedEntries(); dstSeg.set(JAVA_BYTE, FLAGS_BYTE, (byte) ( - (isBigEndian ? 1 << Flags.IS_BIG_ENDIAN.ordinal() : 0) - | (isEmpty_ ? 1 << Flags.IS_EMPTY.ordinal() : 0) + (isEmpty_ ? 1 << Flags.IS_EMPTY.ordinal() : 0) | (count > 0 ? 1 << Flags.HAS_ENTRIES.ordinal() : 0) )); dstSeg.set(JAVA_BYTE, NUM_VALUES_BYTE, (byte) numValues_); @@ -115,12 +112,10 @@ final class DirectArrayOfDoublesCompactSketch extends ArrayOfDoublesCompactSketc dstSeg.set(JAVA_BYTE, FAMILY_ID_BYTE, (byte) Family.TUPLE.getID()); dstSeg.set(JAVA_BYTE, SKETCH_TYPE_BYTE, (byte) SerializerDeserializer.SketchType.ArrayOfDoublesCompactSketch.ordinal()); - final boolean isBigEndian = ByteOrder.nativeOrder().equals(ByteOrder.BIG_ENDIAN); isEmpty_ = isEmpty; final int count = keys.length; dstSeg.set(JAVA_BYTE, FLAGS_BYTE, (byte) ( - (isBigEndian ? 1 << Flags.IS_BIG_ENDIAN.ordinal() : 0) - | (isEmpty_ ? 1 << Flags.IS_EMPTY.ordinal() : 0) + (isEmpty_ ? 1 << Flags.IS_EMPTY.ordinal() : 0) | (count > 0 ? 1 << Flags.HAS_ENTRIES.ordinal() : 0) )); dstSeg.set(JAVA_BYTE, NUM_VALUES_BYTE, (byte) numValues_); @@ -150,11 +145,6 @@ final class DirectArrayOfDoublesCompactSketch extends ArrayOfDoublesCompactSketc throw new SketchesArgumentException("Serial version mismatch. Expected: " + serialVersionUID + ", actual: " + version); } - final boolean isBigEndian = - (seg.get(JAVA_BYTE, FLAGS_BYTE) & (1 << Flags.IS_BIG_ENDIAN.ordinal())) != 0; - if (isBigEndian ^ ByteOrder.nativeOrder().equals(ByteOrder.BIG_ENDIAN)) { - throw new SketchesArgumentException("Byte order mismatch"); - } isEmpty_ = (seg_.get(JAVA_BYTE, FLAGS_BYTE) & (1 << Flags.IS_EMPTY.ordinal())) != 0; thetaLong_ = seg_.get(JAVA_LONG_UNALIGNED, THETA_LONG); @@ -177,11 +167,6 @@ final class DirectArrayOfDoublesCompactSketch extends ArrayOfDoublesCompactSketc throw new SketchesArgumentException("Serial version mismatch. Expected: " + serialVersionUID + ", actual: " + version); } - final boolean isBigEndian = - (seg.get(JAVA_BYTE, FLAGS_BYTE) & (1 << Flags.IS_BIG_ENDIAN.ordinal())) != 0; - if (isBigEndian ^ ByteOrder.nativeOrder().equals(ByteOrder.BIG_ENDIAN)) { - throw new SketchesArgumentException("Byte order mismatch"); - } Util.checkSeedHashes(seg.get(JAVA_SHORT_UNALIGNED, SEED_HASH_SHORT), Util.computeSeedHash(seed)); isEmpty_ = (seg_.get(JAVA_BYTE, FLAGS_BYTE) & (1 << Flags.IS_EMPTY.ordinal())) != 0; thetaLong_ = seg.get(JAVA_LONG_UNALIGNED, THETA_LONG); diff --git a/src/main/java/org/apache/datasketches/tuple/arrayofdoubles/DirectArrayOfDoublesQuickSelectSketch.java b/src/main/java/org/apache/datasketches/tuple/arrayofdoubles/DirectArrayOfDoublesQuickSelectSketch.java index 0bd6ee65f..b34f08f5c 100644 --- a/src/main/java/org/apache/datasketches/tuple/arrayofdoubles/DirectArrayOfDoublesQuickSelectSketch.java +++ b/src/main/java/org/apache/datasketches/tuple/arrayofdoubles/DirectArrayOfDoublesQuickSelectSketch.java @@ -32,7 +32,6 @@ import static org.apache.datasketches.common.Util.computeSeedHash; import static org.apache.datasketches.common.Util.setBits; import java.lang.foreign.MemorySegment; -import java.nio.ByteOrder; import java.util.Arrays; import org.apache.datasketches.common.Family; @@ -107,10 +106,8 @@ class DirectArrayOfDoublesQuickSelectSketch extends ArrayOfDoublesQuickSelectSke seg_.set(JAVA_BYTE, FAMILY_ID_BYTE, (byte) Family.TUPLE.getID()); seg_.set(JAVA_BYTE, SKETCH_TYPE_BYTE, (byte) SerializerDeserializer.SketchType.ArrayOfDoublesQuickSelectSketch.ordinal()); - final boolean isBigEndian = ByteOrder.nativeOrder().equals(ByteOrder.BIG_ENDIAN); seg_.set(JAVA_BYTE, FLAGS_BYTE, (byte) ( - (isBigEndian ? 1 << Flags.IS_BIG_ENDIAN.ordinal() : 0) - | (samplingProbability < 1f ? 1 << Flags.IS_IN_SAMPLING_MODE.ordinal() : 0) + (samplingProbability < 1f ? 1 << Flags.IS_IN_SAMPLING_MODE.ordinal() : 0) | (1 << Flags.IS_EMPTY.ordinal()) )); seg_.set(JAVA_BYTE, NUM_VALUES_BYTE, (byte) numValues); @@ -147,7 +144,7 @@ class DirectArrayOfDoublesQuickSelectSketch extends ArrayOfDoublesQuickSelectSke DirectArrayOfDoublesQuickSelectSketch( final MemorySegment seg, final long seed) { - this(checkSerVer_Endianness(seg), seg, seed); + this(checkSerVer(seg), seg, seed); //SpotBugs CT_CONSTRUCTOR_THROW is false positive. //this construction scheme is compliant with SEI CERT Oracle Coding Standard for Java / OBJ11-J } @@ -173,17 +170,12 @@ class DirectArrayOfDoublesQuickSelectSketch extends ArrayOfDoublesQuickSelectSke setRebuildThreshold(); } - private static final boolean checkSerVer_Endianness(final MemorySegment seg) { + private static final boolean checkSerVer(final MemorySegment seg) { final byte version = seg.get(JAVA_BYTE, SERIAL_VERSION_BYTE); if (version != serialVersionUID) { throw new SketchesArgumentException("Serial version mismatch. Expected: " + serialVersionUID + ", actual: " + version); } - final boolean isBigEndian = - (seg.get(JAVA_BYTE, FLAGS_BYTE) & (1 << Flags.IS_BIG_ENDIAN.ordinal())) != 0; - if (isBigEndian ^ ByteOrder.nativeOrder().equals(ByteOrder.BIG_ENDIAN)) { - throw new SketchesArgumentException("Byte order mismatch"); - } return true; } diff --git a/src/main/java/org/apache/datasketches/tuple/arrayofdoubles/HeapArrayOfDoublesCompactSketch.java b/src/main/java/org/apache/datasketches/tuple/arrayofdoubles/HeapArrayOfDoublesCompactSketch.java index b8f6fd863..f0de95ef4 100644 --- a/src/main/java/org/apache/datasketches/tuple/arrayofdoubles/HeapArrayOfDoublesCompactSketch.java +++ b/src/main/java/org/apache/datasketches/tuple/arrayofdoubles/HeapArrayOfDoublesCompactSketch.java @@ -26,7 +26,6 @@ import static java.lang.foreign.ValueLayout.JAVA_LONG_UNALIGNED; import static java.lang.foreign.ValueLayout.JAVA_SHORT_UNALIGNED; import java.lang.foreign.MemorySegment; -import java.nio.ByteOrder; import java.util.Arrays; import org.apache.datasketches.common.Family; @@ -127,11 +126,6 @@ final class HeapArrayOfDoublesCompactSketch extends ArrayOfDoublesCompactSketch throw new SketchesArgumentException( "Serial version mismatch. Expected: " + serialVersionUID + ", actual: " + version); } - final boolean isBigEndian = - (seg.get(JAVA_BYTE, FLAGS_BYTE) & (1 << Flags.IS_BIG_ENDIAN.ordinal())) != 0; - if (isBigEndian ^ ByteOrder.nativeOrder().equals(ByteOrder.BIG_ENDIAN)) { - throw new SketchesArgumentException("Byte order mismatch"); - } Util.checkSeedHashes(seedHash_, Util.computeSeedHash(seed)); isEmpty_ = (seg.get(JAVA_BYTE, FLAGS_BYTE) & (1 << Flags.IS_EMPTY.ordinal())) != 0; thetaLong_ = seg.get(JAVA_LONG_UNALIGNED, THETA_LONG); @@ -152,7 +146,7 @@ final class HeapArrayOfDoublesCompactSketch extends ArrayOfDoublesCompactSketch return new HeapArrayOfDoublesCompactSketch(keys_.clone(), values_.clone(), thetaLong_, isEmpty_, numValues_, seedHash_); } else { - final byte[] byteArr = this.toByteArray(); + final byte[] byteArr = toByteArray(); MemorySegment.copy(byteArr, 0, dstSeg, JAVA_BYTE, 0, byteArr.length); return new DirectArrayOfDoublesCompactSketch(dstSeg); } @@ -173,10 +167,8 @@ final class HeapArrayOfDoublesCompactSketch extends ArrayOfDoublesCompactSketch seg.set(JAVA_BYTE, SERIAL_VERSION_BYTE, serialVersionUID); seg.set(JAVA_BYTE, FAMILY_ID_BYTE, (byte) Family.TUPLE.getID()); seg.set(JAVA_BYTE, SKETCH_TYPE_BYTE, (byte) SerializerDeserializer.SketchType.ArrayOfDoublesCompactSketch.ordinal()); - final boolean isBigEndian = ByteOrder.nativeOrder().equals(ByteOrder.BIG_ENDIAN); seg.set(JAVA_BYTE, FLAGS_BYTE, (byte) ( - ((isBigEndian ? 1 : 0) << Flags.IS_BIG_ENDIAN.ordinal()) - | ((isEmpty() ? 1 : 0) << Flags.IS_EMPTY.ordinal()) + ((isEmpty() ? 1 : 0) << Flags.IS_EMPTY.ordinal()) | ((count > 0 ? 1 : 0) << Flags.HAS_ENTRIES.ordinal()) )); seg.set(JAVA_BYTE, NUM_VALUES_BYTE, (byte) numValues_); diff --git a/src/main/java/org/apache/datasketches/tuple/arrayofdoubles/HeapArrayOfDoublesQuickSelectSketch.java b/src/main/java/org/apache/datasketches/tuple/arrayofdoubles/HeapArrayOfDoublesQuickSelectSketch.java index beab988d3..9891c9809 100644 --- a/src/main/java/org/apache/datasketches/tuple/arrayofdoubles/HeapArrayOfDoublesQuickSelectSketch.java +++ b/src/main/java/org/apache/datasketches/tuple/arrayofdoubles/HeapArrayOfDoublesQuickSelectSketch.java @@ -31,7 +31,6 @@ import static org.apache.datasketches.common.Util.computeSeedHash; import static org.apache.datasketches.common.Util.exactLog2OfLong; import java.lang.foreign.MemorySegment; -import java.nio.ByteOrder; import java.util.Arrays; import org.apache.datasketches.common.Family; @@ -101,10 +100,6 @@ final class HeapArrayOfDoublesQuickSelectSketch extends ArrayOfDoublesQuickSelec + serialVersionUID + ", actual: " + version); } final byte flags = seg.get(JAVA_BYTE, FLAGS_BYTE); - final boolean isBigEndian = (flags & (1 << Flags.IS_BIG_ENDIAN.ordinal())) > 0; - if (isBigEndian ^ ByteOrder.nativeOrder().equals(ByteOrder.BIG_ENDIAN)) { - throw new SketchesArgumentException("Byte order mismatch"); - } checkSeedHashes(seg.get(JAVA_SHORT_UNALIGNED, SEED_HASH_SHORT), computeSeedHash(seed)); isEmpty_ = (flags & (1 << Flags.IS_EMPTY.ordinal())) > 0; lgNomEntries_ = seg.get(JAVA_BYTE, LG_NOM_ENTRIES_BYTE); @@ -238,10 +233,8 @@ final class HeapArrayOfDoublesQuickSelectSketch extends ArrayOfDoublesQuickSelec seg.set(JAVA_BYTE, FAMILY_ID_BYTE, (byte) Family.TUPLE.getID()); seg.set(JAVA_BYTE, SKETCH_TYPE_BYTE, (byte) SerializerDeserializer.SketchType.ArrayOfDoublesQuickSelectSketch.ordinal()); - final boolean isBigEndian = ByteOrder.nativeOrder().equals(ByteOrder.BIG_ENDIAN); seg.set(JAVA_BYTE, FLAGS_BYTE, (byte)( - (isBigEndian ? 1 << Flags.IS_BIG_ENDIAN.ordinal() : 0) - | (isInSamplingMode() ? 1 << Flags.IS_IN_SAMPLING_MODE.ordinal() : 0) + (isInSamplingMode() ? 1 << Flags.IS_IN_SAMPLING_MODE.ordinal() : 0) | (isEmpty_ ? 1 << Flags.IS_EMPTY.ordinal() : 0) | (count_ > 0 ? 1 << Flags.HAS_ENTRIES.ordinal() : 0) )); diff --git a/src/test/java/org/apache/datasketches/count/CountMinSketchTest.java b/src/test/java/org/apache/datasketches/count/CountMinSketchTest.java index 26fb956be..bb0bef806 100644 --- a/src/test/java/org/apache/datasketches/count/CountMinSketchTest.java +++ b/src/test/java/org/apache/datasketches/count/CountMinSketchTest.java @@ -34,7 +34,6 @@ public class CountMinSketchTest { @Test public void createNewCountMinSketchTest() throws Exception { assertThrows(SketchesArgumentException.class, () -> new CountMinSketch((byte) 5, 1, 123)); - //2^28 buckets and 4 hashes -> a long[] of 8GB! Set JVM -Xmx > 8g assertThrows(SketchesArgumentException.class, () -> new CountMinSketch((byte) 4, (1 << 28), 123)); final byte numHashes = 3; final int numBuckets = 5; @@ -204,7 +203,7 @@ public class CountMinSketchTest { final long seed = 123456; final CountMinSketch c = new CountMinSketch(numHashes, numBuckets, seed); - byte[] b = c.toByteArray(); + final byte[] b = c.toByteArray(); assertThrows(SketchesArgumentException.class, () -> CountMinSketch.deserialize(b, seed - 1)); final CountMinSketch d = CountMinSketch.deserialize(b, seed); @@ -226,10 +225,10 @@ public class CountMinSketchTest { c.update(i, 10*i*i); } - byte[] b = c.toByteArray(); + final byte[] b = c.toByteArray(); assertThrows(SketchesArgumentException.class, () -> CountMinSketch.deserialize(b, seed - 1)); - CountMinSketch d = CountMinSketch.deserialize(b, seed); + final CountMinSketch d = CountMinSketch.deserialize(b, seed); assertEquals(d.getNumHashes_(), c.getNumHashes_()); assertEquals(d.getNumBuckets_(), c.getNumBuckets_()); diff --git a/src/test/java/org/apache/datasketches/hash/MurmurHash3FFM21bTest.java b/src/test/java/org/apache/datasketches/hash/MurmurHash3FFM21bTest.java index 2942c4a0e..867f4a126 100644 --- a/src/test/java/org/apache/datasketches/hash/MurmurHash3FFM21bTest.java +++ b/src/test/java/org/apache/datasketches/hash/MurmurHash3FFM21bTest.java @@ -26,9 +26,7 @@ import static org.testng.Assert.fail; import java.lang.foreign.Arena; import java.lang.foreign.MemorySegment; -import java.nio.ByteOrder; -import org.apache.datasketches.common.MemorySegmentRequest; import org.testng.Assert; import org.testng.annotations.Test; diff --git a/src/test/java/org/apache/datasketches/kll/KllMemorySegmentRequestApp.java b/src/test/java/org/apache/datasketches/kll/KllMemorySegmentRequestApp.java index b2386fd1c..7d875a7e6 100644 --- a/src/test/java/org/apache/datasketches/kll/KllMemorySegmentRequestApp.java +++ b/src/test/java/org/apache/datasketches/kll/KllMemorySegmentRequestApp.java @@ -1,3 +1,22 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + package org.apache.datasketches.kll; import static org.apache.datasketches.kll.KllSketch.getMaxSerializedSizeBytes; diff --git a/src/test/java/org/apache/datasketches/quantiles/ClassicQuantilesMemorySegmentRequestApp.java b/src/test/java/org/apache/datasketches/quantiles/ClassicQuantilesMemorySegmentRequestApp.java index 1537d9847..7556d2be9 100644 --- a/src/test/java/org/apache/datasketches/quantiles/ClassicQuantilesMemorySegmentRequestApp.java +++ b/src/test/java/org/apache/datasketches/quantiles/ClassicQuantilesMemorySegmentRequestApp.java @@ -1,3 +1,22 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + package org.apache.datasketches.quantiles; import static org.testng.Assert.assertEquals; diff --git a/src/test/java/org/apache/datasketches/theta/DirectQuickSelectSketchTest.java b/src/test/java/org/apache/datasketches/theta/DirectQuickSelectSketchTest.java index faebc6aa4..f484b5864 100644 --- a/src/test/java/org/apache/datasketches/theta/DirectQuickSelectSketchTest.java +++ b/src/test/java/org/apache/datasketches/theta/DirectQuickSelectSketchTest.java @@ -23,7 +23,6 @@ import static java.lang.foreign.ValueLayout.JAVA_BYTE; import static java.lang.foreign.ValueLayout.JAVA_LONG_UNALIGNED; import static org.apache.datasketches.common.Family.QUICKSELECT; import static org.apache.datasketches.common.Util.clear; -import static org.apache.datasketches.theta.PreambleUtil.BIG_ENDIAN_FLAG_MASK; import static org.apache.datasketches.theta.PreambleUtil.COMPACT_FLAG_MASK; import static org.apache.datasketches.theta.PreambleUtil.FAMILY_BYTE; import static org.apache.datasketches.theta.PreambleUtil.FLAGS_BYTE; @@ -43,7 +42,6 @@ import static org.testng.Assert.fail; import java.lang.foreign.Arena; import java.lang.foreign.MemorySegment; -import java.nio.ByteOrder; import java.util.Arrays; import org.apache.datasketches.common.Family; @@ -51,11 +49,6 @@ import org.apache.datasketches.common.ResizeFactor; import org.apache.datasketches.common.SketchesArgumentException; import org.apache.datasketches.common.SketchesReadOnlyException; import org.apache.datasketches.common.Util; -import org.apache.datasketches.theta.CompactSketch; -import org.apache.datasketches.theta.DirectQuickSelectSketch; -import org.apache.datasketches.theta.Sketch; -import org.apache.datasketches.theta.Sketches; -import org.apache.datasketches.theta.UpdateSketch; import org.apache.datasketches.thetacommon.HashOperations; import org.testng.annotations.Test; @@ -807,7 +800,7 @@ public class DirectQuickSelectSketchTest { } seg1.set(JAVA_LONG_UNALIGNED, THETA_LONG, Long.MAX_VALUE); //fix theta and seg1.set(JAVA_BYTE, LG_ARR_LONGS_BYTE, (byte) 11); //fix lgArrLongs - final byte badFlags = (byte) (BIG_ENDIAN_FLAG_MASK | COMPACT_FLAG_MASK | READ_ONLY_FLAG_MASK | ORDERED_FLAG_MASK); + final byte badFlags = (byte) (COMPACT_FLAG_MASK | READ_ONLY_FLAG_MASK | ORDERED_FLAG_MASK); seg1.set(JAVA_BYTE, FLAGS_BYTE, badFlags); try { usk2 = DirectQuickSelectSketch.writableWrap(seg1, Util.DEFAULT_UPDATE_SEED); diff --git a/src/test/java/org/apache/datasketches/theta/ReadOnlyMemorySegmentTest.java b/src/test/java/org/apache/datasketches/theta/ReadOnlyMemorySegmentTest.java index cc9672a4e..02e53d259 100644 --- a/src/test/java/org/apache/datasketches/theta/ReadOnlyMemorySegmentTest.java +++ b/src/test/java/org/apache/datasketches/theta/ReadOnlyMemorySegmentTest.java @@ -28,12 +28,6 @@ import java.nio.ByteBuffer; import java.nio.ByteOrder; import org.apache.datasketches.common.SketchesReadOnlyException; -import org.apache.datasketches.theta.Intersection; -import org.apache.datasketches.theta.SetOperation; -import org.apache.datasketches.theta.Sketch; -import org.apache.datasketches.theta.Sketches; -import org.apache.datasketches.theta.Union; -import org.apache.datasketches.theta.UpdateSketch; import org.testng.Assert; import org.testng.annotations.Test; @@ -41,18 +35,18 @@ public class ReadOnlyMemorySegmentTest { @Test public void wrapAndTryUpdatingUpdateSketch() { - UpdateSketch updateSketch = UpdateSketch.builder().build(); + final UpdateSketch updateSketch = UpdateSketch.builder().build(); updateSketch.update(1); - MemorySegment seg = MemorySegment.ofBuffer( + final MemorySegment seg = MemorySegment.ofBuffer( ByteBuffer.wrap(updateSketch.toByteArray()).asReadOnlyBuffer().order(ByteOrder.nativeOrder())); - UpdateSketch sketch = (UpdateSketch) Sketch.wrap(seg); + final UpdateSketch sketch = (UpdateSketch) Sketch.wrap(seg); assertEquals(sketch.getEstimate(), 1.0); assertTrue(seg.isReadOnly()); boolean thrown = false; try { sketch.update(2); - } catch (SketchesReadOnlyException e) { + } catch (final SketchesReadOnlyException e) { thrown = true; } Assert.assertTrue(thrown); @@ -60,34 +54,34 @@ public class ReadOnlyMemorySegmentTest { @Test public void wrapCompactUnorderedSketch() { - UpdateSketch updateSketch = UpdateSketch.builder().build(); + final UpdateSketch updateSketch = UpdateSketch.builder().build(); updateSketch.update(1); - MemorySegment seg = MemorySegment.ofBuffer( + final MemorySegment seg = MemorySegment.ofBuffer( ByteBuffer.wrap(updateSketch.compact(false, null).toByteArray()).asReadOnlyBuffer().order(ByteOrder.nativeOrder())); - Sketch sketch = Sketch.wrap(seg); + final Sketch sketch = Sketch.wrap(seg); assertEquals(sketch.getEstimate(), 1.0); assertTrue(seg.isReadOnly()); } @Test public void wrapCompactOrderedSketch() { - UpdateSketch updateSketch = UpdateSketch.builder().build(); + final UpdateSketch updateSketch = UpdateSketch.builder().build(); updateSketch.update(1); - MemorySegment seg = MemorySegment.ofBuffer(ByteBuffer.wrap(updateSketch.compact().toByteArray()) + final MemorySegment seg = MemorySegment.ofBuffer(ByteBuffer.wrap(updateSketch.compact().toByteArray()) .asReadOnlyBuffer().order(ByteOrder.nativeOrder())); - Sketch sketch = Sketch.wrap(seg); + final Sketch sketch = Sketch.wrap(seg); assertEquals(sketch.getEstimate(), 1.0); assertTrue(seg.isReadOnly()); } @Test public void heapifyUpdateSketch() { - UpdateSketch us1 = UpdateSketch.builder().build(); + final UpdateSketch us1 = UpdateSketch.builder().build(); us1.update(1); - MemorySegment seg = MemorySegment.ofBuffer( + final MemorySegment seg = MemorySegment.ofBuffer( ByteBuffer.wrap(us1.toByteArray()).asReadOnlyBuffer().order(ByteOrder.nativeOrder())); // downcasting is not recommended, for testing only - UpdateSketch us2 = (UpdateSketch) Sketch.heapify(seg); + final UpdateSketch us2 = (UpdateSketch) Sketch.heapify(seg); us2.update(2); assertEquals(us2.getEstimate(), 2.0); assertTrue(seg.isReadOnly()); @@ -95,33 +89,33 @@ public class ReadOnlyMemorySegmentTest { @Test public void heapifyCompactUnorderedSketch() { - UpdateSketch updateSketch = UpdateSketch.builder().build(); + final UpdateSketch updateSketch = UpdateSketch.builder().build(); updateSketch.update(1); - MemorySegment seg = MemorySegment.ofBuffer( + final MemorySegment seg = MemorySegment.ofBuffer( ByteBuffer.wrap(updateSketch.compact(false, null).toByteArray()).asReadOnlyBuffer().order(ByteOrder.nativeOrder())); - Sketch sketch = Sketch.heapify(seg); + final Sketch sketch = Sketch.heapify(seg); assertEquals(sketch.getEstimate(), 1.0); assertTrue(seg.isReadOnly()); } @Test public void heapifyCompactOrderedSketch() { - UpdateSketch updateSketch = UpdateSketch.builder().build(); + final UpdateSketch updateSketch = UpdateSketch.builder().build(); updateSketch.update(1); - MemorySegment seg = MemorySegment.ofBuffer( + final MemorySegment seg = MemorySegment.ofBuffer( ByteBuffer.wrap(updateSketch.compact().toByteArray()).asReadOnlyBuffer().order(ByteOrder.nativeOrder())); - Sketch sketch = Sketch.heapify(seg); + final Sketch sketch = Sketch.heapify(seg); assertEquals(sketch.getEstimate(), 1.0); assertTrue(seg.isReadOnly()); } @Test public void heapifyUnion() { - Union u1 = SetOperation.builder().buildUnion(); + final Union u1 = SetOperation.builder().buildUnion(); u1.update(1); - MemorySegment seg = MemorySegment.ofBuffer( + final MemorySegment seg = MemorySegment.ofBuffer( ByteBuffer.wrap(u1.toByteArray()).asReadOnlyBuffer().order(ByteOrder.nativeOrder())); - Union u2 = (Union) SetOperation.heapify(seg); + final Union u2 = (Union) SetOperation.heapify(seg); u2.update(2); Assert.assertEquals(u2.getResult().getEstimate(), 2.0); assertTrue(seg.isReadOnly()); @@ -129,13 +123,13 @@ public class ReadOnlyMemorySegmentTest { @Test public void wrapAndTryUpdatingUnion() { - Union u1 = SetOperation.builder().buildUnion(); + final Union u1 = SetOperation.builder().buildUnion(); u1.update(1); - MemorySegment seg = MemorySegment.ofBuffer( + final MemorySegment seg = MemorySegment.ofBuffer( ByteBuffer.wrap(u1.toByteArray()).asReadOnlyBuffer().order(ByteOrder.nativeOrder())); - Union u2 = (Union) Sketches.wrapSetOperation(seg); - Union u3 = Sketches.wrapUnion(seg); + final Union u2 = (Union) Sketches.wrapSetOperation(seg); + final Union u3 = Sketches.wrapUnion(seg); Assert.assertEquals(u2.getResult().getEstimate(), 1.0); Assert.assertEquals(u3.getResult().getEstimate(), 1.0); assertTrue(seg.isReadOnly()); @@ -143,33 +137,33 @@ public class ReadOnlyMemorySegmentTest { try { u2.update(2); fail(); - } catch (SketchesReadOnlyException e) { + } catch (final SketchesReadOnlyException e) { //expected } try { u3.update(2); fail(); - } catch (SketchesReadOnlyException e) { + } catch (final SketchesReadOnlyException e) { //expected } } @Test public void heapifyIntersection() { - UpdateSketch us1 = UpdateSketch.builder().build(); + final UpdateSketch us1 = UpdateSketch.builder().build(); us1.update(1); us1.update(2); - UpdateSketch us2 = UpdateSketch.builder().build(); + final UpdateSketch us2 = UpdateSketch.builder().build(); us2.update(2); us2.update(3); - Intersection i1 = SetOperation.builder().buildIntersection(); + final Intersection i1 = SetOperation.builder().buildIntersection(); i1.intersect(us1); i1.intersect(us2); - MemorySegment seg = MemorySegment.ofBuffer( + final MemorySegment seg = MemorySegment.ofBuffer( ByteBuffer.wrap(i1.toByteArray()).asReadOnlyBuffer().order(ByteOrder.nativeOrder())); - Intersection i2 = (Intersection) SetOperation.heapify(seg); + final Intersection i2 = (Intersection) SetOperation.heapify(seg); i2.intersect(us1); Assert.assertEquals(i2.getResult().getEstimate(), 1.0); assertTrue(seg.isReadOnly()); @@ -177,25 +171,25 @@ public class ReadOnlyMemorySegmentTest { @Test public void wrapIntersection() { - UpdateSketch us1 = UpdateSketch.builder().build(); + final UpdateSketch us1 = UpdateSketch.builder().build(); us1.update(1); us1.update(2); - UpdateSketch us2 = UpdateSketch.builder().build(); + final UpdateSketch us2 = UpdateSketch.builder().build(); us2.update(2); us2.update(3); - Intersection i1 = SetOperation.builder().buildIntersection(); + final Intersection i1 = SetOperation.builder().buildIntersection(); i1.intersect(us1); i1.intersect(us2); - MemorySegment seg = MemorySegment.ofBuffer( + final MemorySegment seg = MemorySegment.ofBuffer( ByteBuffer.wrap(i1.toByteArray()).asReadOnlyBuffer().order(ByteOrder.nativeOrder())); - Intersection i2 = (Intersection) SetOperation.wrap(seg); + final Intersection i2 = (Intersection) SetOperation.wrap(seg); Assert.assertEquals(i2.getResult().getEstimate(), 1.0); boolean thrown = false; try { i2.intersect(us1); - } catch (SketchesReadOnlyException e) { + } catch (final SketchesReadOnlyException e) { thrown = true; } Assert.assertTrue(thrown); @@ -210,7 +204,7 @@ public class ReadOnlyMemorySegmentTest { /** * @param s value to print */ - static void println(String s) { + static void println(final String s) { //System.out.println(s); //disable here } diff --git a/src/test/java/org/apache/datasketches/theta/UnionImplTest.java b/src/test/java/org/apache/datasketches/theta/UnionImplTest.java index d96ec04f3..f212b6547 100644 --- a/src/test/java/org/apache/datasketches/theta/UnionImplTest.java +++ b/src/test/java/org/apache/datasketches/theta/UnionImplTest.java @@ -20,7 +20,6 @@ package org.apache.datasketches.theta; import static java.lang.foreign.ValueLayout.JAVA_BYTE; -import static java.lang.foreign.ValueLayout.JAVA_LONG_UNALIGNED; import static org.apache.datasketches.theta.BackwardConversions.convertSerVer3toSerVer1; import static org.apache.datasketches.theta.BackwardConversions.convertSerVer3toSerVer2; import static org.testng.Assert.assertEquals; @@ -28,23 +27,11 @@ import static org.testng.Assert.assertFalse; import static org.testng.Assert.assertTrue; import java.lang.foreign.Arena; -import java.nio.ByteOrder; - import java.lang.foreign.MemorySegment; import org.apache.datasketches.common.MemorySegmentStatus; import org.apache.datasketches.common.SketchesArgumentException; import org.apache.datasketches.common.Util; -import org.apache.datasketches.theta.CompactSketch; -import org.apache.datasketches.theta.PreambleUtil; -import org.apache.datasketches.theta.SetOperation; -import org.apache.datasketches.theta.SetOperationBuilder; -import org.apache.datasketches.theta.Sketch; -import org.apache.datasketches.theta.Sketches; -import org.apache.datasketches.theta.Union; -import org.apache.datasketches.theta.UnionImpl; -import org.apache.datasketches.theta.UpdateSketch; -import org.apache.datasketches.theta.UpdateSketchBuilder; import org.testng.annotations.Test; public class UnionImplTest { diff --git a/tools/testng.xml b/tools/testng.xml index e7f188710..834cca0b8 100644 --- a/tools/testng.xml +++ b/tools/testng.xml @@ -1,4 +1,24 @@ <!DOCTYPE suite SYSTEM "https://testng.org/testng-1.0.dtd"; > + +<!-- +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. +--> + <suite name="AllTestsSuite"> <test name="AllTests"> <packages> --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
