This is an automated email from the ASF dual-hosted git repository. leerho pushed a commit to branch ffm_phase5 in repository https://gitbox.apache.org/repos/asf/datasketches-java.git
commit bd99ec9ae15305e6793152719e521352a1cfa505 Author: Lee Rhodes <[email protected]> AuthorDate: Mon Jul 14 11:17:03 2025 -0700 FFM Phase 5: sketches req, tdigest --- .../datasketches/common/SpecialValueLayouts.java | 114 +++++++++++ .../datasketches/frequencies/ItemsSketch.java | 5 +- .../org/apache/datasketches/req/FloatBuffer.java | 40 ++-- .../org/apache/datasketches/req/ReqCompactor.java | 41 ++-- .../java/org/apache/datasketches/req/ReqSerDe.java | 91 +++++---- .../org/apache/datasketches/req/ReqSketch.java | 100 +++++----- .../apache/datasketches/tdigest/TDigestDouble.java | 217 +++++++++++---------- .../apache/datasketches/req/ReqCompactorTest.java | 27 +-- .../apache/datasketches/req/ReqDebugImplTest.java | 4 + .../datasketches/req/ReqFloatBufferTest.java | 49 ++--- .../datasketches/req/ReqSketchBuilderTest.java | 1 + .../req/ReqSketchCrossLanguageTest.java | 15 +- .../datasketches/req/ReqSketchOtherTest.java | 3 + .../datasketches/req/ReqSketchSortedViewTest.java | 2 + .../org/apache/datasketches/req/ReqSketchTest.java | 19 +- .../tdigest/TDigestCrossLanguageTest.java | 16 +- .../datasketches/tdigest/TDigestDoubleTest.java | 31 +-- 17 files changed, 461 insertions(+), 314 deletions(-) diff --git a/src/main/java/org/apache/datasketches/common/SpecialValueLayouts.java b/src/main/java/org/apache/datasketches/common/SpecialValueLayouts.java new file mode 100644 index 000000000..292d5617f --- /dev/null +++ b/src/main/java/org/apache/datasketches/common/SpecialValueLayouts.java @@ -0,0 +1,114 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.datasketches.common; + +import java.lang.foreign.ValueLayout; +import java.nio.ByteOrder; + +/** + * Value Layouts for Non-native Endianness + */ +public final class SpecialValueLayouts { + + private SpecialValueLayouts() { } + + /** + * The static final for NON <i>ByteOrder.nativeOrder()</i>. + */ + public static final ByteOrder NON_NATIVE_BYTE_ORDER = + (ByteOrder.nativeOrder() == ByteOrder.LITTLE_ENDIAN) ? ByteOrder.BIG_ENDIAN : ByteOrder.LITTLE_ENDIAN; + + //Non-Native Endian Layouts + + /** + * The static final for NON <i>ByteOrder.nativeOrder() char</i>. + */ + public static final ValueLayout.OfChar JAVA_CHAR_UNALIGNED_NON_NATIVE = + ValueLayout.JAVA_CHAR_UNALIGNED.withOrder(NON_NATIVE_BYTE_ORDER); + + /** + * The static final for NON <i>ByteOrder.nativeOrder() double</i>. + */ + public static final ValueLayout.OfDouble JAVA_DOUBLE_UNALIGNED_NON_NATIVE = + ValueLayout.JAVA_DOUBLE_UNALIGNED.withOrder(NON_NATIVE_BYTE_ORDER); + + /** + * The static final for NON <i>ByteOrder.nativeOrder() float</i>. + */ + public static final ValueLayout.OfFloat JAVA_FLOAT_UNALIGNED_NON_NATIVE = + ValueLayout.JAVA_FLOAT_UNALIGNED.withOrder(NON_NATIVE_BYTE_ORDER); + + /** + * The static final for NON <i>ByteOrder.nativeOrder() int</i>. + */ + public static final ValueLayout.OfInt JAVA_INT_UNALIGNED_NON_NATIVE = + ValueLayout.JAVA_INT_UNALIGNED.withOrder(NON_NATIVE_BYTE_ORDER); + + /** + * The static final for NON <i>ByteOrder.nativeOrder() long</i>. + */ + public static final ValueLayout.OfLong JAVA_LONG_UNALIGNED_NON_NATIVE = + ValueLayout.JAVA_LONG_UNALIGNED.withOrder(NON_NATIVE_BYTE_ORDER); + + /** + * The static final for NON <i>ByteOrder.nativeOrder() short</i>. + */ + public static final ValueLayout.OfShort JAVA_SHORT_UNALIGNED_NON_NATIVE = + ValueLayout.JAVA_SHORT_UNALIGNED.withOrder(NON_NATIVE_BYTE_ORDER); + + //Big-Endian Layouts + + /** + * The static final for <i>ByteOrder.BIG_ENDIAN char</i>. + */ + public static final ValueLayout.OfChar JAVA_CHAR_UNALIGNED_BIG_ENDIAN = + ValueLayout.JAVA_CHAR_UNALIGNED.withOrder(ByteOrder.BIG_ENDIAN); + + /** + * The static final for <i>ByteOrder.BIG_ENDIAN double</i>. + */ + public static final ValueLayout.OfDouble JAVA_DOUBLE_UNALIGNED_BIG_ENDIAN = + ValueLayout.JAVA_DOUBLE_UNALIGNED.withOrder(ByteOrder.BIG_ENDIAN); + + /** + * The static final for <i>ByteOrder.BIG_ENDIAN float</i>. + */ + public static final ValueLayout.OfFloat JAVA_FLOAT_UNALIGNED_BIG_ENDIAN = + ValueLayout.JAVA_FLOAT_UNALIGNED.withOrder(ByteOrder.BIG_ENDIAN); + + /** + * The static final for <i>ByteOrder.BIG_ENDIAN int</i>. + */ + public static final ValueLayout.OfInt JAVA_INT_UNALIGNED_BIG_ENDIAN = + ValueLayout.JAVA_INT_UNALIGNED.withOrder(ByteOrder.BIG_ENDIAN); + + /** + * The static final for <i>ByteOrder.BIG_ENDIAN long</i>. + */ + public static final ValueLayout.OfLong JAVA_LONG_UNALIGNED_BIG_ENDIAN = + ValueLayout.JAVA_LONG_UNALIGNED.withOrder(ByteOrder.BIG_ENDIAN); + + /** + * The static final for <i>ByteOrder.BIG_ENDIAN short</i>. + */ + public static final ValueLayout.OfShort JAVA_SHORT_UNALIGNED_BIG_ENDIAN = + ValueLayout.JAVA_SHORT_UNALIGNED.withOrder(ByteOrder.BIG_ENDIAN); + +} diff --git a/src/main/java/org/apache/datasketches/frequencies/ItemsSketch.java b/src/main/java/org/apache/datasketches/frequencies/ItemsSketch.java index bd2835283..6383a26d0 100644 --- a/src/main/java/org/apache/datasketches/frequencies/ItemsSketch.java +++ b/src/main/java/org/apache/datasketches/frequencies/ItemsSketch.java @@ -278,8 +278,9 @@ public class ItemsSketch<T> { //Get countArray final long[] countArray = new long[activeItems]; - final int reqBytes = preBytes + (activeItems * Long.BYTES); //count Arr only + final long reqBytes = (preBytes + ((long)activeItems * Long.BYTES)); //count Arr only checkBounds(0, reqBytes, srcSeg.byteSize()); //check MemorySegment capacity + MemorySegment.copy(srcSeg, JAVA_LONG_UNALIGNED, preBytes, countArray, 0, activeItems); //Get itemArray @@ -530,7 +531,7 @@ public class ItemsSketch<T> { final int preBytes = preLongs << 3; MemorySegment.copy(hashMap.getActiveValues(), 0, seg, JAVA_LONG_UNALIGNED, preBytes, activeItems); - MemorySegment.copy(bytes, 0, seg, JAVA_BYTE, preBytes + (this.getNumActiveItems() << 3), bytes.length); + MemorySegment.copy(bytes, 0, seg, JAVA_BYTE, preBytes + (activeItems << 3), bytes.length); } return outArr; } diff --git a/src/main/java/org/apache/datasketches/req/FloatBuffer.java b/src/main/java/org/apache/datasketches/req/FloatBuffer.java index 86cb20260..1c63906b9 100755 --- a/src/main/java/org/apache/datasketches/req/FloatBuffer.java +++ b/src/main/java/org/apache/datasketches/req/FloatBuffer.java @@ -22,11 +22,11 @@ package org.apache.datasketches.req; import static org.apache.datasketches.common.Util.LS; import static org.apache.datasketches.quantilescommon.QuantileSearchCriteria.INCLUSIVE; +import java.lang.foreign.MemorySegment; import java.util.Arrays; +import org.apache.datasketches.common.positional.PositionalSegment; import org.apache.datasketches.common.SketchesArgumentException; -import org.apache.datasketches.memory.WritableBuffer; -import org.apache.datasketches.memory.WritableMemory; import org.apache.datasketches.quantilescommon.InequalitySearch; import org.apache.datasketches.quantilescommon.QuantileSearchCriteria; @@ -176,7 +176,7 @@ class FloatBuffer { * @return this */ private FloatBuffer ensureSpace(final int space) { - if (count_ + space > capacity_) { + if ((count_ + space) > capacity_) { final int newCap = count_ + space + delta_; ensureCapacity(newCap); } @@ -219,7 +219,7 @@ class FloatBuffer { } final InequalitySearch crit = (searchCrit == INCLUSIVE) ? InequalitySearch.LE : InequalitySearch.LT; final int index = InequalitySearch.find(arr_, low, high, item, crit); - return index == -1 ? 0 : index - low + 1; + return index == -1 ? 0 : (index - low) + 1; } /** @@ -234,8 +234,8 @@ class FloatBuffer { * @return the selected odds from the range */ FloatBuffer getEvensOrOdds(final int startOffset, final int endOffset, final boolean odds) { - final int start = spaceAtBottom_ ? capacity_ - count_ + startOffset : startOffset; - final int end = spaceAtBottom_ ? capacity_ - count_ + endOffset : endOffset; + final int start = spaceAtBottom_ ? (capacity_ - count_) + startOffset : startOffset; + final int end = spaceAtBottom_ ? (capacity_ - count_) + endOffset : endOffset; sort(); final int range = endOffset - startOffset; if ((range & 1) == 1) { @@ -265,7 +265,7 @@ class FloatBuffer { * @return an item given its offset */ float getItem(final int offset) { - final int index = spaceAtBottom_ ? capacity_ - count_ + offset : offset; + final int index = spaceAtBottom_ ? (capacity_ - count_) + offset : offset; return arr_[index]; } @@ -317,11 +317,11 @@ class FloatBuffer { * @return true iff this is exactly equal to that FloatBuffer. */ boolean isEqualTo(final FloatBuffer that) { - if (capacity_ != that.capacity_ - || count_ != that.count_ - || delta_ != that.delta_ - || sorted_ != that.sorted_ - || spaceAtBottom_ != that.spaceAtBottom_) { return false; } + if ((capacity_ != that.capacity_) + || (count_ != that.count_) + || (delta_ != that.delta_) + || (sorted_ != that.sorted_) + || (spaceAtBottom_ != that.spaceAtBottom_)) { return false; } for (int i = 0; i < capacity_; i++) { if (arr_[i] != that.arr_[i]) { return false; } } @@ -354,7 +354,7 @@ class FloatBuffer { int i = capacity_ - count_; int j = bufIn.capacity_ - bufIn.count_; for (int k = tgtStart; k < capacity_; k++) { - if (i < capacity_ && j < bufIn.capacity_) { //both valid + if ((i < capacity_) && (j < bufIn.capacity_)) { //both valid arr_[k] = arr_[i] <= arrIn[j] ? arr_[i++] : arrIn[j++]; } else if (i < capacity_) { //i is valid arr_[k] = arr_[i++]; @@ -368,7 +368,7 @@ class FloatBuffer { int i = count_ - 1; int j = bufInLen - 1; for (int k = totLen; k-- > 0; ) { - if (i >= 0 && j >= 0) { //both valid + if ((i >= 0) && (j >= 0)) { //both valid arr_[k] = arr_[i] >= arrIn[j] ? arr_[i--] : arrIn[j--]; } else if (i >= 0) { //i is valid arr_[k] = arr_[i--]; @@ -401,18 +401,18 @@ class FloatBuffer { byte[] floatsToBytes() { final int bytes = Float.BYTES * count_; final byte[] arr = new byte[bytes]; - final WritableBuffer wbuf = WritableMemory.writableWrap(arr).asWritableBuffer(); + final PositionalSegment posSeg = PositionalSegment.wrap(MemorySegment.ofArray(arr)); if (spaceAtBottom_) { - wbuf.putFloatArray(arr_, capacity_ - count_, count_); + posSeg.setFloatArray(arr_, capacity_ - count_, count_); } else { - wbuf.putFloatArray(arr_, 0, count_); + posSeg.setFloatArray(arr_, 0, count_); } - assert wbuf.getPosition() == bytes; + assert posSeg.getPosition() == bytes; return arr; } /** - * Returns a printable formatted string of the items of this buffer separated by a single space. + * Returns a printable formatted string of the items of this FloatBuffer separated by a single space. * @param fmt The format for each printed item. * @param width the number of items to print per line * @return a printable, formatted string of the items of this buffer. @@ -427,7 +427,7 @@ class FloatBuffer { for (int i = start; i < end; i++) { final float v = arr_[i]; final String str = String.format(fmt, v); - if (i > start && ++cnt % width == 0) { sb.append(LS).append(spaces); } + if ((i > start) && ((++cnt % width) == 0)) { sb.append(LS).append(spaces); } sb.append(str); } return sb.toString(); diff --git a/src/main/java/org/apache/datasketches/req/ReqCompactor.java b/src/main/java/org/apache/datasketches/req/ReqCompactor.java index bdbc919c1..e25c0cb6a 100644 --- a/src/main/java/org/apache/datasketches/req/ReqCompactor.java +++ b/src/main/java/org/apache/datasketches/req/ReqCompactor.java @@ -25,10 +25,10 @@ import static org.apache.datasketches.req.BaseReqSketch.INIT_NUMBER_OF_SECTIONS; import static org.apache.datasketches.req.ReqSketch.MIN_K; import static org.apache.datasketches.req.ReqSketch.NOM_CAP_MULT; +import java.lang.foreign.MemorySegment; import java.util.Random; -import org.apache.datasketches.memory.WritableBuffer; -import org.apache.datasketches.memory.WritableMemory; +import org.apache.datasketches.common.positional.PositionalSegment; import org.apache.datasketches.req.ReqSketch.CompactorReturn; /** @@ -122,7 +122,7 @@ class ReqCompactor { final long compactionRange = computeCompactionRange(secsToCompact); final int compactionStart = (int) (compactionRange & 0xFFFF_FFFFL); //low 32 final int compactionEnd = (int) (compactionRange >>> 32); //high 32 - assert compactionEnd - compactionStart >= 2; + assert (compactionEnd - compactionStart) >= 2; if ((state & 1L) == 1L) { coin = !coin; } //if numCompactions odd, flip coin; else { coin = rand.nextBoolean(); } //random coin flip @@ -137,7 +137,7 @@ class ReqCompactor { buf.trimCount(buf.getCount() - (compactionEnd - compactionStart)); state += 1; ensureEnoughSections(); - cReturn.deltaRetItems = buf.getCount() - startRetItems + promote.getCount(); + cReturn.deltaRetItems = (buf.getCount() - startRetItems) + promote.getCount(); cReturn.deltaNomSize = getNomCapacity() - startNomCap; if (reqDebug != null) { reqDebug.emitCompactionDone(lgWeight); } return promote; @@ -175,7 +175,7 @@ class ReqCompactor { */ int getSerializationBytes() { final int count = buf.getCount(); - return 8 + 4 + 1 + 1 + 2 + 4 + count * Float.BYTES; // 20 + array + return 8 + 4 + 1 + 1 + 2 + 4 + (count * Float.BYTES); // 20 + array } int getNumSections() { @@ -227,9 +227,9 @@ class ReqCompactor { private boolean ensureEnoughSections() { final float szf; final int ne; - if (state >= 1L << numSections - 1 - && sectionSize > MIN_K - && (ne = nearestEven(szf = (float)(sectionSizeFlt / SQRT2))) >= MIN_K) + if ((state >= (1L << (numSections - 1))) + && (sectionSize > MIN_K) + && ((ne = nearestEven(szf = (float)(sectionSizeFlt / SQRT2))) >= MIN_K)) { sectionSizeFlt = szf; sectionSize = ne; @@ -248,9 +248,9 @@ class ReqCompactor { */ private long computeCompactionRange(final int secsToCompact) { final int bufLen = buf.getCount(); - int nonCompact = getNomCapacity() / 2 + (numSections - secsToCompact) * sectionSize; + int nonCompact = (getNomCapacity() / 2) + ((numSections - secsToCompact) * sectionSize); //make compacted region even: - nonCompact = (bufLen - nonCompact & 1) == 1 ? nonCompact + 1 : nonCompact; + nonCompact = ((bufLen - nonCompact) & 1) == 1 ? nonCompact + 1 : nonCompact; final long low = hra ? 0 : nonCompact; final long high = hra ? bufLen - nonCompact : bufLen; return (high << 32) + low; @@ -293,16 +293,16 @@ class ReqCompactor { byte[] toByteArray() { final int bytes = getSerializationBytes(); final byte[] arr = new byte[bytes]; - final WritableBuffer wbuf = WritableMemory.writableWrap(arr).asWritableBuffer(); - wbuf.putLong(state); - wbuf.putFloat(sectionSizeFlt); - wbuf.putByte(lgWeight); - wbuf.putByte(numSections); - wbuf.incrementPosition(2); //pad 2 + final PositionalSegment posSeg = PositionalSegment.wrap(MemorySegment.ofArray(arr)); + posSeg.setLong(state); + posSeg.setFloat(sectionSizeFlt); + posSeg.setByte(lgWeight); + posSeg.setByte(numSections); + posSeg.incrementPosition(2); //pad 2 //buf.sort(); //sort if necessary - wbuf.putInt(buf.getCount()); //count - wbuf.putByteArray(buf.floatsToBytes(), 0, Float.BYTES * buf.getCount()); - assert wbuf.getPosition() == bytes; + posSeg.setInt(buf.getCount()); //count + posSeg.setByteArray(buf.floatsToBytes(), 0, Float.BYTES * buf.getCount()); + assert posSeg.getPosition() == bytes; return arr; } @@ -319,10 +319,9 @@ class ReqCompactor { final int secSz = getSectionSize(); final int numSec = getNumSections(); final long num = getState(); - final String prefix = String.format( + return String.format( " C:%d Len:%d NomSz:%d SecSz:%d NumSec:%d State:%d", h, len, nom, secSz, numSec, num); - return prefix; } } diff --git a/src/main/java/org/apache/datasketches/req/ReqSerDe.java b/src/main/java/org/apache/datasketches/req/ReqSerDe.java index 952749deb..8f343e20a 100644 --- a/src/main/java/org/apache/datasketches/req/ReqSerDe.java +++ b/src/main/java/org/apache/datasketches/req/ReqSerDe.java @@ -23,14 +23,12 @@ import static java.lang.Math.max; import static java.lang.Math.min; import static java.lang.Math.round; +import java.lang.foreign.MemorySegment; import java.util.ArrayList; import java.util.List; import org.apache.datasketches.common.Family; -import org.apache.datasketches.memory.Buffer; -import org.apache.datasketches.memory.Memory; -import org.apache.datasketches.memory.WritableBuffer; -import org.apache.datasketches.memory.WritableMemory; +import org.apache.datasketches.common.positional.PositionalSegment; /** * This class handles serialization and deserialization. @@ -130,24 +128,24 @@ class ReqSerDe { private static final byte SER_VER = 1; private static final byte FAMILY_ID = (byte) Family.REQ.getID(); - static ReqSketch heapify(final Memory mem) { - final Buffer buff = mem.asBuffer(); + static ReqSketch heapify(final MemorySegment seg) { + final PositionalSegment posSeg = PositionalSegment.wrap(seg); //Extract first 8 bytes - final byte preInts = buff.getByte(); - final byte serVer = buff.getByte(); + final byte preInts = posSeg.getByte(); + final byte serVer = posSeg.getByte(); assert serVer == (byte)1; - final byte familyId = buff.getByte(); + final byte familyId = posSeg.getByte(); assert familyId == 17; // Extract flags - final int flags = buff.getByte() & 0xFF; + final int flags = posSeg.getByte() & 0xFF; final boolean empty = (flags & 4) > 0; final boolean hra = (flags & 8) > 0; final boolean rawItems = (flags & 16) > 0; final boolean lvl0Sorted = (flags & 32) > 0; // remainder fields - final int k = buff.getShort() & 0xFFFF; - final int numCompactors = buff.getByte() & 0xFF; - final int numRawItems = buff.getByte() & 0xFF; + final int k = posSeg.getShort() & 0xFFFF; + final int numCompactors = posSeg.getByte() & 0xFF; + final int numRawItems = posSeg.getByte() & 0xFF; // extract different serialization formats final SerDeFormat deserFormat = getDeserFormat(empty, rawItems, numCompactors); switch (deserFormat) { @@ -158,12 +156,12 @@ class ReqSerDe { case RAWITEMS: { assert preInts == 2; final ReqSketch sk = new ReqSketch(k, hra, null); - for (int i = 0; i < numRawItems; i++) { sk.update(buff.getFloat()); } + for (int i = 0; i < numRawItems; i++) { sk.update(posSeg.getFloat()); } return sk; } case EXACT: { assert preInts == 2; - final Compactor compactor = extractCompactor(buff, lvl0Sorted, hra); + final Compactor compactor = extractCompactor(posSeg, lvl0Sorted, hra); //Construct sketch final long totalN = compactor.count; final float minItem = compactor.minItem; @@ -177,14 +175,14 @@ class ReqSerDe { } default: { //ESTIMATION assert preInts == 4; - final long totalN = buff.getLong(); - final float minItem = buff.getFloat(); - final float maxItem = buff.getFloat(); + final long totalN = posSeg.getLong(); + final float minItem = posSeg.getFloat(); + final float maxItem = posSeg.getFloat(); final List<ReqCompactor> compactors = new ArrayList<>(); for (int i = 0; i < numCompactors; i++) { final boolean level0sorted = i == 0 ? lvl0Sorted : true; - final Compactor compactor = extractCompactor(buff, level0sorted, hra); + final Compactor compactor = extractCompactor(posSeg, level0sorted, hra); compactors.add(compactor.reqCompactor); } final ReqSketch sk = new ReqSketch(k, hra, totalN, minItem, maxItem, compactors); @@ -195,17 +193,17 @@ class ReqSerDe { } } - static final Compactor extractCompactor(final Buffer buff, final boolean lvl0Sorted, + static final Compactor extractCompactor(final PositionalSegment posSeg, final boolean lvl0Sorted, final boolean hra) { - final long state = buff.getLong(); - final float sectionSizeFlt = buff.getFloat(); + final long state = posSeg.getLong(); + final float sectionSizeFlt = posSeg.getFloat(); final int sectionSize = round(sectionSizeFlt); - final byte lgWt = buff.getByte(); - final byte numSections = buff.getByte(); - buff.incrementPosition(2); - final int count = buff.getInt(); + final byte lgWt = posSeg.getByte(); + final byte numSections = posSeg.getByte(); + posSeg.incrementPosition(2); + final int count = posSeg.getInt(); final float[] arr = new float[count]; - buff.getFloatArray(arr, 0, count); + posSeg.getFloatArray(arr, 0, count); float minItem = Float.MAX_VALUE; float maxItem = Float.MIN_VALUE; for (int i = 0; i < count; i++) { @@ -267,46 +265,47 @@ class ReqSerDe { final SerDeFormat serDeFormat = getSerFormat(sk); final int bytes = getSerBytes(sk, serDeFormat); final byte[] arr = new byte[bytes]; - final WritableBuffer wbuf = WritableMemory.writableWrap(arr).asWritableBuffer(); + final PositionalSegment posSeg = PositionalSegment.wrap(MemorySegment.ofArray(arr)); + final byte preInts = (byte)(serDeFormat == SerDeFormat.ESTIMATION ? 4 : 2); final byte flags = getFlags(sk); final byte numCompactors = sk.isEmpty() ? 0 : (byte) sk.getNumLevels(); final byte numRawItems = sk.getN() <= 4 ? (byte) sk.getN() : 0; - wbuf.putByte(preInts); - wbuf.putByte(SER_VER); - wbuf.putByte(FAMILY_ID); - wbuf.putByte(flags); - wbuf.putShort((short)sk.getK()); - wbuf.putByte(numCompactors); - wbuf.putByte(numRawItems); + posSeg.setByte(preInts); + posSeg.setByte(SER_VER); + posSeg.setByte(FAMILY_ID); + posSeg.setByte(flags); + posSeg.setShort((short)sk.getK()); + posSeg.setByte(numCompactors); + posSeg.setByte(numRawItems); switch (serDeFormat) { case EMPTY: { - assert wbuf.getPosition() == bytes; + assert posSeg.getPosition() == bytes; return arr; } case RAWITEMS: { final ReqCompactor c0 = sk.getCompactors().get(0); final FloatBuffer fbuf = c0.getBuffer(); - for (int i = 0; i < numRawItems; i++) { wbuf.putFloat(fbuf.getItem(i)); } - assert wbuf.getPosition() == bytes; + for (int i = 0; i < numRawItems; i++) { posSeg.setFloat(fbuf.getItem(i)); } + assert posSeg.getPosition() == bytes; return arr; } case EXACT: { final ReqCompactor c0 = sk.getCompactors().get(0); - wbuf.putByteArray(c0.toByteArray(), 0, c0.getSerializationBytes()); - assert wbuf.getPosition() == bytes; + posSeg.setByteArray(c0.toByteArray(), 0, c0.getSerializationBytes()); + assert posSeg.getPosition() == bytes; return arr; } default: { //Normal Estimation - wbuf.putLong(sk.getN()); - wbuf.putFloat(sk.getMinItem()); - wbuf.putFloat(sk.getMaxItem()); + posSeg.setLong(sk.getN()); + posSeg.setFloat(sk.getMinItem()); + posSeg.setFloat(sk.getMaxItem()); for (int i = 0; i < numCompactors; i++) { final ReqCompactor c = sk.getCompactors().get(i); - wbuf.putByteArray(c.toByteArray(), 0, c.getSerializationBytes()); + posSeg.setByteArray(c.toByteArray(), 0, c.getSerializationBytes()); } - assert wbuf.getPosition() == bytes : wbuf.getPosition() + ", " + bytes; + assert posSeg.getPosition() == bytes : posSeg.getPosition() + ", " + bytes; return arr; } } @@ -318,7 +317,7 @@ class ReqSerDe { return 8; } case RAWITEMS: { - return sk.getCompactors().get(0).getBuffer().getCount() * Float.BYTES + 8; + return (sk.getCompactors().get(0).getBuffer().getCount() * Float.BYTES) + 8; } case EXACT: { return sk.getCompactors().get(0).getSerializationBytes() + 8; diff --git a/src/main/java/org/apache/datasketches/req/ReqSketch.java b/src/main/java/org/apache/datasketches/req/ReqSketch.java index bda7d4b9a..7de815750 100644 --- a/src/main/java/org/apache/datasketches/req/ReqSketch.java +++ b/src/main/java/org/apache/datasketches/req/ReqSketch.java @@ -22,12 +22,12 @@ package org.apache.datasketches.req; import static org.apache.datasketches.common.Util.LS; import static org.apache.datasketches.quantilescommon.QuantileSearchCriteria.INCLUSIVE; +import java.lang.foreign.MemorySegment; import java.util.ArrayList; import java.util.List; import java.util.Random; import org.apache.datasketches.common.SketchesArgumentException; -import org.apache.datasketches.memory.Memory; import org.apache.datasketches.quantilescommon.FloatsSketchSortedView; import org.apache.datasketches.quantilescommon.QuantileSearchCriteria; import org.apache.datasketches.quantilescommon.QuantilesAPI; @@ -123,7 +123,7 @@ public final class ReqSketch extends BaseReqSketch { this.minItem = minItem; this.maxItem = maxItem; this.compactors = compactors; - this.rand = new Random(); + rand = new Random(); } /** @@ -138,9 +138,9 @@ public final class ReqSketch extends BaseReqSketch { ReqSketch(final int k, final boolean highRankAccuracy, final ReqDebug reqDebug) { checkK(k); this.k = k; - this.hra = highRankAccuracy; + hra = highRankAccuracy; this.reqDebug = reqDebug; - this.rand = (reqDebug == null) ? new Random() : new Random(1); + rand = (reqDebug == null) ? new Random() : new Random(1); grow(); } @@ -149,16 +149,16 @@ public final class ReqSketch extends BaseReqSketch { * @param other the other sketch to be deep copied into this one. */ ReqSketch(final ReqSketch other) { - this.k = other.k; - this.hra = other.hra; - this.totalN = other.totalN; - this.retItems = other.retItems; - this.maxNomSize = other.maxNomSize; - this.minItem = other.minItem; - this.maxItem = other.maxItem; - this.reqDebug = other.reqDebug; - this.reqSV = null; - this.rand = (reqDebug == null) ? new Random() : new Random(1); + k = other.k; + hra = other.hra; + totalN = other.totalN; + retItems = other.retItems; + maxNomSize = other.maxNomSize; + minItem = other.minItem; + maxItem = other.maxItem; + reqDebug = other.reqDebug; + reqSV = null; + rand = (reqDebug == null) ? new Random() : new Random(1); for (int i = 0; i < other.getNumLevels(); i++) { compactors.add(new ReqCompactor(other.compactors.get(i))); @@ -169,17 +169,17 @@ public final class ReqSketch extends BaseReqSketch { * Returns a new ReqSketchBuilder * @return a new ReqSketchBuilder */ - public static final ReqSketchBuilder builder() { + public static ReqSketchBuilder builder() { return new ReqSketchBuilder(); } /** - * Returns an ReqSketch on the heap from a Memory image of the sketch. - * @param mem The Memory object holding a valid image of an ReqSketch - * @return an ReqSketch on the heap from a Memory image of the sketch. + * Returns an ReqSketch on the heap from a MemorySegment image of the sketch. + * @param seg The MemorySegment object holding a valid image of an ReqSketch + * @return an ReqSketch on the heap from a MemorySegment image of the sketch. */ - public static ReqSketch heapify(final Memory mem) { - return ReqSerDe.heapify(mem); + public static ReqSketch heapify(final MemorySegment seg) { + return ReqSerDe.heapify(seg); } @Override @@ -199,7 +199,7 @@ public final class ReqSketch extends BaseReqSketch { if (!Float.isFinite(v)) { throw new SketchesArgumentException("Numbers must be finite"); } - if (i < len - 1 && v >= splits[i + 1]) { + if ((i < (len - 1)) && (v >= splits[i + 1])) { throw new SketchesArgumentException( "Numbers must be unique and monotonically increasing"); } @@ -254,7 +254,7 @@ public final class ReqSketch extends BaseReqSketch { @Override public float getQuantile(final double normRank, final QuantileSearchCriteria searchCrit) { if (isEmpty()) { throw new IllegalArgumentException(QuantilesAPI.EMPTY_MSG); } - if (normRank < 0 || normRank > 1.0) { + if ((normRank < 0) || (normRank > 1.0)) { throw new SketchesArgumentException( "Normalized rank must be in the range [0.0, 1.0]: " + normRank); } @@ -379,15 +379,15 @@ public final class ReqSketch extends BaseReqSketch { @Override public ReqSketch merge(final ReqSketch other) { - if (other == null || other.isEmpty()) { return this; } + if ((other == null) || other.isEmpty()) { return this; } if (other.hra != hra) { throw new SketchesArgumentException( "Both sketches must have the same HighRankAccuracy setting."); } totalN += other.totalN; //update min, max items, n - if (Float.isNaN(minItem) || other.minItem < minItem) { minItem = other.minItem; } - if (Float.isNaN(maxItem) || other.maxItem > maxItem) { maxItem = other.maxItem; } + if (Float.isNaN(minItem) || (other.minItem < minItem)) { minItem = other.minItem; } + if (Float.isNaN(maxItem) || (other.maxItem > maxItem)) { maxItem = other.maxItem; } //Grow until self has at least as many compactors as other while (getNumLevels() < other.getNumLevels()) { grow(); } //Merge the items in all height compactors @@ -447,12 +447,12 @@ public final class ReqSketch extends BaseReqSketch { if (item < minItem) { minItem = item; } if (item > maxItem) { maxItem = item; } } - final FloatBuffer buf = compactors.get(0).getBuffer(); - buf.append(item); + final FloatBuffer fbuf = compactors.get(0).getBuffer(); + fbuf.append(item); retItems++; totalN++; if (retItems >= maxNomSize) { - buf.sort(); + fbuf.sort(); compress(); } reqSV = null; @@ -519,7 +519,7 @@ public final class ReqSketch extends BaseReqSketch { } private static void checkK(final int k) { - if ((k & 1) > 0 || k < 4 || k > 1024) { + if (((k & 1) > 0) || (k < 4) || (k > 1024)) { throw new SketchesArgumentException( "<i>K</i> must be even and in the range [4, 1024]: " + k ); } @@ -533,11 +533,11 @@ public final class ReqSketch extends BaseReqSketch { final int compNomCap = c.getNomCapacity(); if (compRetItems >= compNomCap) { - if (h + 1 >= getNumLevels()) { //at the top? + if ((h + 1) >= getNumLevels()) { //at the top? if (reqDebug != null) { reqDebug.emitMustAddCompactor(); } grow(); //add a level, increases maxNomSize } - final FloatBuffer promoted = c.compact(cReturn, this.rand); + final FloatBuffer promoted = c.compact(cReturn, rand); compactors.get(h + 1).getBuffer().mergeSortIn(promoted); retItems += cReturn.deltaRetItems; maxNomSize += cReturn.deltaNomSize; @@ -550,7 +550,7 @@ public final class ReqSketch extends BaseReqSketch { private void grow() { final byte lgWeight = (byte)getNumLevels(); - if (lgWeight == 0 && reqDebug != null) { reqDebug.emitStart(this); } + if ((lgWeight == 0) && (reqDebug != null)) { reqDebug.emitStart(this); } compactors.add(new ReqCompactor(lgWeight, hra, k, reqDebug)); maxNomSize = computeMaxNomSize(); if (reqDebug != null) { reqDebug.emitNewCompactor(lgWeight); } @@ -564,7 +564,7 @@ public final class ReqSketch extends BaseReqSketch { return reqSV; } - private final FloatsSketchSortedView refreshSortedView() { + private FloatsSketchSortedView refreshSortedView() { if (reqSV == null) { final CreateSortedView csv = new CreateSortedView(); reqSV = csv.getSV(); @@ -587,11 +587,11 @@ public final class ReqSketch extends BaseReqSketch { int count = 0; for (int i = 0; i < numComp; i++) { final ReqCompactor c = compactors.get(i); - final FloatBuffer bufIn = c.getBuffer(); - final long bufWeight = 1 << c.getLgWeight(); - final int bufInLen = bufIn.getCount(); - mergeSortIn(bufIn, bufWeight, count, getHighRankAccuracyMode()); - count += bufInLen; + final FloatBuffer fbufIn = c.getBuffer(); + final long fbufWeight = 1 << c.getLgWeight(); + final int fbufInLen = fbufIn.getCount(); + mergeSortIn(fbufIn, fbufWeight, count, getHighRankAccuracyMode()); + count += fbufInLen; } createCumulativeNativeRanks(); return new FloatsSketchSortedView(quantiles, cumWeights, ReqSketch.this); @@ -602,33 +602,33 @@ public final class ReqSketch extends BaseReqSketch { * the ultimate array size has already been set. However, this must simultaneously deal with * sorting the base FloatBuffer as well. * - * @param bufIn given FloatBuffer. If not sorted it will be sorted here. - * @param bufWeight associated weight of input FloatBuffer + * @param fbufIn given FloatBuffer. If not sorted it will be sorted here. + * @param fbufWeight associated weight of input FloatBuffer * @param count tracks number of items inserted into the class arrays */ - private void mergeSortIn(final FloatBuffer bufIn, final long bufWeight, final int count, final boolean hra) { - if (!bufIn.isSorted()) { bufIn.sort(); } - final float[] arrIn = bufIn.getArray(); //may be larger than its item count. - final int bufInLen = bufIn.getCount(); - final int totLen = count + bufInLen; + private void mergeSortIn(final FloatBuffer fbufIn, final long fbufWeight, final int count, final boolean hra) { + if (!fbufIn.isSorted()) { fbufIn.sort(); } + final float[] arrIn = fbufIn.getArray(); //may be larger than its item count. + final int fbufInLen = fbufIn.getCount(); + final int totLen = count + fbufInLen; int i = count - 1; - int j = bufInLen - 1; - int h = hra ? bufIn.getCapacity() - 1 : bufInLen - 1; + int j = fbufInLen - 1; + int h = hra ? fbufIn.getCapacity() - 1 : fbufInLen - 1; for (int k = totLen; k-- > 0; ) { - if (i >= 0 && j >= 0) { //both valid + if ((i >= 0) && (j >= 0)) { //both valid if (quantiles[i] >= arrIn[h]) { quantiles[k] = quantiles[i]; cumWeights[k] = cumWeights[i--]; //not yet natRanks, just individual wts } else { quantiles[k] = arrIn[h--]; j--; - cumWeights[k] = bufWeight; + cumWeights[k] = fbufWeight; } } else if (i >= 0) { //i is valid quantiles[k] = quantiles[i]; cumWeights[k] = cumWeights[i--]; } else if (j >= 0) { //j is valid quantiles[k] = arrIn[h--]; j--; - cumWeights[k] = bufWeight; + cumWeights[k] = fbufWeight; } else { break; } diff --git a/src/main/java/org/apache/datasketches/tdigest/TDigestDouble.java b/src/main/java/org/apache/datasketches/tdigest/TDigestDouble.java index d44a439d1..d127ccc9e 100644 --- a/src/main/java/org/apache/datasketches/tdigest/TDigestDouble.java +++ b/src/main/java/org/apache/datasketches/tdigest/TDigestDouble.java @@ -19,18 +19,19 @@ package org.apache.datasketches.tdigest; +import static org.apache.datasketches.common.SpecialValueLayouts.JAVA_DOUBLE_UNALIGNED_BIG_ENDIAN; +import static org.apache.datasketches.common.SpecialValueLayouts.JAVA_FLOAT_UNALIGNED_BIG_ENDIAN; +import static org.apache.datasketches.common.SpecialValueLayouts.JAVA_INT_UNALIGNED_BIG_ENDIAN; +import static org.apache.datasketches.common.SpecialValueLayouts.JAVA_SHORT_UNALIGNED_BIG_ENDIAN; import static org.apache.datasketches.common.Util.LS; -import java.nio.ByteOrder; +import java.lang.foreign.MemorySegment; import java.util.Arrays; import org.apache.datasketches.common.Family; +import org.apache.datasketches.common.positional.PositionalSegment; import org.apache.datasketches.common.SketchesArgumentException; import org.apache.datasketches.common.SketchesStateException; -import org.apache.datasketches.memory.Buffer; -import org.apache.datasketches.memory.Memory; -import org.apache.datasketches.memory.WritableBuffer; -import org.apache.datasketches.memory.WritableMemory; import org.apache.datasketches.quantilescommon.QuantilesAPI; import org.apache.datasketches.quantilescommon.QuantilesUtil; @@ -51,13 +52,13 @@ public final class TDigestDouble { private final short k_; private double minValue_; private double maxValue_; - private int centroidsCapacity_; + private final int centroidsCapacity_; private int numCentroids_; - private double[] centroidMeans_; - private long[] centroidWeights_; + private final double[] centroidMeans_; + private final long[] centroidWeights_; private long centroidsWeight_; private int numBuffered_; - private double[] bufferValues_; + private final double[] bufferValues_; private static final int BUFFER_MULTIPLIER = 4; @@ -99,7 +100,7 @@ public final class TDigestDouble { */ public void update(final double value) { if (Double.isNaN(value)) { return; } - if (numBuffered_ == centroidsCapacity_ * BUFFER_MULTIPLIER) { compress(); } + if (numBuffered_ == (centroidsCapacity_ * BUFFER_MULTIPLIER)) { compress(); } bufferValues_[numBuffered_] = value; numBuffered_++; minValue_ = Math.min(minValue_, value); @@ -142,7 +143,7 @@ public final class TDigestDouble { * @return true if TDigest has not seen any data */ public boolean isEmpty() { - return numCentroids_ == 0 && numBuffered_ == 0; + return (numCentroids_ == 0) && (numBuffered_ == 0); } /** @@ -181,16 +182,16 @@ public final class TDigestDouble { if (Double.isNaN(value)) { throw new SketchesArgumentException("Operation is undefined for Nan"); } if (value < minValue_) { return 0; } if (value > maxValue_) { return 1; } - if (numCentroids_ + numBuffered_ == 1) { return 0.5; } + if ((numCentroids_ + numBuffered_) == 1) { return 0.5; } compress(); // side effect // left tail final double firstMean = centroidMeans_[0]; if (value < firstMean) { - if (firstMean - minValue_ > 0) { + if ((firstMean - minValue_) > 0) { if (value == minValue_) { return 0.5 / centroidsWeight_; } - return (1.0 + (value - minValue_) / (firstMean - minValue_) * (centroidWeights_[0] / 2.0 - 1.0)); + return (1.0 + (((value - minValue_) / (firstMean - minValue_)) * ((centroidWeights_[0] / 2.0) - 1.0))); } return 0; // should never happen } @@ -198,10 +199,10 @@ public final class TDigestDouble { // right tail final double lastMean = centroidMeans_[numCentroids_ - 1]; if (value > lastMean) { - if (maxValue_ - lastMean > 0) { - if (value == maxValue_) { return 1.0 - 0.5 / centroidsWeight_; } - return 1.0 - ((1.0 + (maxValue_ - value) / (maxValue_ - lastMean) - * (centroidWeights_[numCentroids_ - 1] / 2.0 - 1.0)) / centroidsWeight_); + if ((maxValue_ - lastMean) > 0) { + if (value == maxValue_) { return 1.0 - (0.5 / centroidsWeight_); } + return 1.0 - ((1.0 + (((maxValue_ - value) / (maxValue_ - lastMean)) + * ((centroidWeights_[numCentroids_ - 1] / 2.0) - 1.0))) / centroidsWeight_); } return 1; // should never happen } @@ -211,7 +212,7 @@ public final class TDigestDouble { int upper = BinarySearch.upperBound(centroidMeans_, lower, numCentroids_, value); if (upper == 0) { throw new SketchesStateException("upper == begin in getRank()"); } if (value < centroidMeans_[lower]) { lower--; } - if (upper == numCentroids_ || !(centroidMeans_[upper - 1] < value)) { upper--; } + if ((upper == numCentroids_) || (centroidMeans_[upper - 1] >= value)) { upper--; } double weightBelow = 0; int i = 0; @@ -222,11 +223,11 @@ public final class TDigestDouble { while (i != upper) { weightDelta += centroidWeights_[i++]; } weightDelta -= centroidWeights_[lower] / 2.0; weightDelta += centroidWeights_[upper] / 2.0; - if (centroidMeans_[upper] - centroidMeans_[lower] > 0) { - return (weightBelow + weightDelta * (value - centroidMeans_[lower]) - / (centroidMeans_[upper] - centroidMeans_[lower])) / centroidsWeight_; + if ((centroidMeans_[upper] - centroidMeans_[lower]) > 0) { + return (weightBelow + ((weightDelta * (value - centroidMeans_[lower])) + / (centroidMeans_[upper] - centroidMeans_[lower]))) / centroidsWeight_; } - return (weightBelow + weightDelta / 2.0) / centroidsWeight_; + return (weightBelow + (weightDelta / 2.0)) / centroidsWeight_; } /** @@ -237,7 +238,7 @@ public final class TDigestDouble { public double getQuantile(final double rank) { if (isEmpty()) { throw new SketchesStateException(QuantilesAPI.EMPTY_MSG); } if (Double.isNaN(rank)) { throw new SketchesArgumentException("Operation is undefined for Nan"); } - if (rank < 0 || rank > 1) { throw new SketchesArgumentException("Normalized rank must be within [0, 1]"); } + if ((rank < 0) || (rank > 1)) { throw new SketchesArgumentException("Normalized rank must be within [0, 1]"); } compress(); // side effect @@ -246,40 +247,41 @@ public final class TDigestDouble { // at least 2 centroids final double weight = rank * centroidsWeight_; if (weight < 1) { return minValue_; } - if (weight > centroidsWeight_ - 1.0) { return maxValue_; } + if (weight > (centroidsWeight_ - 1.0)) { return maxValue_; } final double firstWeight = centroidWeights_[0]; - if (firstWeight > 1 && weight < firstWeight / 2.0) { - return minValue_ + (weight - 1.0) / (firstWeight / 2.0 - 1.0) * (centroidMeans_[0] - minValue_); + if ((firstWeight > 1) && (weight < (firstWeight / 2.0))) { + return minValue_ + (((weight - 1.0) / ((firstWeight / 2.0) - 1.0)) * (centroidMeans_[0] - minValue_)); } final double lastWeight = centroidWeights_[numCentroids_ - 1]; - if (lastWeight > 1 && centroidsWeight_ - weight <= lastWeight / 2.0) { - return maxValue_ + (centroidsWeight_ - weight - 1.0) / (lastWeight / 2.0 - 1.0) * (maxValue_ - centroidMeans_[numCentroids_ - 1]); + if ((lastWeight > 1) && ((centroidsWeight_ - weight) <= (lastWeight / 2.0))) { + return maxValue_ + (((centroidsWeight_ - weight - 1.0) / ((lastWeight / 2.0) - 1.0)) + * (maxValue_ - centroidMeans_[numCentroids_ - 1])); } // interpolate between extremes double weightSoFar = firstWeight / 2.0; - for (int i = 0; i < numCentroids_ - 1; i++) { + for (int i = 0; i < (numCentroids_ - 1); i++) { final double dw = (centroidWeights_[i] + centroidWeights_[i + 1]) / 2.0; - if (weightSoFar + dw > weight) { + if ((weightSoFar + dw) > weight) { // the target weight is between centroids i and i+1 double leftWeight = 0; if (centroidWeights_[i] == 1) { - if (weight - weightSoFar < 0.5) { return centroidMeans_[i]; } + if ((weight - weightSoFar) < 0.5) { return centroidMeans_[i]; } leftWeight = 0.5; } double rightWeight = 0; if (centroidWeights_[i + 1] == 1) { - if (weightSoFar + dw - weight <= 0.5) { return centroidMeans_[i + 1]; } + if (((weightSoFar + dw) - weight) <= 0.5) { return centroidMeans_[i + 1]; } rightWeight = 0.5; } final double w1 = weight - weightSoFar - leftWeight; - final double w2 = weightSoFar + dw - weight - rightWeight; + final double w2 = (weightSoFar + dw) - weight - rightWeight; return weightedAverage(centroidMeans_[i], w1, centroidMeans_[i + 1], w2); } weightSoFar += dw; } - final double w1 = weight - centroidsWeight_ - centroidWeights_[numCentroids_ - 1] / 2.0; - final double w2 = centroidWeights_[numCentroids_ - 1] / 2.0 - w1; + final double w1 = weight - centroidsWeight_ - (centroidWeights_[numCentroids_ - 1] / 2.0); + final double w2 = (centroidWeights_[numCentroids_ - 1] / 2.0) - w1; return weightedAverage(centroidWeights_[numCentroids_ - 1], w1, maxValue_, w2); } @@ -321,7 +323,7 @@ public final class TDigestDouble { QuantilesUtil.checkDoublesSplitPointsOrder(splitPoints); final int len = splitPoints.length + 1; final double[] ranks = new double[len]; - for (int i = 0; i < len - 1; i++) { + for (int i = 0; i < (len - 1); i++) { ranks[i] = getRank(splitPoints[i]); } ranks[len - 1] = 1.0; @@ -334,8 +336,8 @@ public final class TDigestDouble { */ int getSerializedSizeBytes() { compress(); // side effect - return getPreambleLongs() * Long.BYTES - + (isEmpty() ? 0 : (isSingleValue() ? Double.BYTES : 2 * Double.BYTES + (Double.BYTES + Long.BYTES) * numCentroids_)); + return (getPreambleLongs() * Long.BYTES) + + (isEmpty() ? 0 : (isSingleValue() ? Double.BYTES : (2 * Double.BYTES) + ((Double.BYTES + Long.BYTES) * numCentroids_))); } /** @@ -345,100 +347,101 @@ public final class TDigestDouble { public byte[] toByteArray() { compress(); // side effect final byte[] bytes = new byte[getSerializedSizeBytes()]; - final WritableBuffer wbuf = WritableMemory.writableWrap(bytes).asWritableBuffer(); - wbuf.putByte((byte) getPreambleLongs()); - wbuf.putByte(SERIAL_VERSION); - wbuf.putByte((byte) Family.TDIGEST.getID()); - wbuf.putShort(k_); - wbuf.putByte((byte) ( + final PositionalSegment posSeg = PositionalSegment.wrap(MemorySegment.ofArray(bytes)); + posSeg.setByte((byte) getPreambleLongs()); + posSeg.setByte(SERIAL_VERSION); + posSeg.setByte((byte) Family.TDIGEST.getID()); + posSeg.setShort(k_); + posSeg.setByte((byte) ( (isEmpty() ? 1 << Flags.IS_EMPTY.ordinal() : 0) | (isSingleValue() ? 1 << Flags.IS_SINGLE_VALUE.ordinal() : 0) | (reverseMerge_ ? 1 << Flags.REVERSE_MERGE.ordinal() : 0) )); - wbuf.putShort((short) 0); // unused + posSeg.setShort((short) 0); // unused if (isEmpty()) { return bytes; } if (isSingleValue()) { - wbuf.putDouble(minValue_); + posSeg.setDouble(minValue_); return bytes; } - wbuf.putInt(numCentroids_); - wbuf.putInt(0); // unused - wbuf.putDouble(minValue_); - wbuf.putDouble(maxValue_); + posSeg.setInt(numCentroids_); + posSeg.setInt(0); // unused + posSeg.setDouble(minValue_); + posSeg.setDouble(maxValue_); for (int i = 0; i < numCentroids_; i++) { - wbuf.putDouble(centroidMeans_[i]); - wbuf.putLong(centroidWeights_[i]); + posSeg.setDouble(centroidMeans_[i]); + posSeg.setLong(centroidWeights_[i]); } return bytes; } /** - * Deserialize TDigest from a given memory. - * Supports reading format of the reference implementation (autodetected). - * @param mem instance of Memory + * Deserialize TDigest from a given MemorySegment. + * Supports reading format of the reference implementation (auto-detected). + * @param seg instance of MemorySegment * @return an instance of TDigest */ - public static TDigestDouble heapify(final Memory mem) { - return heapify(mem, false); + public static TDigestDouble heapify(final MemorySegment seg) { + return heapify(seg, false); } /** - * Deserialize TDigest from a given memory. Supports reading compact format + * Deserialize TDigest from a given MemorySegment. Supports reading compact format * with (float, int) centroids as opposed to (double, long) to represent (mean, weight). - * Supports reading format of the reference implementation (autodetected). - * @param mem instance of Memory + * Supports reading format of the reference implementation (auto-detected). + * @param seg instance of MemorySegment * @param isFloat if true the input represents (float, int) format * @return an instance of TDigest */ - public static TDigestDouble heapify(final Memory mem, final boolean isFloat) { - final Buffer buff = mem.asBuffer(); - final byte preambleLongs = buff.getByte(); - final byte serialVersion = buff.getByte(); - final byte sketchType = buff.getByte(); + public static TDigestDouble heapify(final MemorySegment seg, final boolean isFloat) { + final PositionalSegment posSeg = PositionalSegment.wrap(seg); + final byte preambleLongs = posSeg.getByte(); + final byte serialVersion = posSeg.getByte(); + final byte sketchType = posSeg.getByte(); + if (sketchType != (byte) Family.TDIGEST.getID()) { - if (preambleLongs == 0 && serialVersion == 0 && sketchType == 0) { return heapifyCompat(mem); } + if ((preambleLongs == 0) && (serialVersion == 0) && (sketchType == 0)) { return heapifyCompat(seg); } throw new SketchesArgumentException("Sketch type mismatch: expected " + Family.TDIGEST.getID() + ", actual " + sketchType); } if (serialVersion != SERIAL_VERSION) { throw new SketchesArgumentException("Serial version mismatch: expected " + SERIAL_VERSION + ", actual " + serialVersion); } - final short k = buff.getShort(); - final byte flagsByte = buff.getByte(); + final short k = posSeg.getShort(); + final byte flagsByte = posSeg.getByte(); final boolean isEmpty = (flagsByte & (1 << Flags.IS_EMPTY.ordinal())) > 0; final boolean isSingleValue = (flagsByte & (1 << Flags.IS_SINGLE_VALUE.ordinal())) > 0; final byte expectedPreambleLongs = isEmpty || isSingleValue ? PREAMBLE_LONGS_EMPTY_OR_SINGLE : PREAMBLE_LONGS_MULTIPLE; if (preambleLongs != expectedPreambleLongs) { throw new SketchesArgumentException("Preamble longs mismatch: expected " + expectedPreambleLongs + ", actual " + preambleLongs); } - buff.getShort(); // unused + posSeg.getShort(); // unused if (isEmpty) { return new TDigestDouble(k); } final boolean reverseMerge = (flagsByte & (1 << Flags.REVERSE_MERGE.ordinal())) > 0; if (isSingleValue) { final double value; if (isFloat) { - value = buff.getFloat(); + value = posSeg.getFloat(); } else { - value = buff.getDouble(); + value = posSeg.getDouble(); } return new TDigestDouble(reverseMerge, k, value, value, new double[] {value}, new long[] {1}, 1, null); } - final int numCentroids = buff.getInt(); - buff.getInt(); // unused + final int numCentroids = posSeg.getInt(); + posSeg.getInt(); // unused final double min; final double max; if (isFloat) { - min = buff.getFloat(); - max = buff.getFloat(); + min = posSeg.getFloat(); + max = posSeg.getFloat(); } else { - min = buff.getDouble(); - max = buff.getDouble(); + min = posSeg.getDouble(); + max = posSeg.getDouble(); } final double[] means = new double[numCentroids]; final long[] weights = new long[numCentroids]; long totalWeight = 0; for (int i = 0; i < numCentroids; i++) { - means[i] = isFloat ? buff.getFloat() : buff.getDouble(); - weights[i] = isFloat ? buff.getInt() : buff.getLong(); + means[i] = isFloat ? posSeg.getFloat() : posSeg.getDouble(); + weights[i] = isFloat ? posSeg.getInt() : posSeg.getLong(); totalWeight += weights[i]; } return new TDigestDouble(reverseMerge, k, min, max, means, weights, totalWeight, null); @@ -446,41 +449,41 @@ public final class TDigestDouble { // compatibility with the format of the reference implementation // default byte order of ByteBuffer is used there, which is big endian - private static TDigestDouble heapifyCompat(final Memory mem) { - final Buffer buff = mem.asBuffer(ByteOrder.BIG_ENDIAN); - final int type = buff.getInt(); - if (type != COMPAT_DOUBLE && type != COMPAT_FLOAT) { + private static TDigestDouble heapifyCompat(final MemorySegment seg) { + int offset = 0; + final int type = seg.get(JAVA_INT_UNALIGNED_BIG_ENDIAN, offset); offset += Integer.BYTES; + if ((type != COMPAT_DOUBLE) && (type != COMPAT_FLOAT)) { throw new SketchesArgumentException("unexpected compatibility type " + type); } if (type == COMPAT_DOUBLE) { // compatibility with asBytes() - final double min = buff.getDouble(); - final double max = buff.getDouble(); - final short k = (short) buff.getDouble(); - final int numCentroids = buff.getInt(); + final double min = seg.get(JAVA_DOUBLE_UNALIGNED_BIG_ENDIAN, offset); offset += Double.BYTES; + final double max = seg.get(JAVA_DOUBLE_UNALIGNED_BIG_ENDIAN, offset); offset += Double.BYTES; + final short k = (short) seg.get(JAVA_DOUBLE_UNALIGNED_BIG_ENDIAN, offset); offset += Double.BYTES; + final int numCentroids = seg.get(JAVA_INT_UNALIGNED_BIG_ENDIAN, offset); offset += Integer.BYTES; final double[] means = new double[numCentroids]; final long[] weights = new long[numCentroids]; long totalWeight = 0; for (int i = 0; i < numCentroids; i++) { - weights[i] = (long) buff.getDouble(); - means[i] = buff.getDouble(); + weights[i] = (long) seg.get(JAVA_DOUBLE_UNALIGNED_BIG_ENDIAN, offset); offset += Double.BYTES; + means[i] = seg.get(JAVA_DOUBLE_UNALIGNED_BIG_ENDIAN, offset); offset += Double.BYTES; totalWeight += weights[i]; } return new TDigestDouble(false, k, min, max, means, weights, totalWeight, null); } - // COMPAT_FLOAT: compatibility with asSmallBytes() - final double min = buff.getDouble(); // reference implementation uses doubles for min and max - final double max = buff.getDouble(); - final short k = (short) buff.getFloat(); + // COMPAT_FLOAT: compatibility with asSmallBytes(), reference implementation uses doubles for min and max + final double min = seg.get(JAVA_DOUBLE_UNALIGNED_BIG_ENDIAN, offset); offset += Double.BYTES; + final double max = seg.get(JAVA_DOUBLE_UNALIGNED_BIG_ENDIAN, offset);offset += Double.BYTES; + final short k = (short) seg.get(JAVA_FLOAT_UNALIGNED_BIG_ENDIAN, offset); offset += Float.BYTES; // reference implementation stores capacities of the array of centroids and the buffer as shorts // they can be derived from k in the constructor - buff.getInt(); // unused - final int numCentroids = buff.getShort(); + seg.get(JAVA_INT_UNALIGNED_BIG_ENDIAN, offset); offset += Integer.BYTES; + final int numCentroids = seg.get(JAVA_SHORT_UNALIGNED_BIG_ENDIAN, offset); offset += Short.BYTES; final double[] means = new double[numCentroids]; final long[] weights = new long[numCentroids]; long totalWeight = 0; for (int i = 0; i < numCentroids; i++) { - weights[i] = (long) buff.getFloat(); - means[i] = buff.getFloat(); + weights[i] = (long) seg.get(JAVA_FLOAT_UNALIGNED_BIG_ENDIAN, offset); offset += Float.BYTES; + means[i] = seg.get(JAVA_FLOAT_UNALIGNED_BIG_ENDIAN, offset); offset += Float.BYTES; totalWeight += weights[i]; } return new TDigestDouble(false, k, min, max, means, weights, totalWeight, null); @@ -541,14 +544,14 @@ public final class TDigestDouble { maxValue_ = max; if (k < 10) { throw new SketchesArgumentException("k must be at least 10"); } final int fudge = k < 30 ? 30 : 10; - centroidsCapacity_ = k_ * 2 + fudge; + centroidsCapacity_ = (k_ * 2) + fudge; centroidMeans_ = new double[centroidsCapacity_]; centroidWeights_ = new long[centroidsCapacity_]; bufferValues_ = new double[centroidsCapacity_ * BUFFER_MULTIPLIER]; numCentroids_ = 0; numBuffered_ = 0; centroidsWeight_ = weight; - if (means != null && weights != null) { + if ((means != null) && (weights != null)) { System.arraycopy(means, 0, centroidMeans_, 0, means.length); System.arraycopy(weights, 0, centroidWeights_, 0, weights.length); numCentroids_ = means.length; @@ -579,16 +582,16 @@ public final class TDigestDouble { while (current != num) { final double proposedWeight = centroidWeights_[numCentroids_ - 1] + weights[current]; boolean addThis = false; - if (current != 1 && current != num - 1) { + if ((current != 1) && (current != (num - 1))) { final double q0 = weightSoFar / centroidsWeight_; final double q2 = (weightSoFar + proposedWeight) / centroidsWeight_; final double normalizer = ScaleFunction.normalizer(k_ * 2, centroidsWeight_); - addThis = proposedWeight <= centroidsWeight_ * Math.min(ScaleFunction.max(q0, normalizer), ScaleFunction.max(q2, normalizer)); + addThis = proposedWeight <= (centroidsWeight_ * Math.min(ScaleFunction.max(q0, normalizer), ScaleFunction.max(q2, normalizer))); } if (addThis) { // merge into existing centroid centroidWeights_[numCentroids_ - 1] += weights[current]; - centroidMeans_[numCentroids_ - 1] += (values[current] - centroidMeans_[numCentroids_ - 1]) - * weights[current] / centroidWeights_[numCentroids_ - 1]; + centroidMeans_[numCentroids_ - 1] += ((values[current] - centroidMeans_[numCentroids_ - 1]) + * weights[current]) / centroidWeights_[numCentroids_ - 1]; } else { // copy to a new centroid weightSoFar += centroidWeights_[numCentroids_ - 1]; centroidMeans_[numCentroids_] = values[current]; @@ -622,7 +625,7 @@ public final class TDigestDouble { */ private static final class ScaleFunction { static double max(final double q, final double normalizer) { - return q * (1 - q) / normalizer; + return (q * (1 - q)) / normalizer; } static double normalizer(final double compression, final double n) { @@ -630,11 +633,11 @@ public final class TDigestDouble { } static double z(final double compression, final double n) { - return 4 * Math.log(n / compression) + 24; + return (4 * Math.log(n / compression)) + 24; } } private static double weightedAverage(final double x1, final double w1, final double x2, final double w2) { - return (x1 * w1 + x2 * w2) / (w1 + w2); + return ((x1 * w1) + (x2 * w2)) / (w1 + w2); } } diff --git a/src/test/java/org/apache/datasketches/req/ReqCompactorTest.java b/src/test/java/org/apache/datasketches/req/ReqCompactorTest.java index 87720536b..5839f0a58 100644 --- a/src/test/java/org/apache/datasketches/req/ReqCompactorTest.java +++ b/src/test/java/org/apache/datasketches/req/ReqCompactorTest.java @@ -22,8 +22,13 @@ package org.apache.datasketches.req; import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertTrue; -import org.apache.datasketches.memory.Buffer; -import org.apache.datasketches.memory.Memory; +import java.lang.foreign.MemorySegment; + +import org.apache.datasketches.common.positional.PositionalSegment; +import org.apache.datasketches.req.FloatBuffer; +import org.apache.datasketches.req.ReqCompactor; +import org.apache.datasketches.req.ReqSerDe; +import org.apache.datasketches.req.ReqSketch; import org.apache.datasketches.req.ReqSerDe.Compactor; import org.testng.annotations.Test; @@ -63,10 +68,10 @@ public class ReqCompactorTest { final int nomCap = 2 * 3 * k; final int cap = 2 * nomCap; final int delta = nomCap; - final FloatBuffer buf = c1.getBuffer(); + final FloatBuffer fbuf = c1.getBuffer(); for (int i = 1; i <= nomCap; i++) { - buf.append(i); //compactor doesn't have a direct update() method + fbuf.append(i); //compactor doesn't have a direct update() method } final float minV = 1; final float maxV = nomCap; @@ -76,11 +81,11 @@ public class ReqCompactorTest { final long state = c1.getState(); final int lgWeight = c1.getLgWeight(); final boolean c1hra = c1.isHighRankAccuracy(); - final boolean sorted = buf.isSorted(); + final boolean sorted = fbuf.isSorted(); final byte[] c1ser = c1.toByteArray(); //now deserialize - final Buffer buff = Memory.wrap(c1ser).asBuffer(); - final Compactor compactor = ReqSerDe.extractCompactor(buff, sorted, c1hra); + final PositionalSegment posSeg = PositionalSegment.wrap(MemorySegment.ofArray(c1ser)); + final Compactor compactor = ReqSerDe.extractCompactor(posSeg, sorted, c1hra); final ReqCompactor c2 = compactor.reqCompactor; assertEquals(compactor.minItem, minV); assertEquals(compactor.maxItem, maxV); @@ -91,9 +96,9 @@ public class ReqCompactorTest { assertEquals(c2.getState(), state); assertEquals(c2.getLgWeight(), lgWeight); assertEquals(c2.isHighRankAccuracy(), c1hra); - final FloatBuffer buf2 = c2.getBuffer(); - assertEquals(buf2.getCapacity(), cap); - assertEquals(buf2.getDelta(), delta); - assertTrue(buf.isEqualTo(buf2)); + final FloatBuffer fbuf2 = c2.getBuffer(); + assertEquals(fbuf2.getCapacity(), cap); + assertEquals(fbuf2.getDelta(), delta); + assertTrue(fbuf.isEqualTo(fbuf2)); } } diff --git a/src/test/java/org/apache/datasketches/req/ReqDebugImplTest.java b/src/test/java/org/apache/datasketches/req/ReqDebugImplTest.java index 5e757ce0f..da19aa9ea 100644 --- a/src/test/java/org/apache/datasketches/req/ReqDebugImplTest.java +++ b/src/test/java/org/apache/datasketches/req/ReqDebugImplTest.java @@ -21,6 +21,10 @@ package org.apache.datasketches.req; import java.util.List; +import org.apache.datasketches.req.ReqCompactor; +import org.apache.datasketches.req.ReqDebug; +import org.apache.datasketches.req.ReqSketch; + /** * The implementation of the ReqDebug interface. The current levels are * implemented: diff --git a/src/test/java/org/apache/datasketches/req/ReqFloatBufferTest.java b/src/test/java/org/apache/datasketches/req/ReqFloatBufferTest.java index 49820e094..7a7f77d05 100644 --- a/src/test/java/org/apache/datasketches/req/ReqFloatBufferTest.java +++ b/src/test/java/org/apache/datasketches/req/ReqFloatBufferTest.java @@ -19,14 +19,17 @@ package org.apache.datasketches.req; +import static java.lang.foreign.ValueLayout.JAVA_FLOAT_UNALIGNED; import static org.apache.datasketches.quantilescommon.QuantileSearchCriteria.EXCLUSIVE; import static org.apache.datasketches.quantilescommon.QuantileSearchCriteria.INCLUSIVE; import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertTrue; import static org.testng.Assert.fail; +import java.lang.foreign.MemorySegment; + import org.apache.datasketches.common.SketchesArgumentException; -import org.apache.datasketches.memory.WritableMemory; +import org.apache.datasketches.req.FloatBuffer; import org.testng.annotations.Test; /** @@ -59,7 +62,7 @@ public class ReqFloatBufferTest { private static void checkGetEvensOrOddsImpl(final boolean odds, final boolean spaceAtBottom) { final int cap = 16; final FloatBuffer buf = new FloatBuffer(cap, 0, spaceAtBottom); - for (int i = 0; i < cap/2; i++) { + for (int i = 0; i < (cap/2); i++) { buf.append(i); } final FloatBuffer out = buf.getEvensOrOdds(0, cap/2, odds); @@ -147,7 +150,7 @@ public class ReqFloatBufferTest { } private static void iterateValues(final FloatBuffer buf, final int len) { - for (float v = 0.5f; v <= len + 0.5f; v += 0.5f) { + for (float v = 0.5f; v <= (len + 0.5f); v += 0.5f) { checkCountWithCriteria(buf, v); } } @@ -265,29 +268,29 @@ public class ReqFloatBufferTest { } private static void checkSerDeImpl(final boolean hra) { - final FloatBuffer buf = new FloatBuffer(100, 100, hra); - for (int i = 0; i <= 100; i++) { buf.append(i); } - final int capacity = buf.getCapacity(); - final int count = buf.getCount(); - final int delta = buf.getDelta(); - final boolean sorted = buf.isSorted(); - final boolean sab = buf.isSpaceAtBottom(); - assertEquals(buf.getItemFromIndex(100), 100.0f); - assertEquals(buf.getItemFromIndex(hra ? 199 : 1), 1.0f); - assertEquals(buf.isSpaceAtBottom(), hra); + final FloatBuffer fbuf = new FloatBuffer(100, 100, hra); + for (int i = 0; i <= 100; i++) { fbuf.append(i); } + final int capacity = fbuf.getCapacity(); + final int count = fbuf.getCount(); + final int delta = fbuf.getDelta(); + final boolean sorted = fbuf.isSorted(); + final boolean sab = fbuf.isSpaceAtBottom(); + assertEquals(fbuf.getItemFromIndex(100), 100.0f); + assertEquals(fbuf.getItemFromIndex(hra ? 199 : 1), 1.0f); + assertEquals(fbuf.isSpaceAtBottom(), hra); //uses the serialization method - final WritableMemory wmem = WritableMemory.writableWrap(buf.floatsToBytes()); + final MemorySegment wseg = MemorySegment.ofArray(fbuf.floatsToBytes()); final float[] farr2 = new float[101]; - wmem.getFloatArray(0, farr2, 0, 101); + MemorySegment.copy(wseg, JAVA_FLOAT_UNALIGNED, 0, farr2, 0, 101); //uses the deserialization method - final FloatBuffer buf2 = FloatBuffer.reconstruct(farr2, count, capacity, delta, sorted, sab); - assertEquals(buf2.getCapacity(), capacity); - assertEquals(buf2.getCount(), count); - assertEquals(buf2.getDelta(), delta); - assertEquals(buf2.isSorted(), sorted); - assertEquals(buf2.getItemFromIndex(100), 100.0f); - assertEquals(buf2.getItemFromIndex(hra ? 199 : 1), 1.0f); - assertEquals(buf2.isSpaceAtBottom(), sab); + final FloatBuffer fbuf2 = FloatBuffer.reconstruct(farr2, count, capacity, delta, sorted, sab); + assertEquals(fbuf2.getCapacity(), capacity); + assertEquals(fbuf2.getCount(), count); + assertEquals(fbuf2.getDelta(), delta); + assertEquals(fbuf2.isSorted(), sorted); + assertEquals(fbuf2.getItemFromIndex(100), 100.0f); + assertEquals(fbuf2.getItemFromIndex(hra ? 199 : 1), 1.0f); + assertEquals(fbuf2.isSpaceAtBottom(), sab); } static void print(final Object o) { System.out.print(o.toString()); } diff --git a/src/test/java/org/apache/datasketches/req/ReqSketchBuilderTest.java b/src/test/java/org/apache/datasketches/req/ReqSketchBuilderTest.java index 8e33b03e2..e21405f70 100644 --- a/src/test/java/org/apache/datasketches/req/ReqSketchBuilderTest.java +++ b/src/test/java/org/apache/datasketches/req/ReqSketchBuilderTest.java @@ -22,6 +22,7 @@ package org.apache.datasketches.req; import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertTrue; +import org.apache.datasketches.req.ReqSketchBuilder; import org.testng.annotations.Test; /** diff --git a/src/test/java/org/apache/datasketches/req/ReqSketchCrossLanguageTest.java b/src/test/java/org/apache/datasketches/req/ReqSketchCrossLanguageTest.java index d541db074..e126f9b46 100644 --- a/src/test/java/org/apache/datasketches/req/ReqSketchCrossLanguageTest.java +++ b/src/test/java/org/apache/datasketches/req/ReqSketchCrossLanguageTest.java @@ -26,11 +26,12 @@ import static org.apache.datasketches.common.TestUtil.javaPath; import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertTrue; +import java.lang.foreign.MemorySegment; import java.io.IOException; import java.nio.file.Files; -import org.apache.datasketches.memory.Memory; import org.apache.datasketches.quantilescommon.QuantilesFloatsSketchIterator; +import org.apache.datasketches.req.ReqSketch; import org.testng.annotations.Test; /** @@ -42,9 +43,11 @@ public class ReqSketchCrossLanguageTest { @Test(groups = {GENERATE_JAVA_FILES}) public void generateBinariesForCompatibilityTesting() throws IOException { final int[] nArr = {0, 1, 10, 100, 1000, 10_000, 100_000, 1_000_000}; - for (int n: nArr) { + for (final int n: nArr) { final ReqSketch sk = ReqSketch.builder().build(); - for (int i = 1; i <= n; i++) sk.update(i); + for (int i = 1; i <= n; i++) { + sk.update(i); + } Files.newOutputStream(javaPath.resolve("req_float_n" + n + "_java.sk")).write(sk.toByteArray()); } } @@ -52,16 +55,16 @@ public class ReqSketchCrossLanguageTest { @Test(groups = {CHECK_CPP_FILES}) public void deserializeFromCpp() throws IOException { final int[] nArr = {0, 1, 10, 100, 1000, 10000, 100000, 1000000}; - for (int n: nArr) { + for (final int n: nArr) { final byte[] bytes = Files.readAllBytes(cppPath.resolve("req_float_n" + n + "_cpp.sk")); - final ReqSketch sk = ReqSketch.heapify(Memory.wrap(bytes)); + final ReqSketch sk = ReqSketch.heapify(MemorySegment.ofArray(bytes)); assertTrue(n == 0 ? sk.isEmpty() : !sk.isEmpty()); assertTrue(n > 10 ? sk.isEstimationMode() : !sk.isEstimationMode()); assertEquals(sk.getN(), n); if (n > 0) { assertEquals(sk.getMinItem(), 1); assertEquals(sk.getMaxItem(), n); - QuantilesFloatsSketchIterator it = sk.iterator(); + final QuantilesFloatsSketchIterator it = sk.iterator(); long weight = 0; while(it.next()) { assertTrue(it.getQuantile() >= sk.getMinItem()); diff --git a/src/test/java/org/apache/datasketches/req/ReqSketchOtherTest.java b/src/test/java/org/apache/datasketches/req/ReqSketchOtherTest.java index 46411d846..48b6e582e 100644 --- a/src/test/java/org/apache/datasketches/req/ReqSketchOtherTest.java +++ b/src/test/java/org/apache/datasketches/req/ReqSketchOtherTest.java @@ -33,6 +33,9 @@ import static org.testng.Assert.fail; import org.apache.datasketches.common.SketchesArgumentException; import org.apache.datasketches.quantilescommon.FloatsSortedView; import org.apache.datasketches.quantilescommon.InequalitySearch; +import org.apache.datasketches.req.BaseReqSketch; +import org.apache.datasketches.req.ReqSketch; +import org.apache.datasketches.req.ReqSketchBuilder; import org.testng.annotations.Test; /** diff --git a/src/test/java/org/apache/datasketches/req/ReqSketchSortedViewTest.java b/src/test/java/org/apache/datasketches/req/ReqSketchSortedViewTest.java index 003a53c3b..dbd7d9ffe 100644 --- a/src/test/java/org/apache/datasketches/req/ReqSketchSortedViewTest.java +++ b/src/test/java/org/apache/datasketches/req/ReqSketchSortedViewTest.java @@ -27,6 +27,8 @@ import static org.testng.Assert.assertTrue; import org.apache.datasketches.quantilescommon.FloatsSortedView; import org.apache.datasketches.quantilescommon.FloatsSortedViewIterator; +import org.apache.datasketches.req.ReqSketch; +import org.apache.datasketches.req.ReqSketchBuilder; import org.testng.annotations.Test; /** diff --git a/src/test/java/org/apache/datasketches/req/ReqSketchTest.java b/src/test/java/org/apache/datasketches/req/ReqSketchTest.java index f38dc7f64..98cece3f9 100644 --- a/src/test/java/org/apache/datasketches/req/ReqSketchTest.java +++ b/src/test/java/org/apache/datasketches/req/ReqSketchTest.java @@ -26,13 +26,16 @@ import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertFalse; import static org.testng.Assert.fail; +import java.lang.foreign.MemorySegment; + import org.apache.datasketches.common.SketchesArgumentException; -import org.apache.datasketches.memory.Memory; import org.apache.datasketches.quantilescommon.FloatsSketchSortedView; import org.apache.datasketches.quantilescommon.FloatsSortedViewIterator; import org.apache.datasketches.quantilescommon.QuantileSearchCriteria; import org.apache.datasketches.quantilescommon.QuantilesFloatsSketchIterator; import org.apache.datasketches.quantilescommon.QuantilesUtil; +import org.apache.datasketches.req.ReqSketch; +import org.apache.datasketches.req.ReqSketchBuilder; import org.testng.annotations.Test; /** @@ -124,8 +127,8 @@ public class ReqSketchTest { final String dfmt = "%10.2f%10.6f" + LS; final String sfmt = "%10s%10s" + LS; if (iDebug > 0) { printf(sfmt, "Value", "Rank"); } - float va = 0; - double ranka = 0; + final float va = 0; + final double ranka = 0; for (int i = 0; i < spArr.length; i++) { final float v = spArr[i]; final double trueRank = trueRanks[i]; @@ -269,7 +272,7 @@ public class ReqSketchTest { @Test public void checkSerDe() { final int k = 12; - final int exact = 2 * 3 * k - 1; + final int exact = (2 * 3 * k) - 1; checkSerDeImpl(12, false, 0); checkSerDeImpl(12, true, 0); checkSerDeImpl(12, false, 4); @@ -286,13 +289,13 @@ public class ReqSketchTest { sk1.update(i); } final byte[] sk1Arr = sk1.toByteArray(); - final Memory mem = Memory.wrap(sk1Arr); - final ReqSketch sk2 = ReqSketch.heapify(mem); + final MemorySegment seg = MemorySegment.ofArray(sk1Arr); + final ReqSketch sk2 = ReqSketch.heapify(seg); assertEquals(sk2.getNumRetained(), sk1.getNumRetained()); assertEquals(sk1.isEmpty(), sk2.isEmpty()); if (sk2.isEmpty()) { - try { sk2.getMinItem(); fail(); } catch (IllegalArgumentException e) {} - try { sk2.getMaxItem(); fail(); } catch (IllegalArgumentException e) {} + try { sk2.getMinItem(); fail(); } catch (final IllegalArgumentException e) {} + try { sk2.getMaxItem(); fail(); } catch (final IllegalArgumentException e) {} } else { assertEquals(sk2.getMinItem(), sk1.getMinItem()); assertEquals(sk2.getMaxItem(), sk1.getMaxItem()); diff --git a/src/test/java/org/apache/datasketches/tdigest/TDigestCrossLanguageTest.java b/src/test/java/org/apache/datasketches/tdigest/TDigestCrossLanguageTest.java index 8bdb16bb4..8475b5bd4 100644 --- a/src/test/java/org/apache/datasketches/tdigest/TDigestCrossLanguageTest.java +++ b/src/test/java/org/apache/datasketches/tdigest/TDigestCrossLanguageTest.java @@ -26,10 +26,10 @@ import static org.apache.datasketches.common.TestUtil.javaPath; import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertTrue; +import java.lang.foreign.MemorySegment; import java.io.IOException; import java.nio.file.Files; -import org.apache.datasketches.memory.Memory; import org.testng.annotations.Test; public class TDigestCrossLanguageTest { @@ -37,9 +37,9 @@ public class TDigestCrossLanguageTest { @Test(groups = {CHECK_CPP_FILES}) public void deserializeFromCppDouble() throws IOException { final int[] nArr = {0, 1, 10, 100, 1000, 10_000, 100_000, 1_000_000}; - for (int n: nArr) { + for (final int n: nArr) { final byte[] bytes = Files.readAllBytes(cppPath.resolve("tdigest_double_n" + n + "_cpp.sk")); - final TDigestDouble td = TDigestDouble.heapify(Memory.wrap(bytes)); + final TDigestDouble td = TDigestDouble.heapify(MemorySegment.ofArray(bytes)); assertTrue(n == 0 ? td.isEmpty() : !td.isEmpty()); assertEquals(td.getTotalWeight(), n); if (n > 0) { @@ -59,9 +59,9 @@ public class TDigestCrossLanguageTest { @Test(groups = {CHECK_CPP_FILES}) public void deserializeFromCppFloat() throws IOException { final int[] nArr = {0, 1, 10, 100, 1000, 10_000, 100_000, 1_000_000}; - for (int n: nArr) { + for (final int n: nArr) { final byte[] bytes = Files.readAllBytes(cppPath.resolve("tdigest_float_n" + n + "_cpp.sk")); - final TDigestDouble td = TDigestDouble.heapify(Memory.wrap(bytes), true); + final TDigestDouble td = TDigestDouble.heapify(MemorySegment.ofArray(bytes), true); assertTrue(n == 0 ? td.isEmpty() : !td.isEmpty()); assertEquals(td.getTotalWeight(), n); if (n > 0) { @@ -81,9 +81,11 @@ public class TDigestCrossLanguageTest { @Test(groups = {GENERATE_JAVA_FILES}) public void generateForCppDouble() throws IOException { final int[] nArr = {0, 1, 10, 100, 1000, 10_000, 100_000, 1_000_000}; - for (int n: nArr) { + for (final int n: nArr) { final TDigestDouble td = new TDigestDouble((short) 100); - for (int i = 1; i <= n; i++) td.update(i); + for (int i = 1; i <= n; i++) { + td.update(i); + } Files.newOutputStream(javaPath.resolve("tdigest_double_n" + n + "_java.sk")).write(td.toByteArray()); } } diff --git a/src/test/java/org/apache/datasketches/tdigest/TDigestDoubleTest.java b/src/test/java/org/apache/datasketches/tdigest/TDigestDoubleTest.java index e1c6914c7..3a2bb2e37 100644 --- a/src/test/java/org/apache/datasketches/tdigest/TDigestDoubleTest.java +++ b/src/test/java/org/apache/datasketches/tdigest/TDigestDoubleTest.java @@ -24,9 +24,10 @@ import static org.testng.Assert.assertFalse; import static org.testng.Assert.assertThrows; import static org.testng.Assert.assertTrue; +import java.lang.foreign.MemorySegment; + import org.apache.datasketches.common.SketchesStateException; import org.apache.datasketches.common.TestUtil; -import org.apache.datasketches.memory.Memory; import org.testng.annotations.Test; public class TDigestDoubleTest { @@ -66,7 +67,9 @@ public class TDigestDoubleTest { public void manyValues() { final TDigestDouble td = new TDigestDouble(); final int n = 10000; - for (int i = 0; i < n; i++) td.update(i); + for (int i = 0; i < n; i++) { + td.update(i); + } // System.out.println(td.toString(true)); // td.compress(); // System.out.println(td.toString(true)); @@ -77,10 +80,10 @@ public class TDigestDoubleTest { assertEquals(td.getRank(0), 0, 0.0001); assertEquals(td.getRank(n / 4), 0.25, 0.0001); assertEquals(td.getRank(n / 2), 0.5, 0.0001); - assertEquals(td.getRank(n * 3 / 4), 0.75, 0.0001); + assertEquals(td.getRank((n * 3) / 4), 0.75, 0.0001); assertEquals(td.getRank(n), 1); assertEquals(td.getQuantile(0), 0); - assertEquals(td.getQuantile(0.5), n / 2, n / 2 * 0.03); + assertEquals(td.getQuantile(0.5), n / 2, (n / 2) * 0.03); assertEquals(td.getQuantile(0.9), n * 0.9, n * 0.9 * 0.01); assertEquals(td.getQuantile(0.95), n * 0.95, n * 0.95 * 0.01); assertEquals(td.getQuantile(1), n - 1); @@ -113,9 +116,9 @@ public class TDigestDoubleTest { final int n = 10000; final TDigestDouble td1 = new TDigestDouble(); final TDigestDouble td2 = new TDigestDouble(); - for (int i = 0; i < n / 2; i++) { + for (int i = 0; i < (n / 2); i++) { td1.update(i); - td2.update(n / 2 + i); + td2.update((n / 2) + i); } td1.merge(td2); assertEquals(td1.getTotalWeight(), n); @@ -128,7 +131,7 @@ public class TDigestDoubleTest { public void serializeDeserializeEmpty() { final TDigestDouble td1 = new TDigestDouble(); final byte[] bytes = td1.toByteArray(); - final TDigestDouble td2 = TDigestDouble.heapify(Memory.wrap(bytes)); + final TDigestDouble td2 = TDigestDouble.heapify(MemorySegment.ofArray(bytes)); assertEquals(td2.getK(), td1.getK()); assertEquals(td2.getTotalWeight(), td1.getTotalWeight()); assertEquals(td2.isEmpty(), td1.isEmpty()); @@ -137,9 +140,11 @@ public class TDigestDoubleTest { @Test public void serializeDeserializeNonEmpty() { final TDigestDouble td1 = new TDigestDouble(); - for (int i = 0; i < 10000; i++) td1.update(i); + for (int i = 0; i < 10000; i++) { + td1.update(i); + } final byte[] bytes = td1.toByteArray(); - final TDigestDouble td2 = TDigestDouble.heapify(Memory.wrap(bytes)); + final TDigestDouble td2 = TDigestDouble.heapify(MemorySegment.ofArray(bytes)); assertEquals(td2.getK(), td1.getK()); assertEquals(td2.getTotalWeight(), td1.getTotalWeight()); assertEquals(td2.isEmpty(), td1.isEmpty()); @@ -152,7 +157,7 @@ public class TDigestDoubleTest { @Test public void deserializeFromReferenceImplementationDouble() { final byte[] bytes = TestUtil.getResourceBytes("tdigest_ref_k100_n10000_double.sk"); - final TDigestDouble td = TDigestDouble.heapify(Memory.wrap(bytes)); + final TDigestDouble td = TDigestDouble.heapify(MemorySegment.ofArray(bytes)); final int n = 10000; assertEquals(td.getK(), 100); assertEquals(td.getTotalWeight(), n); @@ -161,14 +166,14 @@ public class TDigestDoubleTest { assertEquals(td.getRank(0), 0, 0.0001); assertEquals(td.getRank(n / 4), 0.25, 0.0001); assertEquals(td.getRank(n / 2), 0.5, 0.0001); - assertEquals(td.getRank(n * 3 / 4), 0.75, 0.0001); + assertEquals(td.getRank((n * 3) / 4), 0.75, 0.0001); assertEquals(td.getRank(n), 1); } @Test public void deserializeFromReferenceImplementationFloat() { final byte[] bytes = TestUtil.getResourceBytes("tdigest_ref_k100_n10000_float.sk"); - final TDigestDouble td = TDigestDouble.heapify(Memory.wrap(bytes)); + final TDigestDouble td = TDigestDouble.heapify(MemorySegment.ofArray(bytes)); final int n = 10000; assertEquals(td.getK(), 100); assertEquals(td.getTotalWeight(), n); @@ -177,7 +182,7 @@ public class TDigestDoubleTest { assertEquals(td.getRank(0), 0, 0.0001); assertEquals(td.getRank(n / 4), 0.25, 0.0001); assertEquals(td.getRank(n / 2), 0.5, 0.0001); - assertEquals(td.getRank(n * 3 / 4), 0.75, 0.0001); + assertEquals(td.getRank((n * 3) / 4), 0.75, 0.0001); assertEquals(td.getRank(n), 1); } } --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
