Jackie-Jiang commented on code in PR #12945:
URL: https://github.com/apache/pinot/pull/12945#discussion_r1583694418
##########
pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/index/ForwardIndexConfig.java:
##########
@@ -31,30 +31,46 @@
import org.apache.pinot.spi.config.table.FieldConfig;
import org.apache.pinot.spi.config.table.FieldConfig.CompressionCodec;
import org.apache.pinot.spi.config.table.IndexConfig;
+import org.apache.pinot.spi.utils.DataSizeUtils;
public class ForwardIndexConfig extends IndexConfig {
public static final int DEFAULT_RAW_WRITER_VERSION = 2;
- public static final ForwardIndexConfig DISABLED = new
ForwardIndexConfig(true, null, null, null, null, null);
+ public static final int DEFAULT_TARGET_MAX_CHUNK_SIZE = 1024 * 1024; // 1MB
+ public static final int DEFAULT_TARGET_DOCS_PER_CHUNK = 1000;
+ public static final ForwardIndexConfig DISABLED =
+ new ForwardIndexConfig(true, null, null, null, null, null, null, null);
public static final ForwardIndexConfig DEFAULT = new Builder().build();
@Nullable
private final CompressionCodec _compressionCodec;
private final boolean _deriveNumDocsPerChunk;
private final int _rawIndexWriterVersion;
+ private final String _targetMaxChunkSize;
+ private final int _targetDocsPerChunk;
@Nullable
private final ChunkCompressionType _chunkCompressionType;
@Nullable
private final DictIdCompressionType _dictIdCompressionType;
public ForwardIndexConfig(@Nullable Boolean disabled, @Nullable
CompressionCodec compressionCodec,
- @Nullable Boolean deriveNumDocsPerChunk, @Nullable Integer
rawIndexWriterVersion) {
+ @Nullable Boolean deriveNumDocsPerChunk, @Nullable Integer
rawIndexWriterVersion,
+ @Nullable String targetMaxChunkSize, @Nullable Integer
targetDocsPerChunk) {
super(disabled);
_deriveNumDocsPerChunk = Boolean.TRUE.equals(deriveNumDocsPerChunk);
_rawIndexWriterVersion = rawIndexWriterVersion == null ?
DEFAULT_RAW_WRITER_VERSION : rawIndexWriterVersion;
_compressionCodec = compressionCodec;
+ if (targetMaxChunkSize != null && !(_deriveNumDocsPerChunk ||
_rawIndexWriterVersion == 4)) {
+ throw new IllegalStateException(
+ "targetMaxChunkSize should only be used when deriveNumDocsPerChunk
is true or rawIndexWriterVersion is 4");
+ }
+ _targetMaxChunkSize =
+ targetMaxChunkSize == null ?
DataSizeUtils.fromBytes(DEFAULT_TARGET_MAX_CHUNK_SIZE) : targetMaxChunkSize;
+ _targetDocsPerChunk =
Review Comment:
Suggest allowing negative value for this config to turn it off and only
honor the max chunk size.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
