This is an automated email from the ASF dual-hosted git repository.
capistrant pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/druid.git
The following commit(s) were added to refs/heads/master by this push:
new e6b0ce9a4c5 enhance the cluster compaction config view to allow
configuring mostFragmentedFirst more easily for operators (#19242)
e6b0ce9a4c5 is described below
commit e6b0ce9a4c579a63f258396c18acf801fd5dee73
Author: Lucas Capistrant <[email protected]>
AuthorDate: Wed Apr 1 15:05:35 2026 -0500
enhance the cluster compaction config view to allow configuring
mostFragmentedFirst more easily for operators (#19242)
---
.../compaction-dynamic-config-completions.ts | 46 ++++++++-
.../compaction-dynamic-config.tsx | 115 +++++++++++++++++++--
2 files changed, 151 insertions(+), 10 deletions(-)
diff --git
a/web-console/src/dialogs/compaction-dynamic-config-dialog/compaction-dynamic-config-completions.ts
b/web-console/src/dialogs/compaction-dynamic-config-dialog/compaction-dynamic-config-completions.ts
index 1fa4b7b2416..8019d9695d3 100644
---
a/web-console/src/dialogs/compaction-dynamic-config-dialog/compaction-dynamic-config-completions.ts
+++
b/web-console/src/dialogs/compaction-dynamic-config-dialog/compaction-dynamic-config-completions.ts
@@ -76,16 +76,49 @@ export const COMPACTION_DYNAMIC_CONFIG_COMPLETIONS:
JsonCompletionRule[] = [
{ value: '2147483647', documentation: 'Maximum value (default)' },
],
},
- // compactionPolicy object properties
+ // compactionPolicy object properties (newestSegmentFirst)
{
path: '$.compactionPolicy',
isObject: true,
+ condition: obj => !obj.type || obj.type === 'newestSegmentFirst',
completions: [
+ { value: 'type', documentation: 'Policy type' },
+ { value: 'priorityDatasource', documentation: 'Datasource to prioritize
for compaction' },
+ ],
+ },
+ // compactionPolicy object properties (mostFragmentedFirst)
+ {
+ path: '$.compactionPolicy',
+ isObject: true,
+ condition: obj => obj.type === 'mostFragmentedFirst',
+ completions: [
+ { value: 'type', documentation: 'Policy type' },
+ { value: 'priorityDatasource', documentation: 'Datasource to prioritize
for compaction' },
{
- value: 'type',
- documentation: 'Policy type (currently only newestSegmentFirst is
supported)',
+ value: 'minUncompactedCount',
+ documentation:
+ 'Minimum number of uncompacted segments in an interval to be
eligible for compaction (default: 100)',
+ },
+ {
+ value: 'minUncompactedBytes',
+ documentation:
+ 'Minimum total bytes of uncompacted segments in an interval
(default: 10 MiB)',
+ },
+ {
+ value: 'maxAverageUncompactedBytesPerSegment',
+ documentation:
+ 'Maximum average size of uncompacted segments eligible for
compaction (default: 2 GiB)',
+ },
+ {
+ value: 'minUncompactedBytesPercentForFullCompaction',
+ documentation:
+ 'Threshold percentage of uncompacted bytes below which minor
compaction is used (default: 0)',
+ },
+ {
+ value: 'minUncompactedRowsPercentForFullCompaction',
+ documentation:
+ 'Threshold percentage of uncompacted rows below which minor
compaction is used (default: 0)',
},
- { value: 'priorityDatasource', documentation: 'Datasource to prioritize
for compaction' },
],
},
// compactionPolicy.type values
@@ -96,6 +129,11 @@ export const COMPACTION_DYNAMIC_CONFIG_COMPLETIONS:
JsonCompletionRule[] = [
value: 'newestSegmentFirst',
documentation: 'Prioritizes segments with more recent intervals for
compaction',
},
+ {
+ value: 'mostFragmentedFirst',
+ documentation:
+ 'Experimental: Prioritizes intervals with the largest number of
small uncompacted segments',
+ },
],
},
// useSupervisors values
diff --git
a/web-console/src/druid-models/compaction-dynamic-config/compaction-dynamic-config.tsx
b/web-console/src/druid-models/compaction-dynamic-config/compaction-dynamic-config.tsx
index 7d3c766c8aa..061ece15246 100644
---
a/web-console/src/druid-models/compaction-dynamic-config/compaction-dynamic-config.tsx
+++
b/web-console/src/druid-models/compaction-dynamic-config/compaction-dynamic-config.tsx
@@ -19,12 +19,24 @@
import { Code } from '@blueprintjs/core';
import type { Field } from '../../components';
-import { deepGet } from '../../utils';
+import { deepGet, deepSet } from '../../utils';
+
+export type CompactionPolicy =
+ | { type: 'newestSegmentFirst'; priorityDatasource?: string | null }
+ | {
+ type: 'mostFragmentedFirst';
+ priorityDatasource?: string | null;
+ minUncompactedCount?: number;
+ minUncompactedBytes?: number;
+ maxAverageUncompactedBytesPerSegment?: number;
+ minUncompactedBytesPercentForFullCompaction?: number;
+ minUncompactedRowsPercentForFullCompaction?: number;
+ };
export interface CompactionDynamicConfig {
compactionTaskSlotRatio: number;
maxCompactionTaskSlots: number;
- compactionPolicy: { type: 'newestSegmentFirst'; priorityDatasource?: string
| null };
+ compactionPolicy: CompactionPolicy;
useSupervisors: boolean;
engine: 'native' | 'msq';
storeCompactionStatePerSegment: boolean;
@@ -74,18 +86,36 @@ export const COMPACTION_DYNAMIC_CONFIG_FIELDS:
Field<CompactionDynamicConfig>[]
name: 'compactionPolicy.type',
label: 'Compaction search policy',
type: 'string',
- suggestions: ['newestSegmentFirst'],
+ suggestions: ['newestSegmentFirst', 'mostFragmentedFirst'],
info: (
<>
- Currently, the only supported policy is
<Code>newestSegmentFirst</Code>, which prioritizes
- segments with more recent intervals for compaction.
+ <p>
+ <Code>newestSegmentFirst</Code> prioritizes segments with more
recent intervals for
+ compaction.
+ </p>
+ <p>
+ <Code>mostFragmentedFirst</Code> (experimental) prioritizes
intervals with the largest
+ number of small uncompacted segments, favoring cluster stability
over query performance on
+ newer intervals.
+ </p>
</>
),
+ adjustment: config => {
+ const policyType = deepGet(config, 'compactionPolicy.type');
+ const priorityDatasource = deepGet(config,
'compactionPolicy.priorityDatasource');
+ const newPolicy: Record<string, any> = { type: policyType };
+ if (priorityDatasource) {
+ newPolicy.priorityDatasource = priorityDatasource;
+ }
+ return deepSet(config, 'compactionPolicy', newPolicy);
+ },
},
{
name: 'compactionPolicy.priorityDatasource',
type: 'string',
- defined: config => deepGet(config, 'compactionPolicy.type') ===
'newestSegmentFirst',
+ defined: config =>
+ deepGet(config, 'compactionPolicy.type') === 'newestSegmentFirst' ||
+ deepGet(config, 'compactionPolicy.type') === 'mostFragmentedFirst',
placeholder: '(none)',
info: (
<>
@@ -95,6 +125,79 @@ export const COMPACTION_DYNAMIC_CONFIG_FIELDS:
Field<CompactionDynamicConfig>[]
</>
),
},
+ {
+ name: 'compactionPolicy.minUncompactedCount',
+ label: 'Min uncompacted segment count',
+ type: 'number',
+ defaultValue: 100,
+ min: 1,
+ defined: config => deepGet(config, 'compactionPolicy.type') ===
'mostFragmentedFirst',
+ info: (
+ <>
+ Minimum number of uncompacted segments that must be present in an
interval to make it
+ eligible for compaction.
+ </>
+ ),
+ },
+ {
+ name: 'compactionPolicy.minUncompactedBytes',
+ label: 'Min uncompacted bytes',
+ type: 'size-bytes',
+ defaultValue: 10485760,
+ defined: config => deepGet(config, 'compactionPolicy.type') ===
'mostFragmentedFirst',
+ info: (
+ <>
+ Minimum total bytes of uncompacted segments that must be present in an
interval to make it
+ eligible for compaction. Default: 10 MiB.
+ </>
+ ),
+ },
+ {
+ name: 'compactionPolicy.maxAverageUncompactedBytesPerSegment',
+ label: 'Max avg uncompacted segment size',
+ type: 'size-bytes',
+ defaultValue: 2147483648,
+ defined: config => deepGet(config, 'compactionPolicy.type') ===
'mostFragmentedFirst',
+ info: (
+ <>
+ Maximum average size of uncompacted segments in an interval eligible
for compaction.
+ Intervals where the average uncompacted segment is larger than this
are skipped. Default: 2
+ GiB.
+ </>
+ ),
+ },
+ {
+ name: 'compactionPolicy.minUncompactedBytesPercentForFullCompaction',
+ label: 'Min uncompacted bytes % for full compaction',
+ type: 'number',
+ defaultValue: 0,
+ min: 0,
+ max: 99,
+ defined: config => deepGet(config, 'compactionPolicy.type') ===
'mostFragmentedFirst',
+ info: (
+ <>
+ Threshold percentage of uncompacted bytes to total bytes. When the
ratio of uncompacted
+ bytes falls below this threshold, minor compaction is used instead of
full compaction. Set
+ to 0 (default) to always use full compaction.
+ </>
+ ),
+ },
+ {
+ name: 'compactionPolicy.minUncompactedRowsPercentForFullCompaction',
+ label: 'Min uncompacted rows % for full compaction',
+ type: 'number',
+ defaultValue: 0,
+ min: 0,
+ max: 99,
+ defined: config => deepGet(config, 'compactionPolicy.type') ===
'mostFragmentedFirst',
+ info: (
+ <>
+ Threshold percentage of uncompacted rows to total rows. When the ratio
of uncompacted rows
+ falls below this threshold, minor compaction is used instead of full
compaction. Set to 0
+ (default) to always use full compaction.
+ </>
+ ),
+ },
{
name: 'storeCompactionStatePerSegment',
label: 'Legacy: Persist last compaction state in segments',
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
